From 7212ba5441ab61588e1abe0c7ee6468743461ea7 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Thu, 5 Jan 2023 17:54:07 -0800 Subject: [PATCH 01/43] work up the static file changes to support honoring pyproject.toml as our targeting mechanism. --- sdk/core/azure-common/pyproject.toml | 6 +++++ .../azure-core-experimental/pyproject.toml | 5 ++++ .../pyproject.toml | 5 ++++ .../pyproject.toml | 5 ++++ sdk/core/azure-core/pyproject.toml | 8 +++++++ sdk/core/azure-mgmt-core/pyproject.toml | 5 ++++ .../pyproject.toml | 6 +++++ .../ci_tools/environment_exclusions.py | 19 --------------- .../ci_tools/parsing/parse_functions.py | 24 ++++++++++++++++++- tools/azure-sdk-tools/setup.py | 3 ++- .../tests/test_parse_functionality.py | 16 ++++++++++++- 11 files changed, 80 insertions(+), 22 deletions(-) create mode 100644 sdk/core/azure-common/pyproject.toml create mode 100644 sdk/core/azure-core-experimental/pyproject.toml create mode 100644 sdk/core/azure-core-tracing-opencensus/pyproject.toml create mode 100644 sdk/core/azure-core-tracing-opentelemetry/pyproject.toml create mode 100644 sdk/core/azure-core/pyproject.toml create mode 100644 sdk/core/azure-mgmt-core/pyproject.toml create mode 100644 sdk/core/azure-servicemanagement-legacy/pyproject.toml diff --git a/sdk/core/azure-common/pyproject.toml b/sdk/core/azure-common/pyproject.toml new file mode 100644 index 000000000000..3c6a4b58d6af --- /dev/null +++ b/sdk/core/azure-common/pyproject.toml @@ -0,0 +1,6 @@ +[tool.azure-sdk-build] +type_check_samples = false +verifytypes = false +pyright = false +mypy = false +pylint = false \ No newline at end of file diff --git a/sdk/core/azure-core-experimental/pyproject.toml b/sdk/core/azure-core-experimental/pyproject.toml new file mode 100644 index 000000000000..4010cf355786 --- /dev/null +++ b/sdk/core/azure-core-experimental/pyproject.toml @@ -0,0 +1,5 @@ +[tool.azure-sdk-build] +type_check_samples = false +verifytypes = false +pyright = false +mypy = false \ No newline at end of file diff --git a/sdk/core/azure-core-tracing-opencensus/pyproject.toml b/sdk/core/azure-core-tracing-opencensus/pyproject.toml new file mode 100644 index 000000000000..4010cf355786 --- /dev/null +++ b/sdk/core/azure-core-tracing-opencensus/pyproject.toml @@ -0,0 +1,5 @@ +[tool.azure-sdk-build] +type_check_samples = false +verifytypes = false +pyright = false +mypy = false \ No newline at end of file diff --git a/sdk/core/azure-core-tracing-opentelemetry/pyproject.toml b/sdk/core/azure-core-tracing-opentelemetry/pyproject.toml new file mode 100644 index 000000000000..4010cf355786 --- /dev/null +++ b/sdk/core/azure-core-tracing-opentelemetry/pyproject.toml @@ -0,0 +1,5 @@ +[tool.azure-sdk-build] +type_check_samples = false +verifytypes = false +pyright = false +mypy = false \ No newline at end of file diff --git a/sdk/core/azure-core/pyproject.toml b/sdk/core/azure-core/pyproject.toml new file mode 100644 index 000000000000..c98953fca712 --- /dev/null +++ b/sdk/core/azure-core/pyproject.toml @@ -0,0 +1,8 @@ +[tool.azure-sdk-build] +type_check_samples = false +verifytypes = false +pyright = false +# For test environments or static checks where a check should be run by default, not explicitly disabling will enable the check. +# pylint is enabled by default, so there is no reason for a pylint = true in every pyproject.toml. +# +# For newly added checks that are not enabled by default, packages should opt IN by " = true". \ No newline at end of file diff --git a/sdk/core/azure-mgmt-core/pyproject.toml b/sdk/core/azure-mgmt-core/pyproject.toml new file mode 100644 index 000000000000..a9a644ab469a --- /dev/null +++ b/sdk/core/azure-mgmt-core/pyproject.toml @@ -0,0 +1,5 @@ +[tool.azure-sdk-build] +mypy = false +verifytypes = false +pyright = false +type_check_samples = false diff --git a/sdk/core/azure-servicemanagement-legacy/pyproject.toml b/sdk/core/azure-servicemanagement-legacy/pyproject.toml new file mode 100644 index 000000000000..f6a04e837927 --- /dev/null +++ b/sdk/core/azure-servicemanagement-legacy/pyproject.toml @@ -0,0 +1,6 @@ +[tool.azure-sdk-build] +mypy = false +type_check_samples = false +verifytypes = false +pyright = false +pylint = false \ No newline at end of file diff --git a/tools/azure-sdk-tools/ci_tools/environment_exclusions.py b/tools/azure-sdk-tools/ci_tools/environment_exclusions.py index ace017d65bf0..fcfd6b630942 100644 --- a/tools/azure-sdk-tools/ci_tools/environment_exclusions.py +++ b/tools/azure-sdk-tools/ci_tools/environment_exclusions.py @@ -83,9 +83,6 @@ "azure-confidentialledger", "azure-containerregistry", "azure-mgmt-core", - "azure-core-experimental", - "azure-core-tracing-opencensus", - "azure-core-tracing-opentelemetry", "azure-iot-deviceupdate", "azure-digitaltwins-core", "azure-eventhub-checkpointstoreblob", @@ -139,11 +136,6 @@ "azure-communication-sms", "azure-confidentialledger", "azure-containerregistry", - "azure-core", - "azure-mgmt-core", - "azure-core-experimental", - "azure-core-tracing-opencensus", - "azure-core-tracing-opentelemetry", "azure-cosmos", "azure-developer-devcenter", "azure-iot-deviceupdate", @@ -214,11 +206,6 @@ "azure-communication-sms", "azure-confidentialledger", "azure-containerregistry", - "azure-core", - "azure-mgmt-core", - "azure-core-experimental", - "azure-core-tracing-opencensus", - "azure-core-tracing-opentelemetry", "azure-cosmos", "azure-developer-devcenter", "azure-iot-deviceupdate", @@ -289,11 +276,6 @@ "azure-communication-sms", "azure-confidentialledger", "azure-containerregistry", - "azure-core", - "azure-mgmt-core", - "azure-core-experimental", - "azure-core-tracing-opencensus", - "azure-core-tracing-opentelemetry", "azure-cosmos", "azure-developer-devcenter", "azure-iot-deviceupdate", @@ -340,7 +322,6 @@ "azure-messaging-webpubsubservice", ] - # -------------------------------------------------------------------------------------------------------------------- # DO NOT add packages to the below lists. They are used to omit packages that will never run type checking. IGNORE_FILTER = ["nspkg", "mgmt", "cognitiveservices"] diff --git a/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py b/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py index 7ed9d36924e1..6fb34688f928 100644 --- a/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py +++ b/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py @@ -2,7 +2,15 @@ import ast import textwrap import re -from typing import Dict, List, Tuple + +try: + # py 311 adds this library natively + import tomllib as toml +except: + # otherwise fall back to pypi package tomli + import tomli as toml + +from typing import Dict, List, Tuple, Any # Assumes the presence of setuptools from pkg_resources import ( @@ -74,6 +82,20 @@ def from_path(cls, parse_directory_or_file: str): include_package_data, ) + def get_build_config(self) -> Dict[str, Any]: + toml_file = os.path.join(self.folder, "pyproject.toml") + + if os.path.exists(toml_file): + try: + with open(toml_file, "rb") as f: + toml_dict = toml.load(f) + if "tool" in toml_dict: + tool_configs = toml_dict["tool"] + if "azure-sdk-build" in tool_configs: + return tool_configs["azure-sdk-build"] + except: + return {} + def read_setup_py_content(setup_filename: str) -> str: """ diff --git a/tools/azure-sdk-tools/setup.py b/tools/azure-sdk-tools/setup.py index d8781864f91c..96863b58e3e6 100644 --- a/tools/azure-sdk-tools/setup.py +++ b/tools/azure-sdk-tools/setup.py @@ -20,7 +20,8 @@ "pyopenssl", "python-dotenv", "PyYAML", - "urllib3" + "urllib3", + "tomli" ] setup( diff --git a/tools/azure-sdk-tools/tests/test_parse_functionality.py b/tools/azure-sdk-tools/tests/test_parse_functionality.py index 28127df91a9a..7d3b0d0b8664 100644 --- a/tools/azure-sdk-tools/tests/test_parse_functionality.py +++ b/tools/azure-sdk-tools/tests/test_parse_functionality.py @@ -5,7 +5,7 @@ from unittest.mock import patch package_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) - +repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) def test_parse_require(): test_scenarios = [ @@ -41,6 +41,20 @@ def test_parse_require_with_no_spec(): assert result[1] is None +def test_toml_result(): + package_with_toml = os.path.join(repo_root, "sdk", "core", "azure-core") + + parsed_setup = ParsedSetup.from_path(package_with_toml) + result = parsed_setup.get_build_config() + + expected = { + "type_check_samples": False, + "verifytypes": False, + "pyright": False, + } + + assert(expected == result) + @patch("ci_tools.parsing.parse_functions.read_setup_py_content") def test_sdk_sample_setup(test_patch): test_patch.return_value = """ From 5dc694321a882878101914d3653cf1d9378bdd31 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 6 Jan 2023 11:40:32 -0800 Subject: [PATCH 02/43] add tomli to packaging deps. add pyproject.toml to spelling ignore list --- .vscode/cspell.json | 2 +- eng/tox/tox.ini | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/cspell.json b/.vscode/cspell.json index 7f2053802ebe..c603842d8e60 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -27,7 +27,7 @@ "sdk/**/_generated/**", "**/*requirement*.txt", "**/assets.json", - + "**/pyproject.toml", "sdk/anomalydetector/**", "sdk/applicationinsights/azure-applicationinsights/**", "sdk/batch/azure-batch/**", diff --git a/eng/tox/tox.ini b/eng/tox/tox.ini index 33961f949010..0250361ed96d 100644 --- a/eng/tox/tox.ini +++ b/eng/tox/tox.ini @@ -39,6 +39,7 @@ pkgs = wheel==0.37.0 packaging==20.4 urllib3==1.26.12 + tomli==2.0.1 [testenv] From 42d27519807b69c0e33bd496d4e486306061ef72 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 6 Jan 2023 15:11:44 -0800 Subject: [PATCH 03/43] begin the process of removing OMITTED_PACKAGES in favor of only honoring the inactive tag. simplify discover_targeted_packages to be more readable in preparation for refactor --- sdk/core/azure-common/pyproject.toml | 3 +- sdk/core/azure/pyproject.toml | 7 ++ sdk/cosmos/azure-mgmt-documentdb/setup.py | 2 +- sdk/regionmove/azure-mgmt-regionmove/setup.py | 2 +- sdk/scheduler/azure-mgmt-scheduler/setup.py | 2 +- tools/azure-sdk-tools/ci_tools/functions.py | 72 +++++++++---------- .../ci_tools/parsing/parse_functions.py | 27 ++++--- .../tests/test_parse_functionality.py | 3 + 8 files changed, 67 insertions(+), 51 deletions(-) create mode 100644 sdk/core/azure/pyproject.toml diff --git a/sdk/core/azure-common/pyproject.toml b/sdk/core/azure-common/pyproject.toml index 3c6a4b58d6af..00f07269246c 100644 --- a/sdk/core/azure-common/pyproject.toml +++ b/sdk/core/azure-common/pyproject.toml @@ -3,4 +3,5 @@ type_check_samples = false verifytypes = false pyright = false mypy = false -pylint = false \ No newline at end of file +pylint = false +regression = false \ No newline at end of file diff --git a/sdk/core/azure/pyproject.toml b/sdk/core/azure/pyproject.toml new file mode 100644 index 000000000000..00f07269246c --- /dev/null +++ b/sdk/core/azure/pyproject.toml @@ -0,0 +1,7 @@ +[tool.azure-sdk-build] +type_check_samples = false +verifytypes = false +pyright = false +mypy = false +pylint = false +regression = false \ No newline at end of file diff --git a/sdk/cosmos/azure-mgmt-documentdb/setup.py b/sdk/cosmos/azure-mgmt-documentdb/setup.py index 6921e0e1eb11..e94bb9de1106 100644 --- a/sdk/cosmos/azure-mgmt-documentdb/setup.py +++ b/sdk/cosmos/azure-mgmt-documentdb/setup.py @@ -51,7 +51,7 @@ author_email='ptvshelp@microsoft.com', url='https://github.com/Azure/azure-sdk-for-python', classifiers=[ - 'Development Status :: 4 - Beta', + 'Development Status :: 7 - Inactive', 'Programming Language :: Python', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: 3', diff --git a/sdk/regionmove/azure-mgmt-regionmove/setup.py b/sdk/regionmove/azure-mgmt-regionmove/setup.py index c80db06cec50..d63fb5669455 100644 --- a/sdk/regionmove/azure-mgmt-regionmove/setup.py +++ b/sdk/regionmove/azure-mgmt-regionmove/setup.py @@ -47,7 +47,7 @@ url='https://github.com/Azure/azure-sdk-for-python', keywords="azure, azure sdk", # update with search keywords relevant to the azure service / product classifiers=[ - 'Development Status :: 4 - Beta', + 'Development Status :: 7 - Inactive', 'Programming Language :: Python', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: 3', diff --git a/sdk/scheduler/azure-mgmt-scheduler/setup.py b/sdk/scheduler/azure-mgmt-scheduler/setup.py index 4b0e8374318e..5913934e2c36 100644 --- a/sdk/scheduler/azure-mgmt-scheduler/setup.py +++ b/sdk/scheduler/azure-mgmt-scheduler/setup.py @@ -47,7 +47,7 @@ url='https://github.com/Azure/azure-sdk-for-python', keywords="azure, azure sdk", # update with search keywords relevant to the azure service / product classifiers=[ - 'Development Status :: 5 - Production/Stable', + 'Development Status :: 7 - Inactive', 'Programming Language :: Python', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: 3', diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index 6b76b4b68afb..991f80efb0a5 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -13,17 +13,6 @@ from typing import List import logging - -OMITTED_CI_PACKAGES = [ - "azure-mgmt-documentdb", - "azure-servicemanagement-legacy", - "azure-mgmt-scheduler", - "azure", - "azure-mgmt", - "azure-storage", - "azure-monitor", - "azure-mgmt-regionmove", -] MANAGEMENT_PACKAGE_IDENTIFIERS = [ "mgmt", "azure-cognitiveservices", @@ -33,7 +22,9 @@ "azure-synapse", "azure-ai-anomalydetector", ] + META_PACKAGES = ["azure", "azure-mgmt", "azure-keyvault"] + REGRESSION_EXCLUDED_PACKAGES = [ "azure-common", ] @@ -113,6 +104,22 @@ def str_to_bool(input_string: str) -> bool: return False +def glob_packages(glob_str: str, target_root_dir: str): + if glob_string: + individual_globs = glob_string.split(",") + else: + individual_globs = "azure-*" + collected_top_level_directories = [] + + for glob_string in individual_globs: + globbed = glob.glob(os.path.join(target_root_dir, glob_string, "setup.py")) + glob.glob( + os.path.join(target_root_dir, "sdk/*/", glob_string, "setup.py") + ) + collected_top_level_directories.extend([os.path.dirname(p) for p in globbed]) + + # deduplicate, in case we have double coverage from the glob strings. Example: "azure-mgmt-keyvault,azure-mgmt-*" + return list(set([collected_top_level_directories])) + def discover_targeted_packages( glob_string: str, target_root_dir: str, @@ -130,40 +137,27 @@ def discover_targeted_packages( :param str filter_type: One a string representing a filter function as a set of options. Options [ "Build", "Docs", "Regression", "Omit_management" ] Defaults to "Build". :param bool compatibility_filter: Enables or disables compatibility filtering of found packages. If the invoking python executable does not match a found package's specifiers, the package will be omitted. Defaults to True. """ - if glob_string: - individual_globs = glob_string.split(",") - else: - individual_globs = "azure-*" - collected_top_level_directories = [] + + # glob the starting package set + collected_packages = glob_packages(glob_string, target_root_dir) - for glob_string in individual_globs: - globbed = glob.glob(os.path.join(target_root_dir, glob_string, "setup.py")) + glob.glob( - os.path.join(target_root_dir, "sdk/*/", glob_string, "setup.py") - ) - collected_top_level_directories.extend([os.path.dirname(p) for p in globbed]) + # apply the additional contains filter + collected_packages = [pkg for pkg in collected_packages if additional_contains_filter in pkg] - # deduplicate, in case we have double coverage from the glob strings. Example: "azure-mgmt-keyvault,azure-mgmt-*" - collected_directories = list(set([p for p in collected_top_level_directories if additional_contains_filter in p])) - pkg_set_ci_filtered = collected_directories - - # if we have individually queued this specific package, it's obvious that we want to build it specifically - # in this case, do not honor the omission list - if len(collected_directories) == 1: - if compatibility_filter: - pkg_set_ci_filtered = filter_for_compatibility(collected_directories) + # filter for compatiblity + if compatibility_filter: + collected_packages = filter_for_compatibility(collected_packages) - # however, if there are multiple packages being built, we should honor the omission list and NOT build the omitted - # packages - else: - allowed_package_set = remove_omitted_packages(collected_directories) - if compatibility_filter: - pkg_set_ci_filtered = filter_for_compatibility(allowed_package_set) + # apply package-specific exclusions only if we have gotten more than one + # todo: remove this after updating the pyproject exclusion + if len(collected_packages) > 1: + collected_packages = remove_omitted_packages(collected_packages) # Apply filter based on filter type. for e.g. Docs, Regression, Management pkg_set_ci_filtered = list(filter(omit_function_dict.get(filter_type, omit_build), pkg_set_ci_filtered)) logging.info("Target packages after filtering by CI Type: {}".format(pkg_set_ci_filtered)) logging.info( - "Package(s) omitted by CI filter: {}".format(list(set(collected_directories) - set(pkg_set_ci_filtered))) + "Package(s) omitted by CI filter: {}".format(list(set(collected_packages) - set(pkg_set_ci_filtered))) ) return sorted(pkg_set_ci_filtered) @@ -217,7 +211,7 @@ def get_package_from_repo(pkg_name: str, repo_root: str = None) -> ParsedSetup: return None -def get_version_from_repo(pkg_name: str, repo_root: str = None): +def get_version_from_repo(pkg_name: str, repo_root: str = None) -> str: pkg_info = get_package_from_repo(pkg_name, repo_root) if pkg_info: # Remove dev build part if version for this package is already updated to dev build @@ -235,7 +229,7 @@ def get_version_from_repo(pkg_name: str, repo_root: str = None): exit(1) -def get_base_version(pkg_name): +def get_base_version(pkg_name: str) -> str: root_dir = discover_repo_root() # find version for the package from source. This logic should be revisited to find version from devops feed glob_path = os.path.join(root_dir, "sdk", "*", pkg_name, "setup.py") diff --git a/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py b/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py index 6fb34688f928..7f78a8e97be8 100644 --- a/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py +++ b/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py @@ -37,23 +37,26 @@ def __init__( self, name: str, version: str, - python_requires: List[str], + python_requires: str, requires: List[str], is_new_sdk: bool, setup_filename: str, name_space: str, - package_data: Dict, + package_data: Dict[str, Any], include_package_data: bool, + classifiers: List[str], ): self.name: str = name self.version: str = version - self.python_requires: List[str] = python_requires + self.python_requires: str = python_requires self.requires: List[str] = requires self.is_new_sdk: bool = is_new_sdk self.setup_filename: str = setup_filename - self.namespace = name_space - self.package_data = package_data - self.include_package_data = include_package_data + self.namespace: str = name_space + self.package_data: Dict[str, Any] = package_data + self.include_package_data: bool = include_package_data + self.classifiers: List[str] = classifiers + self.folder = os.path.dirname(self.setup_filename) @classmethod @@ -68,6 +71,7 @@ def from_path(cls, parse_directory_or_file: str): name_space, package_data, include_package_data, + classifiers, ) = parse_setup(parse_directory_or_file) return cls( @@ -80,6 +84,7 @@ def from_path(cls, parse_directory_or_file: str): name_space, package_data, include_package_data, + classifiers, ) def get_build_config(self) -> Dict[str, Any]: @@ -106,7 +111,7 @@ def read_setup_py_content(setup_filename: str) -> str: return content -def parse_setup(setup_filename: str) -> Tuple[str, str, List[str], List[str], bool, str]: +def parse_setup(setup_filename: str) -> Tuple[str, str, str, List[str], bool, str, str, Dict[str, Any], bool, List[str]]: """ Used to evaluate a setup.py (or a directory containing a setup.py) and return a tuple containing: ( @@ -118,7 +123,8 @@ def parse_setup(setup_filename: str) -> Tuple[str, str, List[str], List[str], bo , , , - + , + ) """ if not setup_filename.endswith("setup.py"): @@ -187,6 +193,10 @@ def setup(*args, **kwargs): if "include_package_data" in kwargs: include_package_data = kwargs["include_package_data"] + classifiers = [] + if "classifiers" in kwargs: + classifiers = kwargs["classifiers"] + is_new_sdk = name in NEW_REQ_PACKAGES or any(map(lambda x: (parse_require(x)[0] in NEW_REQ_PACKAGES), requires)) return ( @@ -199,6 +209,7 @@ def setup(*args, **kwargs): name_space, package_data, include_package_data, + classifiers ) diff --git a/tools/azure-sdk-tools/tests/test_parse_functionality.py b/tools/azure-sdk-tools/tests/test_parse_functionality.py index 7d3b0d0b8664..dbf0ddf3f152 100644 --- a/tools/azure-sdk-tools/tests/test_parse_functionality.py +++ b/tools/azure-sdk-tools/tests/test_parse_functionality.py @@ -128,3 +128,6 @@ def test_sdk_sample_setup(test_patch): assert "pytyped" in result.package_data assert result.include_package_data == True assert result.folder == package_root + assert len(result.classifiers) > 0 + assert result.classifiers[0] == "Development Status :: 5 - Production/Stable" + assert result.classifiers[5] == "Programming Language :: Python :: 3.8" \ No newline at end of file From 3f31011e5a4ca6a67df91b7eddabe08b19b579f3 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 6 Jan 2023 15:31:16 -0800 Subject: [PATCH 04/43] now honoring the Inactive classifier over the omission list --- tools/azure-sdk-tools/ci_tools/functions.py | 30 +++++++++++++-------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index 991f80efb0a5..032aa04eb8bc 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -13,6 +13,8 @@ from typing import List import logging +INACTIVE_CLASSIFIER = "Development Status :: 7 - Inactive" + MANAGEMENT_PACKAGE_IDENTIFIERS = [ "mgmt", "azure-cognitiveservices", @@ -57,7 +59,7 @@ } -def filter_for_compatibility(package_set: List[str]) -> List[str]: +def apply_compatibility_filter(package_set: List[str]) -> List[str]: """ This function takes in a set of paths to python packages. It returns the set filtered by compatibility with the currently running python executable. If a package is unsupported by the executable, it will be omitted from the returned list. @@ -104,7 +106,7 @@ def str_to_bool(input_string: str) -> bool: return False -def glob_packages(glob_str: str, target_root_dir: str): +def glob_packages(glob_str: str, target_root_dir: str) -> List[str]: if glob_string: individual_globs = glob_string.split(",") else: @@ -120,6 +122,15 @@ def glob_packages(glob_str: str, target_root_dir: str): # deduplicate, in case we have double coverage from the glob strings. Example: "azure-mgmt-keyvault,azure-mgmt-*" return list(set([collected_top_level_directories])) +def apply_business_filter(collected_packages: List[str], filter_type: str) -> List[str]: + pkg_set_ci_filtered = list(filter(omit_function_dict.get(filter_type, omit_build), pkg_set_ci_filtered)) + + logging.info("Target packages after filtering by CI Type: {}".format(pkg_set_ci_filtered)) + logging.info( + "Package(s) omitted by CI filter: {}".format(list(set(collected_packages) - set(pkg_set_ci_filtered))) + ) + return pkg_set_ci_filtered + def discover_targeted_packages( glob_string: str, target_root_dir: str, @@ -144,9 +155,9 @@ def discover_targeted_packages( # apply the additional contains filter collected_packages = [pkg for pkg in collected_packages if additional_contains_filter in pkg] - # filter for compatiblity + # filter for compatiblity, this means excluding a package that doesn't support py36 when we are running a py36 executable if compatibility_filter: - collected_packages = filter_for_compatibility(collected_packages) + collected_packages = apply_compatibility_filter(collected_packages) # apply package-specific exclusions only if we have gotten more than one # todo: remove this after updating the pyproject exclusion @@ -154,17 +165,14 @@ def discover_targeted_packages( collected_packages = remove_omitted_packages(collected_packages) # Apply filter based on filter type. for e.g. Docs, Regression, Management - pkg_set_ci_filtered = list(filter(omit_function_dict.get(filter_type, omit_build), pkg_set_ci_filtered)) - logging.info("Target packages after filtering by CI Type: {}".format(pkg_set_ci_filtered)) - logging.info( - "Package(s) omitted by CI filter: {}".format(list(set(collected_packages) - set(pkg_set_ci_filtered))) - ) - return sorted(pkg_set_ci_filtered) + collected_packages = apply_business_filter(collected_packages, filter_type) + + return sorted(collected_packages) def remove_omitted_packages(collected_directories): packages = [ - package_dir for package_dir in collected_directories if os.path.basename(package_dir) not in OMITTED_CI_PACKAGES + pkg for pkg in packages if INACTIVE_CLASSIFIER not in ParsedSetup.from_path(pkg).classifiers ] return packages From f4248f914c22608c49e4dc61f884f31f22391e5e Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 6 Jan 2023 16:20:50 -0800 Subject: [PATCH 05/43] apply black --- tools/azure-sdk-tools/ci_tools/functions.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index 032aa04eb8bc..d485ce417349 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -122,15 +122,15 @@ def glob_packages(glob_str: str, target_root_dir: str) -> List[str]: # deduplicate, in case we have double coverage from the glob strings. Example: "azure-mgmt-keyvault,azure-mgmt-*" return list(set([collected_top_level_directories])) + def apply_business_filter(collected_packages: List[str], filter_type: str) -> List[str]: pkg_set_ci_filtered = list(filter(omit_function_dict.get(filter_type, omit_build), pkg_set_ci_filtered)) logging.info("Target packages after filtering by CI Type: {}".format(pkg_set_ci_filtered)) - logging.info( - "Package(s) omitted by CI filter: {}".format(list(set(collected_packages) - set(pkg_set_ci_filtered))) - ) + logging.info("Package(s) omitted by CI filter: {}".format(list(set(collected_packages) - set(pkg_set_ci_filtered)))) return pkg_set_ci_filtered + def discover_targeted_packages( glob_string: str, target_root_dir: str, @@ -148,7 +148,7 @@ def discover_targeted_packages( :param str filter_type: One a string representing a filter function as a set of options. Options [ "Build", "Docs", "Regression", "Omit_management" ] Defaults to "Build". :param bool compatibility_filter: Enables or disables compatibility filtering of found packages. If the invoking python executable does not match a found package's specifiers, the package will be omitted. Defaults to True. """ - + # glob the starting package set collected_packages = glob_packages(glob_string, target_root_dir) @@ -162,7 +162,7 @@ def discover_targeted_packages( # apply package-specific exclusions only if we have gotten more than one # todo: remove this after updating the pyproject exclusion if len(collected_packages) > 1: - collected_packages = remove_omitted_packages(collected_packages) + collected_packages = apply_inactive_filter(collected_packages) # Apply filter based on filter type. for e.g. Docs, Regression, Management collected_packages = apply_business_filter(collected_packages, filter_type) @@ -170,10 +170,8 @@ def discover_targeted_packages( return sorted(collected_packages) -def remove_omitted_packages(collected_directories): - packages = [ - pkg for pkg in packages if INACTIVE_CLASSIFIER not in ParsedSetup.from_path(pkg).classifiers - ] +def apply_inactive_filter(collected_directories: List[str]) -> List[str]: + packages = [pkg for pkg in packages if INACTIVE_CLASSIFIER not in ParsedSetup.from_path(pkg).classifiers] return packages From bb7fd3a14f95d933d513bcb1b1d1223dad00eb4d Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 6 Jan 2023 16:33:33 -0800 Subject: [PATCH 06/43] we now honor the inactive classifier --- scripts/devops_tasks/common_tasks.py | 15 +-------------- scripts/devops_tasks/test_regression.py | 5 ++--- tools/azure-sdk-tools/ci_tools/functions.py | 15 +++++++++++++-- 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/scripts/devops_tasks/common_tasks.py b/scripts/devops_tasks/common_tasks.py index 2becb325a420..9d0c82a56374 100644 --- a/scripts/devops_tasks/common_tasks.py +++ b/scripts/devops_tasks/common_tasks.py @@ -31,7 +31,7 @@ from packaging.version import Version from packaging.version import parse -from ci_tools.functions import MANAGEMENT_PACKAGE_IDENTIFIERS, lambda_filter_azure_pkg +from ci_tools.functions import MANAGEMENT_PACKAGE_IDENTIFIERS, lambda_filter_azure_pkg, str_to_bool from ci_tools.parsing import parse_require, ParsedSetup DEV_REQ_FILE = "dev_requirements.txt" @@ -41,8 +41,6 @@ logging.getLogger().setLevel(logging.INFO) - - def log_file(file_location, is_error=False): with open(file_location, "r") as file: for line in file: @@ -81,17 +79,6 @@ def clean_coverage(coverage_dir): raise -def str_to_bool(input_string): - if isinstance(input_string, bool): - return input_string - elif input_string.lower() in ("true", "t", "1"): - return True - elif input_string.lower() in ("false", "f", "0"): - return False - else: - return False - - def run_check_call( command_array, working_directory, diff --git a/scripts/devops_tasks/test_regression.py b/scripts/devops_tasks/test_regression.py index cddfd35afdbe..9d8847bb2ffc 100644 --- a/scripts/devops_tasks/test_regression.py +++ b/scripts/devops_tasks/test_regression.py @@ -23,8 +23,7 @@ find_whl, find_tools_packages, get_installed_packages, - extend_dev_requirements, - str_to_bool, + extend_dev_requirements ) from git_helper import ( @@ -34,7 +33,7 @@ clone_repo, ) -from ci_tools.functions import discover_targeted_packages +from ci_tools.functions import discover_targeted_packages, str_to_bool from ci_tools.parsing import ParsedSetup AZURE_GLOB_STRING = "azure*" diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index d485ce417349..e0bae2671a14 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -128,6 +128,7 @@ def apply_business_filter(collected_packages: List[str], filter_type: str) -> Li logging.info("Target packages after filtering by CI Type: {}".format(pkg_set_ci_filtered)) logging.info("Package(s) omitted by CI filter: {}".format(list(set(collected_packages) - set(pkg_set_ci_filtered)))) + return pkg_set_ci_filtered @@ -160,7 +161,6 @@ def discover_targeted_packages( collected_packages = apply_compatibility_filter(collected_packages) # apply package-specific exclusions only if we have gotten more than one - # todo: remove this after updating the pyproject exclusion if len(collected_packages) > 1: collected_packages = apply_inactive_filter(collected_packages) @@ -170,8 +170,19 @@ def discover_targeted_packages( return sorted(collected_packages) +def is_package_active(package_path: str): + disabled = INACTIVE_CLASSIFIER in ParsedSetup.from_path(package_path).classifiers + + override_value = os.getenv(f"ENABLE_{os.path.basename(package_path).upper()}", None) + + if override_value: + return str_to_bool(override_value) + else: + return disabled + + def apply_inactive_filter(collected_directories: List[str]) -> List[str]: - packages = [pkg for pkg in packages if INACTIVE_CLASSIFIER not in ParsedSetup.from_path(pkg).classifiers] + packages = [pkg for pkg in packages if is_package_active(pkg)] return packages From b19f4f49c2575662a939db7a4625662afacb79b1 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 6 Jan 2023 17:22:43 -0800 Subject: [PATCH 07/43] preparing to test --- sdk/core/azure-mgmt/pyproject.toml | 8 ++++ sdk/core/azure/pyproject.toml | 3 +- tools/azure-sdk-tools/ci_tools/functions.py | 5 ++- .../integration/test_package_discovery.py | 45 +++++++++++++++++++ .../tests/test_parse_functionality.py | 14 ------ .../azure-sdk-tools/tests/test_pypi_client.py | 4 -- 6 files changed, 59 insertions(+), 20 deletions(-) create mode 100644 sdk/core/azure-mgmt/pyproject.toml create mode 100644 tools/azure-sdk-tools/tests/integration/test_package_discovery.py diff --git a/sdk/core/azure-mgmt/pyproject.toml b/sdk/core/azure-mgmt/pyproject.toml new file mode 100644 index 000000000000..a7ed650ffeeb --- /dev/null +++ b/sdk/core/azure-mgmt/pyproject.toml @@ -0,0 +1,8 @@ +[tool.azure-sdk-build] +type_check_samples = false +verifytypes = false +pyright = false +mypy = false +pylint = false +regression = false +sphinx = false \ No newline at end of file diff --git a/sdk/core/azure/pyproject.toml b/sdk/core/azure/pyproject.toml index 00f07269246c..a7ed650ffeeb 100644 --- a/sdk/core/azure/pyproject.toml +++ b/sdk/core/azure/pyproject.toml @@ -4,4 +4,5 @@ verifytypes = false pyright = false mypy = false pylint = false -regression = false \ No newline at end of file +regression = false +sphinx = false \ No newline at end of file diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index e0bae2671a14..ff507875a14d 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -17,9 +17,9 @@ MANAGEMENT_PACKAGE_IDENTIFIERS = [ "mgmt", + "nspkg", "azure-cognitiveservices", "azure-servicefabric", - "nspkg", "azure-keyvault", "azure-synapse", "azure-ai-anomalydetector", @@ -44,6 +44,7 @@ and os.path.basename(x) not in META_PACKAGES and os.path.basename(x) not in REGRESSION_EXCLUDED_PACKAGES ) + omit_docs = lambda x: "nspkg" not in x and os.path.basename(x) not in META_PACKAGES omit_build = lambda x: x # Dummy lambda to match omit type lambda_filter_azure_pkg = lambda x: x.startswith("azure") and "-nspkg" not in x @@ -169,6 +170,8 @@ def discover_targeted_packages( return sorted(collected_packages) +def is_package_check_enabled(package_path: str, check: str) -> bool: + pass # todo def is_package_active(package_path: str): disabled = INACTIVE_CLASSIFIER in ParsedSetup.from_path(package_path).classifiers diff --git a/tools/azure-sdk-tools/tests/integration/test_package_discovery.py b/tools/azure-sdk-tools/tests/integration/test_package_discovery.py new file mode 100644 index 000000000000..4b175abfb087 --- /dev/null +++ b/tools/azure-sdk-tools/tests/integration/test_package_discovery.py @@ -0,0 +1,45 @@ +import os + +from ci_tools.parsing import ParsedSetup +from ci_tools.functions import discover_targeted_packages + + +repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..")) +core_service_root = os.path.join(repo_root, "sdk", "core") + +def test_toml_result(): + package_with_toml = os.path.join(core_service_root, "azure-core") + + parsed_setup = ParsedSetup.from_path(package_with_toml) + result = parsed_setup.get_build_config() + + expected = { + "type_check_samples": False, + "verifytypes": False, + "pyright": False, + } + + assert(expected == result) + +def test_discovery(): + pass + +def test_discovery_omit_mgmt(): + pass + +def test_discovery_omit_build(): + pass + +def test_discovery_omit_regression(): + pass + +def test_discovery_honors_contains_filter(): + pass + +def test_discovery_honors_override(): + os.environ["ENABLE_AZURE-COMMON"] = "true" + os.environ["ENABLE_AZURE-SERVICEMANAGEMENT-LEGACY"] = "false" + + results = discover_targeted_packages("azure*", repo_root) + + diff --git a/tools/azure-sdk-tools/tests/test_parse_functionality.py b/tools/azure-sdk-tools/tests/test_parse_functionality.py index dbf0ddf3f152..78f53ecc30c5 100644 --- a/tools/azure-sdk-tools/tests/test_parse_functionality.py +++ b/tools/azure-sdk-tools/tests/test_parse_functionality.py @@ -41,20 +41,6 @@ def test_parse_require_with_no_spec(): assert result[1] is None -def test_toml_result(): - package_with_toml = os.path.join(repo_root, "sdk", "core", "azure-core") - - parsed_setup = ParsedSetup.from_path(package_with_toml) - result = parsed_setup.get_build_config() - - expected = { - "type_check_samples": False, - "verifytypes": False, - "pyright": False, - } - - assert(expected == result) - @patch("ci_tools.parsing.parse_functions.read_setup_py_content") def test_sdk_sample_setup(test_patch): test_patch.return_value = """ diff --git a/tools/azure-sdk-tools/tests/test_pypi_client.py b/tools/azure-sdk-tools/tests/test_pypi_client.py index cde13d5b1eb0..cb2de3dfa154 100644 --- a/tools/azure-sdk-tools/tests/test_pypi_client.py +++ b/tools/azure-sdk-tools/tests/test_pypi_client.py @@ -31,10 +31,6 @@ def test_package_version_retrieve(self): assert result["info"]["name"] == "azure-core" assert result["info"]["release_url"] == "https://pypi.org/project/azure-core/1.8.0/" - @pytest.mark.skipif( - os.environ.get("TF_BUILD", "None") == True, - reason=f"This test isn't worth recording and could be flaky. Skipping in CI.", - ) @patch("pypi_tools.pypi.sys") def test_package_filter_for_compatibility(self, mock_sys): mock_sys.version_info = (2, 7, 0) From c0721c564af3118d1949fa618d68be68759511c8 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 6 Jan 2023 17:43:59 -0800 Subject: [PATCH 08/43] need to avoid the runtime error from the attempted parse --- tools/azure-sdk-tools/ci_tools/functions.py | 5 ++-- .../ci_tools/parsing/parse_functions.py | 29 +++++++++++-------- .../integration/test_package_discovery.py | 9 +++++- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index ff507875a14d..544d740db638 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -107,7 +107,7 @@ def str_to_bool(input_string: str) -> bool: return False -def glob_packages(glob_str: str, target_root_dir: str) -> List[str]: +def glob_packages(glob_string: str, target_root_dir: str) -> List[str]: if glob_string: individual_globs = glob_string.split(",") else: @@ -121,7 +121,7 @@ def glob_packages(glob_str: str, target_root_dir: str) -> List[str]: collected_top_level_directories.extend([os.path.dirname(p) for p in globbed]) # deduplicate, in case we have double coverage from the glob strings. Example: "azure-mgmt-keyvault,azure-mgmt-*" - return list(set([collected_top_level_directories])) + return list(set(collected_top_level_directories)) def apply_business_filter(collected_packages: List[str], filter_type: str) -> List[str]: @@ -173,6 +173,7 @@ def discover_targeted_packages( def is_package_check_enabled(package_path: str, check: str) -> bool: pass # todo + def is_package_active(package_path: str): disabled = INACTIVE_CLASSIFIER in ParsedSetup.from_path(package_path).classifiers diff --git a/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py b/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py index 7f78a8e97be8..98300774e5a2 100644 --- a/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py +++ b/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py @@ -88,20 +88,25 @@ def from_path(cls, parse_directory_or_file: str): ) def get_build_config(self) -> Dict[str, Any]: - toml_file = os.path.join(self.folder, "pyproject.toml") - - if os.path.exists(toml_file): - try: - with open(toml_file, "rb") as f: - toml_dict = toml.load(f) - if "tool" in toml_dict: - tool_configs = toml_dict["tool"] - if "azure-sdk-build" in tool_configs: - return tool_configs["azure-sdk-build"] - except: + return get_build_config(self.folder) + +def get_build_config(package: str) -> Dict[str, Any]: + if package.lower().endswith("setup.py"): + package = os.path.dirname(package) + + toml_file = os.path.join(package, "pyproject.toml") + + if os.path.exists(toml_file): + try: + with open(toml_file, "rb") as f: + toml_dict = toml.load(f) + if "tool" in toml_dict: + tool_configs = toml_dict["tool"] + if "azure-sdk-build" in tool_configs: + return tool_configs["azure-sdk-build"] + except: return {} - def read_setup_py_content(setup_filename: str) -> str: """ Get setup.py content, returns a string. diff --git a/tools/azure-sdk-tools/tests/integration/test_package_discovery.py b/tools/azure-sdk-tools/tests/integration/test_package_discovery.py index 4b175abfb087..357216568c45 100644 --- a/tools/azure-sdk-tools/tests/integration/test_package_discovery.py +++ b/tools/azure-sdk-tools/tests/integration/test_package_discovery.py @@ -22,7 +22,14 @@ def test_toml_result(): assert(expected == result) def test_discovery(): - pass + results = discover_targeted_packages("azure-core", core_service_root) + + # if in a set, this should be empty + non_empty_results = discover_targeted_packages("azure-servicemanagement-legacy", core_service_root) + + assert len(results) > 1 + assert len(non_empty_results) == 1 + def test_discovery_omit_mgmt(): pass From 83b9c5565c91992f6b3830027eb1adb909e8da2c Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 6 Jan 2023 17:51:04 -0800 Subject: [PATCH 09/43] find a workaround for the package parsing of inactive packages --- tools/azure-sdk-tools/ci_tools/functions.py | 13 +++++++++---- .../tests/integration/test_package_discovery.py | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index 544d740db638..996ad1fb0b36 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -74,7 +74,12 @@ def apply_compatibility_filter(package_set: List[str]) -> List[str]: running_major_version = Version(".".join([str(v[0]), str(v[1]), str(v[2])])) for pkg in package_set: - spec_set = SpecifierSet(ParsedSetup.from_path(pkg).python_requires) + try: + spec_set = SpecifierSet(ParsedSetup.from_path(pkg).python_requires) + except RuntimeError as e: + logging.error(f"Unable to parse metadata for package {pkg}, omitting from build.") + continue + pkg_specs_override = TEST_COMPATIBILITY_MAP.get(os.path.basename(pkg), None) if pkg_specs_override: @@ -125,7 +130,7 @@ def glob_packages(glob_string: str, target_root_dir: str) -> List[str]: def apply_business_filter(collected_packages: List[str], filter_type: str) -> List[str]: - pkg_set_ci_filtered = list(filter(omit_function_dict.get(filter_type, omit_build), pkg_set_ci_filtered)) + pkg_set_ci_filtered = list(filter(omit_function_dict.get(filter_type, omit_build), collected_packages)) logging.info("Target packages after filtering by CI Type: {}".format(pkg_set_ci_filtered)) logging.info("Package(s) omitted by CI filter: {}".format(list(set(collected_packages) - set(pkg_set_ci_filtered)))) @@ -185,8 +190,8 @@ def is_package_active(package_path: str): return disabled -def apply_inactive_filter(collected_directories: List[str]) -> List[str]: - packages = [pkg for pkg in packages if is_package_active(pkg)] +def apply_inactive_filter(collected_packages: List[str]) -> List[str]: + packages = [pkg for pkg in collected_packages if is_package_active(pkg)] return packages diff --git a/tools/azure-sdk-tools/tests/integration/test_package_discovery.py b/tools/azure-sdk-tools/tests/integration/test_package_discovery.py index 357216568c45..ea1cf4c15136 100644 --- a/tools/azure-sdk-tools/tests/integration/test_package_discovery.py +++ b/tools/azure-sdk-tools/tests/integration/test_package_discovery.py @@ -22,7 +22,7 @@ def test_toml_result(): assert(expected == result) def test_discovery(): - results = discover_targeted_packages("azure-core", core_service_root) + results = discover_targeted_packages("azure*", core_service_root) # if in a set, this should be empty non_empty_results = discover_targeted_packages("azure-servicemanagement-legacy", core_service_root) From 0becd05a74e4d54a984695a1fe727a3b890971ad Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 6 Jan 2023 18:25:49 -0800 Subject: [PATCH 10/43] tests now working --- tools/azure-sdk-tools/ci_tools/functions.py | 5 +- tools/azure-sdk-tools/dev_requirements.txt | 2 +- .../integration/test_package_discovery.py | 60 +++++++++++++------ .../tests/test_servicemetadata.py | 2 +- 4 files changed, 46 insertions(+), 23 deletions(-) diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index 996ad1fb0b36..ecb57ab5920e 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -175,9 +175,6 @@ def discover_targeted_packages( return sorted(collected_packages) -def is_package_check_enabled(package_path: str, check: str) -> bool: - pass # todo - def is_package_active(package_path: str): disabled = INACTIVE_CLASSIFIER in ParsedSetup.from_path(package_path).classifiers @@ -187,7 +184,7 @@ def is_package_active(package_path: str): if override_value: return str_to_bool(override_value) else: - return disabled + return not disabled def apply_inactive_filter(collected_packages: List[str]) -> List[str]: diff --git a/tools/azure-sdk-tools/dev_requirements.txt b/tools/azure-sdk-tools/dev_requirements.txt index 252afe5ae37a..fc7e3b122c21 100644 --- a/tools/azure-sdk-tools/dev_requirements.txt +++ b/tools/azure-sdk-tools/dev_requirements.txt @@ -1 +1 @@ --e ../azure-devtools \ No newline at end of file +-e ../azure-devtools["ci_tools"] \ No newline at end of file diff --git a/tools/azure-sdk-tools/tests/integration/test_package_discovery.py b/tools/azure-sdk-tools/tests/integration/test_package_discovery.py index ea1cf4c15136..508343247801 100644 --- a/tools/azure-sdk-tools/tests/integration/test_package_discovery.py +++ b/tools/azure-sdk-tools/tests/integration/test_package_discovery.py @@ -6,10 +6,12 @@ repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..")) core_service_root = os.path.join(repo_root, "sdk", "core") +storage_service_root = os.path.join(repo_root, "sdk", "storage") + def test_toml_result(): package_with_toml = os.path.join(core_service_root, "azure-core") - + parsed_setup = ParsedSetup.from_path(package_with_toml) result = parsed_setup.get_build_config() @@ -19,34 +21,58 @@ def test_toml_result(): "pyright": False, } - assert(expected == result) + assert expected == result + def test_discovery(): - results = discover_targeted_packages("azure*", core_service_root) - - # if in a set, this should be empty - non_empty_results = discover_targeted_packages("azure-servicemanagement-legacy", core_service_root) + results = discover_targeted_packages("azure*", core_service_root) - assert len(results) > 1 - assert len(non_empty_results) == 1 + # if in a set, this should be empty + non_empty_results = discover_targeted_packages("azure-servicemanagement-legacy", core_service_root) + + assert len(results) > 1 + assert len(non_empty_results) == 1 def test_discovery_omit_mgmt(): - pass + results = discover_targeted_packages("azure*", storage_service_root, filter_type="Omit_management") + + assert [os.path.basename(result) for result in results] == [ + "azure-storage-blob", + "azure-storage-blob-changefeed", + "azure-storage-file-datalake", + "azure-storage-file-share", + "azure-storage-queue" + ] def test_discovery_omit_build(): - pass + results = discover_targeted_packages("azure*", core_service_root, filter_type="Build") + + assert [os.path.basename(result) for result in results] == [ + "azure-core", + "azure-core-experimental", + "azure-core-tracing-opencensus", + "azure-core-tracing-opentelemetry", + "azure-mgmt-core", + ] + +def test_discovery_single_package(): + results = discover_targeted_packages("azure-servicemanagement-legacy", core_service_root, filter_type="Build") + + assert [os.path.basename(result) for result in results] == [ + "azure-servicemanagement-legacy", + ] def test_discovery_omit_regression(): - pass + pass -def test_discovery_honors_contains_filter(): - pass -def test_discovery_honors_override(): - os.environ["ENABLE_AZURE-COMMON"] = "true" - os.environ["ENABLE_AZURE-SERVICEMANAGEMENT-LEGACY"] = "false" +def test_discovery_honors_contains_filter(): + pass - results = discover_targeted_packages("azure*", repo_root) +def test_discovery_honors_override(): + os.environ["ENABLE_AZURE-COMMON"] = "true" + os.environ["ENABLE_AZURE-SERVICEMANAGEMENT-LEGACY"] = "false" + results = discover_targeted_packages("azure*", repo_root) diff --git a/tools/azure-sdk-tools/tests/test_servicemetadata.py b/tools/azure-sdk-tools/tests/test_servicemetadata.py index f8a701b11264..b23a52b652c4 100644 --- a/tools/azure-sdk-tools/tests/test_servicemetadata.py +++ b/tools/azure-sdk-tools/tests/test_servicemetadata.py @@ -5,7 +5,7 @@ from pathlib import Path import pytest -from packaging_tools.auto_codegen import update_servicemetadata +from packaging_tools.generate_utils import update_servicemetadata """ From ace22bac363793d7c026004c2d397ba901319e82 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 6 Jan 2023 19:31:24 -0800 Subject: [PATCH 11/43] cut over to using the check override for regression. pretty much done with functions.py. time to finish work in environment_exclusions.py --- sdk/core/azure-common/pyproject.toml | 3 +- sdk/core/azure-mgmt/pyproject.toml | 3 +- .../pyproject.toml | 3 +- sdk/core/azure/pyproject.toml | 3 +- tools/azure-sdk-tools/ci_tools/functions.py | 23 ++++++++++++-- .../ci_tools/parsing/__init__.py | 2 ++ .../ci_tools/parsing/parse_functions.py | 1 + .../integration/test_package_discovery.py | 30 +++++++++++++++++-- 8 files changed, 59 insertions(+), 9 deletions(-) diff --git a/sdk/core/azure-common/pyproject.toml b/sdk/core/azure-common/pyproject.toml index 00f07269246c..482c53ffc174 100644 --- a/sdk/core/azure-common/pyproject.toml +++ b/sdk/core/azure-common/pyproject.toml @@ -4,4 +4,5 @@ verifytypes = false pyright = false mypy = false pylint = false -regression = false \ No newline at end of file +regression = false +black = false \ No newline at end of file diff --git a/sdk/core/azure-mgmt/pyproject.toml b/sdk/core/azure-mgmt/pyproject.toml index a7ed650ffeeb..f09a5b522bde 100644 --- a/sdk/core/azure-mgmt/pyproject.toml +++ b/sdk/core/azure-mgmt/pyproject.toml @@ -5,4 +5,5 @@ pyright = false mypy = false pylint = false regression = false -sphinx = false \ No newline at end of file +sphinx = false +black = false \ No newline at end of file diff --git a/sdk/core/azure-servicemanagement-legacy/pyproject.toml b/sdk/core/azure-servicemanagement-legacy/pyproject.toml index f6a04e837927..9dcba289df80 100644 --- a/sdk/core/azure-servicemanagement-legacy/pyproject.toml +++ b/sdk/core/azure-servicemanagement-legacy/pyproject.toml @@ -3,4 +3,5 @@ mypy = false type_check_samples = false verifytypes = false pyright = false -pylint = false \ No newline at end of file +pylint = false +black = false \ No newline at end of file diff --git a/sdk/core/azure/pyproject.toml b/sdk/core/azure/pyproject.toml index a7ed650ffeeb..f09a5b522bde 100644 --- a/sdk/core/azure/pyproject.toml +++ b/sdk/core/azure/pyproject.toml @@ -5,4 +5,5 @@ pyright = false mypy = false pylint = false regression = false -sphinx = false \ No newline at end of file +sphinx = false +black = false \ No newline at end of file diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index ecb57ab5920e..a01f552c6573 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -6,11 +6,11 @@ from ci_tools.variables import discover_repo_root, get_artifact_directory, DEV_BUILD_IDENTIFIER import os, sys, platform, glob, re -from ci_tools.parsing import ParsedSetup +from ci_tools.parsing import ParsedSetup, get_build_config from pypi_tools.pypi import PyPIClient -from typing import List +from typing import List, Any import logging INACTIVE_CLASSIFIER = "Development Status :: 7 - Inactive" @@ -42,7 +42,8 @@ and "mgmt" not in x and os.path.basename(x) not in MANAGEMENT_PACKAGE_IDENTIFIERS and os.path.basename(x) not in META_PACKAGES - and os.path.basename(x) not in REGRESSION_EXCLUDED_PACKAGES + # and os.path.basename(x) not in REGRESSION_EXCLUDED_PACKAGES + and str_to_bool(get_config_setting(x, "regression", True)) == True ) omit_docs = lambda x: "nspkg" not in x and os.path.basename(x) not in META_PACKAGES @@ -176,6 +177,22 @@ def discover_targeted_packages( return sorted(collected_packages) +def get_config_setting(package_path: str, setting: str, default: Any = True) -> Any: + # we should always take the override if one is present + override_value = os.getenv(f"{os.path.basename(package_path).upper()}_{setting.upper()}", None) + if override_value: + return override_value + + # if no override, check for the config setting in the pyproject.toml + config = get_build_config(package_path) + + if config: + if setting.lower() in config: + return config[setting.lower()] + + return default + + def is_package_active(package_path: str): disabled = INACTIVE_CLASSIFIER in ParsedSetup.from_path(package_path).classifiers diff --git a/tools/azure-sdk-tools/ci_tools/parsing/__init__.py b/tools/azure-sdk-tools/ci_tools/parsing/__init__.py index 12b5f7641fac..4d48f7f720d4 100644 --- a/tools/azure-sdk-tools/ci_tools/parsing/__init__.py +++ b/tools/azure-sdk-tools/ci_tools/parsing/__init__.py @@ -5,6 +5,7 @@ get_name_from_specifier, ParsedSetup, read_setup_py_content, + get_build_config ) __all__ = [ @@ -14,4 +15,5 @@ "get_name_from_specifier", "ParsedSetup", "read_setup_py_content", + "get_build_config" ] diff --git a/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py b/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py index 98300774e5a2..7fddfd4a565d 100644 --- a/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py +++ b/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py @@ -107,6 +107,7 @@ def get_build_config(package: str) -> Dict[str, Any]: except: return {} + def read_setup_py_content(setup_filename: str) -> str: """ Get setup.py content, returns a string. diff --git a/tools/azure-sdk-tools/tests/integration/test_package_discovery.py b/tools/azure-sdk-tools/tests/integration/test_package_discovery.py index 508343247801..f13cc52fb094 100644 --- a/tools/azure-sdk-tools/tests/integration/test_package_discovery.py +++ b/tools/azure-sdk-tools/tests/integration/test_package_discovery.py @@ -64,7 +64,24 @@ def test_discovery_single_package(): ] def test_discovery_omit_regression(): - pass + results = discover_targeted_packages("azure*", core_service_root, filter_type="Regression") + + assert [os.path.basename(result) for result in results] == [ + "azure-core", + "azure-core-experimental", + "azure-core-tracing-opencensus", + "azure-core-tracing-opentelemetry" + ] + + storage_results = discover_targeted_packages("azure*", storage_service_root, filter_type="Regression") + + assert [os.path.basename(result) for result in storage_results] == [ + "azure-storage-blob", + "azure-storage-blob-changefeed", + "azure-storage-file-datalake", + "azure-storage-file-share", + "azure-storage-queue" + ] def test_discovery_honors_contains_filter(): @@ -75,4 +92,13 @@ def test_discovery_honors_override(): os.environ["ENABLE_AZURE-COMMON"] = "true" os.environ["ENABLE_AZURE-SERVICEMANAGEMENT-LEGACY"] = "false" - results = discover_targeted_packages("azure*", repo_root) + results = discover_targeted_packages("azure*", core_service_root) + + assert [os.path.basename(result) for result in results] == [ + "azure-common", + "azure-core", + "azure-core-experimental", + "azure-core-tracing-opencensus", + "azure-core-tracing-opentelemetry", + "azure-mgmt-core", + ] From 743fbad2eeddf07d0871cad3ee11b46e07caadc1 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 9 Jan 2023 14:11:57 -0800 Subject: [PATCH 12/43] we now filter the environments BEFORE invoking tox. this will significantly reduce workloads during CI runs --- eng/tox/run_mypy.py | 7 ++-- eng/tox/run_pylint.py | 4 +- eng/tox/run_pyright.py | 7 ++-- eng/tox/run_verifytypes.py | 4 +- scripts/devops_tasks/tox_harness.py | 2 +- .../ci_tools/environment_exclusions.py | 39 ++++++++++++------- 6 files changed, 37 insertions(+), 26 deletions(-) diff --git a/eng/tox/run_mypy.py b/eng/tox/run_mypy.py index e79a12c152b6..934782921fdf 100644 --- a/eng/tox/run_mypy.py +++ b/eng/tox/run_mypy.py @@ -15,8 +15,7 @@ from ci_tools.environment_exclusions import ( is_ignored_package, - MYPY_OPT_OUT, - TYPE_CHECK_SAMPLES_OPT_OUT, + is_check_enabled ) logging.getLogger().setLevel(logging.INFO) @@ -36,7 +35,7 @@ args = parser.parse_args() package_name = os.path.basename(os.path.abspath(args.target_package)) - if package_name in MYPY_OPT_OUT or is_ignored_package(package_name): + if is_check_enabled(args.target_package, "mypy", True) or is_ignored_package(package_name): logging.info( f"Package {package_name} opts-out of mypy check. See https://aka.ms/python/typing-guide for information." ) @@ -62,7 +61,7 @@ except CalledProcessError as src_err: src_code_error = src_err - if package_name in TYPE_CHECK_SAMPLES_OPT_OUT: + if is_check_enabled(args.target_package, "type_check_samples", True): logging.info( f"Package {package_name} opts-out of mypy check on samples." ) diff --git a/eng/tox/run_pylint.py b/eng/tox/run_pylint.py index 826766305b56..fc1cacdf4550 100644 --- a/eng/tox/run_pylint.py +++ b/eng/tox/run_pylint.py @@ -14,7 +14,7 @@ import logging import sys -from ci_tools.environment_exclusions import PYLINT_OPT_OUT +from ci_tools.environment_exclusions import is_check_enabled from ci_tools.parsing import ParsedSetup logging.getLogger().setLevel(logging.INFO) @@ -43,7 +43,7 @@ top_level_module = pkg_details.namespace.split('.')[0] - if pkg_details.name not in PYLINT_OPT_OUT: + if is_check_enabled(args.target_package, "pylint"): try: check_call( [ diff --git a/eng/tox/run_pyright.py b/eng/tox/run_pyright.py index 80d5acbf59fe..fad7955730e9 100644 --- a/eng/tox/run_pyright.py +++ b/eng/tox/run_pyright.py @@ -15,8 +15,7 @@ from ci_tools.environment_exclusions import ( is_ignored_package, - PYRIGHT_OPT_OUT, - TYPE_CHECK_SAMPLES_OPT_OUT, + is_check_enabled, ) logging.getLogger().setLevel(logging.INFO) @@ -36,7 +35,7 @@ args = parser.parse_args() package_name = os.path.basename(os.path.abspath(args.target_package)) - if package_name in PYRIGHT_OPT_OUT or is_ignored_package(package_name): + if is_check_enabled(args.target_package, "pyright") or is_ignored_package(package_name): logging.info( f"Package {package_name} opts-out of pyright check. See https://aka.ms/python/typing-guide for information." ) @@ -46,7 +45,7 @@ os.path.join(args.target_package, "azure"), os.path.join(args.target_package, "samples"), ] - if package_name in TYPE_CHECK_SAMPLES_OPT_OUT: + if is_check_enabled(args.target_package, "type_check_samples"): logging.info( f"Package {package_name} opts-out of pyright check on samples." ) diff --git a/eng/tox/run_verifytypes.py b/eng/tox/run_verifytypes.py index 3f11856975b3..195811b2c92f 100644 --- a/eng/tox/run_verifytypes.py +++ b/eng/tox/run_verifytypes.py @@ -16,7 +16,7 @@ import logging import sys -from ci_tools.environment_exclusions import is_ignored_package, VERIFYTYPES_OPT_OUT +from ci_tools.environment_exclusions import is_ignored_package, is_check_enabled logging.getLogger().setLevel(logging.INFO) @@ -98,7 +98,7 @@ def get_type_complete_score(commands, check_pytyped=False): module = package_name.replace("-", ".") setup_path = os.path.abspath(args.target_package) - if package_name in VERIFYTYPES_OPT_OUT or is_ignored_package(package_name): + if is_check_enabled(args.target_package, "type_check_samples") or is_ignored_package(package_name): logging.info( f"{package_name} opts-out of verifytypes check. See https://aka.ms/python/typing-guide for information." ) diff --git a/scripts/devops_tasks/tox_harness.py b/scripts/devops_tasks/tox_harness.py index d11700cc1778..006e0893fa17 100644 --- a/scripts/devops_tasks/tox_harness.py +++ b/scripts/devops_tasks/tox_harness.py @@ -323,7 +323,7 @@ def prep_and_run_tox(targeted_packages: List[str], parsed_args: Namespace, optio inject_custom_reqs(destination_dev_req, parsed_args.injected_packages, package_dir) if parsed_args.tox_env: - filtered_tox_environment_set = filter_tox_environment_string(parsed_args.tox_env, package_name) + filtered_tox_environment_set = filter_tox_environment_string(parsed_args.tox_env, package_dir) if not filtered_tox_environment_set: logging.info( diff --git a/tools/azure-sdk-tools/ci_tools/environment_exclusions.py b/tools/azure-sdk-tools/ci_tools/environment_exclusions.py index fcfd6b630942..197605db8c21 100644 --- a/tools/azure-sdk-tools/ci_tools/environment_exclusions.py +++ b/tools/azure-sdk-tools/ci_tools/environment_exclusions.py @@ -4,7 +4,8 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -import logging +from ci_tools.functions import get_config_setting +import os PYLINT_OPT_OUT = [ "azure-applicationinsights", @@ -345,7 +346,26 @@ ] -def filter_tox_environment_string(namespace_argument: str, package_name: str) -> str: +def is_check_enabled(package_path: str, check: str, default: bool = True) -> bool: + enabled = default + package_name = os.path.basename(package_path) + + # check the original exclusion list for the basename + exclusions_for_env = [] + try: + exclusions_for_env = globals()[f"{check.strip().upper()}_OPT_OUT"] + except Exception as e: + pass + if package_name in exclusions_for_env: + result = False + + # now pull the new pyproject.toml configuration + config = get_config_setting(package_path, check.strip().lower(), True) + + return (config and enabled) + + +def filter_tox_environment_string(namespace_argument: str, package_path: str) -> str: """ Takes an incoming comma separated list of tox environments and package name. Resolves whether or not each given tox environment should run, given comparison to single unified exclusion file in `environment_exclusions`. @@ -353,22 +373,15 @@ def filter_tox_environment_string(namespace_argument: str, package_name: str) -> :param namespace_argument: A namespace argument. :param package_name: The name of the package. This takes the form of a comma separated list: "whl,sdist,mindependency". "whl". "lint,pyright,sphinx". """ + package_name = os.path.basename(package_path) + if namespace_argument: tox_envs = namespace_argument.strip().split(",") filtered_set = [] for tox_env in tox_envs: - exclusions_for_env = [] - try: - exclusions_for_env = globals()[f"{tox_env.strip().upper()}_OPT_OUT"] - except Exception as e: - pass - - if exclusions_for_env: - if package_name in exclusions_for_env: - continue - - filtered_set.append(tox_env) + if is_check_enabled(package_path, tox_env, True): + filtered_set.append(tox_env) return ",".join(filtered_set) return namespace_argument From dae9d5337f1f4561eab7ed8557028abdc9b5a336 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 9 Jan 2023 14:16:59 -0800 Subject: [PATCH 13/43] ensure tests for package discovery are working --- .../tests/integration/test_package_discovery.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/azure-sdk-tools/tests/integration/test_package_discovery.py b/tools/azure-sdk-tools/tests/integration/test_package_discovery.py index f13cc52fb094..bf55520b71ef 100644 --- a/tools/azure-sdk-tools/tests/integration/test_package_discovery.py +++ b/tools/azure-sdk-tools/tests/integration/test_package_discovery.py @@ -85,7 +85,14 @@ def test_discovery_omit_regression(): def test_discovery_honors_contains_filter(): - pass + + storage_results = discover_targeted_packages("azure*", storage_service_root, "file", filter_type="Regression") + + assert [os.path.basename(result) for result in storage_results] == [ + "azure-storage-file-datalake", + "azure-storage-file-share", + ] + def test_discovery_honors_override(): From efe96a17f0919d70c70d0b4dd7dbf9cdf08670b2 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 9 Jan 2023 14:42:48 -0800 Subject: [PATCH 14/43] now we can enable/disable black on a per-package basis --- eng/pipelines/templates/steps/run_black.yml | 2 +- scripts/devops_tasks/validate_formatting.py | 43 +++++++++++++------ .../ci_tools/environment_exclusions.py | 2 +- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/eng/pipelines/templates/steps/run_black.yml b/eng/pipelines/templates/steps/run_black.yml index 91c322ba6c7b..03ed7e8e9880 100644 --- a/eng/pipelines/templates/steps/run_black.yml +++ b/eng/pipelines/templates/steps/run_black.yml @@ -11,7 +11,7 @@ steps: condition: succeededOrFailed() - script: | - pip install black==21.6b0 + pip install black==21.6b0 tools/azure-sdk-tools["build"] displayName: 'Prep Environment' condition: succeededOrFailed() diff --git a/scripts/devops_tasks/validate_formatting.py b/scripts/devops_tasks/validate_formatting.py index ffdf138afb59..35cd91acb904 100644 --- a/scripts/devops_tasks/validate_formatting.py +++ b/scripts/devops_tasks/validate_formatting.py @@ -12,29 +12,40 @@ import subprocess logging.getLogger().setLevel(logging.INFO) +from ci_tools.functions import discover_targeted_packages +from ci_tools.environment_exclusions import is_check_enabled root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..")) sdk_dir = os.path.join(root_dir, "sdk") def run_black(service_dir): + results = [] logging.info("Running black for {}".format(service_dir)) - out = subprocess.Popen([sys.executable, "-m", "black", "-l", "120", "sdk/{}".format(service_dir)], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - cwd = root_dir - ) + discover_targeted_packages("azure*", os.path.join(root_dir, "sdk", service_dir)) + + for package in discover_targeted_packages: + package_name = os.path.basename(package) + + if is_check_enabled(package, "black", True): + out = subprocess.Popen([sys.executable, "-m", "black", "-l", "120", "sdk/{}/{}".format(service_dir)], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd = root_dir + ) - stdout,stderr = out.communicate() + stdout,stderr = out.communicate() - if stderr: - raise RuntimeError("black ran into some trouble during its invocation: " + stderr) + if stderr: + results.append((package_name, stderr)) + raise RuntimeError("black ran into some trouble during its invocation: " + stderr) - if stdout: - if "reformatted" in stdout.decode('utf-8'): - return False + if stdout: + if "reformatted" in stdout.decode('utf-8'): + results.append((package_name, False)) + return False - return True + return results if __name__ == "__main__": @@ -51,7 +62,13 @@ def run_black(service_dir): args = parser.parse_args() if args.validate != "False": - if not run_black(args.service_directory): + results = run_black(args.service_directory) + + if len(results) > 0: + for result in results: + error = "Code needs reformat." if result[1] == False else error + logging.error(f"Black run for {result[0]} ran into an issue: {error}") + raise ValueError("Found difference between formatted code and current commit. Please re-generate with the latest autorest.") else: diff --git a/tools/azure-sdk-tools/ci_tools/environment_exclusions.py b/tools/azure-sdk-tools/ci_tools/environment_exclusions.py index 197605db8c21..82fe9f70cea9 100644 --- a/tools/azure-sdk-tools/ci_tools/environment_exclusions.py +++ b/tools/azure-sdk-tools/ci_tools/environment_exclusions.py @@ -350,7 +350,7 @@ def is_check_enabled(package_path: str, check: str, default: bool = True) -> boo enabled = default package_name = os.path.basename(package_path) - # check the original exclusion list for the basename + # check the original exclusion lists for the package name exclusions_for_env = [] try: exclusions_for_env = globals()[f"{check.strip().upper()}_OPT_OUT"] From 90bb0689aba34ab7fa43c45efc2d8f1377ba94a7 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 9 Jan 2023 14:54:15 -0800 Subject: [PATCH 15/43] swap the positive and negative cases --- eng/tox/run_mypy.py | 4 ++-- eng/tox/run_pyright.py | 4 ++-- eng/tox/run_verifytypes.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/tox/run_mypy.py b/eng/tox/run_mypy.py index 934782921fdf..78e4bdef9d8a 100644 --- a/eng/tox/run_mypy.py +++ b/eng/tox/run_mypy.py @@ -35,7 +35,7 @@ args = parser.parse_args() package_name = os.path.basename(os.path.abspath(args.target_package)) - if is_check_enabled(args.target_package, "mypy", True) or is_ignored_package(package_name): + if not is_check_enabled(args.target_package, "mypy", True) or is_ignored_package(package_name): logging.info( f"Package {package_name} opts-out of mypy check. See https://aka.ms/python/typing-guide for information." ) @@ -61,7 +61,7 @@ except CalledProcessError as src_err: src_code_error = src_err - if is_check_enabled(args.target_package, "type_check_samples", True): + if not is_check_enabled(args.target_package, "type_check_samples", True): logging.info( f"Package {package_name} opts-out of mypy check on samples." ) diff --git a/eng/tox/run_pyright.py b/eng/tox/run_pyright.py index fad7955730e9..02d3324075d2 100644 --- a/eng/tox/run_pyright.py +++ b/eng/tox/run_pyright.py @@ -35,7 +35,7 @@ args = parser.parse_args() package_name = os.path.basename(os.path.abspath(args.target_package)) - if is_check_enabled(args.target_package, "pyright") or is_ignored_package(package_name): + if not is_check_enabled(args.target_package, "pyright") or is_ignored_package(package_name): logging.info( f"Package {package_name} opts-out of pyright check. See https://aka.ms/python/typing-guide for information." ) @@ -45,7 +45,7 @@ os.path.join(args.target_package, "azure"), os.path.join(args.target_package, "samples"), ] - if is_check_enabled(args.target_package, "type_check_samples"): + if not is_check_enabled(args.target_package, "type_check_samples"): logging.info( f"Package {package_name} opts-out of pyright check on samples." ) diff --git a/eng/tox/run_verifytypes.py b/eng/tox/run_verifytypes.py index 195811b2c92f..ef2444b243e0 100644 --- a/eng/tox/run_verifytypes.py +++ b/eng/tox/run_verifytypes.py @@ -98,7 +98,7 @@ def get_type_complete_score(commands, check_pytyped=False): module = package_name.replace("-", ".") setup_path = os.path.abspath(args.target_package) - if is_check_enabled(args.target_package, "type_check_samples") or is_ignored_package(package_name): + if not is_check_enabled(args.target_package, "type_check_samples") or is_ignored_package(package_name): logging.info( f"{package_name} opts-out of verifytypes check. See https://aka.ms/python/typing-guide for information." ) From 545cdda8abb8c7f822c89fe5f1b131db6ff58cae Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 9 Jan 2023 15:15:00 -0800 Subject: [PATCH 16/43] enable packages to enable/disable black on a per-package basis --- scripts/devops_tasks/validate_formatting.py | 8 ++++---- .../tests/integration/test_package_discovery.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/devops_tasks/validate_formatting.py b/scripts/devops_tasks/validate_formatting.py index 35cd91acb904..78177b47af19 100644 --- a/scripts/devops_tasks/validate_formatting.py +++ b/scripts/devops_tasks/validate_formatting.py @@ -22,13 +22,13 @@ def run_black(service_dir): results = [] logging.info("Running black for {}".format(service_dir)) - discover_targeted_packages("azure*", os.path.join(root_dir, "sdk", service_dir)) + discovered_packages = discover_targeted_packages("azure*", os.path.join(root_dir, "sdk", service_dir)) - for package in discover_targeted_packages: + for package in discovered_packages: package_name = os.path.basename(package) - + if is_check_enabled(package, "black", True): - out = subprocess.Popen([sys.executable, "-m", "black", "-l", "120", "sdk/{}/{}".format(service_dir)], + out = subprocess.Popen([sys.executable, "-m", "black", "-l", "120", "sdk/{}/{}".format(service_dir, package_name)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd = root_dir diff --git a/tools/azure-sdk-tools/tests/integration/test_package_discovery.py b/tools/azure-sdk-tools/tests/integration/test_package_discovery.py index bf55520b71ef..1bbc7762033f 100644 --- a/tools/azure-sdk-tools/tests/integration/test_package_discovery.py +++ b/tools/azure-sdk-tools/tests/integration/test_package_discovery.py @@ -96,8 +96,8 @@ def test_discovery_honors_contains_filter(): def test_discovery_honors_override(): - os.environ["ENABLE_AZURE-COMMON"] = "true" - os.environ["ENABLE_AZURE-SERVICEMANAGEMENT-LEGACY"] = "false" + os.environ["ENABLE_AZURE_COMMON"] = "true" + os.environ["ENABLE_AZURE_SERVICEMANAGEMENT_LEGACY"] = "false" results = discover_targeted_packages("azure*", core_service_root) From 81e2c61d6b1add0c0d576cbe81673dbadb3e85d1 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 9 Jan 2023 16:44:46 -0800 Subject: [PATCH 17/43] enable black --- sdk/core/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 6603617297b0..76a666cf0ad8 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -35,6 +35,7 @@ extends: template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: ServiceDirectory: core + ValidateFormatting: true Artifacts: - name: azure-core safeName: azurecore From ad4516c014611acc24655d290aa1b007776bb370 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Mon, 9 Jan 2023 18:02:51 -0800 Subject: [PATCH 18/43] comment update --- scripts/devops_tasks/tox_harness.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/devops_tasks/tox_harness.py b/scripts/devops_tasks/tox_harness.py index 006e0893fa17..3ba93bb0be4c 100644 --- a/scripts/devops_tasks/tox_harness.py +++ b/scripts/devops_tasks/tox_harness.py @@ -322,13 +322,14 @@ def prep_and_run_tox(targeted_packages: List[str], parsed_args: Namespace, optio inject_custom_reqs(destination_dev_req, parsed_args.injected_packages, package_dir) + breakpoint() if parsed_args.tox_env: filtered_tox_environment_set = filter_tox_environment_string(parsed_args.tox_env, package_dir) if not filtered_tox_environment_set: logging.info( f"All requested tox environments for package {package_name} have been excluded by the environment exclusion list." - + " Check file /tools/azure-sdk-tools/ci_tools/environment_exclusions.py" + + f" Check file /tools/azure-sdk-tools/ci_tools/environment_exclusions.py and the pyproject.toml." ) continue From 853fb23b4985acd06c528128ca638e40e1b91ec8 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Tue, 10 Jan 2023 10:18:45 -0800 Subject: [PATCH 19/43] convert logging.info to logging.debug in functions.py --- tools/azure-sdk-tools/ci_tools/functions.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index a01f552c6573..13ed30741eca 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -89,6 +89,9 @@ def apply_compatibility_filter(package_set: List[str]) -> List[str]: if running_major_version in spec_set: collected_packages.append(pkg) + logging.debug("Target packages after applying compatibility filter: {}".format(collected_packages)) + logging.debug("Package(s) omitted by business filter: {}".format(generate_difference(package_set, collected_packages))) + return collected_packages @@ -112,6 +115,8 @@ def str_to_bool(input_string: str) -> bool: else: return False +def generate_difference(original_packages: List[str], filtered_packages: List[str]): + return list(set(original_packages) - set(filtered_packages)) def glob_packages(glob_string: str, target_root_dir: str) -> List[str]: if glob_string: @@ -133,9 +138,9 @@ def glob_packages(glob_string: str, target_root_dir: str) -> List[str]: def apply_business_filter(collected_packages: List[str], filter_type: str) -> List[str]: pkg_set_ci_filtered = list(filter(omit_function_dict.get(filter_type, omit_build), collected_packages)) - logging.info("Target packages after filtering by CI Type: {}".format(pkg_set_ci_filtered)) - logging.info("Package(s) omitted by CI filter: {}".format(list(set(collected_packages) - set(pkg_set_ci_filtered)))) - + logging.debug("Target packages after applying business filter: {}".format(pkg_set_ci_filtered)) + logging.debug("Package(s) omitted by business filter: {}".format(generate_difference(collected_packages, pkg_set_ci_filtered))) + return pkg_set_ci_filtered @@ -207,6 +212,9 @@ def is_package_active(package_path: str): def apply_inactive_filter(collected_packages: List[str]) -> List[str]: packages = [pkg for pkg in collected_packages if is_package_active(pkg)] + logging.debug("Target packages after applying compatibility filter: {}".format(collected_packages)) + logging.debug("Package(s) omitted by business filter: {}".format(generate_difference(collected_packages, packages))) + return packages From d8aa875e6f32c71802e6138b8ff9bbc059c083fc Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Tue, 10 Jan 2023 10:21:02 -0800 Subject: [PATCH 20/43] patch the validate_formatting script to output a bit of additional detail --- scripts/devops_tasks/validate_formatting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/devops_tasks/validate_formatting.py b/scripts/devops_tasks/validate_formatting.py index 78177b47af19..aca9f3475b83 100644 --- a/scripts/devops_tasks/validate_formatting.py +++ b/scripts/devops_tasks/validate_formatting.py @@ -38,12 +38,12 @@ def run_black(service_dir): if stderr: results.append((package_name, stderr)) - raise RuntimeError("black ran into some trouble during its invocation: " + stderr) if stdout: if "reformatted" in stdout.decode('utf-8'): results.append((package_name, False)) - return False + else: + print(f"black succeeded against {package_name}") return results From 4dc25b4f7af5dc16cf6002805a59ab2200ccffe1 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Tue, 10 Jan 2023 18:14:02 -0800 Subject: [PATCH 21/43] left a breakpoint in the code --- scripts/devops_tasks/tox_harness.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/devops_tasks/tox_harness.py b/scripts/devops_tasks/tox_harness.py index 3ba93bb0be4c..922af3b93228 100644 --- a/scripts/devops_tasks/tox_harness.py +++ b/scripts/devops_tasks/tox_harness.py @@ -322,7 +322,6 @@ def prep_and_run_tox(targeted_packages: List[str], parsed_args: Namespace, optio inject_custom_reqs(destination_dev_req, parsed_args.injected_packages, package_dir) - breakpoint() if parsed_args.tox_env: filtered_tox_environment_set = filter_tox_environment_string(parsed_args.tox_env, package_dir) From 39a96aa5d3163160595db9aa7acc4c773e3f7460 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Fri, 13 Jan 2023 13:21:52 -0800 Subject: [PATCH 22/43] remove broken link from changelog --- sdk/core/azure/CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/sdk/core/azure/CHANGELOG.md b/sdk/core/azure/CHANGELOG.md index e91b8608b75b..5a4f8efd63a0 100644 --- a/sdk/core/azure/CHANGELOG.md +++ b/sdk/core/azure/CHANGELOG.md @@ -199,8 +199,6 @@ Thank you to brandondahler, schaefi for their contributions. **Disclaimer** * There is some breaking changes in the Storage client, due to the update the latest API-Version. - There is a storage sample with the new version available here: - https://github.com/Azure-Samples/storage-python-manage * There is some breaking changes in the GraphRbac client, due to the update the latest API-Version. This is an example of change on our tests: https://github.com/Azure/azure-sdk-for-python/commit/b03cae526d9ac46d1b477840f15d3729aa0d939f#diff-296e794143f66af83d1bf2db6eb7a935 From 77e4036f822707a641226e07e5a53b631ce9e2da Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Tue, 17 Jan 2023 13:13:02 -0800 Subject: [PATCH 23/43] Update tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: McCoy Patiño <39780829+mccoyp@users.noreply.github.com> --- tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py b/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py index 7fddfd4a565d..fcb120b226f6 100644 --- a/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py +++ b/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py @@ -105,7 +105,7 @@ def get_build_config(package: str) -> Dict[str, Any]: if "azure-sdk-build" in tool_configs: return tool_configs["azure-sdk-build"] except: - return {} + return {} def read_setup_py_content(setup_filename: str) -> str: From d1d3eabc5befbb9f8b82cdefaec3975fe6242029 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Tue, 17 Jan 2023 13:13:19 -0800 Subject: [PATCH 24/43] Update tools/azure-sdk-tools/ci_tools/functions.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: McCoy Patiño <39780829+mccoyp@users.noreply.github.com> --- tools/azure-sdk-tools/ci_tools/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index 13ed30741eca..4ae634273595 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -43,7 +43,7 @@ and os.path.basename(x) not in MANAGEMENT_PACKAGE_IDENTIFIERS and os.path.basename(x) not in META_PACKAGES # and os.path.basename(x) not in REGRESSION_EXCLUDED_PACKAGES - and str_to_bool(get_config_setting(x, "regression", True)) == True + and str_to_bool(get_config_setting(x, "regression", True)) ) omit_docs = lambda x: "nspkg" not in x and os.path.basename(x) not in META_PACKAGES From 1601fe5c9fcf460a945861104b3f0ec602f3df8d Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Tue, 17 Jan 2023 13:13:46 -0800 Subject: [PATCH 25/43] Update tools/azure-sdk-tools/ci_tools/functions.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: McCoy Patiño <39780829+mccoyp@users.noreply.github.com> --- tools/azure-sdk-tools/ci_tools/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index 4ae634273595..cd422758dad9 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -90,7 +90,7 @@ def apply_compatibility_filter(package_set: List[str]) -> List[str]: collected_packages.append(pkg) logging.debug("Target packages after applying compatibility filter: {}".format(collected_packages)) - logging.debug("Package(s) omitted by business filter: {}".format(generate_difference(package_set, collected_packages))) + logging.debug("Package(s) omitted by compatibility filter: {}".format(generate_difference(package_set, collected_packages))) return collected_packages From a4b955ecf4dd500baa5859ef699c68c8dee406f0 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Tue, 17 Jan 2023 13:14:26 -0800 Subject: [PATCH 26/43] Update tools/azure-sdk-tools/ci_tools/functions.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: McCoy Patiño <39780829+mccoyp@users.noreply.github.com> --- tools/azure-sdk-tools/ci_tools/functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index cd422758dad9..c0e0254023d7 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -212,8 +212,8 @@ def is_package_active(package_path: str): def apply_inactive_filter(collected_packages: List[str]) -> List[str]: packages = [pkg for pkg in collected_packages if is_package_active(pkg)] - logging.debug("Target packages after applying compatibility filter: {}".format(collected_packages)) - logging.debug("Package(s) omitted by business filter: {}".format(generate_difference(collected_packages, packages))) + logging.debug("Target packages after applying inactive filter: {}".format(collected_packages)) + logging.debug("Package(s) omitted by inactive filter: {}".format(generate_difference(collected_packages, packages))) return packages From 4df4dc2e17d93032ff7d4076b37e4e717ccd28a3 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Tue, 17 Jan 2023 14:40:31 -0800 Subject: [PATCH 27/43] PR feedback from McCoy --- scripts/devops_tasks/tox_harness.py | 2 +- tools/azure-sdk-tools/ci_tools/functions.py | 1 - tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py | 8 ++++---- tools/azure-sdk-tools/tests/test_parse_functionality.py | 1 - 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/devops_tasks/tox_harness.py b/scripts/devops_tasks/tox_harness.py index 922af3b93228..1a456e2b46a2 100644 --- a/scripts/devops_tasks/tox_harness.py +++ b/scripts/devops_tasks/tox_harness.py @@ -328,7 +328,7 @@ def prep_and_run_tox(targeted_packages: List[str], parsed_args: Namespace, optio if not filtered_tox_environment_set: logging.info( f"All requested tox environments for package {package_name} have been excluded by the environment exclusion list." - + f" Check file /tools/azure-sdk-tools/ci_tools/environment_exclusions.py and the pyproject.toml." + + " Check file /tools/azure-sdk-tools/ci_tools/environment_exclusions.py and the pyproject.toml." ) continue diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index c0e0254023d7..fcf9a20be8d4 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -42,7 +42,6 @@ and "mgmt" not in x and os.path.basename(x) not in MANAGEMENT_PACKAGE_IDENTIFIERS and os.path.basename(x) not in META_PACKAGES - # and os.path.basename(x) not in REGRESSION_EXCLUDED_PACKAGES and str_to_bool(get_config_setting(x, "regression", True)) ) diff --git a/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py b/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py index fcb120b226f6..ca54909c4f2d 100644 --- a/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py +++ b/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py @@ -90,11 +90,11 @@ def from_path(cls, parse_directory_or_file: str): def get_build_config(self) -> Dict[str, Any]: return get_build_config(self.folder) -def get_build_config(package: str) -> Dict[str, Any]: - if package.lower().endswith("setup.py"): - package = os.path.dirname(package) +def get_build_config(package_path: str) -> Dict[str, Any]: + if package_path.lower().endswith("setup.py"): + package_path = os.path.dirname(package_path) - toml_file = os.path.join(package, "pyproject.toml") + toml_file = os.path.join(package_path, "pyproject.toml") if os.path.exists(toml_file): try: diff --git a/tools/azure-sdk-tools/tests/test_parse_functionality.py b/tools/azure-sdk-tools/tests/test_parse_functionality.py index 78f53ecc30c5..cc62eb58d9a5 100644 --- a/tools/azure-sdk-tools/tests/test_parse_functionality.py +++ b/tools/azure-sdk-tools/tests/test_parse_functionality.py @@ -5,7 +5,6 @@ from unittest.mock import patch package_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) -repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) def test_parse_require(): test_scenarios = [ From 46809fb6ce0a8a73bd92c752c8e342a618288939 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Tue, 17 Jan 2023 14:42:53 -0800 Subject: [PATCH 28/43] mgmt team accidentally broke sdk_packaging toml in azure-mgmt --- sdk/core/azure-mgmt/sdk_packaging.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/core/azure-mgmt/sdk_packaging.toml b/sdk/core/azure-mgmt/sdk_packaging.toml index e10a1aae3a66..e7687fdae93b 100644 --- a/sdk/core/azure-mgmt/sdk_packaging.toml +++ b/sdk/core/azure-mgmt/sdk_packaging.toml @@ -1,2 +1,2 @@ [packaging] -auto_update = falsetitle = \ No newline at end of file +auto_update = false \ No newline at end of file From c9ec7e8ea21ceffd77eaf756c0b4e6c3ae470d69 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Tue, 17 Jan 2023 14:45:29 -0800 Subject: [PATCH 29/43] solve cspell errors --- .vscode/cspell.json | 1 + tools/azure-sdk-tools/ci_tools/functions.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/cspell.json b/.vscode/cspell.json index e3a046234102..9a3ffb5f948c 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -98,6 +98,7 @@ "tools/azure-sdk-tools/setup.py" ], "words": [ + "tomli", "qnamaker", "mindependency", "automl", diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index fcf9a20be8d4..13ae1a678904 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -167,7 +167,7 @@ def discover_targeted_packages( # apply the additional contains filter collected_packages = [pkg for pkg in collected_packages if additional_contains_filter in pkg] - # filter for compatiblity, this means excluding a package that doesn't support py36 when we are running a py36 executable + # filter for compatibility, this means excluding a package that doesn't support py36 when we are running a py36 executable if compatibility_filter: collected_packages = apply_compatibility_filter(collected_packages) From c89c6696b51505eb7097ddc3227e27053f2937b4 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Tue, 17 Jan 2023 14:48:49 -0800 Subject: [PATCH 30/43] correct spelling errors in test_servicemetadata --- .vscode/cspell.json | 1 + tools/azure-sdk-tools/tests/test_servicemetadata.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.vscode/cspell.json b/.vscode/cspell.json index 9a3ffb5f948c..1479713fb68b 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -99,6 +99,7 @@ ], "words": [ "tomli", + "sdkrel", "qnamaker", "mindependency", "automl", diff --git a/tools/azure-sdk-tools/tests/test_servicemetadata.py b/tools/azure-sdk-tools/tests/test_servicemetadata.py index b23a52b652c4..cd62466c0763 100644 --- a/tools/azure-sdk-tools/tests/test_servicemetadata.py +++ b/tools/azure-sdk-tools/tests/test_servicemetadata.py @@ -13,9 +13,9 @@ Update metadata file -Update MANIFETS.IN +Update MANIFEST.IN -No need to update MANIFETS.IN +No need to update MANIFEST.IN """ MANIFEST_TEMP = """recursive-include tests *.py *.yaml From 19d8d22760f4960e1e29a505bcd73025dca49f26 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot Date: Tue, 17 Jan 2023 22:53:45 +0000 Subject: [PATCH 31/43] Packaging update of azure-mgmt-scheduler --- .../azure-mgmt-scheduler/MANIFEST.in | 1 + sdk/scheduler/azure-mgmt-scheduler/README.md | 49 ++++++++++++++++--- sdk/scheduler/azure-mgmt-scheduler/setup.py | 10 ++-- 3 files changed, 49 insertions(+), 11 deletions(-) diff --git a/sdk/scheduler/azure-mgmt-scheduler/MANIFEST.in b/sdk/scheduler/azure-mgmt-scheduler/MANIFEST.in index 656d15872263..229cba9c661d 100644 --- a/sdk/scheduler/azure-mgmt-scheduler/MANIFEST.in +++ b/sdk/scheduler/azure-mgmt-scheduler/MANIFEST.in @@ -1,5 +1,6 @@ include _meta.json recursive-include tests *.py *.json +recursive-include samples *.py *.md include *.md include azure/__init__.py include azure/mgmt/__init__.py diff --git a/sdk/scheduler/azure-mgmt-scheduler/README.md b/sdk/scheduler/azure-mgmt-scheduler/README.md index 62a5b62e5b5f..367a45a5095c 100644 --- a/sdk/scheduler/azure-mgmt-scheduler/README.md +++ b/sdk/scheduler/azure-mgmt-scheduler/README.md @@ -8,17 +8,52 @@ For a more complete view of Azure libraries, see the [azure sdk python release]( _Azure SDK Python packages support for Python 2.7 has ended 01 January 2022. For more information and questions, please refer to https://github.com/Azure/azure-sdk-for-python/issues/20691_ -# Usage +## Getting started +### Prerequisites -To learn how to use this package, see the [quickstart guide](https://aka.ms/azsdk/python/mgmt) - -For docs and references, see [Python SDK References](https://docs.microsoft.com/python/api/overview/azure/scheduler) -Code samples for this package can be found at [Scheduler Management](https://docs.microsoft.com/samples/browse/?languages=python&term=Getting%20started%20-%20Managing&terms=Getting%20started%20-%20Managing) on docs.microsoft.com. -Additional code samples for different Azure services are available at [Samples Repo](https://github.com/Azure-Samples/azure-samples-python-management/tree/main/samples/scheduler) +- Python 3.7+ is required to use this package. +- [Azure subscription](https://azure.microsoft.com/free/) +### Install the package -# Provide Feedback +```bash +pip install azure-mgmt-scheduler +pip install azure-identity +``` + +### Authentication + +By default, [Azure Active Directory](https://aka.ms/awps/aad) token authentication depends on correct configure of following environment variables. + +- `AZURE_CLIENT_ID` for Azure client ID. +- `AZURE_TENANT_ID` for Azure tenant ID. +- `AZURE_CLIENT_SECRET` for Azure client secret. + +In addition, Azure subscription ID can be configured via environment variable `AZURE_SUBSCRIPTION_ID`. + +With above configuration, client can be authenticated by following code: + +```python +from azure.identity import DefaultAzureCredential +from azure.mgmt.scheduler import SchedulerManagementClient +import os + +sub_id = os.getenv("AZURE_SUBSCRIPTION_ID") +client = SchedulerManagementClient(credential=DefaultAzureCredential(), subscription_id=sub_id) +``` + +## Examples + + +Code samples for this package can be found at [Scheduler Management](https://docs.microsoft.com/samples/browse/?languages=python&term=Getting%20started%20-%20Managing&terms=Getting%20started%20-%20Managing) on docs.microsoft.com and [Samples Repo](https://aka.ms/azsdk/python/mgmt/samples) + + +## Troubleshooting + +## Next steps + +## Provide Feedback If you encounter any bugs or have suggestions, please file an issue in the [Issues](https://github.com/Azure/azure-sdk-for-python/issues) diff --git a/sdk/scheduler/azure-mgmt-scheduler/setup.py b/sdk/scheduler/azure-mgmt-scheduler/setup.py index 5913934e2c36..18f3c7576823 100644 --- a/sdk/scheduler/azure-mgmt-scheduler/setup.py +++ b/sdk/scheduler/azure-mgmt-scheduler/setup.py @@ -47,7 +47,7 @@ url='https://github.com/Azure/azure-sdk-for-python', keywords="azure, azure sdk", # update with search keywords relevant to the azure service / product classifiers=[ - 'Development Status :: 7 - Inactive', + 'Development Status :: 5 - Production/Stable', 'Programming Language :: Python', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: 3', @@ -55,6 +55,7 @@ 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'License :: OSI Approved :: MIT License', ], zip_safe=False, @@ -69,9 +70,10 @@ 'pytyped': ['py.typed'], }, install_requires=[ - 'msrest>=0.7.1', - 'azure-common~=1.1', - 'azure-mgmt-core>=1.3.2,<2.0.0', + "msrest>=0.7.1", + "azure-common~=1.1", + "azure-mgmt-core>=1.3.2,<2.0.0", + "typing-extensions>=4.3.0; python_version<'3.8.0'", ], python_requires=">=3.7" ) From a65804eca8bead15109a3fe404ccb4f1faf73c0d Mon Sep 17 00:00:00 2001 From: Azure SDK Bot Date: Tue, 17 Jan 2023 22:53:46 +0000 Subject: [PATCH 32/43] Packaging update of azure-mgmt-regionmove --- .../azure-mgmt-regionmove/MANIFEST.in | 4 +- .../azure-mgmt-regionmove/README.md | 49 ++++++++++++++++--- sdk/regionmove/azure-mgmt-regionmove/setup.py | 17 ++++--- 3 files changed, 55 insertions(+), 15 deletions(-) diff --git a/sdk/regionmove/azure-mgmt-regionmove/MANIFEST.in b/sdk/regionmove/azure-mgmt-regionmove/MANIFEST.in index 2c31e8da0cb1..740515101e1a 100644 --- a/sdk/regionmove/azure-mgmt-regionmove/MANIFEST.in +++ b/sdk/regionmove/azure-mgmt-regionmove/MANIFEST.in @@ -1,6 +1,8 @@ include _meta.json -recursive-include tests *.py *.yaml +recursive-include tests *.py *.json +recursive-include samples *.py *.md include *.md include azure/__init__.py include azure/mgmt/__init__.py include LICENSE +include azure/mgmt/regionmove/py.typed diff --git a/sdk/regionmove/azure-mgmt-regionmove/README.md b/sdk/regionmove/azure-mgmt-regionmove/README.md index e2320ec4451b..791ccf5823ae 100644 --- a/sdk/regionmove/azure-mgmt-regionmove/README.md +++ b/sdk/regionmove/azure-mgmt-regionmove/README.md @@ -1,26 +1,59 @@ # Microsoft Azure SDK for Python This is the Microsoft Azure MyService Management Client Library. -This package has been tested with Python 3.6+. +This package has been tested with Python 3.7+. For a more complete view of Azure libraries, see the [azure sdk python release](https://aka.ms/azsdk/python/all). ## _Disclaimer_ _Azure SDK Python packages support for Python 2.7 has ended 01 January 2022. For more information and questions, please refer to https://github.com/Azure/azure-sdk-for-python/issues/20691_ -# Usage +## Getting started +### Prerequisites -To learn how to use this package, see the [quickstart guide](https://aka.ms/azsdk/python/mgmt) +- Python 3.7+ is required to use this package. +- [Azure subscription](https://azure.microsoft.com/free/) +### Install the package - -For docs and references, see [Python SDK References](https://docs.microsoft.com/python/api/overview/azure/) -Code samples for this package can be found at [MyService Management](https://docs.microsoft.com/samples/browse/?languages=python&term=Getting%20started%20-%20Managing&terms=Getting%20started%20-%20Managing) on docs.microsoft.com. -Additional code samples for different Azure services are available at [Samples Repo](https://aka.ms/azsdk/python/mgmt/samples) +```bash +pip install azure-mgmt-regionmove +pip install azure-identity +``` +### Authentication -# Provide Feedback +By default, [Azure Active Directory](https://aka.ms/awps/aad) token authentication depends on correct configure of following environment variables. + +- `AZURE_CLIENT_ID` for Azure client ID. +- `AZURE_TENANT_ID` for Azure tenant ID. +- `AZURE_CLIENT_SECRET` for Azure client secret. + +In addition, Azure subscription ID can be configured via environment variable `AZURE_SUBSCRIPTION_ID`. + +With above configuration, client can be authenticated by following code: + +```python +from azure.identity import DefaultAzureCredential +from azure.mgmt.regionmove import RegionMoveServiceAPI +import os + +sub_id = os.getenv("AZURE_SUBSCRIPTION_ID") +client = RegionMoveServiceAPI(credential=DefaultAzureCredential(), subscription_id=sub_id) +``` + +## Examples + + +Code samples for this package can be found at [MyService Management](https://docs.microsoft.com/samples/browse/?languages=python&term=Getting%20started%20-%20Managing&terms=Getting%20started%20-%20Managing) on docs.microsoft.com and [Samples Repo](https://aka.ms/azsdk/python/mgmt/samples) + + +## Troubleshooting + +## Next steps + +## Provide Feedback If you encounter any bugs or have suggestions, please file an issue in the [Issues](https://github.com/Azure/azure-sdk-for-python/issues) diff --git a/sdk/regionmove/azure-mgmt-regionmove/setup.py b/sdk/regionmove/azure-mgmt-regionmove/setup.py index d63fb5669455..48276b679bb9 100644 --- a/sdk/regionmove/azure-mgmt-regionmove/setup.py +++ b/sdk/regionmove/azure-mgmt-regionmove/setup.py @@ -47,15 +47,15 @@ url='https://github.com/Azure/azure-sdk-for-python', keywords="azure, azure sdk", # update with search keywords relevant to the azure service / product classifiers=[ - 'Development Status :: 7 - Inactive', + 'Development Status :: 4 - Beta', 'Programming Language :: Python', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'License :: OSI Approved :: MIT License', ], zip_safe=False, @@ -65,10 +65,15 @@ 'azure', 'azure.mgmt', ]), + include_package_data=True, + package_data={ + 'pytyped': ['py.typed'], + }, install_requires=[ - 'msrest>=0.6.21', - 'azure-common~=1.1', - 'azure-mgmt-core>=1.3.0,<2.0.0', + "msrest>=0.7.1", + "azure-common~=1.1", + "azure-mgmt-core>=1.3.2,<2.0.0", + "typing-extensions>=4.3.0; python_version<'3.8.0'", ], - python_requires=">=3.6" + python_requires=">=3.7" ) From 08dbc26dcadc09d9785bc37c5e9dc63ceeaf6cba Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Wed, 18 Jan 2023 11:43:32 -0800 Subject: [PATCH 33/43] Revert "Packaging update of azure-mgmt-scheduler" This reverts commit 19d8d22760f4960e1e29a505bcd73025dca49f26. --- .../azure-mgmt-scheduler/MANIFEST.in | 1 - sdk/scheduler/azure-mgmt-scheduler/README.md | 49 +++---------------- sdk/scheduler/azure-mgmt-scheduler/setup.py | 10 ++-- 3 files changed, 11 insertions(+), 49 deletions(-) diff --git a/sdk/scheduler/azure-mgmt-scheduler/MANIFEST.in b/sdk/scheduler/azure-mgmt-scheduler/MANIFEST.in index 229cba9c661d..656d15872263 100644 --- a/sdk/scheduler/azure-mgmt-scheduler/MANIFEST.in +++ b/sdk/scheduler/azure-mgmt-scheduler/MANIFEST.in @@ -1,6 +1,5 @@ include _meta.json recursive-include tests *.py *.json -recursive-include samples *.py *.md include *.md include azure/__init__.py include azure/mgmt/__init__.py diff --git a/sdk/scheduler/azure-mgmt-scheduler/README.md b/sdk/scheduler/azure-mgmt-scheduler/README.md index 367a45a5095c..62a5b62e5b5f 100644 --- a/sdk/scheduler/azure-mgmt-scheduler/README.md +++ b/sdk/scheduler/azure-mgmt-scheduler/README.md @@ -8,52 +8,17 @@ For a more complete view of Azure libraries, see the [azure sdk python release]( _Azure SDK Python packages support for Python 2.7 has ended 01 January 2022. For more information and questions, please refer to https://github.com/Azure/azure-sdk-for-python/issues/20691_ -## Getting started +# Usage -### Prerequisites -- Python 3.7+ is required to use this package. -- [Azure subscription](https://azure.microsoft.com/free/) +To learn how to use this package, see the [quickstart guide](https://aka.ms/azsdk/python/mgmt) + +For docs and references, see [Python SDK References](https://docs.microsoft.com/python/api/overview/azure/scheduler) +Code samples for this package can be found at [Scheduler Management](https://docs.microsoft.com/samples/browse/?languages=python&term=Getting%20started%20-%20Managing&terms=Getting%20started%20-%20Managing) on docs.microsoft.com. +Additional code samples for different Azure services are available at [Samples Repo](https://github.com/Azure-Samples/azure-samples-python-management/tree/main/samples/scheduler) -### Install the package -```bash -pip install azure-mgmt-scheduler -pip install azure-identity -``` - -### Authentication - -By default, [Azure Active Directory](https://aka.ms/awps/aad) token authentication depends on correct configure of following environment variables. - -- `AZURE_CLIENT_ID` for Azure client ID. -- `AZURE_TENANT_ID` for Azure tenant ID. -- `AZURE_CLIENT_SECRET` for Azure client secret. - -In addition, Azure subscription ID can be configured via environment variable `AZURE_SUBSCRIPTION_ID`. - -With above configuration, client can be authenticated by following code: - -```python -from azure.identity import DefaultAzureCredential -from azure.mgmt.scheduler import SchedulerManagementClient -import os - -sub_id = os.getenv("AZURE_SUBSCRIPTION_ID") -client = SchedulerManagementClient(credential=DefaultAzureCredential(), subscription_id=sub_id) -``` - -## Examples - - -Code samples for this package can be found at [Scheduler Management](https://docs.microsoft.com/samples/browse/?languages=python&term=Getting%20started%20-%20Managing&terms=Getting%20started%20-%20Managing) on docs.microsoft.com and [Samples Repo](https://aka.ms/azsdk/python/mgmt/samples) - - -## Troubleshooting - -## Next steps - -## Provide Feedback +# Provide Feedback If you encounter any bugs or have suggestions, please file an issue in the [Issues](https://github.com/Azure/azure-sdk-for-python/issues) diff --git a/sdk/scheduler/azure-mgmt-scheduler/setup.py b/sdk/scheduler/azure-mgmt-scheduler/setup.py index 18f3c7576823..5913934e2c36 100644 --- a/sdk/scheduler/azure-mgmt-scheduler/setup.py +++ b/sdk/scheduler/azure-mgmt-scheduler/setup.py @@ -47,7 +47,7 @@ url='https://github.com/Azure/azure-sdk-for-python', keywords="azure, azure sdk", # update with search keywords relevant to the azure service / product classifiers=[ - 'Development Status :: 5 - Production/Stable', + 'Development Status :: 7 - Inactive', 'Programming Language :: Python', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: 3', @@ -55,7 +55,6 @@ 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', 'License :: OSI Approved :: MIT License', ], zip_safe=False, @@ -70,10 +69,9 @@ 'pytyped': ['py.typed'], }, install_requires=[ - "msrest>=0.7.1", - "azure-common~=1.1", - "azure-mgmt-core>=1.3.2,<2.0.0", - "typing-extensions>=4.3.0; python_version<'3.8.0'", + 'msrest>=0.7.1', + 'azure-common~=1.1', + 'azure-mgmt-core>=1.3.2,<2.0.0', ], python_requires=">=3.7" ) From 359e978a9cb06c006d40b44431fd4a4f1e954d5c Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Wed, 18 Jan 2023 11:43:47 -0800 Subject: [PATCH 34/43] Revert "Packaging update of azure-mgmt-regionmove" This reverts commit a65804eca8bead15109a3fe404ccb4f1faf73c0d. --- .../azure-mgmt-regionmove/MANIFEST.in | 4 +- .../azure-mgmt-regionmove/README.md | 49 +++---------------- sdk/regionmove/azure-mgmt-regionmove/setup.py | 17 +++---- 3 files changed, 15 insertions(+), 55 deletions(-) diff --git a/sdk/regionmove/azure-mgmt-regionmove/MANIFEST.in b/sdk/regionmove/azure-mgmt-regionmove/MANIFEST.in index 740515101e1a..2c31e8da0cb1 100644 --- a/sdk/regionmove/azure-mgmt-regionmove/MANIFEST.in +++ b/sdk/regionmove/azure-mgmt-regionmove/MANIFEST.in @@ -1,8 +1,6 @@ include _meta.json -recursive-include tests *.py *.json -recursive-include samples *.py *.md +recursive-include tests *.py *.yaml include *.md include azure/__init__.py include azure/mgmt/__init__.py include LICENSE -include azure/mgmt/regionmove/py.typed diff --git a/sdk/regionmove/azure-mgmt-regionmove/README.md b/sdk/regionmove/azure-mgmt-regionmove/README.md index 791ccf5823ae..e2320ec4451b 100644 --- a/sdk/regionmove/azure-mgmt-regionmove/README.md +++ b/sdk/regionmove/azure-mgmt-regionmove/README.md @@ -1,59 +1,26 @@ # Microsoft Azure SDK for Python This is the Microsoft Azure MyService Management Client Library. -This package has been tested with Python 3.7+. +This package has been tested with Python 3.6+. For a more complete view of Azure libraries, see the [azure sdk python release](https://aka.ms/azsdk/python/all). ## _Disclaimer_ _Azure SDK Python packages support for Python 2.7 has ended 01 January 2022. For more information and questions, please refer to https://github.com/Azure/azure-sdk-for-python/issues/20691_ -## Getting started +# Usage -### Prerequisites -- Python 3.7+ is required to use this package. -- [Azure subscription](https://azure.microsoft.com/free/) +To learn how to use this package, see the [quickstart guide](https://aka.ms/azsdk/python/mgmt) -### Install the package -```bash -pip install azure-mgmt-regionmove -pip install azure-identity -``` + +For docs and references, see [Python SDK References](https://docs.microsoft.com/python/api/overview/azure/) +Code samples for this package can be found at [MyService Management](https://docs.microsoft.com/samples/browse/?languages=python&term=Getting%20started%20-%20Managing&terms=Getting%20started%20-%20Managing) on docs.microsoft.com. +Additional code samples for different Azure services are available at [Samples Repo](https://aka.ms/azsdk/python/mgmt/samples) -### Authentication -By default, [Azure Active Directory](https://aka.ms/awps/aad) token authentication depends on correct configure of following environment variables. - -- `AZURE_CLIENT_ID` for Azure client ID. -- `AZURE_TENANT_ID` for Azure tenant ID. -- `AZURE_CLIENT_SECRET` for Azure client secret. - -In addition, Azure subscription ID can be configured via environment variable `AZURE_SUBSCRIPTION_ID`. - -With above configuration, client can be authenticated by following code: - -```python -from azure.identity import DefaultAzureCredential -from azure.mgmt.regionmove import RegionMoveServiceAPI -import os - -sub_id = os.getenv("AZURE_SUBSCRIPTION_ID") -client = RegionMoveServiceAPI(credential=DefaultAzureCredential(), subscription_id=sub_id) -``` - -## Examples - - -Code samples for this package can be found at [MyService Management](https://docs.microsoft.com/samples/browse/?languages=python&term=Getting%20started%20-%20Managing&terms=Getting%20started%20-%20Managing) on docs.microsoft.com and [Samples Repo](https://aka.ms/azsdk/python/mgmt/samples) - - -## Troubleshooting - -## Next steps - -## Provide Feedback +# Provide Feedback If you encounter any bugs or have suggestions, please file an issue in the [Issues](https://github.com/Azure/azure-sdk-for-python/issues) diff --git a/sdk/regionmove/azure-mgmt-regionmove/setup.py b/sdk/regionmove/azure-mgmt-regionmove/setup.py index 48276b679bb9..d63fb5669455 100644 --- a/sdk/regionmove/azure-mgmt-regionmove/setup.py +++ b/sdk/regionmove/azure-mgmt-regionmove/setup.py @@ -47,15 +47,15 @@ url='https://github.com/Azure/azure-sdk-for-python', keywords="azure, azure sdk", # update with search keywords relevant to the azure service / product classifiers=[ - 'Development Status :: 4 - Beta', + 'Development Status :: 7 - Inactive', 'Programming Language :: Python', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', 'License :: OSI Approved :: MIT License', ], zip_safe=False, @@ -65,15 +65,10 @@ 'azure', 'azure.mgmt', ]), - include_package_data=True, - package_data={ - 'pytyped': ['py.typed'], - }, install_requires=[ - "msrest>=0.7.1", - "azure-common~=1.1", - "azure-mgmt-core>=1.3.2,<2.0.0", - "typing-extensions>=4.3.0; python_version<'3.8.0'", + 'msrest>=0.6.21', + 'azure-common~=1.1', + 'azure-mgmt-core>=1.3.0,<2.0.0', ], - python_requires=">=3.7" + python_requires=">=3.6" ) From a5ba3f36a8883aed89afc9e3f495dfa8f3c64fdc Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Wed, 18 Jan 2023 11:44:43 -0800 Subject: [PATCH 35/43] revert auto commits to azure-mgmt-scheduler and azure-mgmt-regionmove. disable auto-update. --- sdk/regionmove/azure-mgmt-regionmove/sdk_packaging.toml | 1 + sdk/scheduler/azure-mgmt-scheduler/sdk_packaging.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/sdk/regionmove/azure-mgmt-regionmove/sdk_packaging.toml b/sdk/regionmove/azure-mgmt-regionmove/sdk_packaging.toml index 92335d051e16..58e60f17bd62 100644 --- a/sdk/regionmove/azure-mgmt-regionmove/sdk_packaging.toml +++ b/sdk/regionmove/azure-mgmt-regionmove/sdk_packaging.toml @@ -6,3 +6,4 @@ package_doc_id = "" is_stable = false is_arm = true title = "RegionMoveServiceAPI" +auto_update = false \ No newline at end of file diff --git a/sdk/scheduler/azure-mgmt-scheduler/sdk_packaging.toml b/sdk/scheduler/azure-mgmt-scheduler/sdk_packaging.toml index 0f1a6417619a..af79b133f248 100644 --- a/sdk/scheduler/azure-mgmt-scheduler/sdk_packaging.toml +++ b/sdk/scheduler/azure-mgmt-scheduler/sdk_packaging.toml @@ -5,3 +5,4 @@ package_doc_id = "scheduler" is_stable = true sample_link = "" title = "SchedulerManagementClient" +auto_update = false \ No newline at end of file From eef72c14d56d56521513472f5f3837ca7e0dd07b Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Wed, 18 Jan 2023 12:13:31 -0800 Subject: [PATCH 36/43] add detail to eng_sys_checks document --- doc/eng_sys_checks.md | 51 +++++++++++++++---- .../ci_tools/environment_exclusions.py | 12 +++-- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/doc/eng_sys_checks.md b/doc/eng_sys_checks.md index ca5394b6ed30..920e868be34b 100644 --- a/doc/eng_sys_checks.md +++ b/doc/eng_sys_checks.md @@ -4,7 +4,9 @@ - [Targeting a specific package at build queue time](#targeting-a-specific-package-at-build-queue-time) - [Skipping a tox test environment at build queue time](#skipping-a-tox-test-environment-at-build-queue-time) - [Skipping entire sections of builds](#skipping-entire-sections-of-builds) + - [The pyproject.toml](#the-pyprojecttoml) - [Environment variables important to CI](#environment-variables-important-to-ci) + - [Atomic Overrides](#atomic-overrides) - [Analyze Checks](#analyze-checks) - [MyPy](#mypy) - [Pylint](#pylint) @@ -102,6 +104,28 @@ This is the most useful skip, but the following skip variables are also supporte - `Skip.VerifyDependencies` - Omit checking that a package's dependencies are on PyPI before releasing. +## The pyproject.toml + +Starting with [this pr](https://github.com/Azure/azure-sdk-for-python/pull/28345), which checks apply to which packages are now **established** in a `pyproject.toml`, right next to each package's `setup.py`. This not only allows devs to fine-tune which checks that are applied at a package-level, but also seriously reduces confusion as to which checks apply when. + +We default to **enabling** most of our checks like `pylint`, `mypy`, etc. Due to that, most `pyproject.toml` settings will likely be **disabling** checks. + +Here's an example: + +```toml +# from sdk/core/azure-servicemanagement-legacy/pyproject.toml, which is a legacy package +# as a result, all of these checks are disabled +[tool.azure-sdk-build] +mypy = false +type_check_samples = false +verifytypes = false +pyright = false +pylint = false +black = false +``` + +If a package does not yet have a `pyproject.toml`, creating one with just the section `[tool.azure-sdk-build]` will do no harm to the release of the package in question. + ## Environment variables important to CI There are a few differences from a standard local invocation of `tox `. Primarily, these differences adjust the checks to be friendly to parallel invocation. These adjustments are necessary to prevent random CI crashes. @@ -114,6 +138,22 @@ There are a few differences from a standard local invocation of `tox `. Pri The various tooling abstracted by the environments within `eng/tox/tox.ini` take the above variables into account automatically. +### Atomic Overrides + +Packages with classifier `Development Status :: 7 - Inactive`, are **not** built by default and as such normal `checks` like `mypy` and `pylint` are also not run against them. Older "core" packages like `azure-common` and `azure-servicemanagement-legacy` are present, but excluded from the build due to this restriction. + +To temporarily **override** this restriction, a dev need only set the queue time variable: `ENABLE_PACKAGE_NAME`. The `-` in package names should be replaced by an `_`, as that is how the environment variable will be set on the actual CI machine anyway. + +- `ENABLE_AZURE_COMMON=true` +- `ENABLE_AZURE_SERVICEMANAGEMENT_LEGACY=true` + +This same methodology also applies to _individual checks_ that run during various phases of CI. Developers can use a queue time variable of format `PACKAGE_NAME_CHECK=true/false`. + +The name that you should use is visible based on what the `tox environment` that the check refers to! Here are a few examples of enabling/disabling checks: + +- `AZURE_SERVICEBUS_PYRIGHT=true` <-- enable a check that normally is disabled in pyproject.toml +- `AZURE_CORE_PYLINT=false` <-- disable a check that normally runs + ## Analyze Checks Analyze job in both nightly CI and pull request validation pipeline runs a set of static analysis using external and internal tools. Following are the list of these static analysis. @@ -122,15 +162,8 @@ Analyze job in both nightly CI and pull request validation pipeline runs a set o [`MyPy`](https://pypi.org/project/mypy/) is a static analysis tool that runs type checking of python package. MyPy is an opt-in check for packages. Following are the steps to run `MyPy` locally for a specific package: -1. Add the package name to the end of the [`mypy_hard_failure_packages.py`](https://github.com/Azure/azure-sdk-for-python/blob/main/eng/tox/mypy_hard_failure_packages.py) file: - ```python - MYPY_HARD_FAILURE_OPTED = [ - ..., - "azure-my-package", - ] - ``` -2. Go to root of the package -3. Execute following command: `tox -e mypy -c ../../../eng/tox/tox.ini` +1. Go to root of the package +2. Execute following command: `tox -e mypy -c ../../../eng/tox/tox.ini` ### Pylint diff --git a/tools/azure-sdk-tools/ci_tools/environment_exclusions.py b/tools/azure-sdk-tools/ci_tools/environment_exclusions.py index 92c24c9d7304..746309fef97a 100644 --- a/tools/azure-sdk-tools/ci_tools/environment_exclusions.py +++ b/tools/azure-sdk-tools/ci_tools/environment_exclusions.py @@ -341,6 +341,8 @@ def is_check_enabled(package_path: str, check: str, default: bool = True) -> bool: + if package_path.endswith("setup.py"): + package_path = os.path.dirname(package_path) enabled = default package_name = os.path.basename(package_path) @@ -350,8 +352,9 @@ def is_check_enabled(package_path: str, check: str, default: bool = True) -> boo exclusions_for_env = globals()[f"{check.strip().upper()}_OPT_OUT"] except Exception as e: pass + if package_name in exclusions_for_env: - result = False + enabled = False # now pull the new pyproject.toml configuration config = get_config_setting(package_path, check.strip().lower(), True) @@ -364,9 +367,12 @@ def filter_tox_environment_string(namespace_argument: str, package_path: str) -> Takes an incoming comma separated list of tox environments and package name. Resolves whether or not each given tox environment should run, given comparison to single unified exclusion file in `environment_exclusions`. - :param namespace_argument: A namespace argument. - :param package_name: The name of the package. This takes the form of a comma separated list: "whl,sdist,mindependency". "whl". "lint,pyright,sphinx". + :param namespace_argument: A namespace argument. This takes the form of a comma separated list: "whl,sdist,mindependency". "whl". "lint,pyright,sphinx". + :param package_path: The path to the package. """ + if package_path.endswith("setup.py"): + package_path = os.path.dirname(package_path) + package_name = os.path.basename(package_path) if namespace_argument: From 7aab581385733fca0e095b8b552bdddf9b086b7b Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Wed, 18 Jan 2023 15:17:59 -0800 Subject: [PATCH 37/43] resolve final spelling issue --- .vscode/cspell.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/cspell.json b/.vscode/cspell.json index 1479713fb68b..4c4089cb8227 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -95,7 +95,8 @@ ".gitignore", "tools/azure-sdk-tools/devtools_testutils/fake_credentials.py", "tools/azure-sdk-tools/packaging_tools/**", - "tools/azure-sdk-tools/setup.py" + "tools/azure-sdk-tools/setup.py", + "tools/azure-sdk-tools/tests/test_servicemetadata.py" ], "words": [ "tomli", From 3f3ea5fc55ada5cd9d62f671689d6cb488994356 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Wed, 18 Jan 2023 15:38:08 -0800 Subject: [PATCH 38/43] THIS may be the final cspell update --- .vscode/cspell.json | 1 + doc/eng_sys_checks.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/cspell.json b/.vscode/cspell.json index 4c4089cb8227..84fb693a847d 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -99,6 +99,7 @@ "tools/azure-sdk-tools/tests/test_servicemetadata.py" ], "words": [ + "pyprojecttoml", "tomli", "sdkrel", "qnamaker", diff --git a/doc/eng_sys_checks.md b/doc/eng_sys_checks.md index 920e868be34b..e8bfe6d3228d 100644 --- a/doc/eng_sys_checks.md +++ b/doc/eng_sys_checks.md @@ -151,7 +151,7 @@ This same methodology also applies to _individual checks_ that run during variou The name that you should use is visible based on what the `tox environment` that the check refers to! Here are a few examples of enabling/disabling checks: -- `AZURE_SERVICEBUS_PYRIGHT=true` <-- enable a check that normally is disabled in pyproject.toml +- `AZURE_SERVICEBUS_PYRIGHT=true` <-- enable a check that normally is disabled in `pyproject.toml` - `AZURE_CORE_PYLINT=false` <-- disable a check that normally runs ## Analyze Checks From 374489bd929e29477c1092666c000d8017f9bf88 Mon Sep 17 00:00:00 2001 From: semick-dev Date: Wed, 18 Jan 2023 20:29:44 -0800 Subject: [PATCH 39/43] fix the failure in detect api changes --- eng/scripts/get_package_properties.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/eng/scripts/get_package_properties.py b/eng/scripts/get_package_properties.py index cfc149643d2b..57e0fbe3d245 100644 --- a/eng/scripts/get_package_properties.py +++ b/eng/scripts/get_package_properties.py @@ -17,11 +17,13 @@ if "setup.py" in files: try: parsed = ParsedSetup.from_path(root) - print( - "{0} {1} {2} {3}".format( - parsed.name, parsed.version, parsed.is_new_sdk, os.path.dirname(parsed.setup_filename) + + if is_package_active(parsed.folder): + print( + "{0} {1} {2} {3}".format( + parsed.name, parsed.version, parsed.is_new_sdk, os.path.dirname(parsed.setup_filename) + ) ) - ) except: # Skip setup.py if the package cannot be parsed pass From dd33a067e69dacf03c4b2c806c0a5501718086cd Mon Sep 17 00:00:00 2001 From: semick-dev Date: Wed, 18 Jan 2023 21:23:52 -0800 Subject: [PATCH 40/43] identify issue with . as the package path --- tools/azure-sdk-tools/ci_tools/environment_exclusions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/azure-sdk-tools/ci_tools/environment_exclusions.py b/tools/azure-sdk-tools/ci_tools/environment_exclusions.py index 1afa705c4651..d8409503b269 100644 --- a/tools/azure-sdk-tools/ci_tools/environment_exclusions.py +++ b/tools/azure-sdk-tools/ci_tools/environment_exclusions.py @@ -343,6 +343,10 @@ def is_check_enabled(package_path: str, check: str, default: bool = True) -> bool: if package_path.endswith("setup.py"): package_path = os.path.dirname(package_path) + + if package_path == ".": + package_path = os.getcwd() + enabled = default package_name = os.path.basename(package_path) From 3301048224040b07d5891d5705590016b84fdc5b Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Thu, 19 Jan 2023 09:35:59 -0800 Subject: [PATCH 41/43] patch get_package_properties --- eng/scripts/get_package_properties.py | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/scripts/get_package_properties.py b/eng/scripts/get_package_properties.py index 57e0fbe3d245..5dabe2062bf5 100644 --- a/eng/scripts/get_package_properties.py +++ b/eng/scripts/get_package_properties.py @@ -5,6 +5,7 @@ import re from ci_tools.parsing import ParsedSetup +from ci_tools.functions import is_package_active if __name__ == "__main__": parser = argparse.ArgumentParser(description="Get package version details from the repo") From ed6d9e7f90032b2d1359c2f0a5a1588586b295fd Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Thu, 19 Jan 2023 13:06:27 -0800 Subject: [PATCH 42/43] include inactive packages when assembling artifacts --- .../templates/steps/build-artifacts.yml | 2 +- eng/scripts/get_package_properties.py | 10 +++---- tools/azure-sdk-tools/ci_tools/build.py | 28 ++++++++++++++++--- tools/azure-sdk-tools/ci_tools/functions.py | 4 ++- 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/eng/pipelines/templates/steps/build-artifacts.yml b/eng/pipelines/templates/steps/build-artifacts.yml index d98fcf9f0ff5..07f835526406 100644 --- a/eng/pipelines/templates/steps/build-artifacts.yml +++ b/eng/pipelines/templates/steps/build-artifacts.yml @@ -63,7 +63,7 @@ steps: condition: and(succeeded(),eq(variables['SetDevVersion'],'true')) - pwsh: | - sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --service=${{parameters.ServiceDirectory}} + sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --service=${{parameters.ServiceDirectory}} --inactive displayName: 'Generate Packages' condition: succeededOrFailed() diff --git a/eng/scripts/get_package_properties.py b/eng/scripts/get_package_properties.py index 5dabe2062bf5..c0bea2a83592 100644 --- a/eng/scripts/get_package_properties.py +++ b/eng/scripts/get_package_properties.py @@ -5,7 +5,6 @@ import re from ci_tools.parsing import ParsedSetup -from ci_tools.functions import is_package_active if __name__ == "__main__": parser = argparse.ArgumentParser(description="Get package version details from the repo") @@ -19,12 +18,11 @@ try: parsed = ParsedSetup.from_path(root) - if is_package_active(parsed.folder): - print( - "{0} {1} {2} {3}".format( - parsed.name, parsed.version, parsed.is_new_sdk, os.path.dirname(parsed.setup_filename) - ) + print( + "{0} {1} {2} {3}".format( + parsed.name, parsed.version, parsed.is_new_sdk, os.path.dirname(parsed.setup_filename) ) + ) except: # Skip setup.py if the package cannot be parsed pass diff --git a/tools/azure-sdk-tools/ci_tools/build.py b/tools/azure-sdk-tools/ci_tools/build.py index f7f9de9c1987..35beab2803e6 100644 --- a/tools/azure-sdk-tools/ci_tools/build.py +++ b/tools/azure-sdk-tools/ci_tools/build.py @@ -11,6 +11,7 @@ from ci_tools.versioning.version_set_dev import get_dev_version, format_build_id from ci_tools.logging import initialize_logger, run_logged + def build() -> None: parser = argparse.ArgumentParser( description="""This is the primary entrypoint for the "build" action. This command is used to build any package within the azure-sdk-for-python repository.""", @@ -57,6 +58,17 @@ def build() -> None: ), ) + parser.add_argument( + "--inactive", + default=False, + dest="is_dev_build", + action="store_true", + help=( + "Include inactive packages when assembling artifacts. CI builds will include inactive packages as a way to ensure that the yml" + + " controlled artifacts can be associated with a wheel/sdist." + ), + ) + parser.add_argument( "--produce_apiview_artifact", default=False, @@ -93,7 +105,9 @@ def build() -> None: else: target_dir = repo_root - targeted_packages = discover_targeted_packages(args.glob_string, target_dir, args.package_filter_string) + targeted_packages = discover_targeted_packages( + args.glob_string, target_dir, args.package_filter_string, include_inactive=True + ) artifact_directory = get_artifact_directory(args.distribution_directory) build_id = format_build_id(args.build_id or DEFAULT_BUILD_ID) @@ -129,7 +143,7 @@ def build_packages( new_version = get_dev_version(setup_parsed.version, build_id) logger.log(level=logging.DEBUG, msg=f"{setup_parsed.name}: {setup_parsed.version} -> {new_version}") - + set_version_py(setup_parsed.setup_filename, new_version) set_dev_classifier(setup_parsed.setup_filename, new_version) @@ -150,6 +164,12 @@ def create_package( setup_directory_or_file = os.path.dirname(setup_directory_or_file) if enable_wheel: - run_logged([sys.executable, "setup.py", "bdist_wheel", "-d", dist], prefix="create_wheel", cwd=setup_directory_or_file) + run_logged( + [sys.executable, "setup.py", "bdist_wheel", "-d", dist], prefix="create_wheel", cwd=setup_directory_or_file + ) if enable_sdist: - run_logged([sys.executable, "setup.py", "sdist", "--format", "zip", "-d", dist], prefix="create_sdist", cwd=setup_directory_or_file) + run_logged( + [sys.executable, "setup.py", "sdist", "--format", "zip", "-d", dist], + prefix="create_sdist", + cwd=setup_directory_or_file, + ) diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index 13ae1a678904..58e9cbeee489 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -149,6 +149,7 @@ def discover_targeted_packages( additional_contains_filter: str = "", filter_type: str = "Build", compatibility_filter: bool = True, + include_inactive: bool = False ) -> List[str]: """ During build and test, the set of targeted packages may expand or contract depending on the needs of the invocation. @@ -173,7 +174,8 @@ def discover_targeted_packages( # apply package-specific exclusions only if we have gotten more than one if len(collected_packages) > 1: - collected_packages = apply_inactive_filter(collected_packages) + if not include_inactive: + collected_packages = apply_inactive_filter(collected_packages) # Apply filter based on filter type. for e.g. Docs, Regression, Management collected_packages = apply_business_filter(collected_packages, filter_type) From 7b028ecd1cde8ad07b765cc88f74bbf4cd139e09 Mon Sep 17 00:00:00 2001 From: scbedd <45376673+scbedd@users.noreply.github.com> Date: Thu, 19 Jan 2023 13:58:33 -0800 Subject: [PATCH 43/43] resolve issue with inactive flag turning dev mode on --- tools/azure-sdk-tools/ci_tools/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/azure-sdk-tools/ci_tools/build.py b/tools/azure-sdk-tools/ci_tools/build.py index 35beab2803e6..b0097acdd5ec 100644 --- a/tools/azure-sdk-tools/ci_tools/build.py +++ b/tools/azure-sdk-tools/ci_tools/build.py @@ -61,7 +61,7 @@ def build() -> None: parser.add_argument( "--inactive", default=False, - dest="is_dev_build", + dest="inactive", action="store_true", help=( "Include inactive packages when assembling artifacts. CI builds will include inactive packages as a way to ensure that the yml" @@ -106,7 +106,7 @@ def build() -> None: target_dir = repo_root targeted_packages = discover_targeted_packages( - args.glob_string, target_dir, args.package_filter_string, include_inactive=True + args.glob_string, target_dir, args.package_filter_string, filter_type="Build", compatibility_filter=True, include_inactive=args.inactive ) artifact_directory = get_artifact_directory(args.distribution_directory)