[Packaging] Bump bundled python to 3.8.9 and remove more network SD…

…K APIs for MSI (#17816)

.azure-pipelines/templates/azdev_setup.yml

@@ -21,6 +21,8 @@ steps:
       else
         azdev setup -c $CLI_REPO_PATH -r $CLI_EXT_REPO_PATH --debug
       fi
+      # This helps detect issues in CI if a used SDK API version is deleted by the below script.
+      python $CLI_REPO_PATH/build_scripts/windows/scripts/remove_unused_api_versions.py
     displayName: 'azdev setup'
     env:
       CLI_REPO_PATH: ${{ parameters.CLIRepoPath }}

azure-pipelines.yml

@@ -227,15 +227,20 @@ jobs:
           $InstallArgs += $reinstall_option
         }
         Start-Process "msiexec.exe" -ArgumentList $InstallArgs -Wait -NoNewWindow
-        Get-Content .\install_logs.txt
+        $install_time=Measure-Command {Start-Process "msiexec.exe" -ArgumentList $InstallArgs -Wait -NoNewWindow} | select -expand TotalSeconds
         $installed_version=az version --query '\"azure-cli\"' -o tsv
         if ($installed_version -ne $to_be_installed_version){
           echo "The MSI failed to install."
           Exit 1
         }
+        echo 'Install time(seconds):' $install_time
         az --version
+        # Test bundled pip with extension installation
+        az extension add -n rdbms-connect
         az self-test
 
+        Get-Content .\install_logs.txt
+
 - job: BuildDockerImage
   displayName: Build Docker Image
 

build_scripts/windows/scripts/build.cmd

@@ -9,10 +9,10 @@ if "%CLI_VERSION%"=="" (
     echo Please set the CLI_VERSION environment variable, e.g. 2.0.13
     goto ERROR
 )
-set PYTHON_VERSION=3.6.8
+set PYTHON_VERSION=3.8.9
 
 set WIX_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/msi/wix310-binaries-mirror.zip"
-set PYTHON_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/Python368-32.zip"
+set PYTHON_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/Python389-32.zip"
 set PROPAGATE_ENV_CHANGE_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/util/propagate_env_change.zip"
 
 :: Set up the output directory and temp. directories
@@ -26,7 +26,7 @@ mkdir %ARTIFACTS_DIR%
 set TEMP_SCRATCH_FOLDER=%ARTIFACTS_DIR%\cli_scratch
 set BUILDING_DIR=%ARTIFACTS_DIR%\cli
 set WIX_DIR=%ARTIFACTS_DIR%\wix
-set PYTHON_DIR=%ARTIFACTS_DIR%\Python368-32
+set PYTHON_DIR=%ARTIFACTS_DIR%\Python389-32
 set PROPAGATE_ENV_CHANGE_DIR=%~dp0..\propagate_env_change
 
 set REPO_ROOT=%~dp0..\..\..
@@ -75,15 +75,14 @@ if not exist %PYTHON_DIR% (
     mkdir %PYTHON_DIR%
     pushd %PYTHON_DIR%
     echo Downloading Python.
-    curl -o Python368-32.zip %PYTHON_DOWNLOAD_URL% -k
-    unzip -q Python368-32.zip
+    curl -o Python389-32.zip %PYTHON_DOWNLOAD_URL% -k
+    unzip -q Python389-32.zip
     if %errorlevel% neq 0 goto ERROR
-    del Python368-32.zip
+    del Python389-32.zip
     echo Python downloaded and extracted successfully.
     popd
 )
 set PYTHON_EXE=%PYTHON_DIR%\python.exe
-%PYTHON_EXE% -m pip install --upgrade pip==21.0.1 setuptools==52.0.0
 
 robocopy %PYTHON_DIR% %BUILDING_DIR% /s /NFL /NDL
 
@@ -100,6 +99,7 @@ if %errorlevel% neq 0 goto ERROR
 
 pushd %BUILDING_DIR%
 %BUILDING_DIR%\python.exe %~dp0\patch_models_v2.py
+%BUILDING_DIR%\python.exe %~dp0\remove_unused_api_versions.py
 popd
 
 echo Creating the wbin (Windows binaries) folder that will be added to the path...
@@ -123,11 +123,6 @@ for /f %%a in ('dir /b /s *_py3.*.pyc') do (
 )
 popd
 
-:: Remove unused Network SDK API versions
-pushd %BUILDING_DIR%\Lib\site-packages\azure\mgmt\network
-rmdir /s /q v2016_09_01 v2016_12_01 v2017_03_01 v2017_06_01 v2017_08_01 v2017_09_01 v2017_11_01 v2018_02_01 v2018_04_01 v2018_06_01 v2018_10_01 v2018_12_01 v2019_04_01 v2019_08_01 v2019_09_01 v2019_11_01 v2019_12_01 v2020_03_01
-popd
-
 :: Remove .py and only deploy .pyc files
 pushd %BUILDING_DIR%\Lib\site-packages
 for /f %%f in ('dir /b /s *.pyc') do (

build_scripts/windows/scripts/patch_models_v2.py

@@ -245,7 +245,7 @@ def find_models_to_change(module_name):
         return [
             importlib.import_module('.'+label+'.models', main_module.__name__)
             for (_, label, ispkg) in pkgutil.iter_modules(main_module.__path__)
-            if ispkg
+            if ispkg and label != 'aio'
         ]
 
 

build_scripts/windows/scripts/remove_unused_api_versions.py

@@ -0,0 +1,59 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import logging
+import os
+import re
+import shutil
+import azure.mgmt.network
+
+from azure.cli.core.profiles import AD_HOC_API_VERSIONS, AZURE_API_PROFILES, ResourceType
+
+
+_LOGGER = logging.getLogger(__name__)
+
+
+def remove_unused_network_api_versions():
+    # Hard-coded API versions
+    used_network_api_versions = set(AD_HOC_API_VERSIONS[ResourceType.MGMT_NETWORK].values())
+
+    # API versions in profile
+    for _, profile in AZURE_API_PROFILES.items():
+        if ResourceType.MGMT_NETWORK in profile:
+            used_network_api_versions.add(profile[ResourceType.MGMT_NETWORK])
+
+    # Normalize API version: 2019-02-01 -> v2019_02_01
+    used_network_api_vers = {f"v{api.replace('-','_')}" for api in used_network_api_versions}
+
+    # Network SDK has a set of versions imported in models.py.
+    # Let's keep them before we figure out how to remove a version in all related SDK files.
+    path = azure.mgmt.network.__path__[0]
+    model_file = os.path.join(path, 'models.py')
+    with open(model_file, 'r', encoding='utf-8') as f:
+        content = f.read()
+    for m in re.finditer(r'from \.(v[_\d]*)\.models import \*', content):
+        used_network_api_vers.add(m.group(1))
+
+    _LOGGER.info('Used network API versions:')
+    _LOGGER.info(sorted(used_network_api_vers))
+
+    all_api_vers = {d for d in os.listdir(path) if os.path.isdir(os.path.join(path, d)) and d.startswith('v')}
+    _LOGGER.info('All network API versions:')
+    _LOGGER.info(sorted(all_api_vers))
+
+    remove_api_vers = sorted(all_api_vers - used_network_api_vers)
+    _LOGGER.info('Network API versions that will be removed:')
+    _LOGGER.info(remove_api_vers)
+
+    for ver in remove_api_vers:
+        shutil.rmtree(os.path.join(path, ver))
+
+
+def main():
+    remove_unused_network_api_versions()
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    main()

build_scripts/windows/scripts/setup_msi_test.ps1

@@ -16,6 +16,6 @@ $testsdk = Get-ChildItem -Path $PSScriptRoot\..\..\..\artifacts\build\azure_cli_
 & 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install $PSScriptRoot\..\..\..\artifacts\build\$($testsdk.Name)
 
 $fulltest = Get-ChildItem -Path $PSScriptRoot\..\..\..\artifacts\build\azure_cli_fulltest*.whl | Select-Object Name
-& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install $PSScriptRoot\..\..\..\artifacts\build\$($fulltest.Name)
+& 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' -m pip install --no-deps $PSScriptRoot\..\..\..\artifacts\build\$($fulltest.Name)
 
 & 'C:\Program Files (x86)\Microsoft SDKs\Azure\CLI2\python.exe' $PSScriptRoot\test_msi_package.py

src/azure-cli-core/azure/cli/core/profiles/__init__.py

@@ -5,7 +5,7 @@
 
 #  pylint: disable=unused-import
 from azure.cli.core.profiles._shared import AZURE_API_PROFILES, ResourceType, CustomResourceType, PROFILE_TYPE,\
-    SDKProfile
+    SDKProfile, AD_HOC_API_VERSIONS
 
 
 def get_api_version(cli_ctx, resource_type, as_sdk_profile=False):

src/azure-cli-core/azure/cli/core/profiles/_shared.py

@@ -336,6 +336,19 @@ def default_api_version(self):
 }
 
 
+# We should avoid using ad hoc API versions,
+# use the version in a profile as much as possible.
+AD_HOC_API_VERSIONS = {
+    ResourceType.MGMT_NETWORK: {
+        'vm_default_target_network': '2018-01-01',
+        'nw_connection_monitor': '2019-06-01',
+        'container_network': '2018-08-01',
+        'appservice_network': '2020-04-01',
+        'appservice_ensure_subnet': '2019-02-01'
+    }
+}
+
+
 class _ApiVersions:  # pylint: disable=too-few-public-methods
     def __init__(self, client_type, sdk_profile, post_process):
         self._client_type = client_type

src/azure-cli/azure/cli/command_modules/appservice/access_restrictions.py

@@ -20,7 +20,6 @@
 
 logger = get_logger(__name__)
 
-NETWORK_API_VERSION = '2019-02-01'
 ALLOWED_HTTP_HEADER_NAMES = ['x-forwarded-host', 'x-forwarded-for', 'x-azure-fdid', 'x-fd-healthprobe']
 
 
@@ -164,6 +163,7 @@ def _validate_subnet(cli_ctx, subnet, vnet_name, resource_group_name):
 
 
 def _ensure_subnet_service_endpoint(cli_ctx, subnet_id):
+    from azure.cli.core.profiles import AD_HOC_API_VERSIONS, ResourceType
     subnet_id_parts = parse_resource_id(subnet_id)
     subnet_subscription_id = subnet_id_parts['subscription']
     subnet_resource_group = subnet_id_parts['resource_group']
@@ -175,7 +175,8 @@ def _ensure_subnet_service_endpoint(cli_ctx, subnet_id):
                                  ' Use --ignore-missing-endpoint or -i to'
                                  ' skip validation and manually verify service endpoint.')
 
-    vnet_client = network_client_factory(cli_ctx, api_version=NETWORK_API_VERSION)
+    vnet_client = network_client_factory(cli_ctx, api_version=AD_HOC_API_VERSIONS[ResourceType.MGMT_NETWORK]
+                                         ['appservice_ensure_subnet'])
     subnet_obj = vnet_client.subnets.get(subnet_resource_group, subnet_vnet_name, subnet_name)
     subnet_obj.service_endpoints = subnet_obj.service_endpoints or []
     service_endpoint_exists = False

src/azure-cli/azure/cli/command_modules/appservice/appservice_environment.py

@@ -27,7 +27,6 @@
 
 VERSION_2019_08_01 = "2019-08-01"
 VERSION_2019_10_01 = "2019-10-01"
-VERSION_2020_04_01 = "2020-04-01"
 
 logger = get_logger(__name__)
 
@@ -190,11 +189,12 @@ def _get_resource_client_factory(cli_ctx, api_version=None):
 
 
 def _get_network_client_factory(cli_ctx, api_version=None):
+    from azure.cli.core.profiles import AD_HOC_API_VERSIONS, ResourceType
     client = get_mgmt_service_client(cli_ctx, NetworkManagementClient)
     if api_version:
         client.api_version = api_version
     else:
-        client.api_version = VERSION_2020_04_01
+        client.api_version = AD_HOC_API_VERSIONS[ResourceType.MGMT_NETWORK]['appservice_network']
     return client
 
 

src/azure-cli/azure/cli/command_modules/container/_client_factory.py

@@ -54,4 +54,7 @@ def get_auth_management_client(cli_ctx, scope=None, **_):
 def cf_network(cli_ctx):
     from azure.mgmt.network import NetworkManagementClient
     from azure.cli.core.commands.client_factory import get_mgmt_service_client
-    return get_mgmt_service_client(cli_ctx, NetworkManagementClient, api_version="2018-08-01")
+    from azure.cli.core.profiles import AD_HOC_API_VERSIONS, ResourceType
+    return get_mgmt_service_client(cli_ctx, NetworkManagementClient,
+                                   api_version=AD_HOC_API_VERSIONS[ResourceType.MGMT_NETWORK]
+                                   ['container_network'])

src/azure-cli/azure/cli/command_modules/network/custom.py

@@ -4677,9 +4677,12 @@ def create_nw_connection_monitor(cmd,
                                                               tags,
                                                               do_not_start,
                                                               monitoring_interval)
-        client = get_mgmt_service_client(cmd.cli_ctx,
-                                         ResourceType.MGMT_NETWORK,
-                                         api_version='2019-06-01').connection_monitors
+        from azure.cli.core.profiles._shared import AD_HOC_API_VERSIONS
+        client = get_mgmt_service_client(
+            cmd.cli_ctx,
+            ResourceType.MGMT_NETWORK,
+            api_version=AD_HOC_API_VERSIONS[ResourceType.MGMT_NETWORK]['nw_connection_monitor']
+        ).connection_monitors
     elif any(v2_required_parameter_set):  # V2 creation
         connection_monitor = _create_nw_connection_monitor_v2(cmd,
                                                               location,

src/azure-cli/azure/cli/command_modules/vm/_vm_utils.py

@@ -28,10 +28,10 @@ def get_target_network_api(cli_ctx):
         necessarily latest, network API version is order to avoid having to re-record every test that uses VM create
         (which there are a lot) whenever NRP bumps their API version (which is often)!
     """
-    from azure.cli.core.profiles import get_api_version, ResourceType
+    from azure.cli.core.profiles import get_api_version, ResourceType, AD_HOC_API_VERSIONS
     version = get_api_version(cli_ctx, ResourceType.MGMT_NETWORK)
     if cli_ctx.cloud.profile == 'latest':
-        version = '2018-01-01'
+        version = AD_HOC_API_VERSIONS[ResourceType.MGMT_NETWORK]['vm_default_target_network']
     return version