Properly handle k8s clusters of kind provisioned cluster for connecte…

…dk8s (#7103)

* disabling enable/disable-feature for provisioned clusters

* disable additional unsupported subcommands for provisioned clusters

* update extension version

* update error messages for disabled subcommands for provisioned clsuters

* remove test output

* address PR comments

* address more PR comments

* bump extension version number

src/connectedk8s/HISTORY.rst

@@ -2,6 +2,12 @@
 
 Release History
 ===============
+
+1.6.1
+++++++
+* Added support for reading ARM metadata 2023-11-01.
+* Enable connectedk8s CLI extension to be used for new cluster kind of provisioned clusters.
+
 1.6.0
 ++++++
 

src/connectedk8s/azext_connectedk8s/_client_factory.py

@@ -22,9 +22,9 @@
 
 def cf_connectedk8s(cli_ctx, *_):
     from azext_connectedk8s.vendored_sdks import ConnectedKubernetesClient
-    if os.getenv('AZURE_ACCESS_TOKEN'):
+    if os.getenv(consts.Azure_Access_Token_Variable):
         validate_custom_token()
-        credential = AccessTokenCredential(access_token=os.getenv('AZURE_ACCESS_TOKEN'))
+        credential = AccessTokenCredential(access_token=os.getenv(consts.Azure_Access_Token_Variable))
         return get_mgmt_service_client(cli_ctx, ConnectedKubernetesClient, subscription_id=os.getenv('AZURE_SUBSCRIPTION_ID'), credential=credential)
     return get_mgmt_service_client(cli_ctx, ConnectedKubernetesClient)
 
@@ -35,9 +35,9 @@ def cf_connected_cluster(cli_ctx, _):
 
 def cf_connectedk8s_prev_2022_10_01(cli_ctx, *_):
     from azext_connectedk8s.vendored_sdks.preview_2022_10_01 import ConnectedKubernetesClient
-    if os.getenv('AZURE_ACCESS_TOKEN'):
+    if os.getenv(consts.Azure_Access_Token_Variable):
         validate_custom_token()
-        credential = AccessTokenCredential(access_token=os.getenv('AZURE_ACCESS_TOKEN'))
+        credential = AccessTokenCredential(access_token=os.getenv(consts.Azure_Access_Token_Variable))
         return get_mgmt_service_client(cli_ctx, ConnectedKubernetesClient, subscription_id=os.getenv('AZURE_SUBSCRIPTION_ID'), credential=credential)
     return get_mgmt_service_client(cli_ctx, ConnectedKubernetesClient)
 
@@ -46,10 +46,23 @@ def cf_connected_cluster_prev_2022_10_01(cli_ctx, _):
     return cf_connectedk8s_prev_2022_10_01(cli_ctx).connected_cluster
 
 
+def cf_connectedk8s_prev_2023_11_01(cli_ctx, *_):
+    from azext_connectedk8s.vendored_sdks.preview_2023_11_01 import ConnectedKubernetesClient
+    if os.getenv(consts.Azure_Access_Token_Variable):
+        validate_custom_token()
+        credential = AccessTokenCredential(access_token=os.getenv(consts.Azure_Access_Token_Variable))
+        return get_mgmt_service_client(cli_ctx, ConnectedKubernetesClient, subscription_id=os.getenv('AZURE_SUBSCRIPTION_ID'), credential=credential)
+    return get_mgmt_service_client(cli_ctx, ConnectedKubernetesClient)
+
+
+def cf_connected_cluster_prev_2023_11_01(cli_ctx, _):
+    return cf_connectedk8s_prev_2023_11_01(cli_ctx).connected_cluster
+
+
 def cf_connectedmachine(cli_ctx, subscription_id):
     from azure.mgmt.hybridcompute import HybridComputeManagementClient
-    if os.getenv('AZURE_ACCESS_TOKEN'):
-        credential = AccessTokenCredential(access_token=os.getenv('AZURE_ACCESS_TOKEN'))
+    if os.getenv(consts.Azure_Access_Token_Variable):
+        credential = AccessTokenCredential(access_token=os.getenv(consts.Azure_Access_Token_Variable))
         return get_mgmt_service_client(cli_ctx, HybridComputeManagementClient, subscription_id=subscription_id, credential=credential).private_link_scopes
     return get_mgmt_service_client(cli_ctx, HybridComputeManagementClient, subscription_id=subscription_id).private_link_scopes
 
@@ -60,8 +73,8 @@ def cf_resource_groups(cli_ctx, subscription_id=None):
 
 def _resource_client_factory(cli_ctx, subscription_id=None):
     from azure.mgmt.resource import ResourceManagementClient
-    if os.getenv('AZURE_ACCESS_TOKEN'):
-        credential = AccessTokenCredential(access_token=os.getenv('AZURE_ACCESS_TOKEN'))
+    if os.getenv(consts.Azure_Access_Token_Variable):
+        credential = AccessTokenCredential(access_token=os.getenv(consts.Azure_Access_Token_Variable))
         return get_mgmt_service_client(cli_ctx, ResourceType.MGMT_RESOURCE_RESOURCES, subscription_id=subscription_id, credential=credential)
     return get_mgmt_service_client(cli_ctx, ResourceType.MGMT_RESOURCE_RESOURCES, subscription_id=subscription_id)
 
@@ -75,8 +88,8 @@ def resource_providers_client(cli_ctx, subscription_id=None):
 
 
 def _graph_client_factory(cli_ctx, **_):
-    if os.getenv('AZURE_ACCESS_TOKEN'):
-        credential = AccessTokenCredential(access_token=os.getenv('AZURE_ACCESS_TOKEN'))
+    if os.getenv(consts.Azure_Access_Token_Variable):
+        credential = AccessTokenCredential(access_token=os.getenv(consts.Azure_Access_Token_Variable))
         client = GraphRbacManagementClient(credential, os.getenv('AZURE_TENANT_ID'),
                                            base_url=cli_ctx.cloud.endpoints.active_directory_graph_resource_id)
     else:

src/connectedk8s/azext_connectedk8s/_constants.py

@@ -235,3 +235,5 @@
 HELM_STORAGE_URL = "https://k8connecthelm.azureedge.net"
 HELM_VERSION = 'v3.12.2'
 Download_And_Install_Kubectl_Fault_Type = "Failed to download and install kubectl"
+Azure_Access_Token_Variable = 'AZURE_ACCESS_TOKEN'
+Provisioned_Cluster_Kind = 'provisionedcluster'

src/connectedk8s/azext_connectedk8s/custom.py

@@ -44,7 +44,7 @@
 from azext_connectedk8s._client_factory import cf_resource_groups
 from azext_connectedk8s._client_factory import resource_providers_client
 from azext_connectedk8s._client_factory import get_graph_client_service_principals
-from azext_connectedk8s._client_factory import cf_connected_cluster_prev_2022_10_01
+from azext_connectedk8s._client_factory import cf_connected_cluster_prev_2022_10_01, cf_connected_cluster_prev_2023_11_01
 from azext_connectedk8s._client_factory import cf_connectedmachine
 import azext_connectedk8s._constants as consts
 import azext_connectedk8s._utils as utils
@@ -132,7 +132,7 @@ def create_connectedk8s(cmd, client, resource_group_name, cluster_name, correlat
 
     # Set preview client if latest preview properties are provided.
     if enable_private_link is not None or distribution_version is not None or azure_hybrid_benefit is not None:
-        client = cf_connected_cluster_prev_2022_10_01(cmd.cli_ctx, None)
+        client = cf_connected_cluster_prev_2023_11_01(cmd.cli_ctx, None)
 
     # Checking whether optional extra values file has been provided.
     values_file = utils.get_values_file()
@@ -315,14 +315,23 @@ def create_connectedk8s(cmd, client, resource_group_name, cluster_name, correlat
         configmap_rg_name = configmap.data["AZURE_RESOURCE_GROUP"]
         configmap_cluster_name = configmap.data["AZURE_RESOURCE_NAME"]
         if connected_cluster_exists(client, configmap_rg_name, configmap_cluster_name):
+            preview_cluster_resource = None
+            public_key = None
+
+            try:
+                preview_cluster_resource = get_connectedk8s_2023_11_01(cmd, configmap_rg_name,
+                                        configmap_cluster_name)
+                public_key = preview_cluster_resource.agent_public_key_certificate
+            except Exception as e:  # pylint: disable=broad-except
+                utils.arm_exception_handler(e, consts.Get_ConnectedCluster_Fault_Type, 'Failed to check if connected cluster resource already exists.')
+
             if (configmap_rg_name.lower() == resource_group_name.lower() and
                     configmap_cluster_name.lower() == cluster_name.lower()):
                 # Re-put connected cluster
-                try:
-                    public_key = client.get(configmap_rg_name,
-                                            configmap_cluster_name).agent_public_key_certificate
-                except Exception as e:  # pylint: disable=broad-except
-                    utils.arm_exception_handler(e, consts.Get_ConnectedCluster_Fault_Type, 'Failed to check if connected cluster resource already exists.')
+
+                # If cluster is of kind provisioned cluster, there are several properties that cannot be updated
+                validate_existing_provisioned_cluster_for_reput(preview_cluster_resource, kubernetes_distro, kubernetes_infra, enable_private_link, private_link_scope_resource_id, distribution_version, azure_hybrid_benefit, location)    
+
                 cc = generate_request_payload(location, public_key, tags, kubernetes_distro, kubernetes_infra, enable_private_link, private_link_scope_resource_id, distribution_version, azure_hybrid_benefit)
                 cc_response = create_cc_resource(client, resource_group_name, cluster_name, cc, no_wait)
                 cc_response = LongRunningOperation(cmd.cli_ctx)(cc_response)
@@ -418,6 +427,25 @@ def create_connectedk8s(cmd, client, resource_group_name, cluster_name, correlat
                                custom_locations_oid, helm_client_location, enable_private_link, arm_metadata, onboarding_timeout, container_log_path)
     return put_cc_response
 
+def validate_existing_provisioned_cluster_for_reput(cluster_resource, kubernetes_distro, kubernetes_infra, enable_private_link, private_link_scope_resource_id, distribution_version, azure_hybrid_benefit, location):
+    if (cluster_resource != None) and (cluster_resource.kind.lower() == consts.Provisioned_Cluster_Kind):
+        if azure_hybrid_benefit is not None:
+            raise InvalidArgumentValueError("Updating the 'azure hybrid benefit' property of a Provisioned Cluster is not supported from the Connected Cluster CLI. Please use the 'az aksarc update' CLI command.\nhttps://learn.microsoft.com/en-us/cli/azure/aksarc?view=azure-cli-latest#az-aksarc-update")
+
+        validation_values = [
+            kubernetes_distro,
+            kubernetes_infra,
+            converted_priv_link_value,
+            private_link_scope_resource_id,
+            distribution_version,
+            azure_hybrid_benefit,
+            location,
+        ]
+
+        for value in validation_values:
+            if value is not None:
+                raise InvalidArgumentValueError("Updating the following properties of a Provisioned Cluster are not supported from the Connected Cluster CLI: kubernetes_distro, kubernetes_infra, enable_private_link, private_link_scope_resource_id, distribution_version, azure_hybrid_benefit, location, public_key.\n\nPlease use the 'az aksarc update' CLI command. https://learn.microsoft.com/en-us/cli/azure/aksarc?view=azure-cli-latest#az-aksarc-update")
+
 
 def send_cloud_telemetry(cmd):
     telemetry.add_extension_event('connectedk8s', {'Context.Default.AzureCLI.AzureCloud': cmd.cli_ctx.cloud.name})
@@ -832,14 +860,20 @@ def get_server_address(kube_config, kube_context):
 
 
 def get_connectedk8s(cmd, client, resource_group_name, cluster_name):
-    # Override preview client to show private link properties to customers
-    client = cf_connected_cluster_prev_2022_10_01(cmd.cli_ctx, None)
+    # Override preview client to show private link properties and cluster kind to customers
+    client = cf_connected_cluster_prev_2023_11_01(cmd.cli_ctx, None)
+    return client.get(resource_group_name, cluster_name)
+
+
+def get_connectedk8s_2023_11_01(cmd, resource_group_name, cluster_name):
+    # Override preview client to show private link properties and cluster kind to customers
+    client = cf_connected_cluster_prev_2023_11_01(cmd.cli_ctx, None)
     return client.get(resource_group_name, cluster_name)
 
 
 def list_connectedk8s(cmd, client, resource_group_name=None):
-    # Override preview client to show private link properties to customers
-    client = cf_connected_cluster_prev_2022_10_01(cmd.cli_ctx, None)
+    # Override preview client to show private link properties and cluster kind to customers
+    client = cf_connected_cluster_prev_2023_11_01(cmd.cli_ctx, None)
     if not resource_group_name:
         return client.list_by_subscription()
     return client.list_by_resource_group(resource_group_name)
@@ -859,6 +893,11 @@ def delete_connectedk8s(cmd, client, resource_group_name, cluster_name,
 
     logger.warning("This operation might take a while ...\n")
 
+    # Check if the cluster is of supported type for deletion
+    preview_cluster_resource = get_connectedk8s_2023_11_01(cmd, resource_group_name, cluster_name)
+    if (preview_cluster_resource != None) and (preview_cluster_resource.kind.lower() == consts.Provisioned_Cluster_Kind):
+        raise InvalidArgumentValueError("Deleting a Provisioned Cluster is not supported from the Connected Cluster CLI. Please use the 'az aksarc delete' CLI command.\nhttps://learn.microsoft.com/en-us/cli/azure/aksarc?view=azure-cli-latest#az-aksarc-delete")
+
     # Send cloud information to telemetry
     send_cloud_telemetry(cmd)
 
@@ -1006,6 +1045,12 @@ def update_connected_cluster(cmd, client, resource_group_name, cluster_name, htt
 
     proxy_cert = proxy_cert.replace('\\', r'\\\\')
 
+    # Fetch Connected Cluster for agent version
+    connected_cluster = get_connectedk8s_2023_11_01(cmd, resource_group_name, cluster_name)
+
+    if (connected_cluster != None) and (connected_cluster.kind.lower() == consts.Provisioned_Cluster_Kind):
+        raise InvalidArgumentValueError("Updating a Provisioned Cluster is not supported from the Connected Cluster CLI. Please use the 'az aksarc update' CLI command. https://learn.microsoft.com/en-us/cli/azure/aksarc?view=azure-cli-latest#az-aksarc-update")
+
     # Set preview client as most of the patchable fields are available in preview api-version
     client = cf_connected_cluster_prev_2022_10_01(cmd.cli_ctx, None)
 
@@ -1149,6 +1194,12 @@ def update_connected_cluster(cmd, client, resource_group_name, cluster_name, htt
 
 
 def upgrade_agents(cmd, client, resource_group_name, cluster_name, kube_config=None, kube_context=None, arc_agent_version=None, upgrade_timeout="600"):
+    # Check if cluster supports upgrading
+    connected_cluster = get_connectedk8s_2023_11_01(cmd, resource_group_name, cluster_name)
+
+    if (connected_cluster != None) and (connected_cluster.kind.lower() == consts.Provisioned_Cluster_Kind):
+        raise InvalidArgumentValueError("Upgrading a Provisioned Cluster is not supported from the Connected Cluster CLI. Please use the 'az aksarc upgrade' CLI command. https://learn.microsoft.com/en-us/cli/azure/aksarc?view=azure-cli-latest#az-aksarc-upgrade")
+
     logger.warning("This operation might take a while...\n")
 
     # Send cloud information to telemetry
@@ -1395,7 +1446,11 @@ def enable_features(cmd, client, resource_group_name, cluster_name, features, ku
     enable_cluster_connect, enable_azure_rbac, enable_cl = utils.check_features_to_update(features)
 
     # Check if cluster is private link enabled
-    connected_cluster = get_connectedk8s(cmd, client, resource_group_name, cluster_name)
+    connected_cluster = get_connectedk8s_2023_11_01(cmd, resource_group_name, cluster_name)
+
+    if (connected_cluster != None) and (connected_cluster.kind.lower() == consts.Provisioned_Cluster_Kind):
+        raise InvalidArgumentValueError("Enable feature of a Provisioned Cluster is not supported from the Connected Cluster CLI. For information on how to enable a feature on a Provisioned Cluster using a cluster extension, please refer to: https://learn.microsoft.com/en-us/azure/aks/deploy-extensions-az-cli")
+
     if connected_cluster.private_link_state.lower() == "enabled" and (enable_cluster_connect or enable_cl):
         telemetry.set_exception(exception='Invalid arguments provided', fault_type=consts.Invalid_Argument_Fault_Type,
                                 summary='Invalid arguments provided')
@@ -1520,6 +1575,12 @@ def disable_features(cmd, client, resource_group_name, cluster_name, features, k
     confirmation_message = "Disabling few of the features may adversely impact dependent resources. Learn more about this at https://aka.ms/ArcK8sDependentResources. \n" + "Are you sure you want to disable these features: {}".format(features)
     utils.user_confirmation(confirmation_message, yes)
 
+    # Fetch Connected Cluster for agent version
+    connected_cluster = get_connectedk8s_2023_11_01(cmd, resource_group_name, cluster_name)
+
+    if (connected_cluster != None) and (connected_cluster.kind.lower() == consts.Provisioned_Cluster_Kind):
+        raise InvalidArgumentValueError("Disable feature of a Provisioned Cluster is not supported from the Connected Cluster CLI. For information on how to disable a feature on a Provisioned Cluster using a cluster extension, please refer to: https://learn.microsoft.com/en-us/azure/aks/deploy-extensions-az-cli")
+
     logger.warning("This operation might take a while...\n")
 
     disable_cluster_connect, disable_azure_rbac, disable_cl = utils.check_features_to_update(features)
@@ -1548,9 +1609,6 @@ def disable_features(cmd, client, resource_group_name, cluster_name, features, k
 
     release_namespace = validate_release_namespace(client, cluster_name, resource_group_name, kube_config, kube_context, helm_client_location)
 
-    # Fetch Connected Cluster for agent version
-    connected_cluster = get_connectedk8s(cmd, client, resource_group_name, cluster_name)
-
     kubernetes_properties = {'Context.Default.AzureCLI.KubernetesVersion': kubernetes_version}
 
     if hasattr(connected_cluster, 'distribution') and (connected_cluster.distribution is not None):

src/connectedk8s/azext_connectedk8s/vendored_sdks/preview_2023_11_01/__init__.py

@@ -0,0 +1,24 @@
+# coding=utf-8
+# --------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# Code generated by Microsoft (R) AutoRest Code Generator.
+# Changes may cause incorrect behavior and will be lost if the code is regenerated.
+# --------------------------------------------------------------------------
+
+from ._connected_kubernetes_client import ConnectedKubernetesClient
+from ._version import VERSION
+
+__version__ = VERSION
+
+try:
+    from ._patch import __all__ as _patch_all
+    from ._patch import *  # type: ignore # pylint: disable=unused-wildcard-import
+except ImportError:
+    _patch_all = []
+from ._patch import patch_sdk as _patch_sdk
+
+__all__ = ["ConnectedKubernetesClient"]
+__all__.extend([p for p in _patch_all if p not in __all__])
+
+_patch_sdk()