Update methodology of getting endpoints for cloud environment (#704)

* Update methodology of getting endpoints for cloud environment

Will now use arm to get endpoints for each environment
Allow user to get endpoints by listing the arm url, useful if cloud is not global, cn, or usgov
Update documentation
Deprecate de/Germany since the cloud is also deprecated
Create offline endpoints for global, cn and usgov for mocking
Remove hardcoded urls that may touch only global endpoints

* Update cache to lru_cache for backwards compat, update keyvault_settings __init__

* update to use httpx

* use lru_cache instead of cache

* update kql driver to get az config first, update tests that use AzureCloudConfig

* updates based on warnings, update tests

* Updates

Update get_subscription_metadata within ce_common

this will first retrieve a 401 error to get the tenant id from the response, then will try the request again by getting an access token. This way we can get all of the metadata for the subscription

fix typing error for get_managed_tenant_id within ce_common

Add tenant_id to azure_auth

add tenant_id to keyvault_settings

* Test and linting fixes

* Force cloud mapper offline for unit-tests

* add try except for get_subscription_metadata

* Fix test and mypy failures

---------

Co-authored-by: Christopher Cianelli <ccianelli@microsoft.com>
Co-authored-by: Ian Hellen <ianhelle@microsoft.com>

docs/source/getting_started/SettingsEditor.rst

@@ -598,12 +598,15 @@ to the Azure global cloud.
 The Azure clouds supported are:
 
 - **cn** - China
-- **de** - Germany
 - **usgov** - US Government
 
+de - Germany has been deprecated and is no longer supported.
+
 Configuring MSTICPy to use one of these clouds will cause the following
 components to use the Authority and API endpoint URLs specific to that cloud.
 
+The ``resource_manager_url`` setting allows you to specify the Azure Resource Manager Url to use. This is only needed if you are using a cloud outside of global, usgov, and cn. This will override the cloud and its associated Authority and API endpoint URLs.
+
 These components include:
 
 - Microsoft Sentinel data provider
@@ -946,7 +949,7 @@ and other providers loaded in order to find the pivot functions that it
 will attach to entities. For more information see `pivot
 functions <https://msticpy.readthedocs.io/en/latest/data_analysis/PivotFunctions.html>`__
 
-Some components do not require any parameters (e.g. TILookup and Pivot).
+Some components do not require any parameters (e.g. TILookup and Pivot).
 Others do support or require additional settings:
 
 **GeoIpLookup**

docs/source/getting_started/msticpyconfig.rst

@@ -241,6 +241,10 @@ Possible credential types (``auth_methods``) are:
    credentials will fail. We have found Azure CLI to be reliable
    and maintains authentication tokens between notebook sessions.
 
+The ``resource_manager_url`` setting allows you to specify the Azure Resource Manager Url to use. This is only needed if you are using a cloud outside of global, usgov, cn, and de. Example: https://management.azure.com
+
+.. warning:: Setting resource_manager_url  will overwrite the cloud setting. For example, if you set the cloud to be global and then set the resource_manager_url to be https://management.usgovcloudapi.net then the cloud will utilize the usgov endpoints which maybe incorrect for your needs.
+
 .. code:: yaml
 
     Azure:

msticpy/_version.py

@@ -1,2 +1,2 @@
 """Version file."""
-VERSION = "2.7.0"
+VERSION = "2.7.0.pre1"

msticpy/auth/azure_auth.py

@@ -82,6 +82,7 @@ def az_connect(
     # Use auth_methods param or configuration defaults
     data_provs = get_provider_settings(config_section="DataProviders")
     auth_methods = auth_methods or az_cloud_config.auth_methods
+    tenant_id = tenant_id or az_cloud_config.tenant_id
 
     # Ignore AzCLI settings except for authentication creds for EnvCred
     az_cli_config = data_provs.get("AzureCLI")
@@ -105,7 +106,7 @@ def az_connect(
     )
     sub_client = SubscriptionClient(
         credential=credentials.modern,
-        base_url=az_cloud_config.endpoints.resource_manager,  # type: ignore
+        base_url=az_cloud_config.resource_manager,  # type: ignore
         credential_scopes=[az_cloud_config.token_uri],
     )
     if not sub_client:
@@ -169,12 +170,10 @@ def fallback_devicecode_creds(
     """
     cloud = cloud or kwargs.pop("region", AzureCloudConfig().cloud)
     az_config = AzureCloudConfig(cloud)
-    aad_uri = az_config.endpoints.active_directory
-    tenant_id = tenant_id or AzureCloudConfig().tenant_id
+    aad_uri = az_config.authority_uri
+    tenant_id = tenant_id or az_config.tenant_id
     creds = DeviceCodeCredential(authority=aad_uri, tenant_id=tenant_id)
-    legacy_creds = CredentialWrapper(
-        creds, resource_id=AzureCloudConfig(cloud).token_uri
-    )
+    legacy_creds = CredentialWrapper(creds, resource_id=az_config.token_uri)
     if not creds:
         raise CloudError("Could not obtain credentials.")
 

msticpy/auth/azure_auth_core.py

@@ -146,7 +146,10 @@ def _build_certificate_client(
         )
         return None
     return CertificateCredential(
-        authority=aad_uri, tenant_id=tenant_id, client_id=client_id, **kwargs  # type: ignore
+        authority=aad_uri,
+        tenant_id=tenant_id,  # type: ignore
+        client_id=client_id,
+        **kwargs,  # type: ignore
     )
 
 
@@ -246,7 +249,7 @@ def _az_connect_core(
     # Create the auth methods with the specified cloud region
     cloud = cloud or kwargs.pop("region", AzureCloudConfig().cloud)
     az_config = AzureCloudConfig(cloud)
-    aad_uri = az_config.endpoints.active_directory
+    aad_uri = az_config.authority_uri
     logger.info("az_connect_core - using %s cloud and endpoint: %s", cloud, aad_uri)
 
     tenant_id = tenant_id or az_config.tenant_id
@@ -276,9 +279,7 @@ def _az_connect_core(
     azure_identity_logger.handlers = [handler]
 
     # Connect to the subscription client to validate
-    legacy_creds = CredentialWrapper(
-        creds, resource_id=AzureCloudConfig(cloud).token_uri
-    )
+    legacy_creds = CredentialWrapper(creds, resource_id=az_config.token_uri)
     if not creds:
         raise MsticpyAzureConfigError(
             "Cannot authenticate with specified credential types.",