Misc updates for 2.3.2 release: (#644)

* Misc updates for 2.3.2 release:

- adding bs4 to mocks for sphinx in conf.py
- updating min scikit-learn version to avoid numpy aliased types removal
- fix to azure_auth_core.py - if clientsecret type is specified it will only get added to chained creds if clientid and clientsecret params are passed
- fixing lookup_ioc so that it supports use of "observable" kw param as well as "ioc" (observable is what used to be the param name, so this ensures better backward compat
- adding some missing API docs

* re-enabling inter-sphinx in conf.py

* Added logging to azure_auth_core.

Fixed pylint warning in figure_dimension.py

* Fix bug when connect=True specified in sentinel_core.py

Added better handling of input params (like tactics) and ensuring that non-json encodable types (like np.ndarray) are converted to compatible Python types.
Adding logging to sentinel_dynamic_summary, sentinel_dynamic_summary_types
Fixing test cases for previously incorrect "sourceInfo" attribute.
Some isort fixes in unrelated modules - process_tree, timeline_common

* Fixing logic of azure_auth_core.py to allow for cred_types that can't initialize.

- Added more informative exception
Updating keyvault_client.py to use modern credentials
Added utility function to save a variable to a notebook cell + magic equivalent in ipython.py and nbmagics.py

* Added bandit suppression for use of pickle

conda/conda-reqs.txt

@@ -35,7 +35,7 @@ pyjwt>=2.3.0
 python-dateutil>=2.8.1
 pytz>=2019.2
 pyyaml>=3.13
-scikit-learn>=0.20.2
+scikit-learn>=1.0.0
 scipy
 setuptools>=40.6.3
 statsmodels

docs/source/api/msticpy.context.tiproviders.pulsedive.rst

@@ -0,0 +1,7 @@
+msticpy.context.tiproviders.pulsedive module
+============================================
+
+.. automodule:: msticpy.context.tiproviders.pulsedive
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/api/msticpy.context.tiproviders.rst

@@ -20,6 +20,7 @@ Submodules
    msticpy.context.tiproviders.kql_base
    msticpy.context.tiproviders.mblookup
    msticpy.context.tiproviders.open_page_rank
+   msticpy.context.tiproviders.pulsedive
    msticpy.context.tiproviders.result_severity
    msticpy.context.tiproviders.riskiq
    msticpy.context.tiproviders.ti_http_provider

docs/source/api/msticpy.init.logging.rst

@@ -0,0 +1,7 @@
+msticpy.init.logging module
+===========================
+
+.. automodule:: msticpy.init.logging
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/api/msticpy.init.rst

@@ -23,6 +23,7 @@ Submodules
 
    msticpy.init.azure_ml_tools
    msticpy.init.azure_synapse_tools
+   msticpy.init.logging
    msticpy.init.mp_pandas_accessors
    msticpy.init.nbinit
    msticpy.init.nbmagics

docs/source/api/msticpy.vis.figure_dimension.rst

@@ -0,0 +1,7 @@
+msticpy.vis.figure\_dimension module
+====================================
+
+.. automodule:: msticpy.vis.figure_dimension
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/api/msticpy.vis.rst

@@ -15,6 +15,7 @@ Submodules
    msticpy.vis.code_view
    msticpy.vis.data_viewer
    msticpy.vis.entity_graph_tools
+   msticpy.vis.figure_dimension
    msticpy.vis.foliummap
    msticpy.vis.matrix_plot
    msticpy.vis.mordor_browser

docs/source/conf.py

@@ -238,6 +238,7 @@
     "azure.storage.blob",
     "azure.storage",
     "bokeh",
+    "bs4",
     "dnspython",
     "dns",
     "folium",

msticpy/auth/azure_auth_core.py

@@ -14,7 +14,6 @@
 from typing import List, Optional, Tuple, Union
 
 from azure.common.credentials import get_cli_profile
-from azure.common.exceptions import CloudError
 from azure.identity import (
     AzureCliCredential,
     AzurePowerShellCredential,
@@ -39,9 +38,15 @@
 __version__ = VERSION
 __author__ = "Pete Bryan"
 
+logger = logging.getLogger(__name__)
 
 AzCredentials = namedtuple("AzCredentials", ["legacy", "modern"])
 
+_HELP_URI = (
+    "https://msticpy.readthedocs.io/en/latest/"
+    "getting_started/AzureAuthentication.html"
+)
+
 
 # pylint: disable=too-few-public-methods
 class AzureCredEnvNames:
@@ -64,6 +69,7 @@ def _build_env_client(
         and AzureCredEnvNames.AZURE_CLIENT_SECRET not in os.environ
     ):
         # avoid creating env credential if require envs not set.
+        logger.info("'env' credential requested but required env vars not set")
         return None
     return EnvironmentCredential(authority=aad_uri)  # type: ignore
 
@@ -113,12 +119,13 @@ def _build_device_code_client(
 
 def _build_client_secret_client(
     tenant_id: Optional[str] = None, aad_uri: Optional[str] = None, **kwargs
-) -> ClientSecretCredential:
+) -> Optional[ClientSecretCredential]:
     """Build a credential from Client Secret."""
     client_id = kwargs.pop("client_id", None)
     client_secret = kwargs.pop("client_secret", None)
     if not client_secret or not client_id:
-        raise MsticpyAzureConfigError("Client secret and client id are required.")
+        logger.info("'clientsecret' credential requested but no params supplied")
+        return None
     return ClientSecretCredential(
         authority=aad_uri,
         tenant_id=tenant_id,  # type: ignore
@@ -130,11 +137,14 @@ def _build_client_secret_client(
 
 def _build_certificate_client(
     tenant_id: Optional[str] = None, aad_uri: Optional[str] = None, **kwargs
-) -> CertificateCredential:
+) -> Optional[CertificateCredential]:
     """Build a credential from Certificate."""
     client_id = kwargs.pop("client_id", None)
     if not client_id:
-        raise MsticpyAzureConfigError("Client id required.")
+        logger.info(
+            "'certificate' credential requested but client_id param not supplied"
+        )
+        return None
     return CertificateCredential(
         authority=aad_uri, tenant_id=tenant_id, client_id=client_id, **kwargs  # type: ignore
     )
@@ -213,7 +223,7 @@ def _az_connect_core(
 
     Raises
     ------
-    CloudError
+    MsticpyAzureConfigError
         If chained token credential creation fails.
 
     See Also
@@ -228,6 +238,7 @@ def _az_connect_core(
     - Azure CLI (if an active session is logged on)
     - Managed Service Identity
     - Interactive browser logon
+    - and others - see list_auth_methods for full list.
     If the authentication is successful both ADAL (legacy) and
     MSAL (modern) credential types are returned.
 
@@ -236,7 +247,15 @@ def _az_connect_core(
     cloud = cloud or kwargs.pop("region", AzureCloudConfig().cloud)
     az_config = AzureCloudConfig(cloud)
     aad_uri = az_config.endpoints.active_directory
-    tenant_id = tenant_id or AzureCloudConfig().tenant_id
+    logger.info("az_connect_core - using %s cloud and endpoint: %s", cloud, aad_uri)
+
+    tenant_id = tenant_id or az_config.tenant_id
+    auth_methods = auth_methods or az_config.auth_methods
+    logger.info(
+        "TenantId:  %s, requested auth methods: %s",
+        tenant_id,
+        ", ".join(auth_methods or ["none"]),
+    )
     creds = kwargs.pop("credential", None)
     if not creds:
         creds = _build_chained_creds(
@@ -247,23 +266,33 @@ def _az_connect_core(
         )
 
     # Filter and replace error message when credentials not found
+    azure_identity_logger = logging.getLogger("azure.identity")
     handler = logging.StreamHandler(sys.stdout)
     if silent:
         handler.addFilter(_filter_all_warnings)
     else:
         handler.addFilter(_filter_credential_warning)
-    logging.basicConfig(level=logging.WARNING, handlers=[handler])
+    azure_identity_logger.setLevel(logging.WARNING)
+    azure_identity_logger.handlers = [handler]
 
     # Connect to the subscription client to validate
     legacy_creds = CredentialWrapper(
         creds, resource_id=AzureCloudConfig(cloud).token_uri
     )
     if not creds:
-        raise CloudError("Could not obtain credentials.")
+        raise MsticpyAzureConfigError(
+            "Cannot authenticate with specified credential types.",
+            "At least one valid authentication method required.",
+            help_uri=_HELP_URI,
+            title="Authentication failure",
+        )
 
     return AzCredentials(legacy_creds, creds)
 
 
+az_connect_core = _az_connect_core
+
+
 def _build_chained_creds(
     aad_uri,
     requested_clients: Union[List[str], None] = None,
@@ -289,72 +318,57 @@ def _build_chained_creds(
 
     Raises
     ------
-    CloudError
+    MsticpyAzureConfigError
         If the chained credential creation fails.
 
     """
     # Create the chained credential
     if not requested_clients:
         requested_clients = ["env", "cli", "msi", "interactive"]
-    clients = [
-        _CLIENTS[client](tenant_id=tenant_id, aad_uri=aad_uri, **kwargs)  # type: ignore
-        for client in requested_clients
-    ]
-    if not clients:
-        raise MsticpyAzureConfigError(
-            "At least one valid authentication method required."
+        logger.info("No auth methods requested defaulting to: %s", requested_clients)
+    cred_list = []
+    invalid_cred_types: List[str] = []
+    unusable_cred_type: List[str] = []
+    for cred_type in requested_clients:  # type: ignore[union-attr]
+        if cred_type not in _CLIENTS:
+            invalid_cred_types.append(cred_type)
+            logger.info("Unknown authentication type requested: %s", cred_type)
+            continue
+        cred_client = _CLIENTS[cred_type](  # type: ignore[operator]
+            tenant_id=tenant_id, aad_uri=aad_uri, **kwargs
         )
-    return ChainedTokenCredential(*clients)  # type: ignore
-
-
-class _AzCachedConnect:
-    """Singleton class caching Azure credentials."""
-
-    _instance = None
-
-    def __new__(cls):
-        """Override new to check and return existing instance."""
-        if cls._instance is None:
-            cls._instance = super(_AzCachedConnect, cls).__new__(cls)
-            cls.connect.__doc__ = _az_connect_core.__doc__
-        return cls._instance
-
-    def __init__(self):
-        """Initialize the class."""
-        self.az_credentials: Optional[AzCredentials] = None
-        self.cred_cloud: str = self.current_cloud
-
-    @property
-    def current_cloud(self) -> str:
-        """Return current cloud."""
-        return AzureCloudConfig().cloud
-
-    def connect(self, *args, **kwargs):
-        """Call az_connect_core if token is not present or expired."""
-        if self.az_credentials is None:
-            self.az_credentials = _az_connect_core(*args, **kwargs)
-            return self.az_credentials
-        # Check expiry
-        if (
-            datetime.utcfromtimestamp(
-                self.az_credentials.modern.get_token(
-                    AzureCloudConfig().token_uri
-                ).expires_on
+        if cred_client is not None:
+            cred_list.append(cred_client)
+        else:
+            unusable_cred_type.append(cred_type)
+    logger.info(
+        "Cred types added to chained credential: %s",
+        ", ".join(cred.__class__.__name__ for cred in cred_list if cred is not None),
+    )
+    if not cred_list:
+        exception_args = [
+            "Cannot authenticate - no valid credential types.",
+            "At least one valid authentication method required.",
+            f"Configured auth_types: {','.join(requested_clients)}",
+        ]
+        if invalid_cred_types:
+            exception_args.append(
+                f"Unrecognized auth_types: {','.join(invalid_cred_types)}",
             )
-            <= datetime.utcnow()
-        ):
-            self.az_credentials = _az_connect_core(*args, **kwargs)
-        # Check changed cloud
-        if self.cred_cloud != kwargs.get(
-            "cloud", kwargs.get("region", self.current_cloud)
-        ):
-            self.az_credentials = _az_connect_core(*args, **kwargs)
-        return self.az_credentials
-
-
-# externally callable function using the class above
-# _AZ_CACHED_CONNECT = _AzCachedConnect()
-az_connect_core = _az_connect_core
+        if unusable_cred_type:
+            exception_args.extend(
+                [
+                    "The following auth types could not be used due to",
+                    "missing parameters or environment variables:",
+                    ",".join(invalid_cred_types),
+                ]
+            )
+        raise MsticpyAzureConfigError(
+            *exception_args,
+            help_uri=_HELP_URI,
+            title="Authentication failure",
+        )
+    return ChainedTokenCredential(*cred_list)
 
 
 def only_interactive_cred(chained_cred: ChainedTokenCredential):
@@ -423,7 +437,7 @@ def check_cli_credentials() -> Tuple[AzureCliStatus, Optional[str]]:
     except ImportError:
         # Azure CLI not installed
         return AzureCliStatus.CLI_NOT_INSTALLED, None
-    except Exception as ex:  # pylint: disable=broad-except
+    except Exception as ex:  # pylint: disable=broad-except, broad-exception-caught
         if "AADSTS70043: The refresh token has expired" in str(ex):
             message = (
                 "Azure CLI was detected but the token has expired. "

msticpy/auth/keyvault_client.py

@@ -307,12 +307,10 @@ def set_secret(self, secret_name: str, value: Any) -> KeyVaultSecret:
         return self.kv_client.set_secret(name=secret_name, value=value)
 
 
-# pylint: disable=too-many-instance-attributes
 @export
 class BHKeyVaultMgmtClient:
     """Core KeyVault Management client."""
 
-    # pylint: disable=too-many-arguments
     def __init__(
         self,
         tenant_id: str = None,
@@ -377,8 +375,6 @@ def __init__(
         self.resource_group = resource_group or self.settings.get("resourcegroup")
         self.azure_region = azure_region or self.settings.get("azureregion")
 
-    # pylint: enable=too-many-arguments
-
     def list_vaults(self) -> List[str]:
         """
         Return a list of vaults for the subscription.
@@ -389,7 +385,9 @@ def list_vaults(self) -> List[str]:
             Vault names
 
         """
-        mgmt = KeyVaultManagementClient(self.auth_client.legacy, self.subscription_id)
+        mgmt = KeyVaultManagementClient(self.auth_client.modern, self.subscription_id)
+        # vaults.list does not require api_version or filter parameters
+        # pylint: disable=no-value-for-parameter
         return [v.name for v in mgmt.vaults.list()]
 
     def get_vault_uri(self, vault_name: str) -> str:
@@ -407,7 +405,7 @@ def get_vault_uri(self, vault_name: str) -> str:
             Vault URI.
 
         """
-        mgmt = KeyVaultManagementClient(self.auth_client.legacy, self.subscription_id)
+        mgmt = KeyVaultManagementClient(self.auth_client.modern, self.subscription_id)
         try:
             vault = mgmt.vaults.get(self.resource_group, vault_name)
         except (CloudError, ResourceNotFoundError) as cloud_err:
@@ -447,7 +445,7 @@ def create_vault(self, vault_name: str) -> Vault:
                 "Please add ResourceGroup to the KeyVault section of msticpyconfig.yaml",
                 title="missing ResourceGroup value.",
             )
-        mgmt = KeyVaultManagementClient(self.auth_client.legacy, self.subscription_id)
+        mgmt = KeyVaultManagementClient(self.auth_client.modern, self.subscription_id)
         return mgmt.vaults.create_or_update(
             self.resource_group, vault_name, parameters
         ).result()
@@ -481,9 +479,6 @@ def _get_params(self):
         return parameters
 
 
-# pylint: enable=too-many-instance-attributes
-
-
 def _user_oid(token) -> str:
     """
     Return the user Object ID.