Removing azure-mgmt (#42)

* Removing azure-mgmt and adding azure-mgmt-subscription and azure-mgmt-resource

* GeoLite2  url and archive extraction changes

* typo fix for printing mising packages

* replace dummy key for Maxmind geolitelookup

* black formatting

* Cleaned redundent elements

* Changed ti_provider_settings to generic provider_settings module

Added config support for GeoIP providers
GeoIP classes try to obtain API key from config if not supplied
Prevented GeoLiteLookup from instantiating on package load
Added missing pytz and pyyaml packages to requirements.txt and setup.py.

Note: syslog_utils test still failing due to missing geoip database. Will fix by adding secret to piplelines.

* Added config options for geoip.py classes. Will try to read config by default.

Refactored pkg_config and provider_settings to consolidate global functionality in the former.
Fixed tests to copy with missing GeoIP database
Added msticpyconfig-test.yaml for build-time tests (requires env vars to be set)

* Fxing some linting errors

Co-authored-by: Ashwin Patil <ashwin-patil@users.noreply.github.com>
Co-authored-by: Pete Bryan <peter.bryan@microsoft.com>

msticpy/_version.py

@@ -1,2 +1,2 @@
 """Version file."""
-VERSION = "0.2.8"
+VERSION = "0.3.0"

msticpy/data/drivers/odata_driver.py

@@ -136,7 +136,7 @@ def connect(self, connection_str: str = None, **kwargs):
         return json_response
 
     # pylint: disable=too-many-branches
-    def query_with_results(
+    def query_with_results(  # noqa: MC0001
         self, query: str, **kwargs
     ) -> Tuple[pd.DataFrame, Any]:  # noqa: MC0001
         """

msticpy/data/query_source.py

@@ -7,7 +7,7 @@
 from datetime import datetime, timedelta
 from numbers import Number
 import re
-from typing import Dict, List, Tuple, Optional, Union
+from typing import Dict, List, Tuple, Optional, Union, Any
 from collections import ChainMap
 
 from .._version import VERSION
@@ -222,25 +222,8 @@ def create_query(self, **kwargs) -> str:
         # template has been supplied
         for p_name, settings in self.params.items():
             param_value = param_dict[p_name]
-            # special case of datetime specified as a number - we
-            # interpret this as an offset from utcnow
             if settings["type"] == "datetime":
-                if isinstance(param_value, datetime):
-                    param_dict[p_name] = param_value
-                elif isinstance(param_value, Number):
-                    param_dict[p_name] = datetime.utcnow() + timedelta(
-                        param_value  # type: ignore
-                    )
-                else:
-                    # If the param appears to be ISO datetime format transform to datetime object
-                    try:
-                        param_dict[p_name] = datetime.strptime(
-                            param_value, "%Y-%m-%d %H:%M:%S.%f"
-                        )
-                    except ValueError:
-                        tm_delta = self._parse_timedelta(str(param_value))
-                        param_dict[p_name] = datetime.utcnow() + tm_delta
-
+                param_dict[p_name] = self._convert_datetime(param_value)
             if settings["type"] == "list":
                 param_dict[p_name] = self._parse_param_list(param_value)
             # if the parameter requires custom formatting
@@ -257,6 +240,23 @@ def create_query(self, **kwargs) -> str:
 
         return self._query.format(**param_dict)
 
+    def _convert_datetime(self, param_value: Any) -> datetime:
+        if isinstance(param_value, datetime):
+            return param_value
+        if isinstance(param_value, Number):
+            # datetime specified as a number - we
+            # interpret this as an offset from utcnow
+            return datetime.utcnow() + timedelta(
+                param_value  # type: ignore
+            )
+        try:
+            # Try to parse ISO datetime string format transform to datetime object
+            return datetime.strptime(param_value, "%Y-%m-%d %H:%M:%S.%f")
+        except (ValueError, TypeError):
+            # If none of these, assume a time delta
+            tm_delta = self._parse_timedelta(str(param_value))
+            return datetime.utcnow() + tm_delta
+
     @staticmethod
     def _parse_timedelta(time_range: str = "0") -> timedelta:
         """Parse time period string and return equivalent timedelta."""

msticpy/nbtools/pkg_config.py

@@ -18,8 +18,9 @@
 """
 import os
 import sys
+import warnings
 from pathlib import Path
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 
 import pkg_resources
 import yaml
@@ -137,5 +138,66 @@ def _get_top_module():
     return top_module
 
 
+def get_settings(
+    conf_group: Optional[Dict[str, Any]], name_map: Optional[Dict[str, str]] = None
+) -> Dict[Any, Any]:
+    """
+    Lookup configuration values config, environment or KeyVault.
+
+    Parameters
+    ----------
+    conf_group : Optional[Dict[str, Any]]
+        The configuration dictionary
+    name_map : Optional[Dict[str, str]], optional
+        Optional mapping to re-write setting names,
+        by default None
+
+    Returns
+    -------
+    Dict[Any, Any]
+        Dictionary of resolved settings
+
+    Raises
+    ------
+    NotImplementedError
+        Keyvault storage is not yet implemented
+
+    """
+    if not conf_group:
+        return {}
+    setting_dict: Dict[str, Any] = conf_group.copy()
+
+    for arg_name, arg_value in conf_group.items():
+        target_name = arg_name
+        if name_map:
+            target_name = name_map.get(target_name, target_name)
+
+        if isinstance(arg_value, str):
+            setting_dict[target_name] = arg_value
+        elif isinstance(arg_value, dict):
+            try:
+                setting_dict[target_name] = _fetch_setting(arg_value)  # type: ignore
+            except NotImplementedError:
+                warnings.warn(
+                    f"Setting type for setting {arg_value} not yet implemented. "
+                )
+    return setting_dict
+
+
+def _fetch_setting(config_setting: Dict[str, Any]) -> Optional[str]:
+    """Return required value for indirect settings (e.g. getting env var)."""
+    if "EnvironmentVar" in config_setting:
+        env_value = os.environ.get(config_setting["EnvironmentVar"])
+        if not env_value:
+            warnings.warn(
+                f"Environment variable {config_setting['EnvironmentVar']} "
+                + " was not set"
+            )
+        return env_value
+    if "KeyVaultURI" in config_setting:
+        raise NotImplementedError("Keyvault support not yet implemented.")
+    return None
+
+
 # read initial config when first imported.
 refresh_config()

msticpy/nbtools/utility.py

@@ -121,10 +121,10 @@ def get_nb_query_param(nb_url_search: str, param: str) -> Optional[str]:
 
     Parameters
     ----------
-        nb_url_search: str
-            The URL search string
-        param: str
-            The parameter name to search for
+    nb_url_search: str
+        The URL search string
+    param: str
+        The parameter name to search for
 
     Returns
     -------
@@ -253,7 +253,7 @@ def check_and_install_missing_packages(required_packages, notebook=True, user=Tr
     if not missing_packages:
         print("All packages are already installed")
     else:
-        print("Missing packages to be installed:: ", *required_packages, sep=" ")
+        print("Missing packages to be installed:: ", *missing_packages, sep=" ")
         if notebook:
             pkgbar = tqdm_notebook(missing_packages, desc="Installing...", unit="bytes")
         else:
@@ -327,3 +327,7 @@ def md_warn(string: str):
 
 class MsticpyException(Exception):
     """Default exception class for msticpy."""
+
+
+class MsticpyConfigException(Exception):
+    """Configuration exception class for msticpy."""