Ianhelle kql and nbinit nice errs 2020 09 08 (#96)

* Error in pkg_config validate when no config sections are populated.

* Splunk and data_providers fixes

Added query_browser module
Added method to tilookup and data_providers modules to access respective browser widgets.

* Reverting a commit that created circular dependency

* Reverting query_browser since it introduced a circular dependency that broke things

* Query browser

* Adding registered widgets to nbwidgets

Changed QueryTimes to use registered widgets
Added simple text entry with registration
Changed the way that browse_queries gets added as attrib of QueryProvider to prevent circular dependency

* Linting and test errors

* Fixing mypy errors, incorrect annotation in query_source

Updated Splunk queries to use datetime type for parameters.
Added splunklib to mypy.ini

* Fixing time format bug in timeline

* Adding tooltip formatting fix to timeseries.py

* Adding OptionButtons control

* NotebookWidgets notebook update for new widgets

* Adding some extra checks for null entries in msticpyconfig in pkg_config

Making nbinit skip but report any exceptions while validating msticpyconfig
Change dataproviders to use custom paths outside of the package
Fix to entity entityschema
Fix to kql_driver to handle running if not in IPython.
Updated version file to 0.7.0

* Update to OptionButtons with timeout in nbwidgets

Updates to NotebookWidgets.ipynb

* Adding auto-display parameter to SelectSubset widget

Add version param to test-pypi-test-pkg.cmd help.

* Documentation for additional Widgets.

Unit test for query_browser.

* Data obfuscation and time series period extraction

* Fixing timeseries tests

Updates to nbinit.py and kql_driver.py to provider friendlier error messages.

Fixes for spelling and linting warnings

Updated data queries list

* Updating version

* Removing more linting errors and spelling errors.

Updating requirements.txt to avoid conflicts with msal and azure-identity

* Formatting using updated black v20

Fixing conda-reqs-pip.txt requirements
Updating msal req to ~=1.0.1

* Fixing two more inconsistent requirements in conda-reqs.txt

* Incorrect version for msal

conda/conda-reqs-pip.txt

@@ -1,6 +1,6 @@
 azure-cli-core>=2.5.0
 azure-core>=1.2.2
-azure-identity>=1.3.0
+azure-identity==1.3.1
 azure-keyvault-secrets>=4.0.0
 azure-mgmt-compute>=4.6.2
 azure-mgmt-keyvault>=2.0.0

conda/conda-reqs.txt

@@ -2,7 +2,7 @@ adal>=1.2.2
 attrs>=18.2.0
 azure-common>=1.1.18
 azure-mgmt-network>=2.7.0
-azure-mgmt-resource>=2.2.0,<=10.1.0
+azure-mgmt-resource>=2.2.0
 beautifulsoup4>=4.6.3
 bokeh>=1.4.0
 cryptography>=2.8
@@ -17,7 +17,7 @@ msrest>=0.6.0
 msrestazure>=0.6.0
 networkx>=2.2
 numpy>=1.15.4
-pandas>=0.25.0,<=1.0.5
+pandas>=0.25.0
 python-dateutil>=2.8.1
 pytz>=2019.2
 pyyaml>=3.13

msticpy/_version.py

@@ -1,2 +1,2 @@
 """Version file."""
-VERSION = "0.7.1"
+VERSION = "0.8.0"

msticpy/common/exceptions.py

@@ -16,7 +16,7 @@
 
 
 # placeholder for pkg_config.get_config - this function is
-# overwritten by msticp.common.pkg_config
+# overwritten by msticpy.common.pkg_config
 def _get_config(setting_path: str):
     del setting_path
     return True
@@ -237,7 +237,7 @@ def __init__(
 
         """
         mssg = (
-            "Please verfiy that a valid KeyVault section has been configured"
+            "Please verify that a valid KeyVault section has been configured"
             + "in your msticpyconfig.yaml."
         )
         add_args = [*args, mssg]
@@ -261,7 +261,7 @@ def __init__(
 
         """
         mssg = (
-            "Please verfiy that the item using this secret is properly"
+            "Please verify that the item using this secret is properly"
             + " configured in in your msticpyconfig.yaml."
         )
         add_args = [*args, mssg]
@@ -314,6 +314,16 @@ class MsticpyNoDataSourceError(MsticpyUserError):
     )
 
 
+class MsticpyDataQueryError(MsticpyUserError):
+    """Exception class for data query errors."""
+
+    DEF_HELP_URI = (
+        "Query failed",
+        "https://msticpy.readthedocs.io/en/latest/DataAcquisition.html"
+        + "#querying-and-importing-data",
+    )
+
+
 class MsticpyConnectionError(MsticpyUserError):
     """Exception class for KqlConnection errors."""
 

msticpy/common/keyvault_client.py

@@ -100,11 +100,11 @@ def __init__(self):
         """Initialize new instance of KeyVault Settings."""
         try:
             kv_config = config.get_config("KeyVault")
-        except KeyError:
+        except KeyError as err:
             raise MsticpyKeyVaultConfigError(
                 "No KeyVault section found in msticpyconfig.yaml",
                 title="missing Key Vault configuration",
-            )
+            ) from err
         norm_settings = {key.casefold(): val for key, val in kv_config.items()}
         self.__dict__.update(norm_settings)
         if "authority_uri" in self:
@@ -294,9 +294,9 @@ def __init__(
         """
         self.name = name
         self.debug = kwargs.pop("debug", False)
-        self.settings: KeyVaultSettings = kwargs.pop(
-            "settings", None
-        ) or KeyVaultSettings()
+        self.settings: KeyVaultSettings = (
+            kwargs.pop("settings", None) or KeyVaultSettings()
+        )
         self.tenant_id = tenant_id or self.settings.get("tenantid")
         if not self.tenant_id:
             raise MsticpyKeyVaultConfigError(
@@ -767,7 +767,7 @@ def get_secret(self, secret_name: str) -> Any:
                 f"Secret name {secret_name} could not be found in {self.vault_uri}",
                 f"Provider returned: {err}",
                 title=f"secret {secret_name} not found.",
-            )
+            ) from err
         if secret_bundle.value is None or not secret_bundle.value:
             if self.debug:
                 print(
@@ -919,7 +919,7 @@ def get_vault_uri(self, vault_name: str) -> str:
                 + " in your configuration",
                 f"Error returned from provider was {cloud_err}",
                 title="Key Vault vault '{vault_name}' not found.",
-            )
+            ) from cloud_err
         return vault.properties.vault_uri
 
     def create_vault(self, vault_name: str) -> Vault:

msticpy/common/pkg_config.py

@@ -160,7 +160,7 @@ def _read_config_file(config_file: str) -> Dict[str, Any]:
                     "The following error was encountered",
                     str(yml_err),
                     title="config file could not be read",
-                )
+                ) from yml_err
     return {}
 
 

msticpy/data/azure_data.py

@@ -131,7 +131,7 @@ def connect(self, client_id: str = None, tenant_id: str = None, secret: str = No
                     f"{key_name} is missing from AzureCLI section in your",
                     "configuration.",
                     title="missing f{key_name} settings for AzureCLI.",
-                )
+                ) from key_err
         # Create credentials and connect to the subscription client to validate
         self.credentials = ServicePrincipalCredentials(
             client_id=client_id, secret=secret, tenant=tenant_id
@@ -409,23 +409,23 @@ def _get_api(
             try:
                 namespace = resource_id.split("/")[6]
                 service = resource_id.split("/")[7]
-            except IndexError:
+            except IndexError as idx_err:
                 raise MsticpyResourceException(
                     "Provided Resource ID isn't in the correct format.",
                     "It should look like:",
                     "/subscriptions/SUB_ID/resourceGroups/RESOURCE_GROUP/"
                     + "providers/NAMESPACE/SERVICE_NAME/RESOURCE_NAME ",
-                )
+                ) from idx_err
 
         elif resource_provider is not None:
             try:
                 namespace = resource_provider.split("/")[0]
                 service = resource_provider.split("/")[1]
-            except IndexError:
+            except IndexError as idx_err:
                 raise MsticpyResourceException(
                     "Provided Resource Provider isn't in the correct format.",
                     "It should look like: NAMESPACE/SERVICE_NAME",
-                )
+                ) from idx_err
         else:
             raise ValueError(
                 "Please provide an resource ID or resource provider namespace"

msticpy/data/data_obfus.py

@@ -323,25 +323,26 @@ def obfuscate_df(
 
     out_df = data.copy()
     print("obfuscating columns:")
-    for col in data.columns:
-        if col not in col_map:
+    for col_name in data.columns:
+        if col_name not in col_map:
             continue
-        col_type = col_map.get(col, "str")
-        print(col, end=", ")
+        col_type = col_map.get(col_name, "str")
+        print(col_name, end=", ")
         map_func = MAP_FUNCS.get(col_type)
         try:
             if map_func == "null":
-                data[col] = None
-            # pylint: disable=cell-var-from-loop
+                data[col_name] = None
             elif map_func is not None and callable(map_func):
-                out_df[col] = out_df.apply(lambda x: map_func(x[col]), axis=1)
+                out_df[col_name] = out_df.apply(
+                    lambda x, col=col_name, func=map_func: func(x[col]), axis=1
+                )
             else:
-                out_df[col] = out_df.apply(
-                    lambda x: hash_item(x[col], col_type), axis=1
+                out_df[col_name] = out_df.apply(
+                    lambda x, col=col_name, c_type=col_type: hash_item(x[col], c_type),
+                    axis=1,
                 )
-        # pylint: enable=cell-var-from-loop
         except Exception as err:
-            print(col, str(err))
+            print(col_name, str(err))
             raise
 
     print("\ndone")
@@ -393,8 +394,8 @@ def check_obfuscation(
         print("\n".join(obfuscated))
         print("====== End Check =====")
         return None
-    else:
-        return unchanged, obfuscated
+
+    return unchanged, obfuscated
 
 
 @pd.api.extensions.register_dataframe_accessor("mp_obf")

msticpy/data/data_providers.py

@@ -297,7 +297,7 @@ def browse_queries(query_provider, **kwargs):
         Other Parameters
         ----------------
         kwargs :
-            passed to SelectItem constuctor.
+            passed to SelectItem constructor.
 
         Returns
         -------
@@ -386,7 +386,7 @@ def _add_service_queries(
 
         # For now, just add all of the functions again (with any connect-time acquired
         # queries) - we could be more efficient than this but unless there are 1000s of
-        # queries it should not be noticable.
+        # queries it should not be noticeable.
         self._add_query_functions()
 
     @classmethod

msticpy/data/drivers/kql_driver.py

@@ -21,6 +21,7 @@
     MsticpyNoDataSourceError,
     MsticpyNotConnectedError,
     MsticpyKqlConnectionError,
+    MsticpyDataQueryError,
 )
 from ...common.utility import export
 from ..._version import VERSION
@@ -205,13 +206,15 @@ def query_with_results(self, query: str, **kwargs) -> Tuple[pd.DataFrame, Any]:
                     print("Warning - query returned partial results.")
                 return data_frame, result
 
-        print("Warning - query did not complete successfully.")
+        # Query failed
+        err_args = []
         if hasattr(result, "completion_query_info"):
-            print(
-                result.completion_query_info.get("StatusDescription"),
-                f"(code: {result.completion_query_info['StatusCode']})",
-            )
-        return None, result
+            err_desc = result.completion_query_info.get("StatusDescription")
+            err_desc = f"StatusDescription {err_desc}"
+            err_code = f"(err_code: {result.completion_query_info.get('StatusCode')})"
+            err_args = [err_desc, err_code]
+        err_args.append(f"Query:\n{query}")
+        raise MsticpyDataQueryError(*err_args)
 
     def _load_kql_magic(self):
         """Load KqlMagic if not loaded."""

msticpy/data/drivers/splunk_driver.py

@@ -103,19 +103,19 @@ def connect(self, connection_str: str = None, **kwargs):
                 f"Authentication error connecting to Splunk: {err}",
                 title="Splunk connection",
                 help_uri="https://msticpy.readthedocs.io/en/latest/DataProviders.html",
-            )
+            ) from err
         except HTTPError as err:
             raise MsticpyConnectionError(
                 f"Communication error connecting to Splunk: {err}",
                 title="Splunk connection",
                 help_uri="https://msticpy.readthedocs.io/en/latest/DataProviders.html",
-            )
+            ) from err
         except Exception as err:
             raise MsticpyConnectionError(
                 f"Error connecting to Splunk: {err}",
                 title="Splunk connection",
                 help_uri="https://msticpy.readthedocs.io/en/latest/DataProviders.html",
-            )
+            ) from err
         self._connected = True
         print("connected")
 

msticpy/data/queries/kql_mdatp_hunting.yaml

@@ -382,7 +382,7 @@ sources:
       uri: "https://github.com/microsoft/WindowsDefenderATP-Hunting-Queries/blob/master/Lateral%20Movement/ServiceAccountsPerformingRemotePS.txt"
     parameters:
   accessibility_persistence:
-    description: This query looks for persistence or priviledge escalation done using Windows Accessibility features.
+    description: This query looks for persistence or privilege escalation done using Windows Accessibility features.
     metadata:
     args:
       query: '

msticpy/data/queries/kql_sent_alert.yaml

@@ -97,7 +97,7 @@ sources:
         type: datetime
         default: 0  # fake default to prevent this being viewed as required
   list_related_alerts:
-    description: Retrieves list of alerts with a common host, acount or process
+    description: Retrieves list of alerts with a common host, account or process
     metadata:
     args:
       query: '

msticpy/data/queries/kql_sent_winevent_logon.yaml

@@ -160,7 +160,7 @@ sources:
         type: str
         default: 'SecurityEvent | where EventID == 4625'
   list_all_logons_by_host:
-    description: Retrevies all failed or successful logons to a host
+    description: account all failed or successful logons to a host
     args:
       query: '
         {table}

msticpy/data/uploaders/loganalytics_uploader.py

@@ -127,11 +127,11 @@ def _post_data(self, body: str, table_name: str):
         }
         try:
             response = requests.post(uri, data=body, headers=headers)
-        except requests.ConnectionError:
+        except requests.ConnectionError as req_err:
             raise MsticpyConnectionError(
                 "Unable to connect to workspace, ensure your Workspace ID is correct.",
                 title="Unable to connect to Workspace",
-            )
+            ) from req_err
         if self._debug is True:
             print(f"Upload response code: {response.status_code}")
         if response.status_code < 200 or response.status_code > 299:
@@ -172,7 +172,7 @@ def upload_df(self, data: pd.DataFrame, table_name: Any, **kwargs):
             print(f"Upload to {table_name} complete")
 
     def upload_file(
-        self, file_path: str, table_name: str = None, delim: str = ",", **kwargs,
+        self, file_path: str, table_name: str = None, delim: str = ",", **kwargs
     ):
         """
         Upload a seperated value file to Log Analytics.
@@ -214,9 +214,7 @@ def upload_folder(
         t_name = bool(table_name)
         input_files = Path(folder_path).glob(ext)
         # pylint: disable=unnecessary-comprehension
-        input_files = [
-            path for path in input_files  # type: ignore
-        ]
+        input_files = [path for path in input_files]  # type: ignore
         # pylint: enable=unnecessary-comprehension
         progress = tqdm(total=len(list(input_files)), desc="Files", position=0)
         for path in input_files:

msticpy/data/uploaders/splunk_uploader.py

@@ -164,11 +164,11 @@ def upload_file(  # type: ignore
         path = Path(file_path)
         try:
             data = pd.read_csv(path, delimiter=delim)
-        except (ParserError, UnicodeDecodeError):
+        except (ParserError, UnicodeDecodeError) as parse_err:
             raise MsticpyUserError(
                 "The file specified is not a seperated value file.",
                 "Incorrect file type.",
-            )
+            ) from parse_err
 
         if not table_name:
             table_name = path.stem
@@ -210,24 +210,18 @@ def upload_folder(  # type: ignore
         """
         host = kwargs.get("host", None)
         glob_pat = kwargs.get("glob", "*")
-        ext = glob_pat
         t_name = bool(table_name)
-        input_files = Path(folder_path).glob(ext)
-        # pylint: disable=unnecessary-comprehension
-        input_files = [
-            path for path in input_files  # type: ignore
-        ]
-        # pylint: enable=unnecessary-comprehension
+        input_files = Path(folder_path).glob(glob_pat)
         f_progress = tqdm(total=len(list(input_files)), desc="Files", position=0)
         for path in input_files:
             try:
                 data = pd.read_csv(path, delimiter=delim)
-            except (ParserError, UnicodeDecodeError):
+            except (ParserError, UnicodeDecodeError) as parse_err:
                 raise MsticpyUserError(
                     "The file specified is not a seperated value file.",
                     title="Incorrect file type.",
-                )
-            if t_name is False:
+                ) from parse_err
+            if not t_name:
                 table_name = path.stem
             self._post_data(
                 data=data,

msticpy/nbtools/entityschema.py

@@ -226,7 +226,7 @@ def description_str(self) -> str:
         """
         return self.Type
 
-    # pylint: disable=bad-continuation, too-many-branches
+    # pylint: disable=too-many-branches
     @classmethod
     def instantiate_entity(  # noqa: C901
         cls, raw_entity: Mapping[str, Any]