Ianhelle/mde proctree fixes 2021 12 16 (#239)

* Changes:

- Removed checking of DataFamily in data_query_reader.py
- Changed default behavior of kql_driver.py to use Azure authentication
- Change security_base.py to use entities.OSFamily rather than query_defns.DataFamily
- Added ensure_df_datetimes function to auto-convert specified columns from string to datetime
- Added unit test test_query_defns.py
- Added ensure_df_datetime to timeline.py
- Added ensure_df_datetime to timeline_duration.py
- Added ensure_df_datetime to proc_tree_build_mde.py
- Added ensure_df_datetime to proc_tree_build_winlx.py
- Changed mpconfig_defaults.yaml to include "msi"
- All other changes are switching "Microsoft Sentinel" for "Azure Sentinel" in UI/messages.
-

* Linting errors

* Updated black in .pre-commit-config.yaml

Black formatting in ce_common and compound_ctrls
Using returned schema in mdatp_driver.py to auto-format datetimes to pandas timestamp
Returning full response from query_with_results in odata_driver.py
Some fixes in convert_mde_schema_to_internal - field naming and using tz-aware Unknown time value
Moved the proc tree schema code from proc_tree_builder.py to proc_tree_schema.py
Add mocked az_connect to test_kql_driver.py tests - since the change to defaulting to AZ auth in the kql_driver means that tests fail.
Added test case for MDE public data - mde_proc_pub.pkl

* Missing import causes notebook test to fail in proc_tree_builder.py

Linting error in test_kql_driver.py

* Adding tooltips to settings editor buttons
- also changing button text for simple setting editor to "Update"
(from "Save")
- changing order of checks in base64unpack to let you specify UTF-16
decoding
- updating version to 1.5.1

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ repos:
       - id: trailing-whitespace
         args: [--markdown-linebreak-ext=md]
   - repo: https://github.com/ambv/black
-    rev: 20.8b1
+    rev: 21.12b0
     hooks:
       - id: black
         language: python
@@ -36,4 +36,4 @@ repos:
         entry: python -m tools.create_reqs_all
         pass_filenames: False
         language: python
-        types: [python]
+        types: [python]

msticpy/_version.py

@@ -1,2 +1,2 @@
 """Version file."""
-VERSION = "1.5.0"
+VERSION = "1.5.1"

msticpy/analysis/anomalous_sequence/utils/cmds_params_values.py

@@ -107,9 +107,10 @@ def compute_counts(  # noqa MC0001  # nosec
                 pars = dict.fromkeys(pars)
             for par, val in pars.items():
                 param_counts[par] += 1
-                value_counts[val] += 1
                 cmd_param_counts[cmd.name][par] += 1
-                param_value_counts[par][val] += 1
+                if val:
+                    value_counts[val] += 1
+                    param_value_counts[par][val] += 1
         seq2_counts[prev][end_token] += 1
         seq1_counts[end_token] += 1
 

msticpy/common/azure_auth_core.py

@@ -49,7 +49,7 @@ def default_auth_methods() -> List[str]:
             return az_settings["auth_methods"]
     except KeyError:
         pass  # no Azure section in config
-    return ["env", "cli", "msi", "interactive"]
+    return ["cli", "msi", "interactive"]
 
 
 class AzureCloudConfig:

msticpy/common/exceptions.py

@@ -354,7 +354,7 @@ class MsticpyKqlConnectionError(MsticpyUserError):
     """Exception class for KqlConnection errors."""
 
     DEF_HELP_URI = (
-        "Connecting to Azure Sentinel",
+        "Connecting to Microsoft Sentinel",
         "https://msticpy.readthedocs.io/en/latest/data_acquisition/DataProviders.html"
         + "#connecting-to-an-azure-sentinel-workspace",
     )
@@ -402,7 +402,7 @@ class MsticpyAzureConnectionError(MsticpyUserError):
     """Exception class for Azure Connection errors."""
 
     DEF_HELP_URI = (
-        "Connecting to Azure Sentinel",
+        "Connecting to Microsoft Sentinel",
         "https://msticpy.readthedocs.io/en/latest/data_acquisition/AzureData.html"
         + "#instantiating-and-connecting-with-an-azure-data-connector",
     )

msticpy/common/wsconfig.py

@@ -35,7 +35,7 @@
 ]
 
 _NO_CONFIG_WARN = [
-    "Could not find Azure Sentinel settings in msticpyconfig.yaml or a config.json ",
+    "Could not find Microsoft Sentinel settings in msticpyconfig.yaml or a config.json ",
     "(in the current directory or via a MSTICPYCONFIG variable.)",
     "We found the file '{config_file}' and will use this.",
     "We recommend using an explicit msticpyconfig.yaml specified using the",
@@ -54,7 +54,7 @@
 _NO_CONFIG_ERR = [
     "Could not find msticpyconfig.yaml or config.json.",
     "The 'config.json' file is created when you launch notebooks from "
-    "Azure Sentinel. If you have copied the notebook to another location "
+    "Microsoft Sentinel. If you have copied the notebook to another location "
     "or folder you will need to copy this configuration file.",
     "Alternatively, we recommend using an explicit msticpyconfig.yaml"
     "and adding your Workspace and Tenant IDs to that file.",
@@ -261,10 +261,10 @@ def list_workspaces(cls) -> Dict:
 
     def prompt_for_ws(self):
         """Display an interactive prompt for Workspace details."""
-        md_warn("No Azure Sentinel configuration found.")
+        md_warn("No Microsoft Sentinel configuration found.")
         md(
             "Please enter the workspace ID and tenant Id"
-            + " to allow connection to the Azure Sentinel workspace."
+            + " to allow connection to the Microsoft Sentinel workspace."
         )
         ws_id_wgt = widgets.Text(description="Workspace Id:", **WIDGET_DEFAULTS)
         ten_id_wgt = widgets.Text(description="Tenant Id:", **WIDGET_DEFAULTS)
@@ -286,7 +286,7 @@ def update_tnt(chg):
                 "data_acquisition/DataProviders.html#"
                 "connecting-to-an-azure-sentinel-workspace' "
                 "target='_blank' rel='noopener noreferrer'>"
-                "Connecting to an Azure Sentinel Workspace</a>"
+                "Connecting to a Microsoft Sentinel Workspace</a>"
             ),
         )
 

msticpy/config/ce_azure_sentinel.py

@@ -20,12 +20,12 @@
 
 # pylint: disable=too-many-ancestors
 class CEAzureSentinel(CEItemsBase):
-    """Azure Sentinel Workspaces editor component."""
+    """Microsoft Sentinel Workspaces editor component."""
 
-    _DESCRIPTION = "Azure Sentinel workspace settings"
+    _DESCRIPTION = "Microsoft Sentinel workspace settings"
     _COMP_PATH = "AzureSentinel.Workspaces"
     _HELP_TEXT = """
-    Supply the parameters for your Azure Sentinel workspaces here.<br>
+    Supply the parameters for your Microsoft Sentinel workspaces here.<br>
 
     You can get all of these (apart from 'TenantID') from your workspace portal.
     Navigate to "Settings" (on the left side of the screen), then click the
@@ -50,7 +50,7 @@ class CEAzureSentinel(CEItemsBase):
     can be a duplicate of another named entry.
     """
     _HELP_URI = {
-        "Connecting to an Azure Sentinel Workspace": (
+        "Connecting to a Microsoft Sentinel Workspace": (
             "https://msticpy.readthedocs.io/en/latest/data_acquisition/"
             + "DataProviders.html#connecting-to-an-azure-sentinel-workspace"
         ),

msticpy/config/ce_common.py

@@ -47,7 +47,6 @@ def print_debug(*args):
             return
         print(*args)
 
-
 else:
 
     def print_debug(*args):

msticpy/config/comp_edit.py

@@ -149,9 +149,21 @@ class CompEditItemButtons:
     def __init__(self):
         """Initialize the class."""
         btn_layout = widgets.Layout(width="100px")
-        self.btn_add = widgets.Button(description="Add", layout=btn_layout)
-        self.btn_del = widgets.Button(description="Delete", layout=btn_layout)
-        self.btn_save = widgets.Button(description="Update", layout=btn_layout)
+        self.btn_add = widgets.Button(
+            description="Add",
+            tooltip="Add provider/item in drop-down to your settings.",
+            layout=btn_layout,
+        )
+        self.btn_del = widgets.Button(
+            description="Delete",
+            tooltip="Remove selected provider/item from your settings.",
+            layout=btn_layout,
+        )
+        self.btn_save = widgets.Button(
+            description="Update",
+            tooltip="Confirms updates to the settings changes",
+            layout=btn_layout,
+        )
         self.layout = widgets.HBox([self.btn_add, self.btn_del, self.btn_save])
 
 
@@ -189,7 +201,9 @@ def __init__(self, description: str):
         """Initialize the class. Set a label with `description` as content."""
         super().__init__(description=description)
         self.edit_frame = widgets.VBox(layout=self.border_layout("100%"))
-        self.btn_save = widgets.Button(description="Save")
+        self.btn_save = widgets.Button(
+            description="Update", tooltip="Confirms updates to the settings changes"
+        )
         self.container = widgets.VBox(
             [self.edit_frame, self.btn_save], layout=self.no_border_layout("100%")
         )

msticpy/config/compound_ctrls.py

@@ -288,7 +288,6 @@ def _set_kv_secret_value(
         except Exception as err:  # pylint: disable=broad-except
             return False, str(err), None
 
-
 else:
 
     def _set_kv_secret_value(

msticpy/config/mp_config_edit.py

@@ -29,7 +29,7 @@ class MpConfigEdit(CompEditDisplayMixin):
     """Msticpy Configuration helper class."""
 
     _TAB_DEFINITIONS = {
-        "AzureSentinel": CEAzureSentinel,
+        "MicrosoftSentinel": CEAzureSentinel,
         "TI Providers": CETIProviders,
         "Data Providers": CEDataProviders,
         "GeoIP Providers": CEOtherProviders,
@@ -99,9 +99,15 @@ def __init__(
             value=self.current_config_file,
             layout=widgets.Layout(width="75%"),
         )
-        self.btn_save = widgets.Button(description="Save Settings")
+        self.btn_save = widgets.Button(
+            description="Save Settings",
+            tooltip="Save current settings to your config file.",
+        )
         self.btn_save.on_click(self._save_file)
-        self.btn_validate = widgets.Button(description="Validate Settings")
+        self.btn_validate = widgets.Button(
+            description="Validate Settings",
+            tooltip="Run basic sanity checks on current settings.",
+        )
         self.btn_validate.on_click(self._validate_config)
         self.cb_backup = widgets.Checkbox(description="Create backup", value=False)
         vbox = widgets.VBox(

msticpy/config/mp_config_file.py

@@ -72,7 +72,9 @@ def __init__(
         self.settings = settings or {}
 
         self.kv_client: Any = None
-        self.mp_config_def_path = os.environ.get("MSTICPYCONFIG", "")
+        self.mp_config_def_path = os.environ.get(
+            "MSTICPYCONFIG", "./msticpyconfig.yaml"
+        )
         self._current_file = None
 
         # Set up controls

msticpy/data/azure_sentinel.py

@@ -26,7 +26,7 @@
 
 
 class AzureSentinel(AzureData):
-    """Class for returning key Azure Sentinel elements."""
+    """Class for returning key Microsoft Sentinel elements."""
 
     def __init__(self, connect: bool = False, cloud: Optional[str] = None):
         """
@@ -82,7 +82,7 @@ def set_default_subscription(self, subscription_id: str):
 
     def get_sentinel_workspaces(self, sub_id: str = None) -> Dict[str, str]:
         """
-        Return a list of Azure Sentinel workspaces in a Subscription.
+        Return a list of Microsoft Sentinel workspaces in a Subscription.
 
         Parameters
         ----------
@@ -102,7 +102,7 @@ def get_sentinel_workspaces(self, sub_id: str = None) -> Dict[str, str]:
             config = self._check_config(["subscription_id"])
             sub_id = config["subscription_id"]
 
-        print("Finding Azure Sentinel Workspaces...")
+        print("Finding Microsoft Sentinel Workspaces...")
         res = self.get_resources(sub_id=sub_id)  # type: ignore
         # handle no results
         if isinstance(res, pd.DataFrame) and not res.empty:
@@ -124,7 +124,7 @@ def get_sentinel_workspaces(self, sub_id: str = None) -> Dict[str, str]:
 
             return workspaces_dict
 
-        print(f"No Azure Sentinel workspaces in {sub_id}")
+        print(f"No Microsoft Sentinel workspaces in {sub_id}")
         return {}
 
     def set_default_workspace(
@@ -166,7 +166,7 @@ def get_hunting_queries(
         ws_name: str = None,
     ) -> pd.DataFrame:
         """
-        Return all hunting queries in an Azure Sentinel workspace.
+        Return all hunting queries in a Microsoft Sentinel workspace.
 
         Parameters
         ----------
@@ -212,7 +212,7 @@ def get_alert_rules(
         ws_name: str = None,
     ) -> pd.DataFrame:
         """
-        Return all Azure Sentinel alert rules for a workspace.
+        Return all Microsoft Sentinel alert rules for a workspace.
 
         Parameters
         ----------

msticpy/data/data_query_reader.py

@@ -8,7 +8,7 @@
 from pathlib import Path
 import yaml
 
-from .query_defns import DataFamily, DataEnvironment
+from .query_defns import DataEnvironment
 from .._version import VERSION
 
 __version__ = VERSION
@@ -127,11 +127,3 @@ def _validate_data_categories(query_def_dict: Dict):
         or not query_def_dict["metadata"]["data_families"]
     ):
         raise ValueError("Imported file has no data families defined")
-
-    for fam in query_def_dict["metadata"]["data_families"]:
-        if not DataFamily.parse(fam):
-            raise ValueError(
-                f"Unknown data family {fam} in metadata. ",
-                "Valid values are\n",
-                ", ".join(f.name for f in DataFamily),
-            )

msticpy/data/drivers/kql_driver.py

@@ -137,8 +137,10 @@ def connect(self, connection_str: Optional[str] = None, **kwargs):  # noqa: MC00
             )
         if "kqlmagic_args" in kwargs:
             connection_str = connection_str + " " + kwargs["kqlmagic_args"]
-        if "mp_az_auth" in kwargs and "try_token" not in kwargs:
-            self._set_az_auth_option(kwargs["mp_az_auth"])
+        # Default to using Azure Auth if possible.
+        mp_az_auth = kwargs.pop("mp_az_auth", "default")
+        if mp_az_auth and "try_token" not in kwargs:
+            self._set_az_auth_option(mp_az_auth)
         self.current_connection = connection_str
         kql_err_setting = self._get_kql_option("short_errors")
         self._connected = False
@@ -376,7 +378,7 @@ def _raise_kql_error(self, ex):
         kql_err = json.loads(ex.args[0]).get("error")
         if kql_err.get("code") == "WorkspaceNotFoundError":
             ex_mssgs = [
-                "The workspace ID used to connect to Azure Sentinel could not be found.",
+                "The workspace ID used to connect to Microsoft Sentinel could not be found.",
                 "Please check that this is a valid workspace for your subscription",
             ]
             ws_match = re.search(self._WS_RGX, self.current_connection, re.IGNORECASE)

msticpy/data/drivers/mdatp_driver.py

@@ -8,7 +8,7 @@
 import pandas as pd
 
 from .odata_driver import OData, QuerySource
-from ..query_defns import DataEnvironment
+from ..query_defns import DataEnvironment, ensure_df_datetimes
 from ...common.azure_auth import AzureCloudConfig
 from ...common.utility import export
 from ..._version import VERSION
@@ -76,7 +76,21 @@ def query(
 
         """
         del query_source, kwargs
-        return self.query_with_results(query, body=True, api_end=self.api_suffix)[0]
+        data, response = self.query_with_results(
+            query, body=True, api_end=self.api_suffix
+        )
+        if isinstance(data, pd.DataFrame):
+            # If we got a schema we should convert the DateTimes to pandas datetimes
+            if "Schema" not in response:
+                return data
+            date_fields = [
+                field["Name"]
+                for field in response["Schema"]
+                if field["Type"] == "DateTime"
+            ]
+            data = ensure_df_datetimes(data, columns=date_fields)
+            return data
+        return response
 
 
 def _select_api_uris(data_environment):

msticpy/data/drivers/odata_driver.py

@@ -225,7 +225,7 @@ def query_with_results(  # noqa: MC0001
         if not result:
             print("Warning - query did not return any results.")
             return None, json_response
-        return pd.json_normalize(result), result
+        return pd.json_normalize(result), json_response
 
     # pylint: enable=too-many-branches