- Typo corrections in queries (#684)

* - Typo corrections in queries

- Added queries for Identity table (on prem AD)
- Added a pandas accessor for the panel data viewer
- Fixed a bug in iocextract causing confusion between defanged emails and domains with "at" in them.
- Also made the operation on defanged IoCs more consistent so can be set at the class level or
  during the function call to search for or validate an IoC type.

* Make defanged=True by default
Changing behavior back to previous default.
Adding test case for name[at]domain

msticpy/common/utility/format.py

@@ -114,7 +114,7 @@ def defang_ioc(ioc: str, ioc_type: str = None) -> str:
     """
     de_fanged = ioc
     if ioc_type == "email":
-        de_fanged = de_fanged.replace("@", "AT")
+        de_fanged = de_fanged.replace("@", "[AT]")
     elif ioc_type == "url":
         de_fanged = de_fanged.replace("http", "hXXp").replace("ftp", "fXp")
     return de_fanged.replace(".", "[.]")
@@ -141,7 +141,7 @@ def refang_ioc(ioc: str, ioc_type: str = None) -> str:
     """
     re_fanged = ioc
     if ioc_type == "email":
-        re_fanged = re_fanged.replace("AT", "@")
+        re_fanged = re_fanged.replace("[AT]", "@")
     elif ioc_type == "url":
         re_fanged = re_fanged.replace("hXXp", "http").replace("fXp", "ftp")
     return re_fanged.replace("[.]", ".")

msticpy/data/queries/m365d/kql_m365_alerts.yaml

@@ -1,7 +1,7 @@
 metadata:
   version: 1
-  description: MDATP Queries
-  data_environments: [MDATP, MDE, M365D]
+  description: M365D Queries
+  data_environments: [MDATP, MDE, M365D, LogAnalytics]
   data_families: [M365D]
   tags: ["alert"]
 defaults:
@@ -45,12 +45,14 @@ sources:
         | join kind=inner (AlertEvidence
         | where Timestamp >= datetime({start})
         | where Timestamp <= datetime({end})) on AlertId
-        )
+        | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1,
+          AttackTechniques1, ServiceSource1, DetectionSource1, SHA1,
+          SourceSystem1, Type1
         {add_query_items}"
       uri: None
     parameters:
   host_alerts:
-    description: Lists alerts by for a specified hostname
+    description: Lists alerts associated with host/device name
     metadata:
     args:
       query: '
@@ -60,6 +62,9 @@ sources:
         | join kind=inner (AlertEvidence
         | where Timestamp >= datetime({start})
         | where Timestamp <= datetime({end})) on AlertId
+        | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1,
+          AttackTechniques1, ServiceSource1, DetectionSource1, SHA1,
+          SourceSystem1, Type1
         | where EntityType =~ "Machine"
         | where DeviceName has "{host_name}"
         {add_query_items}'
@@ -78,6 +83,9 @@ sources:
         | join kind=inner (AlertEvidence
         | where Timestamp >= datetime({start})
         | where Timestamp <= datetime({end})) on AlertId
+        | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1,
+          AttackTechniques1, ServiceSource1, DetectionSource1, SHA1,
+          SourceSystem1, Type1
         | where EntityType =~ "Ip"
         | where RemoteIP has "{ip_address}"
         {add_query_items}'
@@ -96,6 +104,9 @@ sources:
         | join kind=inner (AlertEvidence
         | where Timestamp >= datetime({start})
         | where Timestamp <= datetime({end})) on AlertId
+        | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1,
+          AttackTechniques1, ServiceSource1, DetectionSource1, SHA1,
+          SourceSystem1, Type1
         | where EntityType =~ "Url"
         | where RemoteUrl has "{url}"
         {add_query_items}'
@@ -114,6 +125,9 @@ sources:
         | join kind=inner (AlertEvidence
         | where Timestamp >= datetime({start})
         | where Timestamp <= datetime({end})) on AlertId
+        | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1,
+          AttackTechniques1, ServiceSource1, DetectionSource1, SHA1,
+          SourceSystem1, Type1
         | where EntityType in~ ("File", "Process")
         | where SHA1 has "{file_hash}"
         {add_query_items}'
@@ -134,6 +148,9 @@ sources:
         | join kind=inner (AlertEvidence
         | where Timestamp >= datetime({start})
         | where Timestamp <= datetime({end})) on AlertId
+        | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1,
+          AttackTechniques1, ServiceSource1, DetectionSource1, SHA1,
+          SourceSystem1, Type1
         | where EntityType in~ ("File", "Process")
         | where SHA256 has "{file_hash}"
         {add_query_items}'
@@ -154,6 +171,9 @@ sources:
         | join kind=inner (AlertEvidence
         | where Timestamp >= datetime({start})
         | where Timestamp <= datetime({end})) on AlertId
+        | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1,
+          AttackTechniques1, ServiceSource1, DetectionSource1, SHA1,
+          SourceSystem1, Type1
         | where EntityType =~ "Process"
         | where FileName =~ "{file_name}"
         {add_query_items}'
@@ -174,6 +194,9 @@ sources:
         | join kind=inner (AlertEvidence
         | where Timestamp >= datetime({start})
         | where Timestamp <= datetime({end})) on AlertId
+        | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1,
+          AttackTechniques1, ServiceSource1, DetectionSource1, SHA1,
+          SourceSystem1, Type1
         | where EntityType =~ "User"
         | where AccountName =~ "{account_name}"
         {add_query_items}'
@@ -192,6 +215,9 @@ sources:
         | join kind=inner (AlertEvidence
         | where Timestamp >= datetime({start})
         | where Timestamp <= datetime({end})) on AlertId
+        | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1,
+          AttackTechniques1, ServiceSource1, DetectionSource1, SHA1,
+          SourceSystem1, Type1
         | where EntityType =~ "RegistryValue"
         | where RegistryKey =~ "{key_name}"
         {add_query_items}'
@@ -210,6 +236,9 @@ sources:
         | join kind=inner (AlertEvidence
         | where Timestamp >= datetime({start})
         | where Timestamp <= datetime({end})) on AlertId
+        | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1,
+          AttackTechniques1, ServiceSource1, DetectionSource1, SHA1,
+          SourceSystem1, Type1
         | where EntityType =~ "MailMessage"
         | where NetworkMessageId =~ "{message_id}"
         {add_query_items}'
@@ -228,6 +257,9 @@ sources:
         | join kind=inner (AlertEvidence
         | where Timestamp >= datetime({start})
         | where Timestamp <= datetime({end})) on AlertId
+        | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1,
+          AttackTechniques1, ServiceSource1, DetectionSource1, SHA1,
+          SourceSystem1, Type1
         | where EntityType =~ "Mailbox"
         | where AccountUpn =~ "{mailbox}"
         {add_query_items}'

msticpy/data/queries/m365d/kql_m365_identity.yaml

@@ -0,0 +1,74 @@
+metadata:
+  version: 1
+  description: M365D User Queries
+  data_environments: [MDATP, MDE, M365D, LogAnalytics]
+  data_families: [IdentityOnPrem]
+  tags: ["user", "account"]
+defaults:
+  metadata:
+    data_source: "user_events"
+  parameters:
+    start:
+      description: Query start time
+      type: datetime
+    end:
+      description: Query end time
+      type: datetime
+    add_query_items:
+      description: Additional query clauses
+      type: str
+      default: ""
+    time_column:
+      description: The name of the column detailing the time the event was generated.
+      type: str
+      default: "Timestamp"
+    table:
+      description: Table name
+      type: str
+      default: IdentityLogonEvents
+sources:
+  logons_for_account:
+    description: Return all Active Directory on-premises user logons for user name
+    metadata:
+    args:
+      query: '
+        {table}
+        | where {time_column} >= datetime({start})
+        | where {time_column} <= datetime({end})
+        | where AccountUpn has "{account_name}"
+        {add_query_items}'
+      uri: None
+    parameters:
+      account_name:
+        description: Name or part name of user/UPN
+        type: str
+  logons_for_ip:
+    description: Return all Active Directory on-premises user logons for ip address
+    metadata:
+    args:
+      query: '
+        {table}
+        | where {time_column} >= datetime({start})
+        | where {time_column} <= datetime({end})
+        | where IPAddress == "{ip_address}"
+        {add_query_items}'
+      uri: None
+    parameters:
+      ip_address:
+        description: Source IP address of logons
+        type: str
+  logons_for_host:
+    description: Return all Active Directory on-premises user logons for host/device name
+    metadata:
+    args:
+      query: '
+        {table}
+        | where {time_column} >= datetime({start})
+        | where {time_column} <= datetime({end})
+        | where DeviceName has "{host_name}"
+        {add_query_items}'
+      uri: None
+    parameters:
+      host_name:
+        description: The host/device source name for logons
+        type: str

msticpy/data/queries/m365d/kql_m365_network.yaml

@@ -39,7 +39,7 @@ sources:
       uri: None
     parameters:
   host_connections:
-    description: Returns connections by for a specified hostname
+    description: Returns connections by a specified hostname
     metadata:
     args:
       query: '

msticpy/init/mp_pandas_accessors.py

@@ -29,6 +29,8 @@ def __init__(self, pandas_obj):
         self._df = pandas_obj
         self._ioc = IoCExtract()
 
+        self._data_viewer_class: Any = None
+
     def b64extract(self, column: str, **kwargs) -> pd.DataFrame:
         """
         Base64-decode strings taken from a pandas dataframe.
@@ -241,3 +243,55 @@ def whois(self, ip_column, **kwargs):
 
         """
         return get_whois_df(data=self._df, ip_column=ip_column, **kwargs)
+
+    def view(self, **kwargs):
+        """
+        Initialize the DataViewer class.
+
+        Parameters
+        ----------
+        data : pd.DataFrame
+            The DataFrame to view
+        selected_cols : List[str], optional
+            Initial subset of columns to show, by default None (all cols)
+
+        Other Parameters
+        ----------------
+        selectable : Union[bool, str], optional
+            Whether rows should be selectable, by default "checkbox"
+        show_index : bool, optional
+            If True show the DataFrame index as a column, by default  True.
+        show_tenant_id : bool, optional
+            If True show the TenantId column, by default  True.
+        max_col_width : int, optional
+            Sets the maximum column width to display, by default 500
+        detail_cols : List[str]
+            List of columns for which details are displayed in collapsible
+            field beneath each table row.
+        kwargs :
+            Other keyword arguments are passed to the panel
+            Tabulator control.
+
+        Notes
+        -----
+        Main attributes:
+        value - original dataframe
+        selected - indexes of currently selected rows
+        selected_dataframe - currently selected rows
+        current_view - current dataframe after filtering and sorting
+        selection - indexes of currently selected rows
+
+        See Also
+        --------
+        Tabulator - https://panel.holoviz.org/reference/widgets/Tabulator.html
+
+        """
+        if self._data_viewer_class is None:
+            try:
+                # pylint: disable=import-outside-toplevel
+                from ..vis.data_viewer_panel import DataViewer
+            except ImportError:
+                print("This component needs the panel package.")
+                return self._df
+            self._data_viewer_class = DataViewer
+        return self._data_viewer_class(data=self._df, **kwargs)