Skip to content

Commit

Permalink
Merge pull request microsoft#293 from microsoft/pebryan/2022-1-18_Sec…
Browse files Browse the repository at this point in the history 
…urityAlertEntity

SentinelAlert entity creation
  • Loading branch information
@petebryan
petebryan committed Feb 1, 2022
2 parents 0fc0d9b + 89b4ee6 commit 4f8a60a
Show file tree
Hide file tree
Showing 11 changed files with 354 additions and 94 deletions.
1 change: 1 addition & 0 deletions msticpy/datamodel/entities/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
"iotdevice": IoTDevice,
"ip": IpAddress,
"networkconnection": NetworkConnection,
"network-connection": NetworkConnection,
"mailbox": Mailbox,
"mail-message": MailMessage,
"mailmessage": MailMessage,
Expand Down
203 changes: 128 additions & 75 deletions msticpy/datamodel/entities/alert.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,15 @@ class Alert(Entity):
Attributes
----------
DisplayName : str
AlertDisplayName : str
Alert DisplayName
CompromisedEntity : str
Alert CompromisedEntity
Count : int
Alert Count
StartTime : datetime
StartTimeUtc : datetime
Alert StartTime
EndTime : datetime
EndTimeUtc : datetime
Alert EndTime
Severity : str
Alert Severity
Expand Down Expand Up @@ -83,8 +83,8 @@ def __init__(
self.DisplayName: Optional[str] = None
self.CompromisedEntity: Optional[str] = None
self.Count: Any = None
self.StartTime: Optional[datetime] = None
self.EndTime: Optional[datetime] = None
self.StartTimeUtc: Optional[datetime] = None
self.EndTimeUtc: Optional[datetime] = None
self.Severity: Any = None
self.SystemAlertIds: List[str] = []
self.AlertType: Optional[str] = None
Expand All @@ -95,14 +95,16 @@ def __init__(
if src_entity:
self._create_from_ent(src_entity)

if isinstance(src_event, pd.Series) or src_event:
if isinstance(src_event, pd.Series) and not src_event.empty:
self._create_from_event(src_event)

def _create_from_ent(self, src_entity): # noqa: MC0001
if "StartTime" in src_entity or "TimeGenerated" in src_entity:
self.TimeGenerated = src_entity["StartTime"] or src_entity["TimeGenerated"]
self.TimeGeneratedUtc = (
src_entity["StartTime"] or src_entity["TimeGenerated"]
)
if "EndTime" in src_entity:
self.EndTime = src_entity["EndTime"]
self.EndTimeUtc = src_entity["EndTime"]
if "StartTime" in src_entity:
self.StartTime = src_entity["StartTime"]
if "AlertDisplayName" in src_entity:
Expand All @@ -124,28 +126,6 @@ def _create_from_ent(self, src_entity): # noqa: MC0001
self.Entities = self._create_entities(ents)
self._add_additional_data(src_entity)

def _extract_entities(self, src_row): # noqa: MC0001
input_entities = []
if isinstance(src_row.Entities, str):
try:
ext_props = json.loads(src_row["Entities"])
for item in ext_props:
for k, v in item.items():
if isinstance(v, dict) and "$ref" in v.keys():
item[k] = [x for x in ext_props if x["$id"] == v["$ref"]][0]
input_entities.append(item)
except json.JSONDecodeError:
pass
if isinstance(src_row.ExtendedProperties, str):
try:
ext_props = json.loads(src_row["ExtendedProperties"])
for ent, val in ext_props.items():
if ent in ["IpAddress", "Username"]:
input_entities.append({"Entity": val, "Type": ent})
except json.JSONDecodeError:
pass
return input_entities

@property
def description_str(self) -> str:
"""Return Entity Description."""
Expand All @@ -161,7 +141,6 @@ def name_str(self) -> str:

def _add_additional_data(self, src_entity: Mapping[str, Any]):
"""Populate additional alert properties."""
entity_props = set(self.__dict__.keys()) | {"AlertDisplayName", "SystemAlertId"}
if isinstance(src_entity, dict):
prop_list = src_entity.items()
elif type(src_entity).__name__ == "SecurityAlert":
Expand All @@ -174,21 +153,23 @@ def _add_additional_data(self, src_entity: Mapping[str, Any]):
return

for prop_name, prop in prop_list:
if prop_name not in entity_props:
if prop_name not in self._entity_schema:
self.AdditionalData[prop_name] = prop
elif prop_name not in self.__dict__:
self.__dict__[prop_name] = prop
else:
continue

def _create_from_event(self, src_event):
"""Create Alert from an alert event."""
self.TimeGenerated = src_event.get("StartTime", src_event.get("TimeGenerated"))
self.DisplayName = src_event.get("DisplayName", src_event.get("Name"))
self.CompromisedEntity = src_event.get("CompromisedEntity")
self.StartTime = src_event.get("StartTime")
self.EndTime = src_event.get("EndTime")
self.Severity = src_event.get("AlertSeverity")
self.AlertDisplayName = src_event.get(
"AlertDisplayName", src_event.get("DisplayName", src_event.get("Name"))
)
self.StartTimeUtc = src_event.get("StartTimeUtc", src_event.get("StartTime"))
self.EndTimeUtc = src_event.get("EndTimeUtc", src_event.get("EndTime"))
self.Severity = src_event.get("Severity", src_event.get("AlertSeverity"))
self.SystemAlertIds = src_event.get("SystemAlertId", src_event.get("ID"))
self.AlertType = src_event.get("AlertType")
self.VendorName = src_event.get("VendorName")
self.ProviderName = src_event.get("ProviderName")
if isinstance(src_event["Entities"], str):
try:
ents = _extract_entities(json.loads(src_event["Entities"]))
Expand All @@ -197,6 +178,101 @@ def _create_from_event(self, src_event):
else:
ents = _extract_entities(src_event["Entities"])
self.Entities = self._create_entities(ents)
for ent in self._entity_schema:
if ent not in self.__dict__:
self.__dict__[ent] = src_event.get(ent)
if "ExtendedProperties" in src_event:
ext_props = json.loads(src_event["ExtendedProperties"])
self._add_additional_data(ext_props)

_entity_schema = {
# CompromisedEntity (type String)
"CompromisedEntity": None,
# Count (type Int)
"Count": None,
# StartTimeUtc (type Datetime)
"StartTimeUtc": None,
# EndTimeUtc (type Datetime)
"EndTimeUtc": None,
# Severity (type String)
"Severity": None,
# SystemAlertIds (type String)
"SystemAlertId": None,
# AlertType (type System.String)
"AlertType": None,
# VendorName (type System.String)
"VendorName": None,
# ProviderName (type System.String)
"ProviderName": None,
# List of associated entities (type List)
"Entities": None,
# Time the alert was generated (type String)
"TimeGenerated": None,
# The product that generated the alert (type String)
"ProductName": None,
# The product component that generated the alert (type String)
"ProductComponentName": None,
# The version of the product generating the alert, if relevant (type String)
"ProductVersion": None,
# The time the alert was made available for consumption (type String)
"ProcessingEndTime": None,
# The life cycle status of the alert. This field is optional and all alerts would have the status (type String)
"Status": None,
# The alert provider or product internal life cycle status (type String)
"ProviderAlertStatus": None,
# The confidence level of this alert (type String)
"ConfidenceLevel": None,
# The confidence score of the alert (type Float)
"ConfidenceScore": None,
# The confidence score calculation status (type String)
"ConfidenceScoreStatus": None,
# A list of reasons for the confidence level of this alert (type List)
"ConfidenceReasons": None,
# The kill chain related intent behind the alert (type String)
"Intent": None,
# The kill chain related techniques behind the alert (type List)
"Techniques": None,
# The kill chain related sub-techniques behind the alert (type List)
"SubTechniques": None,
# If the alert is an incident or a regular alert (type Bool)
"IsIncident": None,
# If the alert is in preview (type Bool)
"IsPreview": None,
# Unique id for the specific alert instance set by the provider (type String)
"ProviderAlertId": None,
# Key to correlate multiple alerts together (type String)
"CorrelationKey": None,
# Identifiers of the Investigations created by the provider for the Alert (type List)
"InvestigationIds": None,
# The resource identifiers for this alert (type List)
"ResourceIdentifiers": None,
# Display name of the main entity being reported on (type String)
"CompromisedEntity": None,
# The display name of the alert (type String)
"AlertDisplayName": None,
# Alert description (type String)
"Description": None,
# Description arguments to build up Description field in placeholders (type Dict)
"DescriptionArguments": None,
# SupportingEvidence (type Dict)
"SupportingEvidence": None,
# Manual action items to take to remediate the alert (type List)
"RemediationSteps": None,
# A bag of fields which will be presented to the use (type Dict)
"ExtendedProperties": None,
# A bag for all links related to the alert (type Dict)
"ExtendedLinks": None,
# Metadata associated with the alert (type Dict)
"Metadata": None,
# A list of edges contained in this alert (type Dict)
"Edges": None,
# A direct link to view the specific alert in originating product portal (type String)
"AlertUri": None,
# Used to provide details about an anomaly in the data found by ML algorithms (type Dict)
"Anomaly": None,
# Used to provide details about a policy assocaited with the alert (type Dict)
"AlertPolicy": None,
}

def _create_entities(self, entities):
"""Create alert entities from returned dicts."""
Expand All @@ -218,35 +294,7 @@ def _create_entities(self, entities):
new_ents.append(ent_obj)
return new_ents

_entity_schema = {
# DisplayName (type System.String)
"DisplayName": None,
# CompromisedEntity (type System.String)
"CompromisedEntity": None,
# Count (type System.Nullable`1[System.Int32])
"Count": None,
# StartTimeUtc (type System.Nullable`1[System.DateTime])
"StartTime": None,
# EndTimeUtc (type System.Nullable`1[System.DateTime])
"EndTime": None,
# Severity (type System.Nullable`1
# [Microsoft.Azure.Security.Detection.AlertContracts.V3.Severity])
"Severity": None,
# SystemAlertIds (type System.Collections.Generic.List`1[System.String])
"SystemAlertIds": None,
# AlertType (type System.String)
"AlertType": None,
# VendorName (type System.String)
"VendorName": None,
# ProviderName (type System.String)
"ProviderName": None,
# List of associated entities
"Entities": None,
# Time the alert was generated.
"TimeGenerated": None,
}

def to_html(self, show_entities=False) -> str:
def to_html(self) -> str:
"""Return the item as HTML string."""
return (
"""
Expand Down Expand Up @@ -277,26 +325,31 @@ def _extract_entities(ents: list):
out_ents = []
for entity in ents:
if isinstance(entity, dict) and "$ref" in entity:
out_ents.append(_find_og_ent(entity, base_ents))
out_ents.append(_find_original_entity(entity, base_ents))
else:
for k, val in entity.items():
if isinstance(val, (list, dict)):
if isinstance(val, list):
nested_ents = []
for item in val:
if isinstance(item, dict) and "$ref" in item:
nested_ents.append(_find_og_ent(item, base_ents))
nested_ents.append(
_find_original_entity(item, base_ents)
)
entity[k] = nested_ents
elif isinstance(val, dict) and "$ref" in val:
entity[k] = _find_og_ent(val, base_ents)
entity[k] = _find_original_entity(val, base_ents)
out_ents.append(entity)
return out_ents


def _find_og_ent(ent, base_ents):
def _find_original_entity(ent, base_ents):
"""Find the original entity referenced by $ref entity."""
id = ent["$ref"]
return next(bent for bent in base_ents if ("$id" in bent) and bent["$id"] == id)
try:
id = ent["$ref"]
return next(bent for bent in base_ents if ("$id" in bent) and bent["$id"] == id)
except StopIteration:
return ent


def _generate_base_ents(ents: list) -> list: # noqa: MC0001
Expand Down
30 changes: 19 additions & 11 deletions msticpy/datamodel/entities/entity.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import json
import pprint
import typing
from datetime import datetime
from abc import ABC
from copy import deepcopy
from typing import Any, Dict, List, Mapping, Optional, Type, Union
Expand Down Expand Up @@ -82,7 +83,7 @@ def __init__(self, src_entity: Mapping[str, Any] = None, **kwargs):
"""
super().__init__()
self.TimeGenerated = None
self.TimeGenerated = datetime.utcnow()
self.Type = self._get_entity_type_name(type(self))
# If we have an unknown entity see if we a type passed in
if self.Type == "unknownentity" and "Type" in kwargs:
Expand Down Expand Up @@ -340,10 +341,10 @@ def is_equivalent(self, other: Any) -> bool:
if not isinstance(other, Entity):
return False
return not any(
self.properties[prop] != other.properties[prop]
and self.properties[prop]
and other.properties[prop]
for prop in self.properties
self.__dict__[prop] != other.__dict__[prop]
and self.__dict__[prop]
and other.__dict__[prop]
for prop in self.__dict__ # pylint: disable=consider-using-dict-items
)

def merge(self, other: Any) -> "Entity":
Expand All @@ -369,7 +370,7 @@ def merge(self, other: Any) -> "Entity":
for prop, value in other.properties.items():
if not value:
continue
if not self.properties[prop]:
if not self.__dict__[prop]:
setattr(merged, prop, value)
# Future (ianhelle) - cannot merge ID field
if other.edges:
Expand Down Expand Up @@ -424,7 +425,7 @@ def properties(self) -> dict:
return {
name: value
for name, value in self.__dict__.items()
if not name.startswith("_") and name != "edges"
if not name.startswith("_") and name != "edges" and value
}

@property
Expand Down Expand Up @@ -509,11 +510,18 @@ def _get_entity_type_name(cls, entity_type: Type) -> str:
The V3 serialized name.
"""
name = next(
iter(
(key for key, val in cls.ENTITY_NAME_MAP.items() if val == entity_type)
try:
name = next(
iter(
(
key
for key, val in cls.ENTITY_NAME_MAP.items()
if val == entity_type
)
)
)
)
except StopIteration:
name = None
return name or "unknown"

@property
Expand Down
4 changes: 4 additions & 0 deletions msticpy/datamodel/soc/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,7 @@
# license information.
# --------------------------------------------------------------------------
"""SOC Entity sub-package."""

# flake8: noqa: F401
from .incident import Incident
from .sentinel_alert import SentinelAlert

0 comments on commit 4f8a60a

Please sign in to comment.