Fixed load_events() method for AlarmManager

.github/workflows/test.yml

@@ -2,7 +2,7 @@ name: test
 on:
   schedule:
     # Run complete tests every month
-    - cron:  '0 0 1 * *'
+    - cron:  '0 0 20 * *'
 jobs: 
   test:
     name: ${{ matrix.os }} py${{ matrix.python-version }}

msiempy/alarm.py

@@ -5,7 +5,7 @@
 log = logging.getLogger('msiempy')
 
 from .core import NitroDict, FilteredQueryList
-from .event import Event
+from .event import Event, EventManager
 from .core.utils import regex_match, dehexify
 
 __pdoc__={}
@@ -22,8 +22,8 @@ class AlarmManager(FilteredQueryList):
     `filters` are computed locally - Unlike `msiempy.event.EventManager` filters.  
     - `page_size` : max number of rows per query.  
     - `page_number` : defaulted to 1.
-    - `filters` : `[(field, [values]), (field, [values])]` Filters applied to `msiempy.alarm.Alarm` objects. A single `tuple` is also accepted. 
-    - `event_filters` : `[(field, [values]), (field, [values])]` Filters applied to `msiempy.event.Event` objects. A single `tuple` is also accepted.  
+    - `filters` : `[(field, [values]), (field, [values])]` Filters applied to `msiempy.alarm.Alarm` objects. A single `tuple` is also accepted.  
+    - `event_filters` : `[(field, [values]), (field, [values])]` Filters applied to `msiempy.event.Event` objects. A single `tuple` is also accepted.   
     - `time_range` : Query time range. String representation of a time range.  
     - `start_time` : Query starting time, can be a `string` or a `datetime` object. Parsed with `dateutil`.  
     - `end_time` : Query endding time, can be a `string` or a `datetime` object. Parsed with `dateutil`.  
@@ -204,7 +204,6 @@ def load_data(self, pages=1, **kwargs):
 
         if 'page_number' not in kwargs:
             log.info(str(len(alarms)) + " alarms are matching your filter(s)")
-
 
         self.data=alarms
         return(self)
@@ -253,32 +252,28 @@ def qry_load_data(self, workers=10,
                 )
 
         #Casting to list of Alarms to be able to call load_details etc...        
-        alarm_based_filtered = [Alarm(adict=a) for a in no_filtered_alarms if self._alarm_match(a)]
+        alarm_based_filtered = AlarmManager([a for a in no_filtered_alarms if self._alarm_match(a)])
 
         if alarms_details :
 
             log.info("Getting alarms infos...")
-            alarm_detailed = self.perform(Alarm.load_details,
-                list(alarm_based_filtered),
+            alarm_based_filtered.perform(Alarm.load_details,
                 asynch=True,
                 progress=True,
                 workers=workers)
 
             #Casting to list of Alarms to be able to call load_details etc...        
-            detailed_alarm_based_filtered = [Alarm(adict=a) for a in alarm_detailed if self._alarm_match(a)]
+            detailed_alarm_based_filtered = AlarmManager([ a for a in alarm_based_filtered if self._alarm_match(a) ])
 
             if events_details :
                 log.info("Getting full events infos...")
-                event_detailed = self.perform(Alarm.load_events, 
-                    list(alarm_detailed),
+                detailed_alarm_based_filtered.perform(Alarm.load_events, 
                     func_args=dict(use_query=use_query, extra_fields=extra_fields),
                     asynch=True, 
                     progress=True, 
                     workers=workers)
 
-                filtered_alarms = [Alarm(adict=a) for a in event_detailed if self._event_match(a)]
-            else:
-                filtered_alarms = [Alarm(adict=a) for a in detailed_alarm_based_filtered if self._event_match(a)] 
+            filtered_alarms = AlarmManager([ a for a in detailed_alarm_based_filtered if self._event_match(a)]) 
         else :
             filtered_alarms = alarm_based_filtered
             log.warning('Event filters and some Alarm filters are ignored when `alarms_details is False`')
@@ -287,13 +282,13 @@ def qry_load_data(self, workers=10,
 
     def _alarm_match(self, alarm):
         """
-        Internal filter method that is going to return True if the passed alarm match any alarm related filters.
+        Internal filter method that is going to return True if the passed alarm match all alarm related filters.
         """
         match=True
         for alarm_filter in self._alarm_filters :
             match=False
             try: value = str(alarm[alarm_filter[0]]) #Can only match strings
-            except KeyError: continue
+            except KeyError: break
             for filter_value in alarm_filter[1]:
                 if regex_match(filter_value.lower(), value.lower()):
                     match=True
@@ -304,13 +299,13 @@ def _alarm_match(self, alarm):
 
     def _event_match(self, alarm):
         """
-        Internal filter method that is going to return True if the passed alarm match any event related filters.
+        Internal filter method that is going to return True if the passed alarm match all event related filters.
         """
         match=True
         for event_filter in self._event_filters :
             match=False
             try: value = str(alarm['events'][0][event_filter[0]])
-            except KeyError: continue
+            except KeyError: break
             for filter_value in event_filter[1]:
                 if regex_match(filter_value.lower(), value.lower()) :
                     match=True
@@ -352,7 +347,7 @@ def __init__(self, *arg, **kwargs):
             self.data['id'] = {'value':str(kwargs['id'])}
         # Casting all events to Event object
         if 'events' in self.data and isinstance(self.data['events'], list):
-            self.data['events'] = [Event(d) for d in self.data['events']]
+            self['events']=EventManager(self.data['events'])
 
     POSSIBLE_ALARM_STATUS=[
         ['acknowledged', 'ack',],
@@ -425,24 +420,27 @@ def load_details(self):
         """
         the_id = self.data['id']['value']
         self.data.update(self.data_from_id(the_id))
+        if isinstance(self.data['events'], (list)):
+            self['events']=EventManager(self.data['events'])
         self.data['id']['value']=the_id
-
+        
         return self
 
     def refresh(self):
         """Update the alarm with detailled data loaded from the SIEM.  
         """        
         self.load_details()
 
-    def load_events(self, use_query=False, extra_fields=[]):
+    def load_events(self, use_query=False, extra_fields=[], workers=1):
         """
-        Retreive the complete trigerring Event objects from an Alarm.  
-        This methos is automatically called automatically bu default when calling `load_data()`.   
+        Retreive the complete trigerring Event(s) objects from an Alarm.  
+        This methos is automatically called automatically by default when calling `load_data()`.   
         Arguments:  
 
-        - `use_query` : Uses the query module to retreive common event data. Only works with SIEM v 11.2 or greater.    
+        - `use_query` : Uses the query module to retreive the event(s) data. Only works with SIEM v 11.2 or greater.    
         Default behaviour will call `ipsGetAlertData` to retreive the complete event definition.  
         - `extra_fields` : Only when `use_query=True`. Additionnal event fields to load in the query. See : `msiempy.event.EventManager`  
+        - `workers`: The number of asynchronous workers.  
 
         .. Warning:: On SIEM v10.X This method will only load the details of the first triggering event.  
         """
@@ -459,6 +457,9 @@ def load_events(self, use_query=False, extra_fields=[]):
             #set it as the only item of the event list
             self.data['events']= [ the_first_event ]
 
+        elif isinstance(self.data['events'], (EventManager)):
+            # The list has been loaded from notifyGetTriggeredNotificationDetail
+            self.data['events'].perform(Event.refresh, asynch=True, workers=workers, func_args=dict(use_query=use_query, extra_fields=extra_fields))
         else:
             log.info('The alarm {} ({}) has no events associated'.format(self.data['alarmName'], self.data['triggeredDate']))
             self.data['events']= [ Event() ]

msiempy/core/session.py

@@ -628,10 +628,15 @@ def api_request(self, method, data=None, http='post', callback=None, raw=False,
 
                     return result
 
-        #Hard errors, could retry
-        except requests.exceptions.Timeout as e:
-            log.error(e)
-            raise
+        except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
+
+            if retry>0 :
+                log.warning('An network error occured ({}), retrying api_request()'.format(e))
+                time.sleep(1)
+                return self.api_request(method, data, http, callback, raw, secure, retry=retry-1)
+            else :
+                raise e
+
         except requests.exceptions.TooManyRedirects as e :
             log.error(e)
             raise

msiempy/core/types.py

@@ -290,7 +290,6 @@ def refresh(self):
         """
         Execute refresh function on all items.
         """
-        log.warning("The function NitroList.refresh hasn't been correctly tested")
         self.perform(NitroDict.refresh, message='Refreshing all items...')
 
     def perform(self, func, data=None, func_args=None, confirm=False, asynch=False,  workers=None , progress=False, message=None):

msiempy/event.py

@@ -1188,9 +1188,9 @@ def data_from_id(self, id, use_query=False, extra_fields=[]):
         Arguments:   
 
         - `id` : The event ID. (i.e. : `144128388087414784|747122896`)  
-        - `use_query` : Uses the query module to retreive common event data. Only works with SIEM 11.2 or greater.    
+        - `use_query` (`bool`): Uses the query module to retreive common event data. Only works with SIEM 11.2 or greater.    
         Default behaviour will call `ipsGetAlertData` to retreive the complete event definition.  
-        - `extra_fields` : Only when `use_query=True`. Additionnal event fields to load in the query.  
+        - `extra_fields` (`list`): Only when `use_query=True`. Additionnal event fields to load in the query.  
         """
 
         if use_query == True :
@@ -1217,15 +1217,37 @@ def data_from_id(self, id, use_query=False, extra_fields=[]):
         elif use_query == False :
             return self.nitro.request('get_alert_data', id=id)
 
-    def refresh(self): 
-        """Re-load event's data"""
-        if 'Alert.IPSIDAlertID' in self.data.keys() :
-            # ensure to re-use the query module if that's the case
-            self.data.update(self.data_from_id(self.data['Alert.IPSIDAlertID'], 
-                use_query=True, extra_fields=self.data.keys()))
-        else :
+    def refresh(self, use_query=None, extra_fields=None): 
+        """
+        Re-load event's data.
+
+        Arguments:  
+        - `use_query` (`bool`): Force the use of the query module to retreive the event data. 
+        The default behaviour will use `EventManager` query if an 'Alert.IPSIDAlertID' key is present in the event, 
+        else call `ipsGetAlertData` to get the full details.  
+        - `extra_fields` (`list`): Only when `use_query=True` or the Event is already a query event. Additionnal event fields to load in the query.  
+
+        .. Warning:: Enforce `use_query=True` will reset the Events fields to whatever is passed to `extra_fields`
+
+        Raise `AttributeError` if the event ID has not been found.  
+        """
+        if not self.get_id():
+            raise AttributeError("Can't refresh a Event without an ID: {}".format(self.data))
+        if use_query==None:
+            if 'Alert.IPSIDAlertID' in self.data.keys() :
+                # ensure to re-use the query module if that's the case
+                self.data.update(self.data_from_id(self.data['Alert.IPSIDAlertID'], 
+                    use_query=True, extra_fields=self.data.keys() + extra_fields if extra_fields else [] ))
+            else :
+                the_id = self.get_id()
+                self.data.update(self.data_from_id(the_id))
+        elif use_query:
+            self.data.update(self.data_from_id(self.get_id(), 
+                use_query=True, extra_fields=extra_fields if extra_fields else [] ))
+        else:
             the_id = self.get_id()
             self.data.update(self.data_from_id(the_id))
+
 
 class _QueryFilter(collections.UserDict): 
     """Base class for all SIEM query objects in order to dump the filter as dict.

tests/auth/test_alarm.py

@@ -1,5 +1,6 @@
 import msiempy
 from msiempy.core import NitroList
+from msiempy import EventManager
 import msiempy.alarm
 import unittest
 import pprint
@@ -188,7 +189,7 @@ def test_loading_part_of_the_alarm_details_and_events(self):
 
         for alarm in detailed :
             events = alarm.get('events', 0) # Events should not be zero
-            self.assertIn(type(events), [str, type(None), list] , msg="No events loaded for the alarm after load_details() call")
+            self.assertIn(type(events), [str, type(None), list, EventManager] , msg="No events loaded for the alarm after load_details() call")
 
         detailed_w_events = alarms.perform(msiempy.alarm.Alarm.load_events, 
             data=[alarms[1]], 
@@ -228,7 +229,6 @@ def test_ack_unack(self):
         #     alarms.perform(msiempy.alarm.Alarm.refresh)
         #     alarms.perform(msiempy.alarm.Alarm.refresh)
 
-        alarms.perform(msiempy.alarm.Alarm.refresh)
         print(alarms.get_text(fields=['id','acknowledgedDate','acknowledgedUsername']))
         [ self.assertTrue(len(alarm['acknowledgedDate']) > 0) for alarm in alarms ]
 

tests/auth/test_event.py

@@ -217,7 +217,6 @@ def test_getitem(self):
         [ self.assertFalse(key in an_event) for key in ["Rule.msg","Alert.SrcIP","Alert.DstIP", "Alert.SrcMac","Alert.DstMac","Alert.NormID","Alert.BIN(4)"] ]
         [ self.assertFalse(key in an_event) for key in ["Rule.msg","SrcIP","DstIP", "SrcMac","DstMac","NormID","HostID"] ]
 
-    @unittest.skip("Skipping this test until the test ESM can generate Alarms with event associated ")
     def test_get_id(self):
 
         alarms = AlarmManager(