Fix lookback timezone

Packs/Base/ReleaseNotes/1_31_34.md

@@ -0,0 +1,5 @@
+
+#### Scripts
+##### CommonServerPython
+- Fix look back feature to support timezone aware dates.
+

Packs/Base/Scripts/CommonServerPython/CommonServerPython.py

@@ -27,6 +27,7 @@
 from distutils.version import LooseVersion
 from threading import Lock
 from inspect import currentframe
+import pytz
 
 import demistomock as demisto
 import warnings
@@ -37,9 +38,9 @@ def __line__():
     return cf.f_back.f_lineno
 
 
-# 42 - The line offset from the beggining of the file.
+# 43 - The line offset from the beggining of the file.
 _MODULES_LINE_MAPPING = {
-    'CommonServerPython': {'start': __line__() - 42, 'end': float('inf')},
+    'CommonServerPython': {'start': __line__() - 43, 'end': float('inf')},
 }
 
 
@@ -1408,6 +1409,7 @@ class SmartGetDict(dict):
     :rtype: ``SmartGetDict``
 
     """
+
     def get(self, key, default=None):
         res = dict.get(self, key)
         if res is not None:
@@ -10290,6 +10292,9 @@ def get_fetch_run_time_range(last_run, first_fetch, look_back=0, timezone=0, dat
     else:
         last_run_time = dateparser.parse(last_run_time, settings={'TIMEZONE': 'UTC'})
 
+    # # Make sure both times are timezone aware
+    now = now.replace(tzinfo=pytz.UTC)
+    last_run_time = last_run_time.replace(tzinfo=pytz.UTC)
     if look_back > 0:
         if now - last_run_time < timedelta(minutes=look_back):
             last_run_time = now - timedelta(minutes=look_back)

Packs/Base/Scripts/CommonServerPython/CommonServerPython_test.py

@@ -10,6 +10,7 @@
 import dateparser
 from freezegun import freeze_time
 import pytest
+import pytz
 import requests
 from pytest import raises, mark
 
@@ -7529,14 +7530,28 @@ class TestFetchWithLookBack:
         }
     ]
 
-    def example_fetch_incidents(self):
+    INCIDENTS_TIME_AWARE = [
+        {
+            'incident_id': incident.get('incident_id'),
+            'created': incident.get('created') + 'Z'
+        } for incident in INCIDENTS
+    ]
+
+    NEW_INCIDENTS_TIME_AWARE = [
+        {
+            'incident_id': incident.get('incident_id'),
+            'created': incident.get('created') + 'Z'
+        } for incident in NEW_INCIDENTS
+    ]
+
+    def example_fetch_incidents(self, time_aware=False):
         """
         An example fetch for testing
         """
 
         from CommonServerPython import get_fetch_run_time_range, filter_incidents_by_duplicates_and_limit, \
             update_last_run_object
-
+        date_format = '%Y-%m-%dT%H:%M:%S' + ('Z' if time_aware else '')
         incidents = []
 
         params = demisto.params()
@@ -7549,18 +7564,18 @@ def example_fetch_incidents(self):
         fetch_limit = last_run.get('limit') or fetch_limit_param
 
         start_fetch_time, end_fetch_time = get_fetch_run_time_range(last_run=last_run, first_fetch=first_fetch,
-                                                                    look_back=look_back, timezone=time_zone)
+                                                                    look_back=look_back, timezone=time_zone, date_format=date_format)
 
         query = self.build_query(start_fetch_time, end_fetch_time, fetch_limit)
-        incidents_res = self.get_incidents_request(query)
+        incidents_res = self.get_incidents_request(query, date_format)
 
         incidents = filter_incidents_by_duplicates_and_limit(incidents_res=incidents_res, last_run=last_run,
                                                              fetch_limit=fetch_limit_param, id_field='incident_id')
 
         last_run = update_last_run_object(last_run=last_run, incidents=incidents, fetch_limit=fetch_limit_param,
                                           start_fetch_time=start_fetch_time,
                                           end_fetch_time=end_fetch_time, look_back=look_back,
-                                          created_time_field='created', id_field='incident_id')
+                                          created_time_field='created', id_field='incident_id', date_format=date_format)
 
         demisto.setLastRun(last_run)
         return incidents
@@ -7572,10 +7587,12 @@ def build_query(start_time, end_time, limit, return_incidents_by_limit=True):
             query['limit'] = limit
         return query
 
-    def get_incidents_request(self, query):
-        from_time = datetime.strptime(query['from'], '%Y-%m-%dT%H:%M:%S')
-        incidents = [inc for inc in self.INCIDENTS if
-                     datetime.strptime(inc['created'], '%Y-%m-%dT%H:%M:%S') > from_time]
+    def get_incidents_request(self, query, date_format):
+        time_aware = 'Z' in date_format
+        source_incidents = self.INCIDENTS_TIME_AWARE if time_aware else self.INCIDENTS
+        from_time = datetime.strptime(query['from'], date_format)
+        incidents = [inc for inc in source_incidents if
+                     datetime.strptime(inc['created'], date_format) > from_time]
         if query.get('limit') is not None:
             return incidents[:query['limit']]
         return incidents
@@ -7592,6 +7609,17 @@ def set_last_run(self, new_last_run):
          {'limit': 2, 'time': INCIDENTS[2]['created']}),
         ({'limit': 3, 'first_fetch': '2 hours'}, [INCIDENTS[1], INCIDENTS[2], INCIDENTS[3]], [INCIDENTS[4]],
          {'limit': 3, 'time': INCIDENTS[3]['created']}),
+
+        ({'limit': 2, 'first_fetch': '40 minutes'}, [INCIDENTS_TIME_AWARE[2], INCIDENTS_TIME_AWARE[3]], [INCIDENTS_TIME_AWARE[4]],
+         {'limit': 2, 'time': INCIDENTS_TIME_AWARE[3]['created']}),
+        ({'limit': 3, 'first_fetch': '40 minutes'}, [INCIDENTS_TIME_AWARE[2], INCIDENTS_TIME_AWARE[3], INCIDENTS_TIME_AWARE[4]], [],
+         {'limit': 3, 'time': INCIDENTS_TIME_AWARE[4]['created']}),
+        ({'limit': 2, 'first_fetch': '2 hours'}, [INCIDENTS_TIME_AWARE[1], INCIDENTS_TIME_AWARE[2]], [INCIDENTS_TIME_AWARE[3],
+                                                                                                      INCIDENTS_TIME_AWARE[4]],
+         {'limit': 2, 'time': INCIDENTS_TIME_AWARE[2]['created']}),
+        ({'limit': 3, 'first_fetch': '2 hours'}, [INCIDENTS_TIME_AWARE[1], INCIDENTS_TIME_AWARE[2], INCIDENTS_TIME_AWARE[3]],
+         [INCIDENTS_TIME_AWARE[4]],
+         {'limit': 3, 'time': INCIDENTS_TIME_AWARE[3]['created']}),
     ])
     def test_regular_fetch(self, mocker, params, result_phase1, result_phase2, expected_last_run):
         """
@@ -7608,6 +7636,7 @@ def test_regular_fetch(self, mocker, params, result_phase1, result_phase2, expec
             # skip for python 2 - date
             assert True
             return
+        time_aware = 'Z' in expected_last_run['time']
 
         self.LAST_RUN = {}
 
@@ -7617,24 +7646,26 @@ def test_regular_fetch(self, mocker, params, result_phase1, result_phase2, expec
         mocker.patch.object(demisto, 'setLastRun', side_effect=self.set_last_run)
 
         # Run first fetch
-        incidents_phase1 = self.example_fetch_incidents()
+        incidents_phase1 = self.example_fetch_incidents(time_aware)
 
         assert incidents_phase1 == result_phase1
         assert self.LAST_RUN == expected_last_run
 
         # Run second fetch
         mocker.patch.object(demisto, 'getLastRun', return_value=self.LAST_RUN)
-        incidents_phase2 = self.example_fetch_incidents()
+        incidents_phase2 = self.example_fetch_incidents(time_aware)
 
         assert incidents_phase2 == result_phase2
 
     def mock_dateparser(self, date_string, settings):
+        time_aware = isinstance(date_string, str) and 'Z' in date_string
+        date_format = '%Y-%m-%dT%H:%M:%S' + ('Z' if time_aware else '')
         date_arr = date_string.split(' ')
         if len(date_arr) > 1 and date_arr[0].isdigit():
             return datetime(2022, 4, 1, 11, 0, 0) - timedelta(minutes=int(date_arr[0])) if date_arr[1] == 'minutes' \
                 else datetime(2022, 4, 1, 11, 0, 0) - timedelta(hours=int(date_arr[0]))
         return datetime(2022, 4, 1, 11, 0, 0) - (
-                    datetime(2022, 4, 1, 11, 0, 0) - datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%S'))
+                    datetime(2022, 4, 1, 11, 0, 0) - datetime.strptime(date_string, date_format))
 
     @pytest.mark.parametrize(
         'params, result_phase1, result_phase2, result_phase3, expected_last_run_phase1, expected_last_run_phase2, new_incidents, index',
@@ -7667,6 +7698,38 @@ def mock_dateparser(self, date_string, settings):
                     {'found_incident_ids': {3: '', 4: '', 5: '', 7: '', 8: ''}, 'limit': 3},
                     [NEW_INCIDENTS[1], NEW_INCIDENTS[2]], 3
             ),
+
+            (
+                    {'limit': 2, 'first_fetch': '50 minutes', 'look_back': 15}, [INCIDENTS_TIME_AWARE[2], INCIDENTS_TIME_AWARE[3]],
+                    [NEW_INCIDENTS_TIME_AWARE[0], INCIDENTS_TIME_AWARE[4]], [],
+                    {'found_incident_ids': {3: '', 4: ''}, 'limit': 4},
+                    {'found_incident_ids': {3: '', 4: '', 5: '', 6: ''}, 'limit': 6},
+                    [NEW_INCIDENTS_TIME_AWARE[0]], 2
+            ),
+            (
+                    {'limit': 2, 'first_fetch': '20 minutes', 'look_back': 30}, [INCIDENTS_TIME_AWARE[2], INCIDENTS_TIME_AWARE[3]],
+                    [NEW_INCIDENTS_TIME_AWARE[1], NEW_INCIDENTS_TIME_AWARE[2]], [INCIDENTS_TIME_AWARE[4]],
+                    {'found_incident_ids': {3: '', 4: ''}, 'limit': 4},
+                    {'found_incident_ids': {3: '', 4: '', 7: '', 8: ''}, 'limit': 6},
+                    [NEW_INCIDENTS_TIME_AWARE[1], NEW_INCIDENTS_TIME_AWARE[2]], 3
+            ),
+            (
+                    {'limit': 3, 'first_fetch': '181 minutes', 'look_back': 15},
+                    [INCIDENTS_TIME_AWARE[0], INCIDENTS_TIME_AWARE[1], INCIDENTS_TIME_AWARE[2]], [NEW_INCIDENTS_TIME_AWARE[0],
+                                                                                                  INCIDENTS_TIME_AWARE[3],
+                                                                                                  INCIDENTS_TIME_AWARE[4]], [],
+                    {'found_incident_ids': {1: '', 2: '', 3: ''}, 'limit': 6},
+                    {'found_incident_ids': {1: '', 2: '', 3: '', 4: '', 5: '', 6: ''}, 'limit': 9},
+                    [NEW_INCIDENTS_TIME_AWARE[0]], 2
+            ),
+            (
+                    {'limit': 3, 'first_fetch': '20 minutes', 'look_back': 30},
+                    [INCIDENTS_TIME_AWARE[2], INCIDENTS_TIME_AWARE[3], INCIDENTS_TIME_AWARE[4]],
+                    [NEW_INCIDENTS_TIME_AWARE[1], NEW_INCIDENTS_TIME_AWARE[2]], [],
+                    {'found_incident_ids': {3: '', 4: '', 5: ''}, 'limit': 6},
+                    {'found_incident_ids': {3: '', 4: '', 5: '', 7: '', 8: ''}, 'limit': 3},
+                    [NEW_INCIDENTS_TIME_AWARE[1], NEW_INCIDENTS_TIME_AWARE[2]], 3
+            ),
         ])
     def test_fetch_with_look_back(self, mocker, params, result_phase1, result_phase2, result_phase3,
                                   expected_last_run_phase1, expected_last_run_phase2, new_incidents, index):
@@ -7684,9 +7747,9 @@ def test_fetch_with_look_back(self, mocker, params, result_phase1, result_phase2
             # skip for python 2 - date
             assert True
             return
-
+        time_aware = 'Z' in result_phase1[0]['created']
         self.LAST_RUN = {}
-        incidents = self.INCIDENTS[:]
+        incidents = self.INCIDENTS_TIME_AWARE[:] if time_aware else self.INCIDENTS[:] 
 
         mocker.patch.object(CommonServerPython, 'get_current_time', return_value=datetime(2022, 4, 1, 11, 0, 0))
         mocker.patch.object(dateparser, 'parse', side_effect=self.mock_dateparser)
@@ -7695,7 +7758,7 @@ def test_fetch_with_look_back(self, mocker, params, result_phase1, result_phase2
         mocker.patch.object(demisto, 'setLastRun', side_effect=self.set_last_run)
 
         # Run first fetch
-        incidents_phase1 = self.example_fetch_incidents()
+        incidents_phase1 = self.example_fetch_incidents(time_aware)
 
         assert incidents_phase1 == result_phase1
         assert self.LAST_RUN['limit'] == expected_last_run_phase1['limit']
@@ -7704,11 +7767,14 @@ def test_fetch_with_look_back(self, mocker, params, result_phase1, result_phase2
             assert inc['incident_id'] in self.LAST_RUN['found_incident_ids']
 
         next_run_phase1 = self.LAST_RUN['time']
-        self.INCIDENTS = incidents[:index] + new_incidents + incidents[index:]
-
+        source_incidents = incidents[:index] + new_incidents + incidents[index:]
+        if time_aware:
+            self.INCIDENTS_TIME_AWARE = source_incidents
+        else:
+            self.INCIDENTS = source_incidents
         # Run second fetch
         mocker.patch.object(demisto, 'getLastRun', return_value=self.LAST_RUN)
-        incidents_phase2 = self.example_fetch_incidents()
+        incidents_phase2 = self.example_fetch_incidents(time_aware)
 
         assert incidents_phase2 == result_phase2
         assert self.LAST_RUN['limit'] == expected_last_run_phase2['limit']
@@ -7724,12 +7790,15 @@ def test_fetch_with_look_back(self, mocker, params, result_phase1, result_phase2
 
         # Run third fetch
         mocker.patch.object(demisto, 'getLastRun', return_value=self.LAST_RUN)
-        incidents_phase3 = self.example_fetch_incidents()
+        incidents_phase3 = self.example_fetch_incidents(time_aware)
 
         assert incidents_phase3 == result_phase3
 
         # Remove new incidents from self.INCIDENTS
-        self.INCIDENTS = incidents
+        if time_aware:
+            self.INCIDENTS_TIME_AWARE = incidents
+        else:
+            self.INCIDENTS = incidents
 
 
 class TestTracebackLineNumberAdgustment:

Packs/Base/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Base",
     "description": "The base pack for Cortex XSOAR.",
     "support": "xsoar",
-    "currentVersion": "1.31.33",
+    "currentVersion": "1.31.34",
     "author": "Cortex XSOAR",
     "serverMinVersion": "6.0.0",
     "url": "https://www.paloaltonetworks.com/cortex",
@@ -24,4 +24,4 @@
         "xsoar",
         "marketplacev2"
     ]
-}
+}

Packs/DeveloperTools/TestPlaybooks/script-CallTableToMarkdown.yml

@@ -81,6 +81,7 @@ timeout: '0'
 type: python
 subtype: python2
 tags: []
+dockerimage: demisto/python:2.7.18.27799
 tests:
 - No test
 fromversion: 5.0.0

Packs/DeveloperTools/TestPlaybooks/script-TestPyCommonServer.yml

@@ -615,5 +615,6 @@ scripttarget: 0
 runonce: false
 runas: DBotWeakRole
 fromversion: 5.0.0
+dockerimage: demisto/python:2.7.18.27799
 tests:
 - TestCommonPython

pyproject.toml

@@ -23,8 +23,9 @@ pytest-mock = "^3.7.0"
 freezegun = "^1.1.0"
 mock-open = "^1.4.0"
 
+# If mypy requires for a `types-*` package, add it here with `poetry add --group typing <package>`
 [tool.poetry.group.typing.dependencies]
-# If mypy requires for a `typed-*` package, add it here with `poetry add --group typing <package>`
+types-python-dateutil = "^2.8.19.3"
 
 
 [tool.poetry.group.ci]