Xdr ioc to keep troubleshoting

Packs/CortexXDR/Integrations/XDR_iocs/README.md

@@ -170,3 +170,15 @@ There are no input arguments for this command.
 #### Context Output
 
 There is no context output for this command.
+
+
+#### Base Command
+
+`xdr-iocs-to-keep-file`
+#### Input
+
+There are no input arguments for this command.
+
+#### Context Output
+
+There is no context output for this command.

Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.py

@@ -1,6 +1,7 @@
 import demistomock as demisto
 from CommonServerPython import *
 from CommonServerUserPython import *
+from pathlib import Path
 import hashlib
 import secrets
 import string
@@ -167,6 +168,7 @@ def prepare_disable_iocs(iocs: str) -> tuple[str, list]:
 
 
 def create_file_iocs_to_keep(file_path, batch_size: int = 200):
+    demisto.info('Starting create file ioc to keep')
     with open(file_path, 'w') as _file:
         has_iocs = False
         for ioc in (batch.get('value', '') for batch in get_iocs_generator(size=batch_size)):
@@ -312,6 +314,7 @@ def get_temp_file() -> str:
 
 
 def sync(client: Client):
+    demisto.debug("executing sync")
     temp_file_path: str = get_temp_file()
     try:
         create_file_sync(temp_file_path)  # can be empty
@@ -320,13 +323,18 @@ def sync(client: Client):
         client.http_request(path, requests_kwargs)
     finally:
         os.remove(temp_file_path)
-    set_integration_context({'ts': int(datetime.now(timezone.utc).timestamp() * 1000),
-                             'time': datetime.now(timezone.utc).strftime(DEMISTO_TIME_FORMAT),
-                             'iocs_to_keep_time': create_iocs_to_keep_time()})
-    return_outputs('sync with XDR completed.')
+    set_integration_context(
+        {
+            "ts": int(datetime.now(timezone.utc).timestamp() * 1000),
+            "time": datetime.now(timezone.utc).strftime(DEMISTO_TIME_FORMAT),
+        }
+    )
+    set_new_iocs_to_keep_time()
+    return_outputs("sync with XDR completed.")
 
 
 def iocs_to_keep(client: Client):
+    demisto.debug("executing iocs_to_keep")
     if datetime.utcnow().hour not in range(1, 3):
         raise DemistoException('iocs_to_keep runs only between 01:00 and 03:00.')
     temp_file_path: str = get_temp_file()
@@ -335,11 +343,22 @@ def iocs_to_keep(client: Client):
         requests_kwargs: dict = get_requests_kwargs(file_path=temp_file_path)
         path = 'iocs_to_keep'
         client.http_request(path, requests_kwargs)
+        set_new_iocs_to_keep_time()
     finally:
         os.remove(temp_file_path)
     return_outputs('sync with XDR completed.')
 
 
+def get_iocs_to_keep_file():
+    demisto.info('get_iocs_to_keep_file executed')
+    temp_file_path = Path(get_temp_file())
+    try:
+        create_file_iocs_to_keep(temp_file_path)
+        return_results(fileResult('xdr-ioc-to-keep-file', temp_file_path.read_text()))
+    finally:
+        os.remove(temp_file_path)
+
+
 def create_last_iocs_query(from_date, to_date):
     return f'modified:>={from_date} and modified:<{to_date} and ({Client.query})'
 
@@ -373,6 +392,7 @@ def get_indicators(indicators: str) -> list:
 
 
 def tim_insert_jsons(client: Client):
+    # takes our changes and pushes to XDR
     indicators = demisto.args().get('indicator', '')
     validation_errors = []
     if not indicators:
@@ -482,6 +502,7 @@ def xdr_ioc_to_demisto(ioc: dict) -> dict:
 
 
 def get_changes(client: Client):
+    # takes changes from XDR
     from_time: dict = get_integration_context()
     if not from_time:
         raise DemistoException('XDR is not synced.')
@@ -508,34 +529,57 @@ def module_test(client: Client):
 
 def fetch_indicators(client: Client, auto_sync: bool = False):
     if not get_integration_context() and auto_sync:
+        demisto.debug("running sync with first_time=True")
+        # this will happen on the first time we run
         xdr_iocs_sync_command(client, first_time=True)
     else:
+        # this will happen every minute
         get_changes(client)
         if auto_sync:
             tim_insert_jsons(client)
-            if iocs_to_keep_time():
+            # TODO: Remove/comment this log line
+            demisto.debug("checking if iocs_to_keep should run")
+            if is_iocs_to_keep_time():
                 # first_time=False will call iocs_to_keep
+                demisto.debug("running sync with first_time=False")
                 xdr_iocs_sync_command(client)
 
 
 def xdr_iocs_sync_command(client: Client, first_time: bool = False):
     if first_time or not get_integration_context():
+        # the sync is the large operation including the data and the get_integration_context is fill in the sync
         sync(client)
     else:
         iocs_to_keep(client)
 
 
-def iocs_to_keep_time():
-    hour, minute = get_integration_context().get('iocs_to_keep_time', (0, 0))
-    time_now = datetime.now(timezone.utc)
-    return time_now.hour == hour and time_now.min == minute
-
-
-def create_iocs_to_keep_time():
+def set_new_iocs_to_keep_time():
     offset = secrets.randbelow(115)
-    hour, minute, = divmod(offset, 60)
+    hour, minute = divmod(offset, 60)
     hour += 1
-    return hour, minute
+    last_ioc_to_keep = datetime.now(timezone.utc)
+    last_ioc_to_keep = last_ioc_to_keep.replace(hour=hour, minute=minute) + timedelta(
+        days=1
+    )
+    next_iocs_to_keep_time = last_ioc_to_keep.strftime(DEMISTO_TIME_FORMAT)
+    demisto.debug(f"Setting next iocs to keep time to {next_iocs_to_keep_time}.")
+    set_integration_context(
+        get_integration_context()
+        | {"next_iocs_to_keep_time": next_iocs_to_keep_time}
+    )
+
+
+def is_iocs_to_keep_time():
+    next_iocs_to_keep_time = get_integration_context().get("next_iocs_to_keep_time")
+    time_now = datetime.now(timezone.utc)
+    if (
+        next_iocs_to_keep_time is not None
+        and time_now.hour in range(1, 3)
+        and time_now > datetime.strptime(next_iocs_to_keep_time, DEMISTO_TIME_FORMAT).replace(tzinfo=timezone.utc)
+    ):
+        return True
+
+    return False
 
 
 def is_xdr_data(ioc):
@@ -568,16 +612,6 @@ def get_indicator_xdr_score(indicator: str, xdr_server: int):
         return xdr_local
 
 
-def set_sync_time(time: str):
-    date_time_obj = parse(time, settings={'TIMEZONE': 'UTC'})
-    if not date_time_obj:
-        raise ValueError('invalid time format.')
-    set_integration_context({'ts': int(date_time_obj.timestamp() * 1000),
-                             'time': date_time_obj.strftime(DEMISTO_TIME_FORMAT),
-                             'iocs_to_keep_time': create_iocs_to_keep_time()})
-    return_results(f'set sync time to {time} succeeded.')
-
-
 def get_sync_file():
     temp_file_path = get_temp_file()
     try:
@@ -646,12 +680,14 @@ def main():  # pragma: no cover
     demisto.debug(f'Command being called is {command}')
 
     try:
-        if command == 'fetch-indicators':
-            fetch_indicators(client, params.get('autoSync', False))
+        if command == "fetch-indicators":
+            fetch_indicators(client, params.get("autoSync", False))
         elif command == 'xdr-iocs-set-sync-time':
-            set_sync_time(demisto.args()['time'])
-        elif command == 'xdr-iocs-create-sync-file':
+            return_warning('This command is deprecated and is not relevant anymore.')
+        elif command == "xdr-iocs-create-sync-file":
             get_sync_file()
+        elif command == 'xdr-iocs-to-keep-file':
+            get_iocs_to_keep_file()
         elif command in commands:
             commands[command](client)
         elif command == 'xdr-iocs-sync':

Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.yml

@@ -162,6 +162,8 @@ display: Cortex XDR - IOC
 name: Cortex XDR - IOC
 script:
   commands:
+  - description: Create a file with all the ioc's that are going to sync to xdr.
+    name: xdr-iocs-to-keep-file
   - arguments:
     - auto: PREDEFINED
       default: true
@@ -186,6 +188,7 @@ script:
       name: time
       required: true
     description: Set sync time manually (Do not use this command unless you unredstandard the consequences).
+    deprecated: true
     name: xdr-iocs-set-sync-time
   - description: Creates the sync file for the manual process. Run this command when instructed by the XDR support team.
     name: xdr-iocs-create-sync-file

Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs_test.py

@@ -639,20 +639,6 @@ def test_get_sync_file(self, mocker):
         get_sync_file()
         assert return_results_mock.call_args[0][0]['File'] == 'xdr-sync-file'
 
-    def test_set_sync_time(self, mocker):
-        mocker_reurn_results = mocker.patch('XDR_iocs.return_results')
-        mocker_set_context = mocker.patch.object(demisto, 'setIntegrationContext')
-        set_sync_time('2021-11-25T00:00:00')
-        mocker_reurn_results.assert_called_once_with('set sync time to 2021-11-25T00:00:00 succeeded.')
-        call_args = mocker_set_context.call_args[0][0]
-        assert call_args['ts'] == 1637798400000
-        assert call_args['time'] == '2021-11-25T00:00:00Z'
-        assert call_args['iocs_to_keep_time']
-
-    def test_set_sync_time_with_invalid_time(self):
-        with pytest.raises(ValueError, match='invalid time format.'):
-            set_sync_time('test')
-
     @freeze_time('2020-06-03T02:00:00Z')
     def test_iocs_to_keep(self, mocker):
         http_request = mocker.patch.object(Client, 'http_request')
@@ -1011,3 +997,33 @@ def test_create_validation_errors_response(validation_errors, expected_str):
     """
     from XDR_iocs import create_validation_errors_response
     assert expected_str in create_validation_errors_response(validation_errors)
+
+
+@pytest.mark.parametrize('current_time,next_iocs_to_keep_time,should_run_iocs_to_keep', [
+    ('2020-01-01T02:00:00Z', '2020-01-01T01:00:00Z', True),
+    ('2020-01-01T01:05:00Z', '2020-01-01T02:00:00Z', False),
+    ('2020-01-01T04:00:00Z', '2020-01-01T01:00:00Z', False),
+    ('2020-01-02T02:00:00Z', '2020-01-01T01:00:00Z', True),
+    ('2020-01-02T04:00:00Z', '2020-01-01T01:00:00Z', False),
+    ('2020-01-01T01:05:00Z', None, False),
+])
+def test_is_iocs_to_keep_time(current_time, next_iocs_to_keep_time, should_run_iocs_to_keep, mocker):
+    mocker.patch.object(demisto, 'getIntegrationContext', return_value={"next_iocs_to_keep_time": next_iocs_to_keep_time})
+    with freeze_time(current_time):
+        assert is_iocs_to_keep_time() == should_run_iocs_to_keep
+
+
+@pytest.mark.parametrize('random_int,expected_next_time', [
+    (0, '2023-11-16T01:00:00Z'),
+    (40, '2023-11-16T01:40:00Z'),
+    (60, '2023-11-16T02:00:00Z'),
+    (100, '2023-11-16T02:40:00Z'),
+    (115, '2023-11-16T02:55:00Z'),
+])
+@freeze_time('2023-11-15T18:00:00')
+def test_set_new_iocs_to_keep_time(random_int, expected_next_time, mocker):
+    mocker.patch('XDR_iocs.secrets.randbelow', return_value=random_int)
+    mocker.patch.object(demisto, 'getIntegrationContext', return_value={})
+    set_integration_context_mock = mocker.patch.object(demisto, 'setIntegrationContext')
+    set_new_iocs_to_keep_time()
+    set_integration_context_mock.assert_called_once_with({'next_iocs_to_keep_time': expected_next_time})

Packs/CortexXDR/ReleaseNotes/6_0_2.md

@@ -0,0 +1,6 @@
+
+#### Integrations
+
+##### Cortex XDR - IOC
+
+- Improved the fetch implementation to ensure that expired and deleted IOCs are deleted after not more that 24 hours.

Packs/CortexXDR/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Cortex XDR by Palo Alto Networks",
     "description": "Automates Cortex XDR incident response, and includes custom Cortex XDR incident views and layouts to aid analyst investigations.",
     "support": "xsoar",
-    "currentVersion": "6.0.1",
+    "currentVersion": "6.0.2",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",