diff --git a/Packs/CortexXDR/Integrations/XDR_iocs/README.md b/Packs/CortexXDR/Integrations/XDR_iocs/README.md index a78d0e9bee75..58eba1c28ce8 100644 --- a/Packs/CortexXDR/Integrations/XDR_iocs/README.md +++ b/Packs/CortexXDR/Integrations/XDR_iocs/README.md @@ -170,3 +170,15 @@ There are no input arguments for this command. #### Context Output There is no context output for this command. + + +#### Base Command + +`xdr-iocs-to-keep-file` +#### Input + +There are no input arguments for this command. + +#### Context Output + +There is no context output for this command. diff --git a/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.py b/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.py index ce9512b36d6d..cbcce4cffd1c 100644 --- a/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.py +++ b/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.py @@ -1,6 +1,7 @@ import demistomock as demisto from CommonServerPython import * from CommonServerUserPython import * +from pathlib import Path import hashlib import secrets import string @@ -167,6 +168,7 @@ def prepare_disable_iocs(iocs: str) -> tuple[str, list]: def create_file_iocs_to_keep(file_path, batch_size: int = 200): + demisto.info('Starting create file ioc to keep') with open(file_path, 'w') as _file: has_iocs = False for ioc in (batch.get('value', '') for batch in get_iocs_generator(size=batch_size)): @@ -312,6 +314,11 @@ def get_temp_file() -> str: def sync(client: Client): + """ + Sync command is supposed to run only in first run or the integration context is empty. + Creates the initial sync between xdr and xsoar iocs. + """ + demisto.debug("executing sync") temp_file_path: str = get_temp_file() try: create_file_sync(temp_file_path) # can be empty @@ -320,13 +327,23 @@ def sync(client: Client): client.http_request(path, requests_kwargs) finally: os.remove(temp_file_path) - set_integration_context({'ts': int(datetime.now(timezone.utc).timestamp() * 1000), - 'time': datetime.now(timezone.utc).strftime(DEMISTO_TIME_FORMAT), - 'iocs_to_keep_time': create_iocs_to_keep_time()}) - return_outputs('sync with XDR completed.') + set_integration_context( + { + "ts": int(datetime.now(timezone.utc).timestamp() * 1000), + "time": datetime.now(timezone.utc).strftime(DEMISTO_TIME_FORMAT), + } + ) + set_new_iocs_to_keep_time() + return_outputs("sync with XDR completed.") def iocs_to_keep(client: Client): + """ + Creats a file of all the indicators from xsoar we want to keep in XDR. + All the indicators not send to XDR with the file will be deleted from XDR. + This is to sync the expired/deleted/no more under filter IOC. + """ + demisto.debug("executing iocs_to_keep") if datetime.utcnow().hour not in range(1, 3): raise DemistoException('iocs_to_keep runs only between 01:00 and 03:00.') temp_file_path: str = get_temp_file() @@ -335,11 +352,22 @@ def iocs_to_keep(client: Client): requests_kwargs: dict = get_requests_kwargs(file_path=temp_file_path) path = 'iocs_to_keep' client.http_request(path, requests_kwargs) + set_new_iocs_to_keep_time() finally: os.remove(temp_file_path) return_outputs('sync with XDR completed.') +def get_iocs_to_keep_file(): + demisto.info('get_iocs_to_keep_file executed') + temp_file_path = Path(get_temp_file()) + try: + create_file_iocs_to_keep(temp_file_path) + return_results(fileResult('xdr-ioc-to-keep-file', temp_file_path.read_text())) + finally: + os.remove(temp_file_path) + + def create_last_iocs_query(from_date, to_date): return f'modified:>={from_date} and modified:<{to_date} and ({Client.query})' @@ -373,6 +401,7 @@ def get_indicators(indicators: str) -> list: def tim_insert_jsons(client: Client): + # takes our changes and pushes to XDR indicators = demisto.args().get('indicator', '') validation_errors = [] if not indicators: @@ -482,6 +511,7 @@ def xdr_ioc_to_demisto(ioc: dict) -> dict: def get_changes(client: Client): + # takes changes from XDR from_time: dict = get_integration_context() if not from_time: raise DemistoException('XDR is not synced.') @@ -508,34 +538,66 @@ def module_test(client: Client): def fetch_indicators(client: Client, auto_sync: bool = False): if not get_integration_context() and auto_sync: + demisto.debug("running sync with first_time=True") + # this will happen on the first time we run xdr_iocs_sync_command(client, first_time=True) else: + # This will happen every fetch time interval as defined in the integration configuration get_changes(client) if auto_sync: tim_insert_jsons(client) - if iocs_to_keep_time(): + demisto.debug("checking if iocs_to_keep should run") + if is_iocs_to_keep_time(): # first_time=False will call iocs_to_keep + demisto.debug("running sync with first_time=False") xdr_iocs_sync_command(client) def xdr_iocs_sync_command(client: Client, first_time: bool = False): if first_time or not get_integration_context(): + # the sync is the large operation including the data and the get_integration_context is fill in the sync sync(client) else: iocs_to_keep(client) -def iocs_to_keep_time(): - hour, minute = get_integration_context().get('iocs_to_keep_time', (0, 0)) - time_now = datetime.now(timezone.utc) - return time_now.hour == hour and time_now.min == minute - - -def create_iocs_to_keep_time(): +def set_new_iocs_to_keep_time(): offset = secrets.randbelow(115) - hour, minute, = divmod(offset, 60) + hour, minute = divmod(offset, 60) hour += 1 - return hour, minute + last_ioc_to_keep = datetime.now(timezone.utc) + last_ioc_to_keep = last_ioc_to_keep.replace(hour=hour, minute=minute) + timedelta( + days=1 + ) + next_iocs_to_keep_time = last_ioc_to_keep.strftime(DEMISTO_TIME_FORMAT) + demisto.debug(f"Setting next iocs to keep time to {next_iocs_to_keep_time}.") + # This will set the new ioc to keep time in the integration context + set_integration_context( + get_integration_context() + | {"next_iocs_to_keep_time": next_iocs_to_keep_time} + ) + + +def is_iocs_to_keep_time(): + """ + This function checks if this is the time to run the iocs_to_keep command. + In order to remove deleted/expired/filtered indicators. + """ + next_iocs_to_keep_time = get_integration_context().get("next_iocs_to_keep_time") + + if next_iocs_to_keep_time is None: + # This is supposed to happen only in the case of appliying the fixed version on a running instance. + set_new_iocs_to_keep_time() + next_iocs_to_keep_time = get_integration_context().get("next_iocs_to_keep_time") + + time_now = datetime.now(timezone.utc) + if ( + time_now.hour in range(1, 3) + and time_now > datetime.strptime(next_iocs_to_keep_time, DEMISTO_TIME_FORMAT).replace(tzinfo=timezone.utc) + ): + return True + + return False def is_xdr_data(ioc): @@ -568,16 +630,6 @@ def get_indicator_xdr_score(indicator: str, xdr_server: int): return xdr_local -def set_sync_time(time: str): - date_time_obj = parse(time, settings={'TIMEZONE': 'UTC'}) - if not date_time_obj: - raise ValueError('invalid time format.') - set_integration_context({'ts': int(date_time_obj.timestamp() * 1000), - 'time': date_time_obj.strftime(DEMISTO_TIME_FORMAT), - 'iocs_to_keep_time': create_iocs_to_keep_time()}) - return_results(f'set sync time to {time} succeeded.') - - def get_sync_file(): temp_file_path = get_temp_file() try: @@ -646,12 +698,14 @@ def main(): # pragma: no cover demisto.debug(f'Command being called is {command}') try: - if command == 'fetch-indicators': - fetch_indicators(client, params.get('autoSync', False)) + if command == "fetch-indicators": + fetch_indicators(client, params.get("autoSync", False)) elif command == 'xdr-iocs-set-sync-time': - set_sync_time(demisto.args()['time']) - elif command == 'xdr-iocs-create-sync-file': + return_warning('This command is deprecated and is not relevant anymore.') + elif command == "xdr-iocs-create-sync-file": get_sync_file() + elif command == 'xdr-iocs-to-keep-file': + get_iocs_to_keep_file() elif command in commands: commands[command](client) elif command == 'xdr-iocs-sync': diff --git a/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.yml b/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.yml index 8a13b1225d6f..90943878f7ef 100644 --- a/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.yml +++ b/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.yml @@ -162,14 +162,16 @@ display: Cortex XDR - IOC name: Cortex XDR - IOC script: commands: + - description: Create a file with all the IOCs that are going to sync to Cortex XDR. + name: xdr-iocs-to-keep-file - arguments: - - auto: PREDEFINED - default: true - defaultValue: 'false' - description: |- + - description: |- For first sync, set to true. (do NOT run this twice!). name: firstTime + auto: PREDEFINED + default: true + defaultValue: 'false' predefined: - 'true' - 'false' @@ -177,16 +179,17 @@ script: name: xdr-iocs-sync - arguments: - description: IOCs to push. leave empty to push all recently modified IOCs.the indicators. - isArray: true name: indicator + isArray: true description: Push modified IOCs to Cortex XDR. name: xdr-iocs-push - - arguments: + - description: Set sync time manually. (Do not use this command unless you understand the consequences.) + name: xdr-iocs-set-sync-time + arguments: - description: The time of the file creation (use UTC time zone). name: time required: true - description: Set sync time manually (Do not use this command unless you unredstandard the consequences). - name: xdr-iocs-set-sync-time + deprecated: true - description: Creates the sync file for the manual process. Run this command when instructed by the XDR support team. name: xdr-iocs-create-sync-file - arguments: @@ -203,7 +206,7 @@ script: required: true description: Disables IOCs in the XDR server. name: xdr-iocs-disable - dockerimage: demisto/python3:3.10.13.80014 + dockerimage: demisto/python3:3.10.13.80593 feed: true runonce: false script: '-' diff --git a/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs_test.py b/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs_test.py index 02a5e0c97955..243487a7e1b2 100644 --- a/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs_test.py +++ b/Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs_test.py @@ -639,20 +639,6 @@ def test_get_sync_file(self, mocker): get_sync_file() assert return_results_mock.call_args[0][0]['File'] == 'xdr-sync-file' - def test_set_sync_time(self, mocker): - mocker_reurn_results = mocker.patch('XDR_iocs.return_results') - mocker_set_context = mocker.patch.object(demisto, 'setIntegrationContext') - set_sync_time('2021-11-25T00:00:00') - mocker_reurn_results.assert_called_once_with('set sync time to 2021-11-25T00:00:00 succeeded.') - call_args = mocker_set_context.call_args[0][0] - assert call_args['ts'] == 1637798400000 - assert call_args['time'] == '2021-11-25T00:00:00Z' - assert call_args['iocs_to_keep_time'] - - def test_set_sync_time_with_invalid_time(self): - with pytest.raises(ValueError, match='invalid time format.'): - set_sync_time('test') - @freeze_time('2020-06-03T02:00:00Z') def test_iocs_to_keep(self, mocker): http_request = mocker.patch.object(Client, 'http_request') @@ -1011,3 +997,40 @@ def test_create_validation_errors_response(validation_errors, expected_str): """ from XDR_iocs import create_validation_errors_response assert expected_str in create_validation_errors_response(validation_errors) + + +@pytest.mark.parametrize('current_time,next_iocs_to_keep_time,should_run_iocs_to_keep', [ + ('2020-01-01T02:00:00Z', '2020-01-01T01:00:00Z', True), + ('2020-01-01T01:05:00Z', '2020-01-01T02:00:00Z', False), + ('2020-01-01T04:00:00Z', '2020-01-01T01:00:00Z', False), + ('2020-01-02T02:00:00Z', '2020-01-01T01:00:00Z', True), + ('2020-01-02T04:00:00Z', '2020-01-01T01:00:00Z', False), +]) +def test_is_iocs_to_keep_time(current_time, next_iocs_to_keep_time, should_run_iocs_to_keep, mocker): + mocker.patch.object(demisto, 'getIntegrationContext', return_value={"next_iocs_to_keep_time": next_iocs_to_keep_time}) + with freeze_time(current_time): + assert is_iocs_to_keep_time() == should_run_iocs_to_keep + + +def test_is_iocs_to_keep_time_without_integration_context(mocker): + mocker.patch.object(demisto, 'getIntegrationContext', side_effect=[{"next_iocs_to_keep_time": None}, + {"next_iocs_to_keep_time": None}, + {"next_iocs_to_keep_time": '2020-01-02T01:05:00Z'}]) + with freeze_time('2020-01-02T04:00:00Z'): + assert not is_iocs_to_keep_time() + + +@pytest.mark.parametrize('random_int,expected_next_time', [ + (0, '2023-11-16T01:00:00Z'), + (40, '2023-11-16T01:40:00Z'), + (60, '2023-11-16T02:00:00Z'), + (100, '2023-11-16T02:40:00Z'), + (115, '2023-11-16T02:55:00Z'), +]) +@freeze_time('2023-11-15T18:00:00') +def test_set_new_iocs_to_keep_time(random_int, expected_next_time, mocker): + mocker.patch('XDR_iocs.secrets.randbelow', return_value=random_int) + mocker.patch.object(demisto, 'getIntegrationContext', return_value={}) + set_integration_context_mock = mocker.patch.object(demisto, 'setIntegrationContext') + set_new_iocs_to_keep_time() + set_integration_context_mock.assert_called_once_with({'next_iocs_to_keep_time': expected_next_time}) diff --git a/Packs/CortexXDR/ReleaseNotes/6_0_3.md b/Packs/CortexXDR/ReleaseNotes/6_0_3.md new file mode 100644 index 000000000000..882300babeb4 --- /dev/null +++ b/Packs/CortexXDR/ReleaseNotes/6_0_3.md @@ -0,0 +1,7 @@ + +#### Integrations + +##### Cortex XDR - IOC + +- Improved the fetch implementation to ensure that expired and deleted IOCs are deleted after not more than 24 hours. +- Updated the Docker image to: *demisto/python3:3.10.13.80593*. diff --git a/Packs/CortexXDR/pack_metadata.json b/Packs/CortexXDR/pack_metadata.json index 23cf639a2eec..d2ba15a74246 100644 --- a/Packs/CortexXDR/pack_metadata.json +++ b/Packs/CortexXDR/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Cortex XDR by Palo Alto Networks", "description": "Automates Cortex XDR incident response, and includes custom Cortex XDR incident views and layouts to aid analyst investigations.", "support": "xsoar", - "currentVersion": "6.0.2", + "currentVersion": "6.0.3", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "",