From f7b7d5c654b3ab87328911735414aa2a6cd94297 Mon Sep 17 00:00:00 2001 From: Darya Koval <72339940+daryakoval@users.noreply.github.com> Date: Thu, 7 Sep 2023 07:53:37 +0300 Subject: [PATCH] Cs falcon fetch limit issue (#29411) * fixed the parameter that send as a limit * update rn * update test playbook * Update Packs/CrowdStrikeFalcon/ReleaseNotes/1_11_7.md Co-authored-by: Shelly Tzohar <45915502+Shellyber@users.noreply.github.com> * fixing test playbook * adding sort incidents by the ids order; fix time field issue * rename rn * bump version * added unitest * fix rn * save unitest fix * save format * save unitest fix * update docker --------- Co-authored-by: Shelly Tzohar <45915502+Shellyber@users.noreply.github.com> --- .../CrowdStrikeFalcon/CrowdStrikeFalcon.py | 47 +++++++++++++++---- .../CrowdStrikeFalcon/CrowdStrikeFalcon.yml | 34 +++++++------- .../CrowdStrikeFalcon_test.py | 43 ++++++++++++++--- .../CrowdStrikeFalcon/ReleaseNotes/1_11_8.md | 8 ++++ Packs/CrowdStrikeFalcon/pack_metadata.json | 2 +- 5 files changed, 100 insertions(+), 34 deletions(-) create mode 100644 Packs/CrowdStrikeFalcon/ReleaseNotes/1_11_8.md diff --git a/Packs/CrowdStrikeFalcon/Integrations/CrowdStrikeFalcon/CrowdStrikeFalcon.py b/Packs/CrowdStrikeFalcon/Integrations/CrowdStrikeFalcon/CrowdStrikeFalcon.py index 15fc6907ef66..8323ce049b01 100644 --- a/Packs/CrowdStrikeFalcon/Integrations/CrowdStrikeFalcon/CrowdStrikeFalcon.py +++ b/Packs/CrowdStrikeFalcon/Integrations/CrowdStrikeFalcon/CrowdStrikeFalcon.py @@ -529,7 +529,7 @@ def detection_to_incident(detection): incident = { 'name': 'Detection ID: ' + str(detection.get('detection_id')), - 'occurred': str(detection.get('created_timestamp')), + 'occurred': str(detection.get('first_behavior')), 'rawJSON': json.dumps(detection), 'severity': severity_string_to_int(detection.get('max_severity_displayname')) } @@ -1306,7 +1306,7 @@ def get_fetch_detections(last_created_timestamp=None, filter_arg=None, offset: i if filter_arg: params['filter'] = filter_arg elif last_created_timestamp: - params['filter'] = f"created_timestamp:>'{last_created_timestamp}'" + params['filter'] = f"first_behavior:>'{last_created_timestamp}'" elif last_updated_timestamp: params['filter'] = f"date_updated:>'{last_updated_timestamp}'" @@ -2500,6 +2500,21 @@ def migrate_last_run(last_run: dict[str, str] | list[dict]) -> list[dict]: return [updated_last_run_detections, updated_last_run_incidents, {}] +def sort_incidents_summaries_by_ids_order(ids_order, full_incidents, id_field): + """ sort incidents list by the order that ids_order list has + + Args: + ids_order: list of ids + full_incidents: list of incidents + id_field: name of the id field + Returns: + list[dict]: New last run object. + """ + incidents_by_id = {i[id_field]: i for i in full_incidents} + incidents = [incidents_by_id[i] for i in ids_order] + return incidents + + def fetch_incidents(): incidents: list = [] detections: list = [] @@ -2527,7 +2542,7 @@ def fetch_incidents(): incident_type = 'detection' fetch_query = demisto.params().get('fetch_query') if fetch_query: - fetch_query = f"created_timestamp:>'{start_fetch_time}'+{fetch_query}" + fetch_query = f"first_behavior:>'{start_fetch_time}'+{fetch_query}" detections_ids = demisto.get(get_fetch_detections(filter_arg=fetch_query, limit=fetch_limit), 'resources') else: detections_ids = demisto.get(get_fetch_detections(last_created_timestamp=start_fetch_time, limit=fetch_limit), @@ -2536,18 +2551,22 @@ def fetch_incidents(): raw_res = get_detections_entities(detections_ids) if raw_res is not None and "resources" in raw_res: - for detection in demisto.get(raw_res, "resources"): + full_detections = demisto.get(raw_res, "resources") + sorted_detections = sort_incidents_summaries_by_ids_order(ids_order=detections_ids, + full_incidents=full_detections, + id_field='detection_id') + for detection in sorted_detections: detection['incident_type'] = incident_type demisto.debug( f"CrowdStrikeFalconMsg: Detection {detection['detection_id']} " - f"was fetched which was created in {detection['created_timestamp']}") + f"was fetched which was created in {detection['first_behavior']}") incident = detection_to_incident(detection) detections.append(incident) detections = filter_incidents_by_duplicates_and_limit(incidents_res=detections, last_run=current_fetch_info_detections, - fetch_limit=fetch_limit, id_field='name') + fetch_limit=INCIDENTS_PER_FETCH, id_field='name') for detection in detections: occurred = dateparser.parse(detection["occurred"]) @@ -2583,13 +2602,17 @@ def fetch_incidents(): if incidents_ids: raw_res = get_incidents_entities(incidents_ids) if raw_res is not None and "resources" in raw_res: - for incident in demisto.get(raw_res, "resources"): + full_incidents = demisto.get(raw_res, "resources") + sorted_incidents = sort_incidents_summaries_by_ids_order(ids_order=incidents_ids, + full_incidents=full_incidents, + id_field='incident_id') + for incident in sorted_incidents: incident['incident_type'] = incident_type incident_to_context = incident_to_incident_context(incident) incidents.append(incident_to_context) incidents = filter_incidents_by_duplicates_and_limit(incidents_res=incidents, last_run=current_fetch_info_incidents, - fetch_limit=fetch_limit, id_field='name') + fetch_limit=INCIDENTS_PER_FETCH, id_field='name') for incident in incidents: occurred = dateparser.parse(incident["occurred"]) if occurred: @@ -2617,14 +2640,18 @@ def fetch_incidents(): if idp_detections_ids: raw_res = get_idp_detection_entities(idp_detections_ids) if "resources" in raw_res: - for idp_detection in demisto.get(raw_res, "resources"): + full_detections = demisto.get(raw_res, "resources") + sorted_detections = sort_incidents_summaries_by_ids_order(ids_order=idp_detections_ids, + full_incidents=full_detections, + id_field='composite_id') + for idp_detection in sorted_detections: idp_detection['incident_type'] = IDP_DETECTION idp_detection_to_context = idp_detection_to_incident_context(idp_detection) idp_detections.append(idp_detection_to_context) idp_detections = filter_incidents_by_duplicates_and_limit(incidents_res=idp_detections, last_run=current_fetch_info_idp_detections, - fetch_limit=fetch_limit, id_field='name') + fetch_limit=INCIDENTS_PER_FETCH, id_field='name') updated_last_run = update_last_run_object(last_run=current_fetch_info_idp_detections, incidents=idp_detections, fetch_limit=fetch_limit, start_fetch_time=start_fetch_time, end_fetch_time=end_fetch_time, diff --git a/Packs/CrowdStrikeFalcon/Integrations/CrowdStrikeFalcon/CrowdStrikeFalcon.yml b/Packs/CrowdStrikeFalcon/Integrations/CrowdStrikeFalcon/CrowdStrikeFalcon.yml index 57bd70bef54f..8ac65bf1b957 100644 --- a/Packs/CrowdStrikeFalcon/Integrations/CrowdStrikeFalcon/CrowdStrikeFalcon.yml +++ b/Packs/CrowdStrikeFalcon/Integrations/CrowdStrikeFalcon/CrowdStrikeFalcon.yml @@ -412,7 +412,7 @@ script: - description: Any commands run against an offline-queued session will be queued up and executed when the host comes online. name: queue_offline defaultValue: false - - description: A comma-separated list of host agent IDs to run commands for. (Can be retrieved by running the 'cs-falcon-search-device' command.) + - description: A comma-separated list of host agent IDs to run commands for. (Can be retrieved by running the 'cs-falcon-search-device' command.). name: host_ids required: true - description: The type of command to run. @@ -423,7 +423,7 @@ script: required: true - auto: PREDEFINED defaultValue: read - description: 'The scope to run the command for. Possible values are: "read", "write", and "admin". (NOTE: In order to run the CrowdStrike RTR `put` command, it is necessary to pass `scope=admin`.)' + description: 'The scope to run the command for. Possible values are: "read", "write", and "admin". (NOTE: In order to run the CrowdStrike RTR `put` command, it is necessary to pass `scope=admin`.).' name: scope predefined: - read @@ -491,7 +491,7 @@ script: - description: The file entry ID to upload. name: entry_id required: true - description: Uploads a file to the CrowdStrike cloud. (Can be used for the RTR 'put' command.) + description: Uploads a file to the CrowdStrike cloud. (Can be used for the RTR 'put' command.). name: cs-falcon-upload-file - arguments: - description: The ID of the file to delete. (The ID of the file can be retrieved by running the 'cs-falcon-list-files' command). @@ -500,7 +500,7 @@ script: description: Deletes a file based on the provided ID. Can delete only one file at a time. name: cs-falcon-delete-file - arguments: - - description: A comma-separated list of file IDs to get. (The list of file IDs can be retrieved by running the 'cs-falcon-list-files' command.) + - description: A comma-separated list of file IDs to get. (The list of file IDs can be retrieved by running the 'cs-falcon-list-files' command.). name: file_id required: true description: Returns files based on the provided IDs. These files are used for the RTR 'put' command. @@ -531,7 +531,7 @@ script: description: The full name of the file. type: String - contextPath: CrowdStrike.File.Permission - description: 'The permission type of the file. Possible values are: "private", which is used only by the user who uploaded it, "group", which is used by all RTR Admins, and "public", which is used by all active-responders and RTR admins' + description: 'The permission type of the file. Possible values are: "private", which is used only by the user who uploaded it, "group", which is used by all RTR Admins, and "public", which is used by all active-responders and RTR admins.' type: String - contextPath: CrowdStrike.File.SHA256 description: The SHA-256 hash of the file. @@ -595,7 +595,7 @@ script: description: The size of the file in bytes. type: Number - arguments: - - description: A comma-separated list of script IDs to return. (The script IDs can be retrieved by running the 'cs-falcon-list-scripts' command.) + - description: A comma-separated list of script IDs to return. (The script IDs can be retrieved by running the 'cs-falcon-list-scripts' command.). name: script_id required: true description: Returns custom scripts based on the provided ID. Used for the RTR 'runscript' command. @@ -641,7 +641,7 @@ script: description: Whether the user has write access to the script. type: Boolean - arguments: - - description: The script ID to delete. (Script IDs can be retrieved by running the 'cs-falcon-list-scripts' command.) + - description: The script ID to delete. (Script IDs can be retrieved by running the 'cs-falcon-list-scripts' command.). name: script_id required: true description: Deletes a custom-script based on the provided ID. Can delete only one script at a time. @@ -692,7 +692,7 @@ script: - arguments: - description: The name of the script to run. name: script_name - - description: A comma-separated list of host agent IDs to run commands. (The list of host agent IDs can be retrieved by running the 'cs-falcon-search-device' command.) + - description: A comma-separated list of host agent IDs to run commands. (The list of host agent IDs can be retrieved by running the 'cs-falcon-search-device' command.). name: host_ids required: true - description: The PowerShell script code to run. @@ -1290,7 +1290,7 @@ script: description: The identity of the user/process who last updated the IOC. type: string - contextPath: CrowdStrike.NextPageToken - description: A pagination token used with the limit parameter to manage pagination of results + description: A pagination token used with the limit parameter to manage pagination of results. - arguments: - auto: PREDEFINED description: 'The IOC type to retrieve. Possible values are: "sha256", "sha1", "md5", "domain", "ipv4", and "ipv6". Either ioc_id or ioc_type and value must be provided.' @@ -2274,7 +2274,7 @@ script: - description: The ID of the host group. name: host_group_id required: true - - description: A comma-separated list of host agent IDs to run commands. (The list of host agent IDs can be retrieved by running the 'cs-falcon-search-device' command.) + - description: A comma-separated list of host agent IDs to run commands. (The list of host agent IDs can be retrieved by running the 'cs-falcon-search-device' command.). isArray: true name: host_ids required: true @@ -2309,7 +2309,7 @@ script: - description: The ID of the host group. name: host_group_id required: true - - description: A comma-separated list of host agent IDs to run commands. (The list of host agent IDs can be retrieved by running the 'cs-falcon-search-device' command.) + - description: A comma-separated list of host agent IDs to run commands. (The list of host agent IDs can be retrieved by running the 'cs-falcon-search-device' command.). isArray: true name: host_ids required: true @@ -2627,7 +2627,7 @@ script: description: Updates the remote incident or detection with local incident or detection changes. This method is only used for debugging purposes and will not update the current incident or detection. name: update-remote-system - arguments: - - description: Limit the vulnerabilities returned to specific properties. Each value must be enclosed in single quotes and placed immediately after the colon with no space. For example, 'filter=status:'open'+cve.id:['CVE-2013-3900','CVE-2021-1675']' + - description: Limit the vulnerabilities returned to specific properties. Each value must be enclosed in single quotes and placed immediately after the colon with no space. For example, 'filter=status:'open'+cve.id:['CVE-2013-3900','CVE-2021-1675']'. name: filter - description: Unique agent identifier (AID) of a sensor. name: aid @@ -2652,7 +2652,7 @@ script: - description: Type of host a sensor is running on. name: host_type isArray: true - - description: Filter for vulnerabilities based on the number of days since a host last connected to CrowdStrike Falcon. Enter a numeric value from 3 to 45 to indicate the number of days you want to look back. Example- last_seen_within:10 + - description: Filter for vulnerabilities based on the number of days since a host last connected to CrowdStrike Falcon. Enter a numeric value from 3 to 45 to indicate the number of days you want to look back. Example- last_seen_within:10. name: last_seen_within - auto: PREDEFINED description: Indicates if the vulnerability is suppressed by a suppression rule. @@ -3864,7 +3864,7 @@ script: description: A unique identifier for the scan profile used in the scan. type: String - contextPath: CrowdStrike.ODSScanHost.host_id - description: A unique identifier for the host that was scanned + description: A unique identifier for the host that was scanned. type: String - contextPath: CrowdStrike.ODSScanHost.host_scan_id description: A unique identifier for the scan that was performed on the host. @@ -3882,10 +3882,10 @@ script: description: The number of files that were skipped during the scan. type: Number - contextPath: CrowdStrike.ODSScanHost.status - description: The status of the scan. (e.g., "completed", "pending", "cancelled", "running", or "failed") + description: The status of the scan. (e.g., "completed", "pending", "cancelled", "running", or "failed"). type: String - contextPath: CrowdStrike.ODSScanHost.severity - description: A severity score assigned to the scan, ranging from 0 to 100 + description: A severity score assigned to the scan, ranging from 0 to 100. type: Number - contextPath: CrowdStrike.ODSScanHost.started_on description: The date and time when the scan was started. @@ -4375,7 +4375,7 @@ script: - contextPath: CrowdStrike.IDPEntity.EmailAddresses description: The identity entity email address. type: String - dockerimage: demisto/py3-tools:1.0.0.72621 + dockerimage: demisto/py3-tools:1.0.0.73055 isfetch: true ismappable: true isremotesyncin: true diff --git a/Packs/CrowdStrikeFalcon/Integrations/CrowdStrikeFalcon/CrowdStrikeFalcon_test.py b/Packs/CrowdStrikeFalcon/Integrations/CrowdStrikeFalcon/CrowdStrikeFalcon_test.py index c30d28e10b5b..938f5fcc6bef 100644 --- a/Packs/CrowdStrikeFalcon/Integrations/CrowdStrikeFalcon/CrowdStrikeFalcon_test.py +++ b/Packs/CrowdStrikeFalcon/Integrations/CrowdStrikeFalcon/CrowdStrikeFalcon_test.py @@ -2184,10 +2184,13 @@ def set_up_mocks(self, requests_mock, mocker): requests_mock.post(f'{SERVER_URL}/detects/entities/summaries/GET/v1', json={'resources': [{'detection_id': 'ldt:1', 'created_timestamp': '2020-09-04T09:16:11Z', - 'max_severity_displayname': 'Low'}, + 'max_severity_displayname': 'Low', + 'first_behavior': '2020-09-04T09:16:11Z' + }, {'detection_id': 'ldt:2', 'created_timestamp': '2020-09-04T09:20:11Z', - 'max_severity_displayname': 'Low'}]}) + 'max_severity_displayname': 'Low', + 'first_behavior': '2020-09-04T09:16:11Z'}]}) requests_mock.get(f'{SERVER_URL}/incidents/queries/incidents/v1', json={}) requests_mock.post(f'{SERVER_URL}/incidents/entities/incidents/GET/v1', json={}) @@ -2255,11 +2258,16 @@ def test_new_fetch(self, set_up_mocks, mocker, requests_mock): requests_mock.post(f'{SERVER_URL}/detects/entities/summaries/GET/v1', json={'resources': [{'detection_id': 'ldt:1', 'created_timestamp': '2020-09-04T09:16:11Z', - 'max_severity_displayname': 'Low'}]}) + 'max_severity_displayname': 'Low', 'first_behavior': '2020-09-04T09:16:11Z'}, + {'detection_id': 'ldt:2', + 'created_timestamp': '2020-09-04T09:16:11Z', + 'max_severity_displayname': 'Low', 'first_behavior': '2020-09-04T09:16:11Z'} + ]}) from CrowdStrikeFalcon import fetch_incidents fetch_incidents() assert demisto.setLastRun.mock_calls[0][1][0][0] == { - 'time': '2020-09-04T09:16:11Z', 'limit': 2, "found_incident_ids": {'Detection ID: ldt:1': 1599210970}} + 'time': '2020-09-04T09:16:11Z', 'limit': 2, "found_incident_ids": {'Detection ID: ldt:1': 1599210970, + 'Detection ID: ldt:2': 1599210970}} def test_fetch_incident_type(self, set_up_mocks, mocker): """ @@ -2394,12 +2402,14 @@ def test_new_fetch(self, set_up_mocks, mocker, requests_mock): 'offset': 2}, {}]) # Override post to have 1 results so FETCH_LIMIT won't be reached requests_mock.post(f'{SERVER_URL}/incidents/entities/incidents/GET/v1', - json={'resources': [{'incident_id': 'ldt:1', 'start': '2020-09-04T09:16:11Z'}]}) + json={'resources': [{'incident_id': 'ldt:1', 'start': '2020-09-04T09:16:11Z'}, + {'incident_id': 'ldt:2', 'start': '2020-09-04T09:16:11Z'}]}) from CrowdStrikeFalcon import fetch_incidents fetch_incidents() assert demisto.setLastRun.mock_calls[0][1][0][1] == {'time': '2020-09-04T09:16:11Z', 'limit': 2, - 'found_incident_ids': {'Incident ID: ldt:1': 1598462533}} + 'found_incident_ids': {'Incident ID: ldt:1': 1598462533, + 'Incident ID: ldt:2': 1598462533}} def test_incident_type_in_fetch(self, set_up_mocks, mocker): """Tests the addition of incident_type field to the context @@ -5502,3 +5512,24 @@ def test_list_detection_summaries_command_no_results(mocker): mocker.patch('CrowdStrikeFalcon.http_request', return_value=response) res = list_detection_summaries_command() assert res.readable_output == '### CrowdStrike Detections\n**No entries.**\n' + + +def test_sort_incidents_summaries_by_ids_order(): + """ + Test sort incidents in the order by incidents ids + + Given: + - Full incidents response, sorted ids + When: + - Searching for detections using fetch_incidents() + Then: + - The incidents returned in sorted order + """ + from CrowdStrikeFalcon import sort_incidents_summaries_by_ids_order + full_incidents = [{"id": "2", "name": "test2"}, + {"id": "3", "name": "test3"}, + {"id": "1", "name": "test1"}] + res = sort_incidents_summaries_by_ids_order(ids_order=["1", "2", "3"], full_incidents=full_incidents, id_field="id") + assert res == [{"id": "1", "name": "test1"}, {"id": "2", "name": "test2"}, + {"id": "3", "name": "test3"}, + ] diff --git a/Packs/CrowdStrikeFalcon/ReleaseNotes/1_11_8.md b/Packs/CrowdStrikeFalcon/ReleaseNotes/1_11_8.md new file mode 100644 index 000000000000..4896bdd2126d --- /dev/null +++ b/Packs/CrowdStrikeFalcon/ReleaseNotes/1_11_8.md @@ -0,0 +1,8 @@ + +#### Integrations + +##### CrowdStrike Falcon + +- Fixed an issue where the **Max incidents per fetch** parameter did not work properly. +- Updated the Docker image to: *demisto/py3-tools:1.0.0.73055*. + diff --git a/Packs/CrowdStrikeFalcon/pack_metadata.json b/Packs/CrowdStrikeFalcon/pack_metadata.json index db5a73c4b5bb..333712b72797 100644 --- a/Packs/CrowdStrikeFalcon/pack_metadata.json +++ b/Packs/CrowdStrikeFalcon/pack_metadata.json @@ -2,7 +2,7 @@ "name": "CrowdStrike Falcon", "description": "The CrowdStrike Falcon OAuth 2 API (formerly the Falcon Firehose API), enables fetching and resolving detections, searching devices, getting behaviors by ID, containing hosts, and lifting host containment.", "support": "xsoar", - "currentVersion": "1.11.7", + "currentVersion": "1.11.8", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "",