From 89f16e9554e948a3d1fe32ab93ffb5899dea1273 Mon Sep 17 00:00:00 2001 From: Mai Morag <81917647+maimorag@users.noreply.github.com> Date: Wed, 31 Jan 2024 10:32:39 +0200 Subject: [PATCH] 31078 misp (#32237) * adding debug to indicators * adding the unify yml * fixed custom version * Update README.md (#23810) Edit the file to remove duplication of command names in the right pane. * Modeling rules fixes (#24259) * save * save no exit_code * save not fail on test-modeling-rules * remove ciscoasa changes * Update Docker Image To demisto/chromium (#24291) * Updated Metadata Of Pack ExpanseV2 * Added release notes to pack ExpanseV2 * Packs/ExpanseV2/Scripts/ExpanseGenerateIssueMapWidgetScript/ExpanseGenerateIssueMapWidgetScript.yml Docker image update * Update Docker Image To demisto/chromium (#24291) * Updated Metadata Of Pack ExpanseV2 * Added release notes to pack ExpanseV2 * Packs/ExpanseV2/Scripts/ExpanseGenerateIssueMapWidgetScript/ExpanseGenerateIssueMapWidgetScript.yml Docker image update * release note * Delete .gitlab/ci/on-push.yml * Delete Packs/Auditd/ModelingRules/Auditd_1_3/Auditd_1_3_testdata.json * try * fix * fix * adding limit to fetch indicators * adding to docs * fixing limit * fix val * custom integration * fix cr * testing * fix pre-commit * Update Packs/FeedMISP/ReleaseNotes/1_0_31.md Co-authored-by: ShirleyDenkberg <62508050+ShirleyDenkberg@users.noreply.github.com> * changes fix * limit and page size together * fix * fix fetch * fix rl * adding unit tests * fix doc string * fix * not to push * cr notes * cr notes ream me * cr notes - fix docstring test * cr notes - fix command, yml, timestamp * cr notes - fix duplicates * pre commit * pre commit fix * cr notes * fix unit test * Apply suggestions from code review Co-authored-by: EyalPintzov <91007713+eyalpalo@users.noreply.github.com> --------- Co-authored-by: ShirleyDenkberg <62508050+ShirleyDenkberg@users.noreply.github.com> Co-authored-by: Darya Koval <72339940+daryakoval@users.noreply.github.com> Co-authored-by: content-bot <55035720+content-bot@users.noreply.github.com> Co-authored-by: EyalPintzov <91007713+eyalpalo@users.noreply.github.com> --- .../Integrations/FeedMISP/FeedMISP.py | 115 +++------ .../Integrations/FeedMISP/FeedMISP.yml | 5 +- .../Integrations/FeedMISP/FeedMISP_test.py | 239 ++++-------------- Packs/FeedMISP/ReleaseNotes/1_0_31.md | 7 + Packs/FeedMISP/pack_metadata.json | 2 +- 5 files changed, 99 insertions(+), 269 deletions(-) create mode 100644 Packs/FeedMISP/ReleaseNotes/1_0_31.md diff --git a/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP.py b/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP.py index 97dd5a844663..e9ff6a435187 100644 --- a/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP.py +++ b/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP.py @@ -166,7 +166,6 @@ def build_indicators_iterator(attributes: Dict[str, Any], url: Optional[str]) -> except KeyError as err: demisto.debug(str(err)) raise KeyError(f'Could not parse returned data as attributes list. \nError massage: {err}') - demisto.debug(f' Number of indicators: {len(indicators_iterator)}') return indicators_iterator @@ -211,7 +210,7 @@ def handle_file_type_fields(raw_type: str, indicator_obj: Dict[str, Any]) -> Non indicator_obj['fields'][raw_type.upper()] = hash_value -def build_params_dict(tags: List[str], attribute_type: List[str]) -> Dict[str, Any]: +def build_params_dict(tags: List[str], attribute_type: List[str], limit: int, page: int) -> Dict[str, Any]: """ Creates a dictionary in the format required by MISP to be used as a query. Args: @@ -227,6 +226,8 @@ def build_params_dict(tags: List[str], attribute_type: List[str]) -> Dict[str, A 'tags': { 'OR': tags if tags else [], }, + 'limit': limit, + 'page': page } return params @@ -307,82 +308,15 @@ def build_indicator(value_: str, type_: str, raw_data: Dict[str, Any], reputatio return indicator_obj -def update_indicators_iterator(indicators_iterator: List[Dict[str, Any]], - params_dict: Dict[str, Any], - is_fetch: bool) -> Optional[List[Dict[str, Any]]]: - """ - sorts the indicators by their timestamp and returns a list of only new indicators received from MISP - Args: - params_dict: user's params sent to misp - indicators_iterator: list of indicators - is_fetch: flag for wether funciton was called for fetching command or a get - Returns: Sorted list of new indicators - """ - last_run = demisto.getLastRun() - demisto.debug(f"last_run: {last_run}") - indicators_iterator.sort(key=lambda indicator: indicator['value']['timestamp']) - - if last_run is None: - return indicators_iterator - if params_dict != last_run.get('params'): - if is_fetch: - demisto.setLastRun(None) - return indicators_iterator - - last_timestamp = int(last_run.get('timestamp')) - - for index in range(len(indicators_iterator)): - if int(indicators_iterator[index]['value']['timestamp']) > last_timestamp: - return indicators_iterator[index:] - return [] - - -def search_query_indicators_pagination(client: Client, params_dict: Dict[str, Any]) -> Dict[str, Any]: - params_dict['page'] = 1 - response: Dict[str, Dict[str, List]] = {'response': {'Attribute': []}} - search_query_per_page = client.search_query(params_dict).get('response', {}).get('Attribute') - while len(search_query_per_page): - demisto.debug(f'search_query_per_page: {params_dict["page"]} number of indicators: {len(search_query_per_page)}') - response['response']['Attribute'].extend(search_query_per_page) - params_dict['page'] += 1 - search_query_per_page = client.search_query(params_dict).get('response', {}).get('Attribute') - return response - - -def fetch_indicators(client: Client, - tags: List[str], +def build_indicators(response: Dict[str, Any], attribute_type: List[str], - query: Optional[str], tlp_color: Optional[str], url: Optional[str], reputation: Optional[str], - feed_tags: Optional[List], - limit: int = -1, - is_fetch: bool = True) -> List[Dict]: - params_dict = clean_user_query(query) if query else build_params_dict(tags, attribute_type) - if limit and limit not in params_dict: - params_dict['limit'] = limit - response = search_query_indicators_pagination(client, params_dict) if is_fetch else client.search_query(params_dict) - if error_message := response.get('Error'): - raise DemistoException(error_message) + feed_tags: Optional[List]) -> List[Dict]: indicators_iterator = build_indicators_iterator(response, url) - added_indicators_iterator = update_indicators_iterator(indicators_iterator, params_dict, is_fetch) indicators = [] - - if not added_indicators_iterator: - return [] - - if limit > 0: - added_indicators_iterator = added_indicators_iterator[:limit] - - if is_fetch: - # fetching command, need to update last run dict - demisto.setLastRun({ - 'params': params_dict, - 'timestamp': added_indicators_iterator[len(added_indicators_iterator) - 1]['value']['timestamp'] - }) - - for indicator in added_indicators_iterator: + for indicator in indicators_iterator: value_ = indicator['value']['value'] type_ = indicator['type'] raw_type = indicator.pop('raw_type') @@ -522,7 +456,7 @@ def test_module(client: Client) -> str: def get_attributes_command(client: Client, args: Dict[str, str], params: Dict[str, str]) -> CommandResults: - """Wrapper for retrieving indicators from the feed to the war-room. + """ Wrapper for fetching indicators from the feed to the war-room. Args: client: Client object with request args: demisto.args() @@ -537,8 +471,13 @@ def get_attributes_command(client: Client, args: Dict[str, str], params: Dict[st feed_tags = argToList(params.get("feedTags", [])) query = args.get('query', None) attribute_type = argToList(args.get('attribute_type', '')) - indicators = fetch_indicators(client, tags, attribute_type, - query, tlp_color, params.get('url'), reputation, feed_tags, limit, False) + page = arg_to_number(args.get('page')) or 1 + params_dict = clean_user_query(query) if query else build_params_dict(tags=tags, attribute_type=attribute_type, limit=limit, + page=page) + response = client.search_query(params_dict) + if error_message := response.get('Error'): + raise DemistoException(error_message) + indicators = build_indicators(response, attribute_type, tlp_color, params.get('url'), reputation, feed_tags) hr_indicators = [] for indicator in indicators: hr_indicators.append({ @@ -558,13 +497,14 @@ def get_attributes_command(client: Client, args: Dict[str, str], params: Dict[st ) -def fetch_attributes_command(client: Client, params: Dict[str, str]) -> List[Dict]: +def fetch_attributes_command(client: Client, params: Dict[str, str]): """ - Wrapper for fetching indicators from the feed to the Indicators tab. + Fetching indicators from the feed to the Indicators tab. Args: client: Client object with request params: demisto.params() Returns: List of indicators. + """ tlp_color = params.get('tlp_color') reputation = params.get('feedReputation') @@ -572,9 +512,19 @@ def fetch_attributes_command(client: Client, params: Dict[str, str]) -> List[Dic feed_tags = argToList(params.get("feedTags", [])) attribute_types = argToList(params.get('attribute_types', '')) query = params.get('query', None) - indicators = fetch_indicators(client, tags, attribute_types, query, tlp_color, - params.get('url'), reputation, feed_tags) - return indicators + params_dict = clean_user_query(query) if query else build_params_dict(tags=tags, attribute_type=attribute_types, limit=2000, + page=1) + search_query_per_page = client.search_query(params_dict) + while len(search_query_per_page.get("response", {}).get("Attribute", [])): + demisto.debug(f'search_query_per_page number of attributes:\ + {len(search_query_per_page.get("response", {}).get("Attribute", []))}\ + page: {params_dict["page"]}') + indicators = build_indicators(search_query_per_page, attribute_types, tlp_color, params.get('url'), reputation, feed_tags) + demisto.createIndicators(indicators) + params_dict['page'] += 1 + search_query_per_page = client.search_query(params_dict) + if error_message := search_query_per_page.get('Error'): + raise DemistoException(f"Error in API call - check the input parameters and the API Key. Error: {error_message}") def main(): @@ -601,9 +551,8 @@ def main(): elif command == 'misp-feed-get-indicators': return_results(get_attributes_command(client, args, params)) elif command == 'fetch-indicators': - indicators = fetch_attributes_command(client, params) - for iter_ in batch(indicators, batch_size=2000): - demisto.createIndicators(iter_) + fetch_attributes_command(client, params) + else: raise NotImplementedError(f'Command {command} is not implemented.') diff --git a/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP.yml b/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP.yml index c0ff627345b1..75eee6171477 100644 --- a/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP.yml +++ b/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP.yml @@ -127,6 +127,9 @@ script: - name: limit defaultValue: '10' description: The maximum number of results to return. + - name: page + defaultValue: '1' + description: The page number of the results to retrieve. - name: tags description: Attributes having one of the tags, or being an attribute of an event having one of the tags, will be returned. You can enter a comma-separated list of tags, for example ,,. The list of MISP tags can be found in your MISP instance under 'Event Actions'>'List Tags'. - name: attribute_type @@ -139,7 +142,7 @@ script: script: '-' type: python subtype: python3 - dockerimage: demisto/python3:3.10.13.83255 + dockerimage: demisto/python3:3.10.13.86272 fromversion: 5.5.0 tests: - MISPfeed Test diff --git a/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP_test.py b/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP_test.py index cbc62ae484f8..3c3592f51b0c 100644 --- a/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP_test.py +++ b/Packs/FeedMISP/Integrations/FeedMISP/FeedMISP_test.py @@ -4,8 +4,8 @@ from CommonServerPython import DemistoException, ThreatIntel, FeedIndicatorType from FeedMISP import clean_user_query, build_indicators_iterator, \ - handle_file_type_fields, get_galaxy_indicator_type, build_indicators_from_galaxies, update_indicators_iterator, \ - update_indicator_fields, get_ip_type, search_query_indicators_pagination, Client + handle_file_type_fields, get_galaxy_indicator_type, build_indicators_from_galaxies, \ + update_indicator_fields, get_ip_type, Client, fetch_attributes_command def test_build_indicators_iterator_success(): @@ -206,173 +206,6 @@ def test_build_indicators_from_galaxies(): assert galaxy_indicators[0]['type'] == ThreatIntel.ObjectsNames.ATTACK_PATTERN -def test_update_indicators_iterator_first_fetch(mocker): - """ - Given - - Indicators received - When - - First fetch, no last run parameters - Then - - return all indicators - """ - indicators_iterator = [ - { - 'value': {'timestamp': '5'}, - 'type': 'IP', - 'raw_type': 'ip-src', - }, - { - 'value': {'timestamp': '1'}, - 'type': 'IP', - 'raw_type': 'ip-src', - }, - { - 'value': {'timestamp': '3'}, - 'type': 'IP', - 'raw_type': 'ip-src', - }, - ] - query = {'key': 'val'} - mocker.patch.object(demisto, 'getLastRun', return_value=None) - added_indicators_iterator = update_indicators_iterator(indicators_iterator, query, True) - assert added_indicators_iterator == indicators_iterator - - -def test_update_indicators_iterator_timestamp_exists_all_new_indicators_same_query(mocker): - """ - Given - - Indicators received, lastrun has timestamp and query - When - - indicators updated after timestamp and same query as before - Then - - return all indicators - """ - indicators_iterator = [ - { - 'value': {'timestamp': '5'}, - 'type': 'IP', - 'raw_type': 'ip-src', - }, - { - 'value': {'timestamp': '1'}, - 'type': 'IP', - 'raw_type': 'ip-src', - }, - { - 'value': {'timestamp': '3'}, - 'type': 'IP', - 'raw_type': 'ip-src', - }, - ] - query = {'key': 'val'} - mocker.patch.object(demisto, 'getLastRun', return_value={'timestamp': '0', 'params': query}) - added_indicators_iterator = update_indicators_iterator(indicators_iterator, query, True) - assert added_indicators_iterator == indicators_iterator - - -def test_update_indicators_iterator_timestamp_exists_no_new_indicators_same_query(mocker): - """ - Given - - Indicators received, lastrun has the timestamp and query - When - - last run timestamp is bigger then the indicators timestamp and query is the same - Then - - return no indicators - """ - indicators_iterator = [ - { - 'value': {'timestamp': '1'}, - 'type': 'IP', - 'raw_type': 'ip-src', - }, - { - 'value': {'timestamp': '3'}, - 'type': 'IP', - 'raw_type': 'ip-src', - }, - ] - query = {'key': 'val'} - mocker.patch.object(demisto, 'getLastRun', return_value={'timestamp': '4', 'params': query}) - added_indicators_iterator = update_indicators_iterator(indicators_iterator, query, True) - assert not added_indicators_iterator - - -def test_update_indicators_iterator_timestamp_exists_some_new_indicators_same_query(mocker): - """ - Given - - Indicators received, lastrun has the timestamp and query - When - - some indicators has timestamp bigger then the lastrun timestamp - Then - - return indicators which have timestamp bigger then lastrun timestamp - """ - indicators_iterator = [ - { - 'value': {'timestamp': '5'}, - 'type': 'IP', - 'raw_type': 'ip-src', - }, - { - 'value': {'timestamp': '1'}, - 'type': 'IP', - 'raw_type': 'ip-src', - }, - { - 'value': {'timestamp': '3'}, - 'type': 'IP', - 'raw_type': 'ip-src', - }, - ] - query = {'key': 'val'} - mocker.patch.object(demisto, 'getLastRun', return_value={'timestamp': '4', 'params': query}) - added_indicators_iterator = update_indicators_iterator(indicators_iterator, query, True) - assert added_indicators_iterator[0]['value']['timestamp'] == '5' - - -def test_update_indicators_iterator_timestamp_exists_no_indicators_same_query(mocker): - """ - Given - - No indicators received - When - - lastrun has timestamp and query - Then - - return no indicators - """ - indicators_iterator = [] - query = {'key': 'val'} - mocker.patch.object(demisto, 'getLastRun', return_value={'timestamp': '4', 'params': query}) - added_indicators_iterator = update_indicators_iterator(indicators_iterator, query, True) - assert not added_indicators_iterator - - -def test_update_indicators_iterator_indicators_before_timestamp_different_query(mocker): - """ - Given - - Indicators received, lastrun has the timestamp and query - When - - all indicators have smaller timestamp then lastrun but query has changed - Then - - reset lastrun and return all indicators - """ - indicators_iterator = [ - { - 'value': {'timestamp': '1'}, - 'type': 'IP', - 'raw_type': 'ip-src', - }, - { - 'value': {'timestamp': '3'}, - 'type': 'IP', - 'raw_type': 'ip-src', - }, - ] - query = {'key': 'val'} - old_query = {'key': 'old'} - mocker.patch.object(demisto, 'getLastRun', return_value={'timestamp': '4', 'params': old_query}) - added_indicators_iterator = update_indicators_iterator(indicators_iterator, query, True) - assert added_indicators_iterator == indicators_iterator - - @pytest.mark.parametrize( "indicator, feed_tags, expected_calls", [ @@ -450,31 +283,69 @@ def test_get_ip_type(indicator, indicator_type): assert get_ip_type(indicator) == indicator_type -indicators_examples = [ - ({'response': {'Attribute': ['data1', 'data2']}}, ({'response': {'Attribute': []}}), - {'response': {'Attribute': ['data1', 'data2']}}), - ({'response': {'Attribute': []}}, ({'response': {'Attribute': []}}), - {'response': {'Attribute': []}}) - -] +def test_search_query_indicators_pagination(mocker): + """ + Given: + - All relevant arguments for the command + When: + - the fetch_attributes_command function runs + Then: + - Ensure the pagination mechanism return the expected result (good http response is returned) + """ + client = Client(base_url="example", + authorization="auth", + verify=False, + proxy=False, + timeout=60) + returned_result_1 = {'response': + {'Attribute': [{'id': '1', 'event_id': '1', 'object_id': '0', + 'object_relation': None, 'category': 'Payload delivery', + 'type': 'sha256', 'to_ids': True, 'uuid': '5fd0c620', + 'timestamp': '1607517728', 'distribution': '5', 'sharing_group_id': '0', + 'comment': 'malspam', 'deleted': False, 'disable_correlation': False, + 'first_seen': None, 'last_seen': None, + 'value': 'val1', 'Event': {}}, + {'id': '2', 'event_id': '2', 'object_id': '0', + 'object_relation': None, 'category': 'Payload delivery', + 'type': 'sha256', 'to_ids': True, 'uuid': '5fd0c620', + 'timestamp': '1607517728', 'distribution': '5', 'sharing_group_id': '0', + 'comment': 'malspam', 'deleted': False, 'disable_correlation': False, 'first_seen': None, + 'last_seen': None, 'value': 'val2', 'Event': {}}]}} + returned_result_2 = {'response': {'Attribute': []}} + mocker.patch.object(Client, '_http_request', side_effect=[returned_result_1, returned_result_2]) + params_dict = { + 'type': 'attribute', + 'filters': {'category': ['Payload delivery']}, + } + mocker.patch.object(demisto, 'setLastRun') + mocker.patch.object(demisto, 'createIndicators') + fetch_attributes_command(client, params_dict) + indicators = demisto.createIndicators.call_args[0][0] + assert len(indicators) == 2 -@pytest.mark.parametrize('returned_result_1, returned_result_2, expected_result', indicators_examples) -def test_search_query_indicators_pagination(mocker, returned_result_1, returned_result_2, expected_result): +def test_search_query_indicators_pagination_bad_case(mocker): """ Given: - All relevant arguments for the command When: - - the search_query_indicators_pagination function runs + - the fetch_attributes_command function runs Then: - - Ensure the pagination mechanism return the expected result + - Ensure the pagination mechanism raises an error (bad http response is returned) """ + from CommonServerPython import DemistoException client = Client(base_url="example", authorization="auth", verify=False, proxy=False, timeout=60) - mocker.patch.object(Client, '_http_request', side_effect=[returned_result_1, returned_result_2]) - params_dict = {'param1': 'value1'} - result = search_query_indicators_pagination(client, params_dict) - assert result == expected_result + returned_result = {'Error': 'failed api call'} + expected_result = "Error in API call - check the input parameters and the API Key. Error: failed api call" + mocker.patch.object(Client, '_http_request', return_value=returned_result) + params_dict = { + 'type': 'attribute', + 'filters': {'category': ['Payload delivery']} + } + with pytest.raises(DemistoException) as e: + fetch_attributes_command(client, params_dict) + assert str(e.value) == expected_result diff --git a/Packs/FeedMISP/ReleaseNotes/1_0_31.md b/Packs/FeedMISP/ReleaseNotes/1_0_31.md new file mode 100644 index 000000000000..07c240ff415a --- /dev/null +++ b/Packs/FeedMISP/ReleaseNotes/1_0_31.md @@ -0,0 +1,7 @@ + +#### Integrations + +##### MISP Feed +- Added a **page** argument to the ***misp-feed-get-indicators*** command. +- Fixed an issue where ***fetch-indicators*** crushed due to memory overflow. +- Updated the Docker image to: *demisto/python3:3.10.13.86272*. diff --git a/Packs/FeedMISP/pack_metadata.json b/Packs/FeedMISP/pack_metadata.json index 8648f96594e6..3b4056d8a51a 100644 --- a/Packs/FeedMISP/pack_metadata.json +++ b/Packs/FeedMISP/pack_metadata.json @@ -2,7 +2,7 @@ "name": "MISP Feed", "description": "Indicators feed from MISP", "support": "xsoar", - "currentVersion": "1.0.30", + "currentVersion": "1.0.31", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "",