diff --git a/Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/CiscoUmbrellaCloudSecurityv2.py b/Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/CiscoUmbrellaCloudSecurityv2.py index 4ce5292ef292..9abc59ddb2eb 100644 --- a/Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/CiscoUmbrellaCloudSecurityv2.py +++ b/Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/CiscoUmbrellaCloudSecurityv2.py @@ -1,7 +1,7 @@ import enum import http -from typing import Any, TypeVar from collections.abc import Callable +from typing import Any, TypeVar import demistomock as demisto # noqa: F401 from CommonServerPython import * # noqa: F401 @@ -418,7 +418,7 @@ def handle_pagination( page_size: int | None = None, page: int | None = None, **kwargs, -) -> tuple[list[dict[str, Any]] | dict[str, Any], list[dict[str, Any]] | dict[str, Any]]: +) -> tuple[list[dict[str, Any]] | dict[str, Any], dict[str, Any]]: """Handles pagination for the given list_command. Args: @@ -432,7 +432,7 @@ def handle_pagination( **kwargs: Keyword arguments passed down by the CLI to configure the request. Returns: - tuple[list[dict[str, Any]] | dict[str, Any], list[dict[str, Any]] | dict[str, Any]]: + tuple[list[dict[str, Any]] | dict[str, Any], dict[str, Any]]: A tuple containing the list of items and raw responses, or a single item and raw response if page is None. """ if page: @@ -442,9 +442,7 @@ def handle_pagination( return raw_response.get('data', []), raw_response page = 1 - outputs: list[dict[str, Any]] = [] - raw_responses: list[dict[str, Any]] = [] # Keep calling the API until the required amount of items have been met. while limit > 0: @@ -462,7 +460,6 @@ def handle_pagination( if limit < received_items: output = output[:limit] - raw_responses.append(raw_response) outputs += output # If the API returned less than the required amount of items, we're done. @@ -473,10 +470,11 @@ def handle_pagination( limit -= received_items page += 1 + raw_response['meta']['total'] = len(outputs) outputs_result = get_single_or_full_list(outputs) - raw_response_result = get_single_or_full_list(raw_responses) + raw_response['data'] = outputs_result - return outputs_result, raw_response_result + return outputs_result, raw_response @logger @@ -485,7 +483,7 @@ def find_destinations( destination_list_id: str, destinations: set[str] | None = None, destination_ids: set[str] | None = None, -) -> tuple[list[dict[str, Any]] | dict[str, Any], list[dict[str, Any]] | dict[str, Any]]: +) -> tuple[list[dict[str, Any]] | dict[str, Any], dict[str, Any]]: """Fetches and returns destinations and their associated data from a given list_command callable. Args: @@ -500,7 +498,7 @@ def find_destinations( ValueError: If both destinations and destination_ids weren't provided. Returns: - tuple[list[dict[str, Any]] | dict[str, Any], list[dict[str, Any]] | dict[str, Any]]: + tuple[list[dict[str, Any]] | dict[str, Any], dict[str, Any]]: A tuple containing two lists: one with the fetched data, and one with the raw responses from the list command. """ @@ -542,12 +540,11 @@ def filter_out_non_target_items( destinations = set(destinations) if destinations else None destination_ids = set(destination_ids) if destination_ids else None - page = 1 - + page = 0 outputs: list[dict[str, Any]] = [] - raw_responses: list[dict[str, Any]] = [] while destinations or destination_ids: + page += 1 demisto.debug(f'Calling list command with {destination_list_id=}, {page=}, limit={MAX_LIMIT}') raw_response = list_command(destination_list_id=destination_list_id, page=page, limit=MAX_LIMIT) items = raw_response.get('data', []) @@ -556,8 +553,6 @@ def filter_out_non_target_items( demisto.debug(f'The API returned no items for {page=}, stopping') break - raw_responses.append(raw_response) - for item in items: # Called twice to avoid duplicates if an item was given in ID and destination. if filter_out_non_target_items( @@ -577,10 +572,11 @@ def filter_out_non_target_items( demisto.debug(f'These are the last items in the API {page=}, stopping') break + raw_response['meta']['total'] = len(outputs) outputs_result = get_single_or_full_list(outputs) - raw_response_result = get_single_or_full_list(raw_responses) + raw_response['data'] = outputs_result - return outputs_result, raw_response_result + return outputs_result, raw_response @logger diff --git a/Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/CiscoUmbrellaCloudSecurityv2.yml b/Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/CiscoUmbrellaCloudSecurityv2.yml index 43b8a5b1308d..d218165d04e0 100644 --- a/Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/CiscoUmbrellaCloudSecurityv2.yml +++ b/Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/CiscoUmbrellaCloudSecurityv2.yml @@ -483,7 +483,7 @@ script: script: '' type: python subtype: python3 - dockerimage: demisto/python3:3.10.13.72123 + dockerimage: demisto/python3:3.10.13.74666 isfetch: false fromversion: 6.9.0 tests: diff --git a/Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/CiscoUmbrellaCloudSecurityv2_test.py b/Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/CiscoUmbrellaCloudSecurityv2_test.py index 564c77c1733a..fb8439a00df7 100644 --- a/Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/CiscoUmbrellaCloudSecurityv2_test.py +++ b/Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/CiscoUmbrellaCloudSecurityv2_test.py @@ -1,5 +1,6 @@ import json import os +from unittest.mock import patch import CiscoUmbrellaCloudSecurityv2 import CommonServerPython @@ -48,6 +49,7 @@ def mock_client(requests_mock) -> CiscoUmbrellaCloudSecurityv2.Client: ) +@patch('CiscoUmbrellaCloudSecurityv2.MAX_LIMIT', 4) def test_list_destinations_command(requests_mock, mock_client): """ Scenario: @@ -67,25 +69,39 @@ def test_list_destinations_command(requests_mock, mock_client): """ args = { 'destination_list_id': '123', + 'limit': 7, } - response = load_mock_response('destinations.json') - url = CommonServerPython.urljoin(DESTINATION_ENDPOINT, f'{args["destination_list_id"]}/destinations') - requests_mock.get(url=url, json=response) + responses = [] + + for i in range(1, 3): + response = load_mock_response(f'destinations{i}.json') + responses.append(response) + + url: str = CommonServerPython.urljoin( + DESTINATION_ENDPOINT, + f'{args["destination_list_id"]}/destinations?page={i}&limit={4}', + ) + requests_mock.get(url=url, json=response) command_results: CommonServerPython.CommandResults = CiscoUmbrellaCloudSecurityv2.list_destinations_command( mock_client, args ) + expected_outputs = responses[0]['data'] + responses[1]['data'][:3] + response['meta']['total'] = len(expected_outputs) + response['data'] = expected_outputs + assert command_results.outputs_prefix == ( f'{CiscoUmbrellaCloudSecurityv2.INTEGRATION_OUTPUT_PREFIX}.' f'{CiscoUmbrellaCloudSecurityv2.DESTINATION_OUTPUT_PREFIX}' ) assert command_results.outputs_key_field == CiscoUmbrellaCloudSecurityv2.ID_OUTPUTS_KEY_FIELD - assert command_results.outputs == response['data'] + assert command_results.outputs == expected_outputs assert command_results.raw_response == response +@patch('CiscoUmbrellaCloudSecurityv2.MAX_LIMIT', 4) def test_list_destinations_command_fetch_destinations(requests_mock, mock_client): """ Scenario: @@ -106,19 +122,33 @@ def test_list_destinations_command_fetch_destinations(requests_mock, mock_client args = { 'destination_list_id': '123', 'destinations': ['www.LiorSB.com', '1.1.1.1'], - 'destination_ids': ['111', '333'], + 'destination_ids': ['111', '333', '555', '1010'], } - response = load_mock_response('destinations.json') - url = CommonServerPython.urljoin(DESTINATION_ENDPOINT, f'{args["destination_list_id"]}/destinations') - requests_mock.get(url=url, json=response) + responses = [] + + for i in range(1, 4): + response = load_mock_response(f'destinations{i}.json') + responses.append(response) + + url: str = CommonServerPython.urljoin( + DESTINATION_ENDPOINT, + f'{args["destination_list_id"]}/destinations?page={i}&limit={4}', + ) + requests_mock.get(url=url, json=response) command_results: CommonServerPython.CommandResults = CiscoUmbrellaCloudSecurityv2.list_destinations_command( mock_client, args ) - data = response['data'] - expected_outputs = [data[0], data[2], data[3]] + expected_outputs = [ + responses[0]['data'][0], + responses[0]['data'][2], + responses[0]['data'][3], + responses[1]['data'][0], + ] + response['meta']['total'] = len(expected_outputs) + response['data'] = expected_outputs assert command_results.outputs_prefix == ( f'{CiscoUmbrellaCloudSecurityv2.INTEGRATION_OUTPUT_PREFIX}.' diff --git a/Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/test_data/destinations.json b/Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/test_data/destinations1.json similarity index 100% rename from Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/test_data/destinations.json rename to Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/test_data/destinations1.json diff --git a/Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/test_data/destinations2.json b/Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/test_data/destinations2.json new file mode 100644 index 000000000000..c5d82ea9862b --- /dev/null +++ b/Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/test_data/destinations2.json @@ -0,0 +1,41 @@ +{ + "status": { + "code": 200, + "text": "OK" + }, + "meta": { + "page": 2, + "limit": 100, + "total": 4 + }, + "data": [ + { + "id": "555", + "destination": "www.LiorSB1.com", + "type": "domain", + "comment": "Pikachu", + "createdAt": "2023-07-06 04:42:55" + }, + { + "id": "666", + "destination": "www.pokemon1.com", + "type": "domain", + "comment": "Choose", + "createdAt": "2023-07-06 04:42:55" + }, + { + "id": "777", + "destination": "www.serebii1.com", + "type": "domain", + "comment": "I", + "createdAt": "2023-07-06 04:42:55" + }, + { + "id": "8888", + "destination": "www.serebii1.com", + "type": "domain", + "comment": "You", + "createdAt": "2023-07-06 04:42:55" + } + ] +} diff --git a/Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/test_data/destinations3.json b/Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/test_data/destinations3.json new file mode 100644 index 000000000000..7d7b89b92102 --- /dev/null +++ b/Packs/Cisco-umbrella-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/test_data/destinations3.json @@ -0,0 +1,12 @@ +{ + "status": { + "code": 200, + "text": "OK" + }, + "meta": { + "page": 3, + "limit": 100, + "total": 0 + }, + "data": [] +} diff --git a/Packs/Cisco-umbrella-cloud-security/ReleaseNotes/2_0_2.md b/Packs/Cisco-umbrella-cloud-security/ReleaseNotes/2_0_2.md new file mode 100644 index 000000000000..95553c36834c --- /dev/null +++ b/Packs/Cisco-umbrella-cloud-security/ReleaseNotes/2_0_2.md @@ -0,0 +1,3 @@ +#### Integrations +##### Cisco Umbrella Cloud Security v2 +Updated pagination iterations. diff --git a/Packs/Cisco-umbrella-cloud-security/pack_metadata.json b/Packs/Cisco-umbrella-cloud-security/pack_metadata.json index 35696d2842a8..f4bd1695dfba 100644 --- a/Packs/Cisco-umbrella-cloud-security/pack_metadata.json +++ b/Packs/Cisco-umbrella-cloud-security/pack_metadata.json @@ -2,7 +2,7 @@ "name": "Cisco Umbrella cloud security", "description": "Basic integration with Cisco Umbrella that allows you to add domains to destination lists (e.g. global block / allow)", "support": "xsoar", - "currentVersion": "2.0.1", + "currentVersion": "2.0.2", "author": "Cortex XSOAR", "url": "https://www.paloaltonetworks.com/cortex", "email": "", @@ -18,4 +18,4 @@ "marketplacev2" ], "certification": "certified" -} \ No newline at end of file +}