added plus 1 for each iteration in find destination (demisto#29811)

* added plus 1 for each iteration in find destination (demisto#29760) * added plus 1 for each iteration in find destination * added release notes * Update Packs/Cisco-umbrella-cloud-security/ReleaseNotes/2_0_2.md Co-authored-by: ShirleyDenkberg <62508050+ShirleyDenkberg@users.noreply.github.com> * updated docker image tag to latest * updated unit test for pagination functions * removed comments --------- Co-authored-by: ShirleyDenkberg <62508050+ShirleyDenkberg@users.noreply.github.com> * Update 2_0_2.md --------- Co-authored-by: LiorQM <106475467+LiorQM@users.noreply.github.com> Co-authored-by: ShirleyDenkberg <62508050+ShirleyDenkberg@users.noreply.github.com> Co-authored-by: RotemAmit <ramit@paloaltonetworks.com>
xsoar-contrib · Oct 5, 2023 · 5b94f5e · 5b94f5e
1 parent 192b49f
commit 5b94f5e
Show file tree

Hide file tree

Showing 8 changed files with 112 additions and 30 deletions.
diff --git a/...-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/CiscoUmbrellaCloudSecurityv2.py b/...-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/CiscoUmbrellaCloudSecurityv2.py
@@ -1,7 +1,7 @@
 import enum
 import http
-from typing import Any, TypeVar
 from collections.abc import Callable
+from typing import Any, TypeVar
 
 import demistomock as demisto  # noqa: F401
 from CommonServerPython import *  # noqa: F401
@@ -418,7 +418,7 @@ def handle_pagination(
     page_size: int | None = None,
     page: int | None = None,
     **kwargs,
-) -> tuple[list[dict[str, Any]] | dict[str, Any], list[dict[str, Any]] | dict[str, Any]]:
+) -> tuple[list[dict[str, Any]] | dict[str, Any], dict[str, Any]]:
     """Handles pagination for the given list_command.
 
     Args:
@@ -432,7 +432,7 @@ def handle_pagination(
         **kwargs: Keyword arguments passed down by the CLI to configure the request.
 
     Returns:
-        tuple[list[dict[str, Any]] | dict[str, Any], list[dict[str, Any]] | dict[str, Any]]:
+        tuple[list[dict[str, Any]] | dict[str, Any], dict[str, Any]]:
             A tuple containing the list of items and raw responses, or a single item and raw response if page is None.
     """
     if page:
@@ -442,9 +442,7 @@ def handle_pagination(
         return raw_response.get('data', []), raw_response
 
     page = 1
-
     outputs: list[dict[str, Any]] = []
-    raw_responses: list[dict[str, Any]] = []
 
     # Keep calling the API until the required amount of items have been met.
     while limit > 0:
@@ -462,7 +460,6 @@ def handle_pagination(
         if limit < received_items:
             output = output[:limit]
 
-        raw_responses.append(raw_response)
         outputs += output
 
         # If the API returned less than the required amount of items, we're done.
@@ -473,10 +470,11 @@ def handle_pagination(
         limit -= received_items
         page += 1
 
+    raw_response['meta']['total'] = len(outputs)
     outputs_result = get_single_or_full_list(outputs)
-    raw_response_result = get_single_or_full_list(raw_responses)
+    raw_response['data'] = outputs_result
 
-    return outputs_result, raw_response_result
+    return outputs_result, raw_response
 
 
 @logger
@@ -485,7 +483,7 @@ def find_destinations(
     destination_list_id: str,
     destinations: set[str] | None = None,
     destination_ids: set[str] | None = None,
-) -> tuple[list[dict[str, Any]] | dict[str, Any], list[dict[str, Any]] | dict[str, Any]]:
+) -> tuple[list[dict[str, Any]] | dict[str, Any], dict[str, Any]]:
     """Fetches and returns destinations and their associated data from a given list_command callable.
 
     Args:
@@ -500,7 +498,7 @@ def find_destinations(
         ValueError: If both destinations and destination_ids weren't provided.
 
     Returns:
-        tuple[list[dict[str, Any]] | dict[str, Any], list[dict[str, Any]] | dict[str, Any]]:
+        tuple[list[dict[str, Any]] | dict[str, Any], dict[str, Any]]:
             A tuple containing two lists: one with the fetched data,
             and one with the raw responses from the list command.
     """
@@ -542,12 +540,11 @@ def filter_out_non_target_items(
     destinations = set(destinations) if destinations else None
     destination_ids = set(destination_ids) if destination_ids else None
 
-    page = 1
-
+    page = 0
     outputs: list[dict[str, Any]] = []
-    raw_responses: list[dict[str, Any]] = []
 
     while destinations or destination_ids:
+        page += 1
         demisto.debug(f'Calling list command with {destination_list_id=}, {page=}, limit={MAX_LIMIT}')
         raw_response = list_command(destination_list_id=destination_list_id, page=page, limit=MAX_LIMIT)
         items = raw_response.get('data', [])
@@ -556,8 +553,6 @@ def filter_out_non_target_items(
             demisto.debug(f'The API returned no items for {page=}, stopping')
             break
 
-        raw_responses.append(raw_response)
-
         for item in items:
             # Called twice to avoid duplicates if an item was given in ID and destination.
             if filter_out_non_target_items(
@@ -577,10 +572,11 @@ def filter_out_non_target_items(
             demisto.debug(f'These are the last items in the API {page=}, stopping')
             break
 
+    raw_response['meta']['total'] = len(outputs)
     outputs_result = get_single_or_full_list(outputs)
-    raw_response_result = get_single_or_full_list(raw_responses)
+    raw_response['data'] = outputs_result
 
-    return outputs_result, raw_response_result
+    return outputs_result, raw_response
 
 
 @logger

diff --git a/...cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/CiscoUmbrellaCloudSecurityv2.yml b/...cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/CiscoUmbrellaCloudSecurityv2.yml
@@ -483,7 +483,7 @@ script:
   script: ''
   type: python
   subtype: python3
-  dockerimage: demisto/python3:3.10.13.72123
+  dockerimage: demisto/python3:3.10.13.74666
   isfetch: false
 fromversion: 6.9.0
 tests:

diff --git a/...d-security/Integrations/CiscoUmbrellaCloudSecurityv2/CiscoUmbrellaCloudSecurityv2_test.py b/...d-security/Integrations/CiscoUmbrellaCloudSecurityv2/CiscoUmbrellaCloudSecurityv2_test.py
@@ -1,5 +1,6 @@
 import json
 import os
+from unittest.mock import patch
 
 import CiscoUmbrellaCloudSecurityv2
 import CommonServerPython
@@ -48,6 +49,7 @@ def mock_client(requests_mock) -> CiscoUmbrellaCloudSecurityv2.Client:
     )
 
 
+@patch('CiscoUmbrellaCloudSecurityv2.MAX_LIMIT', 4)
 def test_list_destinations_command(requests_mock, mock_client):
     """
     Scenario:
@@ -67,25 +69,39 @@ def test_list_destinations_command(requests_mock, mock_client):
     """
     args = {
         'destination_list_id': '123',
+        'limit': 7,
     }
-    response = load_mock_response('destinations.json')
-    url = CommonServerPython.urljoin(DESTINATION_ENDPOINT, f'{args["destination_list_id"]}/destinations')
 
-    requests_mock.get(url=url, json=response)
+    responses = []
+
+    for i in range(1, 3):
+        response = load_mock_response(f'destinations{i}.json')
+        responses.append(response)
+
+        url: str = CommonServerPython.urljoin(
+            DESTINATION_ENDPOINT,
+            f'{args["destination_list_id"]}/destinations?page={i}&limit={4}',
+        )
+        requests_mock.get(url=url, json=response)
 
     command_results: CommonServerPython.CommandResults = CiscoUmbrellaCloudSecurityv2.list_destinations_command(
         mock_client, args
     )
 
+    expected_outputs = responses[0]['data'] + responses[1]['data'][:3]
+    response['meta']['total'] = len(expected_outputs)
+    response['data'] = expected_outputs
+
     assert command_results.outputs_prefix == (
         f'{CiscoUmbrellaCloudSecurityv2.INTEGRATION_OUTPUT_PREFIX}.'
         f'{CiscoUmbrellaCloudSecurityv2.DESTINATION_OUTPUT_PREFIX}'
     )
     assert command_results.outputs_key_field == CiscoUmbrellaCloudSecurityv2.ID_OUTPUTS_KEY_FIELD
-    assert command_results.outputs == response['data']
+    assert command_results.outputs == expected_outputs
     assert command_results.raw_response == response
 
 
+@patch('CiscoUmbrellaCloudSecurityv2.MAX_LIMIT', 4)
 def test_list_destinations_command_fetch_destinations(requests_mock, mock_client):
     """
     Scenario:
@@ -106,19 +122,33 @@ def test_list_destinations_command_fetch_destinations(requests_mock, mock_client
     args = {
         'destination_list_id': '123',
         'destinations': ['www.LiorSB.com', '1.1.1.1'],
-        'destination_ids': ['111', '333'],
+        'destination_ids': ['111', '333', '555', '1010'],
     }
-    response = load_mock_response('destinations.json')
-    url = CommonServerPython.urljoin(DESTINATION_ENDPOINT, f'{args["destination_list_id"]}/destinations')
 
-    requests_mock.get(url=url, json=response)
+    responses = []
+
+    for i in range(1, 4):
+        response = load_mock_response(f'destinations{i}.json')
+        responses.append(response)
+
+        url: str = CommonServerPython.urljoin(
+            DESTINATION_ENDPOINT,
+            f'{args["destination_list_id"]}/destinations?page={i}&limit={4}',
+        )
+        requests_mock.get(url=url, json=response)
 
     command_results: CommonServerPython.CommandResults = CiscoUmbrellaCloudSecurityv2.list_destinations_command(
         mock_client, args
     )
 
-    data = response['data']
-    expected_outputs = [data[0], data[2], data[3]]
+    expected_outputs = [
+        responses[0]['data'][0],
+        responses[0]['data'][2],
+        responses[0]['data'][3],
+        responses[1]['data'][0],
+    ]
+    response['meta']['total'] = len(expected_outputs)
+    response['data'] = expected_outputs
 
     assert command_results.outputs_prefix == (
         f'{CiscoUmbrellaCloudSecurityv2.INTEGRATION_OUTPUT_PREFIX}.'

diff --git a/...oudSecurityv2/test_data/destinations.json → ...udSecurityv2/test_data/destinations1.json b/...oudSecurityv2/test_data/destinations.json → ...udSecurityv2/test_data/destinations1.json
diff --git a/...lla-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/test_data/destinations2.json b/...lla-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/test_data/destinations2.json
@@ -0,0 +1,41 @@
+{
+    "status": {
+        "code": 200,
+        "text": "OK"
+    },
+    "meta": {
+        "page": 2,
+        "limit": 100,
+        "total": 4
+    },
+    "data": [
+        {
+            "id": "555",
+            "destination": "www.LiorSB1.com",
+            "type": "domain",
+            "comment": "Pikachu",
+            "createdAt": "2023-07-06 04:42:55"
+        },
+        {
+            "id": "666",
+            "destination": "www.pokemon1.com",
+            "type": "domain",
+            "comment": "Choose",
+            "createdAt": "2023-07-06 04:42:55"
+        },
+        {
+            "id": "777",
+            "destination": "www.serebii1.com",
+            "type": "domain",
+            "comment": "I",
+            "createdAt": "2023-07-06 04:42:55"
+        },
+        {
+            "id": "8888",
+            "destination": "www.serebii1.com",
+            "type": "domain",
+            "comment": "You",
+            "createdAt": "2023-07-06 04:42:55"
+        }
+    ]
+}
diff --git a/...lla-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/test_data/destinations3.json b/...lla-cloud-security/Integrations/CiscoUmbrellaCloudSecurityv2/test_data/destinations3.json
@@ -0,0 +1,12 @@
+{
+    "status": {
+        "code": 200,
+        "text": "OK"
+    },
+    "meta": {
+        "page": 3,
+        "limit": 100,
+        "total": 0
+    },
+    "data": []
+}
diff --git a/Packs/Cisco-umbrella-cloud-security/ReleaseNotes/2_0_2.md b/Packs/Cisco-umbrella-cloud-security/ReleaseNotes/2_0_2.md
@@ -0,0 +1,3 @@
+#### Integrations
+##### Cisco Umbrella Cloud Security v2
+Updated pagination iterations.
diff --git a/Packs/Cisco-umbrella-cloud-security/pack_metadata.json b/Packs/Cisco-umbrella-cloud-security/pack_metadata.json
@@ -2,7 +2,7 @@
     "name": "Cisco Umbrella cloud security",
     "description": "Basic integration with Cisco Umbrella that allows you to add domains to destination lists (e.g. global block / allow)",
     "support": "xsoar",
-    "currentVersion": "2.0.1",
+    "currentVersion": "2.0.2",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",
@@ -18,4 +18,4 @@
         "marketplacev2"
     ],
     "certification": "certified"
-}
+}