[Qradar] - add timeout param, update test-module and implement retry …

…for connection errors (#31339)

* add qradar timeout param

* add timeout to client

* add docs and bump rn

* fixes

* refactor test-module

* retry for http_request

* fixes

* elif

* rn and docker image

* rn

* bump rn

* update docker

* logger

* add fetch-interval and set default to 10

* README.md

* readme and udpates

* rn

* arg to number

* update docker-image

* ut

* fix uts

Packs/QRadar/Integrations/QRadar_v3/QRadar_v3.py

@@ -16,7 +16,7 @@
 ''' ADVANCED GLOBAL PARAMETERS '''
 
 FAILURE_SLEEP = 20  # sleep between consecutive failures events fetch
-FETCH_SLEEP = 60  # sleep between fetches
+FETCH_SLEEP = arg_to_number(demisto.params().get("fetch_interval")) or 60  # sleep between fetches
 BATCH_SIZE = 100  # batch size used for offense ip enrichment
 OFF_ENRCH_LIMIT = BATCH_SIZE * 10  # max amount of IPs to enrich per offense
 MAX_WORKERS = 8  # max concurrent workers used for events enriching
@@ -36,6 +36,9 @@
 EVENTS_SEARCH_TRIES = 3  # number of retries for creating a new search
 EVENTS_POLLING_TRIES = 10  # number of retries for events polling
 EVENTS_SEARCH_RETRY_SECONDS = 100  # seconds between retries to create a new search
+CONNECTION_ERRORS_RETRIES = 5  # num of times to retry in case of connection-errors
+CONNECTION_ERRORS_INTERVAL = 1  # num of seconds between each time to send an http-request in case of a connection error.
+
 
 ADVANCED_PARAMETERS_STRING_NAMES = [
     'DOMAIN_ENRCH_FLG',
@@ -359,7 +362,9 @@ class QueryStatus(str, Enum):
 
 class Client(BaseClient):
 
-    def __init__(self, server: str, verify: bool, proxy: bool, api_version: str, credentials: dict):
+    def __init__(
+        self, server: str, verify: bool, proxy: bool, api_version: str, credentials: dict, timeout: Optional[int] = None
+    ):
         username = credentials.get('identifier')
         password = credentials.get('password')
         if username == API_USERNAME:
@@ -370,23 +375,38 @@ def __init__(self, server: str, verify: bool, proxy: bool, api_version: str, cre
             self.base_headers = {'Version': api_version}
         base_url = urljoin(server, '/api')
         super().__init__(base_url=base_url, verify=verify, proxy=proxy, auth=auth)
+        self.timeout = timeout  # type: ignore[assignment]
         self.password = password
         self.server = server
 
     def http_request(self, method: str, url_suffix: str, params: Optional[dict] = None,
                      json_data: Optional[dict] = None, additional_headers: Optional[dict] = None,
                      timeout: Optional[int] = None, resp_type: str = 'json'):
         headers = {**additional_headers, **self.base_headers} if additional_headers else self.base_headers
-        return self._http_request(
-            method=method,
-            url_suffix=url_suffix,
-            params=params,
-            json_data=json_data,
-            headers=headers,
-            error_handler=self.qradar_error_handler,
-            timeout=timeout,
-            resp_type=resp_type
-        )
+        for _time in range(1, CONNECTION_ERRORS_RETRIES + 1):
+            try:
+                return self._http_request(
+                    method=method,
+                    url_suffix=url_suffix,
+                    params=params,
+                    json_data=json_data,
+                    headers=headers,
+                    error_handler=self.qradar_error_handler,
+                    timeout=timeout or self.timeout,
+                    resp_type=resp_type
+                )
+            except (DemistoException, requests.ReadTimeout) as error:
+                demisto.error(f'Error {error} in time {_time}')
+                if (
+                    isinstance(
+                        error, DemistoException
+                    ) and not isinstance(
+                        error.exception, requests.ConnectionError) or _time == CONNECTION_ERRORS_RETRIES
+                ):
+                    raise
+                else:
+                    time.sleep(CONNECTION_ERRORS_INTERVAL)  # pylint: disable=sleep-exists
+        return None
 
     @staticmethod
     def qradar_error_handler(res: requests.Response):
@@ -1734,38 +1754,13 @@ def test_module_command(client: Client, params: dict) -> str:
         - (str): 'ok' if test passed
         - raises DemistoException if something had failed the test.
     """
-    global EVENTS_SEARCH_TRIES, EVENTS_POLLING_TRIES
-    EVENTS_SEARCH_TRIES = 1
-    EVENTS_POLLING_TRIES = 1
     try:
         ctx = get_integration_context()
         print_context_data_stats(ctx, "Test Module")
         is_long_running = params.get('longRunning')
-        mirror_options = params.get('mirror_options', DEFAULT_MIRRORING_DIRECTION)
-        mirror_direction = MIRROR_DIRECTION.get(mirror_options)
-
         if is_long_running:
             validate_long_running_params(params)
-            ip_enrich, asset_enrich = get_offense_enrichment(params.get('enrichment', 'IPs And Assets'))
-            # Try to retrieve the last successfully retrieved offense
-            last_highest_id = max(ctx.get(LAST_FETCH_KEY, 0) - 1, 0)
-            get_incidents_long_running_execution(
-                client=client,
-                offenses_per_fetch=1,
-                user_query=params.get('query', ''),
-                fetch_mode=params.get('fetch_mode', ''),
-                events_columns=params.get('events_columns') or DEFAULT_EVENTS_COLUMNS,
-                events_limit=0,
-                ip_enrich=ip_enrich,
-                asset_enrich=asset_enrich,
-                last_highest_id=last_highest_id,
-                incident_type=params.get('incident_type'),
-                mirror_direction=mirror_direction,
-                first_fetch=params.get('first_fetch', '3 days'),
-                mirror_options=params.get('mirror_options', '')
-            )
-        else:
-            client.offenses_list(range_="items=0-0")
+        client.offenses_list(range_="items=0-0")
         message = 'ok'
     except DemistoException as e:
         err_msg = str(e)
@@ -4211,6 +4206,7 @@ def main() -> None:  # pragma: no cover
     if api_version and float(api_version) < MINIMUM_API_VERSION:
         raise DemistoException(f'API version cannot be lower than {MINIMUM_API_VERSION}')
     credentials = params.get('credentials')
+    timeout = arg_to_number(params.get("timeout"))
 
     try:
 
@@ -4219,7 +4215,9 @@ def main() -> None:  # pragma: no cover
             verify=verify_certificate,
             proxy=proxy,
             api_version=api_version,
-            credentials=credentials)
+            credentials=credentials,
+            timeout=timeout
+        )
         # All command names with or are for supporting QRadar v2 command names for backward compatibility
         if command == 'test-module':
             validate_integration_context()

Packs/QRadar/Integrations/QRadar_v3/QRadar_v3.yml

@@ -69,7 +69,7 @@ configuration:
   name: offenses_per_fetch
   type: 0
   additionalinfo: In case of mirroring with events, this value will be used for mirroring API calls as well, and it is advised to have a small value.
-  defaultvalue: '20'
+  defaultvalue: '10'
   section: Collect
   advanced: true
   required: false
@@ -153,6 +153,22 @@ configuration:
   section: Connect
   advanced: true
   required: false
+- defaultvalue: '60'
+  display: "Timeout for http-requests"
+  name: timeout
+  additionalinfo: The timeout of the HTTP requests sent to the Qradar API (in seconds). If no value is provided, the timeout will be set to 60 seconds.
+  type: 0
+  section: Collect
+  advanced: true
+  required: false
+- defaultvalue: '60'
+  display: "Fetch Incidents Interval"
+  name: fetch_interval
+  additionalinfo: The fetch interval between before each fetch-incidents execution (seconds).
+  type: 0
+  section: Collect
+  advanced: true
+  required: false
 script:
   commands:
   - name: qradar-offenses-list
@@ -2539,7 +2555,7 @@ script:
   script: '-'
   type: python
   subtype: python3
-  dockerimage: demisto/python3:3.10.13.82076
+  dockerimage: demisto/python3:3.10.13.83255
   isremotesyncin: true
   longRunning: true
   isFetchSamples: true

Packs/QRadar/Integrations/QRadar_v3/QRadar_v3_test.py

@@ -5,6 +5,8 @@
 from datetime import datetime
 from collections.abc import Callable
 import copy
+
+import requests
 from requests.exceptions import ReadTimeout
 
 import QRadar_v3  # import module separately for mocker
@@ -114,6 +116,30 @@ def test_get_optional_time_parameter_valid_time_argument(arg, iso_format, epoch_
     assert (get_time_parameter(arg, iso_format=iso_format, epoch_format=epoch_format)) == expected
 
 
+def test_connection_errors_recovers(mocker):
+    """
+    Given:
+     - Connection Error, ReadTimeout error and a success response
+
+    When:
+     - running the http_request method
+
+    Then:
+     - Ensure that success message is printed and recovery for http request happens.
+    """
+    mocker.patch.object(demisto, "error")
+    mocker.patch.object(
+        client,
+        "_http_request",
+        side_effect=[
+            DemistoException(message="error", exception=requests.ConnectionError("error")),
+            requests.ReadTimeout("error"),
+            "success"
+        ]
+    )
+    assert client.http_request(method="GET", url_suffix="url") == "success"
+
+
 @pytest.mark.parametrize('dict_key, inner_keys, expected',
                          [('continent', ['name'], {'ContinentName': 'NorthAmerica'}),
                           ('city', ['name'], {'CityName': 'Mukilteo'}),
@@ -358,7 +384,7 @@ def test_create_single_asset_for_offense_enrichment():
                            None,
                            ([], QueryStatus.ERROR.value))
                           ])
-def test_poll_offense_events_with_retry(requests_mock, status_exception, status_response, results_response, search_id,
+def test_poll_offense_events_with_retry(mocker, requests_mock, status_exception, status_response, results_response, search_id,
                                         expected):
     """
     Given:
@@ -373,6 +399,7 @@ def test_poll_offense_events_with_retry(requests_mock, status_exception, status_
      - Case a: Ensure that expected events are returned.
      - Case b: Ensure that None is returned.
     """
+    mocker.patch.object(demisto, "error")
     context_data = {MIRRORED_OFFENSES_QUERIED_CTX_KEY: {},
                     MIRRORED_OFFENSES_FINISHED_CTX_KEY: {}}
     set_integration_context(context_data)

Packs/QRadar/Integrations/QRadar_v3/README.md

@@ -8,7 +8,7 @@ This integration was integrated and tested with version 14-20 of QRadar v3
 3. Click **Add instance** to create and configure a new integration instance.
 
     | **Parameter** | **Description** | **Required** |
-    | --- | --- | --- |
+    | --- | -- | --- |
     | Server URL | \(e.g., https://1.1.1.1\) | True |
     | Username |  | True |
     | Password |  | True |
@@ -28,7 +28,8 @@ This integration was integrated and tested with version 14-20 of QRadar v3
     | Advanced Parameters | Comma-separated configuration for advanced parameter values. E.g., EVENTS_INTERVAL_SECS=20,FETCH_SLEEP=5 | False |
     | Trust any certificate (not secure) |  | False |
     | Use system proxy settings |  | False |
-
+    | Timeout for http-requests | The timeout of the HTTP requests sent to the Qradar API (in seconds). If no value is provided, the timeout will be set to 60 seconds. | False |
+    | Fetch Incidents Interval | The fetch interval between before each fetch-incidents execution. (seconds) | False |
 4. Click **Test** to validate the URLs, token, and connection.
 
 ## Required Permissions

Packs/QRadar/ReleaseNotes/2_4_45.md

@@ -0,0 +1,11 @@
+
+#### Integrations
+
+##### IBM QRadar v3
+
+- Added the **Timeout for http-requests** integration parameter to avoid read timeouts from Qradar api (seconds).
+- Added the **Fetch Incidents Interval** integration parameter to configure the **fetch-incidents** interval (seconds).
+- Fixed an issue where the test of the integration reached into timeouts.
+- Added support to recover from connection errors in all Qradar commands.
+- Updated the default value of the **Number of offenses to pull per API call** integration parameter to 10.
+- Updated the Docker image to: *demisto/python3:3.10.13.83255*.

Packs/QRadar/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "IBM QRadar",
     "description": "Fetch offenses as incidents and search QRadar",
     "support": "xsoar",
-    "currentVersion": "2.4.44",
+    "currentVersion": "2.4.45",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",