Prevent duplicate incidents

Packs/Claroty/Integrations/Claroty/Claroty.py

@@ -3,7 +3,7 @@
 from CommonServerUserPython import *
 """ IMPORTS """
 from distutils.util import strtobool
-from typing import List, Tuple, Dict, Any, Union
+from typing import Any
 import json
 import requests
 import dateparser
@@ -147,7 +147,7 @@ def get_alerts(self, fields: list, sort_by: dict, filters: list, limit: int = 10
         url_suffix = self._add_extra_params_to_url('ranger/alerts', fields, sort_by, filters, limit, page_number)
         return self._request_with_token(url_suffix, 'GET')
 
-    def get_alert(self, rid: str) -> Union[Dict, str, requests.Response]:
+    def get_alert(self, rid: str) -> dict | str | requests.Response:
         return self._request_with_token(f'ranger/alerts/{rid}', 'GET')
 
     def get_ranger_table_filters(self, table: str) -> dict:
@@ -172,7 +172,7 @@ def resolve_alert(self, selected_alerts: list, filters: dict, resolve_type: int,
         )
 
     @staticmethod
-    def _add_extra_params_to_url(url_suffix: str, fields: list, sort_by: dict, filters: List[Filter], limit: int = 10,
+    def _add_extra_params_to_url(url_suffix: str, fields: list, sort_by: dict, filters: list[Filter], limit: int = 10,
                                  page_number: int = 1) -> str:
         url_suffix += "?fields=" + ',;$'.join(fields)
         url_suffix += f"&page={page_number}&per_page={limit}"
@@ -206,7 +206,7 @@ def test_module(client: Client):
     return 'ok'
 
 
-def get_assets_command(client: Client, args: dict) -> Tuple:
+def get_assets_command(client: Client, args: dict) -> tuple:
     relevant_fields, sort_by, limit = _init_request_values("asset", "id", "asset_limit", args)
     filters = []
 
@@ -252,7 +252,7 @@ def get_assets_command(client: Client, args: dict) -> Tuple:
     )
 
 
-def resolve_alert_command(client: Client, args: dict) -> Tuple:
+def resolve_alert_command(client: Client, args: dict) -> tuple:
     bad_input = False
     selected_alerts_arg = args.get("selected_alerts", [])
     selected_alert_list = selected_alerts_arg.split(",") \
@@ -288,7 +288,7 @@ def resolve_alert_command(client: Client, args: dict) -> Tuple:
     )
 
 
-def get_single_alert_command(client: Client, args: dict) -> Tuple:
+def get_single_alert_command(client: Client, args: dict) -> tuple:
     relevant_fields = get_fields("alert", args.get("fields", "").split(","))
     alert_rid = args.get("alert_rid", None)
     result = client.get_alert(alert_rid)
@@ -305,7 +305,7 @@ def get_single_alert_command(client: Client, args: dict) -> Tuple:
     )
 
 
-def query_alerts_command(client: Client, args: dict) -> Tuple:
+def query_alerts_command(client: Client, args: dict) -> tuple:
     relevant_fields, sort_by, limit = _init_request_values("alert", "timestamp", "alert_limit", args, True)
     filters = []
 
@@ -353,7 +353,7 @@ def query_alerts_command(client: Client, args: dict) -> Tuple:
     )
 
 
-def _add_exclude_resolved_alerts_filters(filters: List[Filter]):
+def _add_exclude_resolved_alerts_filters(filters: list[Filter]):
     if not filters:
         return [Filter("resolved", "false", "exact")]
 
@@ -362,7 +362,7 @@ def _add_exclude_resolved_alerts_filters(filters: List[Filter]):
 
 
 def _init_request_values(obj_name: str, sort_by_default_value: str, limit_arg: str, args: dict,
-                         get_sort_order_arg: bool = False) -> Tuple[List, Dict, int]:
+                         get_sort_order_arg: bool = False) -> tuple[list, dict, int]:
     relevant_fields = get_fields(obj_name, args.get("fields", "").split(","))
 
     sort_order = False
@@ -380,7 +380,7 @@ def _init_request_values(obj_name: str, sort_by_default_value: str, limit_arg: s
     return relevant_fields, sort_by, limit
 
 
-def _parse_alerts_result(alert_result: dict, fields: list) -> List[dict]:
+def _parse_alerts_result(alert_result: dict, fields: list) -> list[dict]:
     if 'objects' not in alert_result:
         return []
     obj = alert_result.get('objects', [])
@@ -433,7 +433,7 @@ def _parse_single_alert(alert_obj, fields: list):
     return parsed_alert_result
 
 
-def _parse_assets_result(assets_result: dict, fields: list) -> Tuple:
+def _parse_assets_result(assets_result: dict, fields: list) -> tuple:
     if 'objects' not in assets_result:
         return [], []
     obj = assets_result.get('objects', [])
@@ -491,10 +491,10 @@ def get_sort(field_to_sort_by: str, order_by_desc: bool = False) -> dict:
 
 
 def get_sort_order(sort_order: str) -> bool:
-    return False if sort_order == "asc" else True
+    return sort_order != "asc"
 
 
-def get_fields(obj_name: str, fields: List[str]) -> list:
+def get_fields(obj_name: str, fields: list[str]) -> list:
     if obj_name == "alert":
         fields.append("resource_id")
         if "all" in fields:
@@ -533,7 +533,7 @@ def transform_filters_labels_to_values(table_filters, filter_name: str, filter_v
 
 def get_severity_filter(severity: str) -> str:
     severity_values = []
-    for severity_key, severity_value in CTD_TO_DEMISTO_SEVERITY.items():
+    for _severity_key, severity_value in CTD_TO_DEMISTO_SEVERITY.items():
         if severity_value >= CTD_TO_DEMISTO_SEVERITY.get(severity, 0):
             severity_values.append(str(severity_value))
     return ",;$".join(severity_values)
@@ -601,22 +601,23 @@ def fetch_incidents(client: Client, last_run, first_fetch_time):
         page_to_query = 1
 
     for item in items:
-        # Make datetime object unaware of timezone for comparison
-        parsed_date = dateparser.parse(item['Timestamp'])
-        assert parsed_date is not None, f"failed parsing {item['Timestamp']}"
-        incident_created_time = parsed_date.replace(tzinfo=None)
-
         # Don't add duplicated incidents
-        # if item["ResourceID"] not in last_run_rids:
-        incident = {
-            'name': item.get('Description', None),
-            'occurred': incident_created_time.strftime(DATE_FORMAT),
-            'severity': CTD_TO_DEMISTO_SEVERITY.get(item.get('Severity', None), None),
-            'rawJSON': json.dumps(item)
-        }
-
-        incidents.append(incident)
-        current_rids.append(item["ResourceID"])
+        rid = item["ResourceID"]
+        if rid not in last_run_rids:
+            # Make datetime object unaware of timezone for comparison
+            parsed_date = dateparser.parse(item['Timestamp'])
+            assert parsed_date is not None, f"failed parsing {item['Timestamp']}"
+            incident_created_time = parsed_date.replace(tzinfo=None)
+
+            incident = {
+                'name': item.get('Description', None),
+                'occurred': incident_created_time.strftime(DATE_FORMAT),
+                'severity': CTD_TO_DEMISTO_SEVERITY.get(item.get('Severity', None), None),
+                'rawJSON': json.dumps(item)
+            }
+
+            incidents.append(incident)
+            current_rids.append(rid)
 
     # If there were no items queried, latest_created_time is the same as last run
     if latest_created_time is None:

Packs/Claroty/Integrations/Claroty/Claroty.yml

@@ -95,7 +95,7 @@ script:
       - Low
       - Medium
       - High
-    - description: Get assets with that include the given insight name
+    - description: Get assets with that include the given insight name.
       name: insight_name
       predefined:
       - ''
@@ -109,7 +109,7 @@ script:
     - defaultValue: '10'
       description: Maximal value of assets to query at once.
       name: asset_limit
-    - description: Get all assets seen last from the given date. Format - YYYY-MM-DDThh:mm:ssZ. Example - 2020-02-02T01:02:03Z
+    - description: Get all assets seen last from the given date. Format - YYYY-MM-DDThh:mm:ssZ. Example - 2020-02-02T01:02:03Z.
       name: assets_last_seen
     description: Gets all assets from CTD. You can apply one or more filters.
     name: claroty-get-assets
@@ -218,7 +218,7 @@ script:
       defaultValue: timestamp
       description: |-
         The field by which to sort the results. The default value is "timestamp".
-        Default sort order is ascending
+        Default sort order is ascending.
       name: sort_by
       predefined:
       - resource_id
@@ -233,11 +233,11 @@ script:
       - timestamp
     - description: Returns alerts that match this alert type.
       name: type
-    - description: The start date from which to get alerts. Format - YYYY-MM-DDThh:mm:ssZ. Example - 2020-02-02T01:02:03Z
+    - description: The start date from which to get alerts. Format - YYYY-MM-DDThh:mm:ssZ. Example - 2020-02-02T01:02:03Z.
       name: date_from
     - auto: PREDEFINED
       defaultValue: asc
-      description: The sorting order of the alerts - descending or ascending
+      description: The sorting order of the alerts - descending or ascending.
       name: sort_order
       predefined:
       - asc
@@ -261,7 +261,7 @@ script:
       description: The alert type.
       type: String
     - contextPath: Claroty.Alert.AlertTypeID
-      description: The alert type int value
+      description: The alert type int value.
       type: Number
     - contextPath: Claroty.Alert.Description
       description: The alert description.
@@ -339,7 +339,7 @@ script:
       description: The alert category.
       type: String
   - arguments:
-    - description: The ResourceId of the Alerts to resolve (in <alert_id>-<site_id> format)
+    - description: The ResourceId of the Alerts to resolve (in <alert_id>-<site_id> format).
       name: selected_alerts
       required: true
     - auto: PREDEFINED
@@ -374,7 +374,7 @@ script:
       - description
       - alert_indicators
       - actionable_assets
-    - description: Resource ID of the desired alert. Expected value - <alert_id>-<site_id>
+    - description: Resource ID of the desired alert. Expected value - <alert_id>-<site_id>.
       name: alert_rid
       required: true
     description: Get a single alert from CTD.
@@ -384,7 +384,7 @@ script:
       description: The alert type.
       type: String
     - contextPath: Claroty.Alert.AlertTypeID
-      description: The alert type int value
+      description: The alert type int value.
       type: Number
     - contextPath: Claroty.Alert.Description
       description: The alert description.
@@ -407,7 +407,7 @@ script:
     - contextPath: Claroty.Alert.Severity
       description: The alert severity.
       type: String
-  dockerimage: demisto/python3:3.10.12.63474
+  dockerimage: demisto/python3:3.10.13.86272
   isfetch: true
   runonce: false
   script: '-'

Packs/Claroty/Integrations/Claroty/Claroty_test.py

@@ -1,3 +1,4 @@
+import json
 import dateparser
 import demistomock as demisto
 from Claroty import Client, fetch_incidents
@@ -137,16 +138,41 @@ def test_claroty_authentication(mocker, requests_mock):
 
 
 def test_claroty_fetch_incidents(mocker, requests_mock):
+    """
+    GIVEN a mock-client configured to return an Alert
+    WHEN the 'fetch_incidents' method is called multiple times
+    THEN the configured alert is returned exactly once. (no duplicate alerts should be received)
+    """
+    def get_rids_from_incidents(incidents: list[dict[str, str]]) -> set[int]:
+        raw_jsons_incidents = [json.loads(item["rawJSON"]) for item in incidents]
+        rids_from_incidents = {item["ResourceID"] for item in raw_jsons_incidents}
+
+        return rids_from_incidents
+
     client = _create_client(mocker, requests_mock, "https://website.com:5000/ranger/alerts", GET_ALERTS_RESPONSE, "GET")
     first_fetch_time = demisto.params().get('fetch_time', '7 days').strip()
     mocker.patch.object(demisto, 'incidents')
-    nextcheck, incidents = fetch_incidents(client, {'lastRun': dateparser.parse("2018-10-24T14:13:20+00:00")}, first_fetch_time)
+    next_check, incidents = fetch_incidents(client, {'lastRun': dateparser.parse("2018-10-24T14:13:20+00:00")}, first_fetch_time)
 
-    assert nextcheck['last_fetch']
+    assert next_check['last_fetch']
     assert isinstance(incidents, list)
     assert incidents[0]['severity'] == 4  # Demisto severity is higher by one (doesn't start at 0)
     assert isinstance(incidents[0]['name'], str)
 
+    rids_from_incidents = get_rids_from_incidents(incidents)
+
+    next_check, second_batch_incidents = fetch_incidents(client, next_check, first_fetch_time)
+    rids_from_second_batch_incidents = get_rids_from_incidents(second_batch_incidents)
+    assert next_check['last_fetch']
+    assert isinstance(second_batch_incidents, list)
+    assert rids_from_second_batch_incidents & rids_from_incidents == set(), "Fetching the same incident multiple times!"
+
+    next_check, third_batch_incidents = fetch_incidents(client, next_check, first_fetch_time)
+    rids_from_third_batch_incidents = get_rids_from_incidents(third_batch_incidents)
+    assert next_check['last_fetch']
+    assert isinstance(third_batch_incidents, list)
+    assert rids_from_third_batch_incidents & rids_from_incidents == set(), "Fetching the same incident multiple times!"
+
 
 def test_claroty_query_alerts(mocker, requests_mock):
     client = _create_client(mocker, requests_mock, "https://website.com:5000/ranger/alerts", GET_ALERTS_RESPONSE, "GET")

Packs/Claroty/ReleaseNotes/1_0_27.md

@@ -0,0 +1,7 @@
+
+#### Integrations
+
+##### Claroty
+
+- Updated the Docker image to: *demisto/python3:3.10.13.86272*.
+- Fixed an issue where **fetch-incidents** returned duplicated incidents.

Packs/Claroty/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Claroty",
     "description": "Use the Claroty CTD to manage assets and alerts.",
     "support": "partner",
-    "currentVersion": "1.0.26",
+    "currentVersion": "1.0.27",
     "author": "Claroty",
     "url": "",
     "email": "support@claroty.com",