XDR IOCs - custom severity (#21153)

* extract severity

* bump version

* use field

* fix, add param

* rn

* not required

* undo change ut

* use text field

* improvements

* change field name

* fix UT?

* map xdr to xsoar severity

* translate both ways

* add critical

* fix UT

* update docker

* fix ut

* fix tag syntax

* more logs

* no translation, more logs

* fix ut

* docs

* Update XDR_iocs.py

typo

* Update XDR_iocs_test.py

typo again

* override_severity, UT

* uppercase severity

* argToList

* Update Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.py

Co-authored-by: Adi Bamberger Edri <72088126+BEAdi@users.noreply.github.com>

* fix param

* remove blank line from eof

* update docker version

* validate severity param

* fix syntax

* fix ut

* better name

* remove file

* improve method

* update docker

* validate severity value

* fix UT

* remove files

* fix rn

* better error messages

Co-authored-by: Dror Avrahami <davrahami@paloaltonetworks.com>
Co-authored-by: darbel <darbel@paloaltonetworks.com>
Co-authored-by: Adi Bamberger Edri <72088126+BEAdi@users.noreply.github.com>

Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.py

@@ -18,6 +18,15 @@
     "HASH": 'File',
     "IP": 'IP'
 }
+xdr_severity_to_demisto: dict[str, str] = {
+    'SEV_010_INFO': 'INFO',
+    'SEV_020_LOW': 'LOW',
+    'SEV_030_MEDIUM': 'MEDIUM',
+    'SEV_040_HIGH': 'HIGH',
+    'SEV_050_CRITICAL': 'CRITICAL',
+    'SEV_090_UNKNOWN': 'UNKNOWN',
+}
+
 xdr_reputation_to_demisto: Dict = {
     'GOOD': 1,
     'SUSPICIOUS': 2,
@@ -43,21 +52,24 @@ def batch_iocs(generator, batch_size=200):
 
 
 class Client:
-    severity: str = ''
+    # All values here are the defaults, which may be changed via params, on main()
     query: str = 'reputation:Bad and (type:File or type:Domain or type:IP)'
+    override_severity: bool = True
+    severity: str = ''  # used when override_severity is True
+    xsoar_severity_field: str = 'sourceoriginalseverity'  # used when override_severity is False
     tag = 'Cortex XDR'
     tlp_color = None
     error_codes: Dict[int, str] = {
         500: 'XDR internal server error.',
-        401: 'Unauthorized access. An issue occurred during authentication. This can indicate an '    # noqa: W504
-             + 'incorrect key, id, or other invalid authentication parameters.',
+        401: 'Unauthorized access. An issue occurred during authentication. '
+             'This can indicate an incorrect key, id, or other invalid authentication parameters.',
         402: 'Unauthorized access. User does not have the required license type to run this API.',
         403: 'Unauthorized access. The provided API key does not have the required RBAC permissions to run this API.',
         404: 'XDR Not found: The provided URL may not be of an active XDR server.',
         413: 'Request entity too large. Please reach out to the XDR support team.'
     }
 
-    def __init__(self, params: Dict):
+    def __init__(self, params: dict):
         self._base_url: str = urljoin(params.get('url'), '/public_api/v1/indicators/')
         self._verify_cert: bool = not params.get('insecure', False)
         self._params = params
@@ -156,7 +168,7 @@ def create_file_sync(file_path, batch_size: int = 200):
 
 
 def get_iocs_generator(size=200, query=None) -> Iterable:
-    query = query if query else Client.query
+    query = query or Client.query
     query = f'expirationStatus:active AND ({query})'
     try:
         for iocs in map(lambda x: x.get('iocs', []), IndicatorsSearcher(size=size, query=query)):
@@ -210,32 +222,46 @@ def demisto_types_to_xdr(_type: str) -> str:
 
 def demisto_ioc_to_xdr(ioc: Dict) -> Dict:
     try:
+        demisto.debug(f'Raw outgoing IOC: {ioc}')
         xdr_ioc: Dict = {
             'indicator': ioc['value'],
-            'severity': Client.severity,
+            'severity': Client.severity,  # default, may be overwritten, see below
             'type': demisto_types_to_xdr(str(ioc['indicator_type'])),
             'reputation': demisto_score_to_xdr.get(ioc.get('score', 0), 'UNKNOWN'),
             'expiration_date': demisto_expiration_to_xdr(ioc.get('expiration'))
         }
         # get last 'IndicatorCommentRegular'
-        comment: Dict = next(filter(lambda x: x.get('type') == 'IndicatorCommentRegular', reversed(ioc.get('comments', []))), {})
+        comment: Dict = next(
+            filter(lambda x: x.get('type') == 'IndicatorCommentRegular', reversed(ioc.get('comments', []))), {}
+        )
         if comment:
             xdr_ioc['comment'] = comment.get('content')
-        if ioc.get('aggregatedReliability'):
-            xdr_ioc['reliability'] = ioc['aggregatedReliability'][0]
-        vendors = demisto_vendors_to_xdr(ioc.get('moduleToFeedMap', {}))
-        if vendors:
+        if aggregated_reliability := ioc.get('aggregatedReliability'):
+            xdr_ioc['reliability'] = aggregated_reliability[0]
+        if vendors := demisto_vendors_to_xdr(ioc.get('moduleToFeedMap', {})):
             xdr_ioc['vendors'] = vendors
 
-        threat_type = ioc.get('CustomFields', {}).get('threattypes', {})
-        if threat_type:
+        custom_fields = ioc.get('CustomFields', {})
+
+        if threat_type := custom_fields.get('threattypes', {}):
             threat_type = threat_type[0] if isinstance(threat_type, list) else threat_type
             threat_type = threat_type.get('threatcategory')
             if threat_type:
                 xdr_ioc['class'] = threat_type
-        if ioc.get('CustomFields', {}).get('xdrstatus') == 'disabled':
+
+        if custom_fields.get('xdrstatus') == 'disabled':
             xdr_ioc['status'] = 'DISABLED'
+
+        if (not Client.override_severity) and (custom_severity := custom_fields.get(Client.xsoar_severity_field)):
+            # Override is True: use Client.severity
+            # Override is False: use the value from the xsoar_severity_field, or Client.severity as default
+            xdr_ioc['severity'] = custom_severity  # NOTE: these do NOT need translation to XDR's 0x0_xxxx_xxxx format
+
+        xdr_ioc['severity'] = validate_fix_severity_value(xdr_ioc['severity'], ioc['value'])
+
+        demisto.debug(f'Processed outgoing IOC: {xdr_ioc}')
         return xdr_ioc
+
     except KeyError as error:
         demisto.debug(f'unexpected IOC format in key: {str(error)}, {str(ioc)}')
         return {}
@@ -293,7 +319,7 @@ def get_indicators(indicators: str) -> List:
     if indicators:
         iocs: list = []
         not_found = []
-        for indicator in indicators.split(','):
+        for indicator in argToList(indicators):
             search_indicators = IndicatorsSearcher()
             data = search_indicators.search_indicators_by_version(value=indicator).get('iocs')
             if data:
@@ -328,7 +354,7 @@ def iocs_command(client: Client):
     indicators = demisto.args().get('indicator', '')
     if command == 'enable':
         path, iocs = prepare_enable_iocs(indicators)
-    else:   # command == 'disable'
+    else:  # command == 'disable'
         path, iocs = prepare_disable_iocs(indicators)
     requests_kwargs: Dict = get_requests_kwargs(_json=iocs)
     client.http_request(url_suffix=path, requests_kwargs=requests_kwargs)
@@ -357,6 +383,8 @@ def xdr_ioc_to_demisto(ioc: Dict) -> Dict:
     indicator = ioc.get('RULE_INDICATOR', '')
     xdr_server_score = int(xdr_reputation_to_demisto.get(ioc.get('REPUTATION'), 0))
     score = get_indicator_xdr_score(indicator, xdr_server_score)
+    severity = Client.severity if Client.override_severity else xdr_severity_to_demisto[ioc['RULE_SEVERITY']]
+
     entry: Dict = {
         "value": indicator,
         "type": xdr_types_to_demisto.get(ioc.get('IOC_TYPE')),
@@ -365,12 +393,14 @@ def xdr_ioc_to_demisto(ioc: Dict) -> Dict:
             "tags": Client.tag,
             "xdrstatus": ioc.get('RULE_STATUS', '').lower(),
             "expirationdate": xdr_expiration_to_demisto(ioc.get('RULE_EXPIRATION_TIME')),
+            Client.xsoar_severity_field: severity,
         },
         "rawJSON": ioc
     }
     if Client.tlp_color:
         entry['fields']['trafficlightprotocol'] = Client.tlp_color
 
+    demisto.debug(f'Processed incoming entry: {entry}')
     return entry
 
 
@@ -380,11 +410,15 @@ def get_changes(client: Client):
         raise DemistoException('XDR is not synced.')
     path, requests_kwargs = prepare_get_changes(from_time['ts'])
     requests_kwargs: Dict = get_requests_kwargs(_json=requests_kwargs)
-    iocs: List = client.http_request(url_suffix=path, requests_kwargs=requests_kwargs).get('reply', [])
-    if iocs:
+    demisto.debug(f'creating http request to {path} with {str(requests_kwargs)}')
+    if iocs := client.http_request(url_suffix=path, requests_kwargs=requests_kwargs).get('reply', []):
         from_time['ts'] = iocs[-1].get('RULE_MODIFY_TIME', from_time) + 1
+        demisto.debug(f'setting integration context with {from_time=}')
         set_integration_context(from_time)
-        demisto.createIndicators(list(map(xdr_ioc_to_demisto, iocs)))
+        demisto_indicators = list(map(xdr_ioc_to_demisto, iocs))
+        for indicator in demisto_indicators:
+            demisto.debug(f'indicator: {indicator}')
+        demisto.createIndicators(demisto_indicators)
 
 
 def module_test(client: Client):
@@ -464,7 +498,7 @@ def set_sync_time(time: str):
     set_integration_context({'ts': int(date_time_obj.timestamp() * 1000),
                              'time': date_time_obj.strftime(DEMISTO_TIME_FORMAT),
                              'iocs_to_keep_time': create_iocs_to_keep_time()})
-    return_results(f'set sync time to {time} seccedded.')
+    return_results(f'set sync time to {time} succeeded.')
 
 
 def get_sync_file():
@@ -477,15 +511,50 @@ def get_sync_file():
         os.remove(temp_file_path)
 
 
-def main():   # pragma: no cover
-    # """
-    # Executes an integration command
-    # """
+def to_cli_name(field_name: str):
+    return field_name.lower().replace(' ', '')
+
+
+def validate_fix_severity_value(severity: str, indicator_value: Optional[str] = None) -> str:
+    """raises error if the value is invalid, returns the value (fixes informational->info)
+
+    Args:
+        severity (str): the severity value, must be of INFO,LOW,MEDIUM,HIGH,CRITICAL,UNKNOWN
+        indicator_value (Optional[str]): displayed in case of error
+
+    Raises:
+        DemistoException: when the value isn't allowed (nor can be fixed automatically)
+
+    Returns:
+        _type_: str, validated severity value
+    """
+    allowed_values = xdr_severity_to_demisto.values()
+    severity_upper = severity.upper()
+
+    if severity_upper == "INFORMATIONAL":
+        severity_upper = "INFO"
+
+    if severity_upper not in allowed_values:
+        prefix = f'indicator {indicator_value}: ' if indicator_value else ''
+        raise DemistoException(f"{prefix}the severity value must be one of {', '.join(allowed_values)} (got {severity})")
+
+    return severity_upper
+
+
+def main():  # pragma: no cover
     params = demisto.params()
-    Client.severity = params.get('severity', '').upper()
-    Client.query = params.get('query', Client.query)
-    Client.tag = params.get('feedTags', params.get('tag', Client.tag))
+    Client.severity = params.get('severity', '')
+    Client.override_severity = argToBoolean(params.get('override_severity', True))
     Client.tlp_color = params.get('tlp_color')
+
+    # In this integration, parameters are set in the *class level*, the defaults are in the class definition.
+    if query := params.get('query'):
+        Client.query = query
+    if tag := (params.get('feedTags') or params.get('tag')):
+        Client.tag = tag
+    if xsoar_severity_field := params.get('xsoar_severity_field'):
+        Client.xsoar_severity_field = to_cli_name(xsoar_severity_field)
+
     client = Client(params)
     commands = {
         'test-module': module_test,

Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs.yml

@@ -21,15 +21,13 @@ configuration:
   required: false
   type: 8
 - defaultvalue: 'true'
-  additionalinfo: When enabled, indicators will be synced from Cortex XSOAR to Cortex
-    XDR. Disable if you prefer to use a playbook to sync indicators.
+  additionalinfo: When enabled, indicators will be synced from Cortex XSOAR to Cortex XDR. Disable if you prefer to use a playbook to sync indicators.
   display: Auto Sync
   name: autoSync
   required: false
   type: 8
-- additionalinfo: Map the severity of each indicator that will be synced to Cortex
-    XDR.
-  display: Cortex XDR Severity
+- additionalinfo: When the `override severity` parameter is set to True, the severity level chosen here will be used for all indicators. Otherwise, the severity level of the indicator will be used.
+  display: Overriding severity value
   hidden: false
   name: severity
   options:
@@ -47,8 +45,7 @@ configuration:
   name: feedTags
   required: false
   type: 0
-- additionalinfo: The query used to collect indicators to sync from Cortex XSOAR to
-    Cortex XDR.
+- additionalinfo: The query used to collect indicators to sync from Cortex XSOAR to Cortex XDR.
   defaultvalue: reputation:Bad and (type:File or type:Domain or type:IP)
   display: Sync Query
   hidden: false
@@ -63,8 +60,7 @@ configuration:
   name: proxy
   required: false
   type: 8
-- additionalinfo: Indicators from this integration instance will be marked with this
-    reputation
+- additionalinfo: Indicators from this integration instance will be marked with this reputation
   display: Indicator Reputation
   name: feedReputation
   options:
@@ -87,8 +83,7 @@ configuration:
   - F - Reliability cannot be judged
   required: true
   type: 15
-- additionalinfo: The Traffic Light Protocol (TLP) designation to apply to indicators
-    fetched from the feed
+- additionalinfo: The Traffic Light Protocol (TLP) designation to apply to indicators fetched from the feed
   display: Traffic Light Protocol Color
   name: tlp_color
   options:
@@ -122,17 +117,24 @@ configuration:
   name: feedFetchInterval
   required: false
   type: 19
-- additionalinfo: When selected, the exclusion list is ignored for indicators from
-    this feed. This means that if an indicator from this feed is on the exclusion
-    list, the indicator might still be added to the system.
+- additionalinfo: The XSOAR indicator field used as severity.
+  display: XSOAR Severity Field
+  name: xsoar_severity_field
+  required: false
+  type: 0
+  defaultvalue: 'sourceoriginalseverity'
+- defaultvalue: 'true'
+  additionalinfo: When enabled, the severity value will be taken from the `severity` parameter, regardless of the IOC severity value. Otherwise, the severity value will be taken according to the `xsoar_severity_field` parameter.
+  display: Override severity
+  name: override_severity
+  required: false
+  type: 8
+- additionalinfo: When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system.
   display: Bypass exclusion list
   name: feedBypassExclusionList
   required: false
   type: 8
-description: Use the Cortex XDR - IOCs feed integration to sync indicators from Cortex
-  XSOAR to Cortex XDR and back to Cortex XSOAR. Cortex XDR is the world's first detection
-  and response app that natively integrates network, endpoint and cloud data to stop
-  sophisticated attacks.
+description: Use the Cortex XDR - IOCs feed integration to sync indicators from Cortex XSOAR to Cortex XDR and back to Cortex XSOAR. Cortex XDR is the world's first detection and response app that natively integrates network, endpoint and cloud data to stop sophisticated attacks.
 display: Cortex XDR - IOC
 name: Cortex XDR - IOC
 script:
@@ -157,8 +159,7 @@ script:
     name: xdr-iocs-sync
   - arguments:
     - default: false
-      description: IOCs to push. leave empty to push all recently modified IOCs.the
-        indicators
+      description: IOCs to push. leave empty to push all recently modified IOCs.the indicators
       isArray: true
       name: indicator
       required: false
@@ -175,13 +176,11 @@ script:
       required: true
       secret: false
     deprecated: false
-    description: Set sync time manually (Do not use this command unless you unredstandard
-      the consequences).
+    description: Set sync time manually (Do not use this command unless you unredstandard the consequences).
     execution: false
     name: xdr-iocs-set-sync-time
   - deprecated: false
-    description: Creates the sync file for the manual process. Run this command when
-      instructed by the XDR support team.
+    description: Creates the sync file for the manual process. Run this command when instructed by the XDR support team.
     execution: false
     name: xdr-iocs-create-sync-file
   - arguments:
@@ -206,7 +205,7 @@ script:
     description: Disables IOCs in the XDR server.
     execution: false
     name: xdr-iocs-disable
-  dockerimage: demisto/python3:3.10.7.33922
+  dockerimage: demisto/python3:3.10.8.36650
   feed: true
   isfetch: false
   longRunning: false

Packs/CortexXDR/Integrations/XDR_iocs/XDR_iocs_description.md

@@ -2,9 +2,11 @@
 Use the Cortex XDR - IOCs feed integration to sync indicators from XSOAR to XDR.
 Cortex XDR is the world's first detection and response app that natively integrates network, endpoint and cloud data to stop sophisticated attacks.
 
+**Note: Only one instance of this integration is supported at a time. If more than one is configured, instances will interrupt one another.**
+
 ### Generate an API Key and API Key ID
 1. In your Cortex XDR platform, go to **Settings**.
-2. Click the **+New Key** button in the top right corner.
+2. Click the **(+) New Key** button in the top-right corner.
 3. Generate a key of type **Advanced** with an **Administrator** role.
 4. Copy and paste the key.
 5. From the ID column, copy the Key ID.
@@ -13,8 +15,13 @@ Cortex XDR is the world's first detection and response app that natively integra
 1. In your Cortex XDR platform, go to **Settings**.
 2. Click the **Copy URL** button in the top right corner.
 
-### Query
-The query to find indicators.
+### Severity
+The integration sync the severity field (indicator field, set as parameter) between XSOAR and XDR.
+
+The severity field must be one of the following: 
+- a textual field, or 
+- a single-select field, accepting the following values: `INFO`,`LOW`,`MEDIUM`,`HIGH`,`CRITICAL`. 
+
+Using an invalid field (or one that is not present in the indicator) may cause fetching to stop working.
 
-### Important
-Do not use more then one instance. Instances will override each other.
+**Note: Due to XSOAR system limitations, once the severity is manually changed within XSOAR, it is excluded from being updated by the fetching process.**