JSON Feed - Remove ports from IPv4s Indicators (#32223)

* add parameter

* Parameter to remove ports

* RN

* yml fixes

* RN

* small fix

* RN

* cover

* Apply suggestions from code review

Co-authored-by: Arad Carmi <62752352+AradCarmi@users.noreply.github.com>

* Bump pack from version FeedMalwareBazaar to 1.0.38.

* Bump pack from version FeedGCPWhitelist to 2.0.38.

* Bump pack from version AccentureCTI_Feed to 1.1.35.

* Bump pack from version FeedAWS to 1.1.49.

* Update FeedJSON.yml

* docker bump

* RN

* Apply suggestions from code review

Doc review

Co-authored-by: ShirleyDenkberg <62508050+ShirleyDenkberg@users.noreply.github.com>

* Fixed test PB

* RN

---------

Co-authored-by: Arad Carmi <62752352+AradCarmi@users.noreply.github.com>
Co-authored-by: Content Bot <bot@demisto.com>
Co-authored-by: ShirleyDenkberg <62508050+ShirleyDenkberg@users.noreply.github.com>

Packs/AccentureCTI_Feed/ReleaseNotes/1_1_35.md

@@ -0,0 +1,6 @@
+
+#### Integrations
+
+##### ACTI Indicator Feed
+
+Improved handling of IPv4 indicators with ports.

Packs/AccentureCTI_Feed/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Accenture CTI Feed",
     "description": "Accenture Cyber Threat Intelligence Feed",
     "support": "partner",
-    "currentVersion": "1.1.34",
+    "currentVersion": "1.1.35",
     "author": "Accenture",
     "url": "https://www.accenture.com/us-en/services/security/cyber-defense",
     "email": "CTI.AcctManagement@accenture.com",

Packs/ApiModules/.secrets-ignore

@@ -1772,3 +1772,4 @@ https://portal.azure.com
 https://portal.azure.us
 https://vault.azure.cn
 https://vault.azure.net
+192.168.1.1:443

Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule.py

@@ -228,7 +228,7 @@ def get_formatted_feed_name(feed_name: str):
     return feed_name
 
 
-def test_module(client: Client, limit) -> str:
+def test_module(client: Client, limit) -> str:  # pragma: no cover
     for feed_name, feed in client.feed_name_to_config.items():
         custom_build_iterator = feed.get('custom_build_iterator')
         if custom_build_iterator:
@@ -239,7 +239,8 @@ def test_module(client: Client, limit) -> str:
 
 
 def fetch_indicators_command(client: Client, indicator_type: str, feedTags: list, auto_detect: bool,
-                             create_relationships: bool = False, limit: int = 0, **kwargs) -> Tuple[List[dict], bool]:
+                             create_relationships: bool = False, limit: int = 0, remove_ports: bool = False,
+                             **kwargs) -> Tuple[List[dict], bool]:
     """
     Fetches the indicators from client.
     :param client: Client of a JSON Feed
@@ -295,7 +296,7 @@ def fetch_indicators_command(client: Client, indicator_type: str, feedTags: list
             indicators.extend(
                 handle_indicator_function(client, item, feed_config, service_name, indicator_type, indicator_field,
                                           use_prefix_flat, feedTags, auto_detect, mapping_function,
-                                          create_relationships, create_relationships_function))
+                                          create_relationships, create_relationships_function, remove_ports))
 
             if limit and len(indicators) >= limit:  # We have a limitation only when get-indicators command is
                 # called, and then we return for each service_name "limit" of indicators
@@ -319,7 +320,8 @@ def indicator_mapping(mapping: Dict, indicator: Dict, attributes: Dict):
 def handle_indicator(client: Client, item: Dict, feed_config: Dict, service_name: str,
                      indicator_type: str, indicator_field: str, use_prefix_flat: bool,
                      feedTags: list, auto_detect: bool, mapping_function: Callable = indicator_mapping,
-                     create_relationships: bool = False, relationships_func: Callable = None) -> List[dict]:
+                     create_relationships: bool = False, relationships_func: Callable = None,
+                     remove_ports: bool = False) -> List[dict]:
     indicator_list = []
     mapping = feed_config.get('mapping')
     take_value_from_flatten = False
@@ -351,7 +353,6 @@ def handle_indicator(client: Client, item: Dict, feed_config: Dict, service_name
         indicator_value = attributes.get(indicator_field)
     indicator['value'] = indicator_value
     attributes['value'] = indicator_value
-
     if mapping:
         mapping_function(mapping, indicator, attributes)
 
@@ -361,6 +362,9 @@ def handle_indicator(client: Client, item: Dict, feed_config: Dict, service_name
     if feed_config.get('rawjson_include_indicator_type'):
         item['_indicator_type'] = current_indicator_type
 
+    if remove_ports and indicator['type'] == 'IP' and indicator['value']:
+        indicator['value'] = indicator['value'].split(':')[0]
+
     indicator['rawJSON'] = item
 
     indicator_list.append(indicator)
@@ -425,7 +429,7 @@ def extract(json_element, prefix_field="", use_prefix=False):
     return fields
 
 
-def feed_main(params, feed_name, prefix):
+def feed_main(params, feed_name, prefix):  # pragma: no cover
     handle_proxy()
     client = Client(**params)
     indicator_type = params.get('indicator_type')
@@ -442,9 +446,10 @@ def feed_main(params, feed_name, prefix):
             return_results(test_module(client, limit))
 
         elif command == 'fetch-indicators':
+            remove_ports = argToBoolean(params.get('remove_ports', False))
             create_relationships = params.get('create_relationships')
             indicators, no_update = fetch_indicators_command(client, indicator_type, feedTags, auto_detect,
-                                                             create_relationships)
+                                                             create_relationships, remove_ports=remove_ports)
 
             # check if the version is higher than 6.5.0 so we can use noUpdate parameter
             if is_demisto_version_ge('6.5.0'):
@@ -463,9 +468,10 @@ def feed_main(params, feed_name, prefix):
                         demisto.createIndicators(b)
 
         elif command == f'{prefix}get-indicators':
-            # dummy command for testing
+            remove_ports = argToBoolean(demisto.args().get('remove_ports', False))
             create_relationships = params.get('create_relationships')
-            indicators, _ = fetch_indicators_command(client, indicator_type, feedTags, auto_detect, create_relationships, limit)
+            indicators, _ = fetch_indicators_command(client, indicator_type, feedTags, auto_detect,
+                                                     create_relationships, limit, remove_ports)
             hr = tableToMarkdown('Indicators', indicators, headers=['value', 'type', 'rawJSON'])
             return_results(CommandResults(readable_output=hr, raw_response=indicators))
 

Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule_test.py

@@ -131,7 +131,7 @@ def test_json_feed_with_config_mapping():
 
 FLAT_LIST_OF_INDICATORS = '''{
     "hooks": [
-    "1.1.1.1",
+    "1.1.1.1:8080",
     "2.2.2.2",
     "3.3.3.3"
     ]
@@ -143,7 +143,8 @@ def test_list_of_indicators_with_no_json_object():
         'Github': {
             'url': 'https://api.github.com/meta',
             'extractor': "hooks",
-            'indicator': None
+            'indicator': None,
+            'remove_ports': "true"
         }
     }
 
@@ -157,7 +158,7 @@ def test_list_of_indicators_with_no_json_object():
         )
 
         indicators, _ = fetch_indicators_command(client=client, indicator_type=None, feedTags=['test'],
-                                                 auto_detect=True)
+                                                 auto_detect=True, remove_ports=True)
         assert len(indicators) == 3
         assert indicators[0].get('value') == '1.1.1.1'
         assert indicators[0].get('type') == 'IP'
@@ -169,7 +170,8 @@ def test_post_of_indicators_with_no_json_object():
         'Github': {
             'url': 'https://api.github.com/meta',
             'extractor': "hooks",
-            'indicator': None
+            'indicator': None,
+            'remove_ports': "false"
         }
     }
 
@@ -186,7 +188,7 @@ def test_post_of_indicators_with_no_json_object():
         indicators, _ = fetch_indicators_command(client=client, indicator_type=None, feedTags=['test'], auto_detect=True)
         assert matcher.last_request.text == 'test=1'
         assert len(indicators) == 3
-        assert indicators[0].get('value') == '1.1.1.1'
+        assert indicators[0].get('value') == '1.1.1.1:8080'
         assert indicators[0].get('type') == 'IP'
         assert indicators[1].get('rawJSON') == {'indicator': '2.2.2.2'}
 
@@ -416,6 +418,53 @@ def test_json_feed_with_config_mapping_with_aws_feed_no_update(mocker):
         assert last_run.call_count == 0
 
 
+@pytest.mark.parametrize('remove_ports, expected_result', [
+    (True, "192.168.1.1"),
+    (False, "192.168.1.1:443")
+])
+def test_remove_ports_threatfox(mocker, remove_ports, expected_result):
+    """
+    Given
+    - Fetch indicators command calling a server with type IPv4 indicators with ports.
+
+    When
+    - Running fetch indicators command
+
+    Then
+    - Ports are either included or removed based on the `remove_ports` parameter.
+    """
+    with open('test_data/threatfox_recent.json') as iocs:
+        iocs = json.load(iocs)
+
+    mocker.patch.object(demisto, 'debug')
+
+    feed_name_to_config = {
+        'THREATFOX': {
+            'url': 'https://threatfox.abuse.ch/export/json/recent/',
+            'extractor': "*[0].ioc_value",
+            'indicator_type': FeedIndicatorType.IP,
+        }
+    }
+    mocker.patch('CommonServerPython.is_demisto_version_ge', return_value=True)
+    mocker.patch('JSONFeedApiModule.is_demisto_version_ge', return_value=True)
+
+    with requests_mock.Mocker() as m:
+        m.get('https://threatfox.abuse.ch/export/json/recent/', json=iocs, status_code=200)
+
+        client = Client(
+            url='https://threatfox.abuse.ch/export/json/recent/',
+            feed_name_to_config=feed_name_to_config,
+            insecure=True
+        )
+
+        indicators = fetch_indicators_command(client=client,
+                                              indicator_type='IP',
+                                              auto_detect=True,
+                                              remove_ports=remove_ports,
+                                              feedTags=["ThreatFox"])
+        assert indicators[0][0]["value"] == expected_result
+
+
 def test_json_feed_with_config_mapping_with_aws_feed_with_update(mocker):
     """
     Given

Packs/ApiModules/Scripts/JSONFeedApiModule/test_data/threatfox_recent.json

@@ -0,0 +1,19 @@
+{
+    "1230586": [
+        {
+            "ioc_value": "192.168.1.1:443",
+            "ioc_type": "ip:port",
+            "threat_type": "botnet_cc",
+            "malware": "win.cobalt_strike",
+            "malware_alias": "Agentemis,BEACON,CobaltStrike,cobeacon",
+            "malware_printable": "Cobalt Strike",
+            "first_seen_utc": "2024-01-14 13:29:02",
+            "last_seen_utc": null,
+            "confidence_level": 100,
+            "reference": "",
+            "tags": "cobaltstrike,cs-watermark-100000",
+            "anonymous": "0",
+            "reporter": "myceliumbroker"
+        }
+    ]
+}

Packs/FeedAWS/ReleaseNotes/1_1_49.md

@@ -0,0 +1,6 @@
+
+#### Integrations
+
+##### AWS Feed
+
+Improved handling of IPv4 indicators with ports.

Packs/FeedAWS/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "AWS Feed",
     "description": "Indicators feed from AWS",
     "support": "xsoar",
-    "currentVersion": "1.1.48",
+    "currentVersion": "1.1.49",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",

Packs/FeedFastly/ReleaseNotes/1_1_27.md

@@ -0,0 +1,6 @@
+
+#### Integrations
+
+##### Fastly Feed
+
+Improved handling of IPv4 indicators with ports.

Packs/FeedFastly/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Fastly Feed",
     "description": "Indicators feed from Fastly",
     "support": "xsoar",
-    "currentVersion": "1.1.26",
+    "currentVersion": "1.1.27",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",

Packs/FeedGCPWhitelist/ReleaseNotes/2_0_38.md

@@ -0,0 +1,6 @@
+
+#### Integrations
+
+##### Google IP Ranges Feed
+
+Improved handling of IPv4 indicators with ports.

Packs/FeedGCPWhitelist/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Google IP Ranges Feed",
     "description": "Use the Google IP Ranges Feed integration to get GCP and Google global IP ranges.",
     "support": "xsoar",
-    "currentVersion": "2.0.37",
+    "currentVersion": "2.0.38",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",

Packs/FeedIntel471/ReleaseNotes/2_1_3.md

@@ -0,0 +1,10 @@
+
+#### Integrations
+
+##### Intel471 Malware Feed (Deprecated)
+
+Improved handling of IPv4 indicators with ports.
+
+##### Intel471 Actors Feed (Deprecated)
+
+Improved handling of IPv4 indicators with ports.

Packs/FeedIntel471/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Intel471 Feed",
     "description": "This content pack fetches actor and malware related indicators from Intel 471. It also fetches watcher alerts.",
     "support": "partner",
-    "currentVersion": "2.1.2",
+    "currentVersion": "2.1.3",
     "author": "Intel 471",
     "url": "https://www.intel471.com",
     "email": "support@intel471.com",

Packs/FeedJSON/Integrations/FeedJSON/FeedJSON.py

@@ -13,14 +13,15 @@ def main():
             'extractor': params.get('extractor'),
             'indicator': params.get('indicator', 'indicator'),
             'rawjson_include_indicator_type': params.get('rawjson_include_indicator_type'),
+            'remove_ports': params.get('remove_ports', False)
         }
     }
     auto_detect = params.get('auto_detect_type')
     indicator_type = params.get('indicator_type')
-    if demisto.command() == 'test-module':  # only fail when doing "Test" to avoid breaking an existing feed
-        if auto_detect and indicator_type:
-            return_error(f'Indicator Type (value: {indicator_type}) should not be set if "Auto detect indicator type" '
-                         'is checked. Either use Auto Detect or set manually the Indicator Type.')
+    if demisto.command() == 'test-module' and auto_detect and indicator_type:
+        # only fail when doing "Test" to avoid breaking an existing feed
+        return_error(f'Indicator Type (value: {indicator_type}) should not be set if "Auto detect indicator type" '
+                     'is checked. Either use Auto Detect or set manually the Indicator Type.')
 
     if not auto_detect:
         if not indicator_type:

Packs/FeedJSON/Integrations/FeedJSON/FeedJSON.yml

@@ -72,21 +72,27 @@ configuration:
   name: auto_detect_type
   type: 8
   required: false
-- additionalinfo: Type of the indicator in the feed. If auto-detect is checked then the value set as Indicator Type will be ignored.
-  display: Indicator Type
+- additionalinfo: Remove ports from IPv4 type indicators in the fetch command (e.g. ,192.168.1.1:8080 -> 192.168.1.1)
+  display: Remove IPv4 Ports
+  name: remove_ports
+  type: 8
+  required: false
+  defaultvalue: 'false'
+- display: Indicator Type
   name: indicator_type
   type: 0
   required: false
+  additionalinfo: Type of the indicator in the feed. If auto-detect is checked then the value set as Indicator Type will be ignored.
 - display: Username
   name: credentials
-  type: 9
   required: false
+  type: 9
 - additionalinfo: JMESPath expression for extracting the indicators. You can use http://jmespath.org/ to identify the proper expression.
   display: JMESPath Extractor
   name: extractor
-  required: true
   type: 0
-- additionalinfo: The JSON attribute that holds the indicator value. Default value is 'indicator'.
+  required: true
+- additionalinfo: "The JSON attribute that holds the indicator value. Default value is 'indicator'."
   display: JSON Indicator Attribute
   name: indicator
   type: 0
@@ -101,11 +107,11 @@ configuration:
   name: headers
   type: 12
   required: false
-- additionalinfo: "When using a custom classifier and mapper with this feed, use this option to include the indicator type in the raw json used for classification and mapping."
-  display: Include indicator type for mapping
+- display: Include indicator type for mapping
   name: rawjson_include_indicator_type
   type: 8
   required: false
+  additionalinfo: "When using a custom classifier and mapper with this feed, use this option to include the indicator type in the raw JSON used for classification and mapping."
 - display: Trust any certificate (not secure)
   name: insecure
   type: 8
@@ -114,7 +120,7 @@ configuration:
   name: proxy
   type: 8
   required: false
-- additionalinfo: When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system.
+- additionalinfo: "When selected, the exclusion list is ignored for indicators from this feed. This means that if an indicator from this feed is on the exclusion list, the indicator might still be added to the system."
   display: Bypass exclusion list
   name: feedBypassExclusionList
   type: 8
@@ -133,9 +139,12 @@ script:
     - defaultValue: '50'
       description: The maximum number of results to return. The default value is 50.
       name: limit
+    - defaultValue: 'false'
+      description: Remove ports from IPv4s.
+      name: remove_ports
     description: Gets the feed indicators.
     name: json-get-indicators
-  dockerimage: demisto/py3-tools:1.0.0.47376
+  dockerimage: demisto/py3-tools:1.0.0.86899
   feed: true
   runonce: false
   script: '-'