AWS Feed - added support for IPv6 (#27141)

* AWS Feed - added support for IPv6 * added tests * cr * tests an rn * validation * rn * cr * di
demisto · Jun 14, 2023 · 4156d18 · 4156d18
1 parent 41afeaf
commit 4156d18
Show file tree

Hide file tree

Showing 20 changed files with 255 additions and 48 deletions.
diff --git a/Packs/AccentureCTI_Feed/ReleaseNotes/1_1_14.md b/Packs/AccentureCTI_Feed/ReleaseNotes/1_1_14.md
@@ -0,0 +1,6 @@
+
+#### Integrations
+
+##### ACTI Indicator Feed
+
+- Enhanced the ***JSONFeedApiModule*** to support fetching multiple indicator types in the ***AWS Feed*** integration. The change has no impact on this integration.
diff --git a/Packs/AccentureCTI_Feed/pack_metadata.json b/Packs/AccentureCTI_Feed/pack_metadata.json
@@ -2,7 +2,7 @@
     "name": "Accenture CTI Feed",
     "description": "Accenture Cyber Threat Intelligence Feed",
     "support": "partner",
-    "currentVersion": "1.1.13",
+    "currentVersion": "1.1.14",
     "author": "Accenture",
     "url": "https://www.accenture.com/us-en/services/security/cyber-defense",
     "email": "CTI.AcctManagement@accenture.com",

diff --git a/Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule.py b/Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule.py
@@ -111,11 +111,13 @@ def build_iterator(self, feed: dict, feed_name: str, **kwargs) -> Tuple[List, bo
         url = feed.get('url', self.url)
 
         if is_demisto_version_ge('6.5.0'):
+            prefix_feed_name = get_formatted_feed_name(feed_name)  # Support for AWS feed
+
             # Set the If-None-Match and If-Modified-Since headers
             # if we have etag or last_modified values in the context, with server version higher than 6.5.0.
             last_run = demisto.getLastRun()
-            etag = demisto.get(last_run, f'{feed_name}.etag')
-            last_modified = demisto.get(last_run, f'{feed_name}.last_modified')
+            etag = last_run.get(prefix_feed_name, {}).get('etag') or last_run.get(feed_name, {}).get('etag')
+            last_modified = last_run.get(prefix_feed_name, {}).get('last_modified') or last_run.get(feed_name, {}).get('last_modified')  # noqa: E501
 
             if etag:
                 self.headers['If-None-Match'] = etag
@@ -147,6 +149,7 @@ def build_iterator(self, feed: dict, feed_name: str, **kwargs) -> Tuple[List, bo
         try:
             r.raise_for_status()
             if r.content:
+                demisto.debug(f'JSON: found content for {feed_name}')
                 data = r.json()
                 result = jmespath.search(expression=feed.get('extractor'), data=data)
 
@@ -170,7 +173,6 @@ def get_no_update_value(response: requests.Response, feed_name: str) -> bool:
         boolean with the value for noUpdate argument.
         The value should be False if the response was modified.
     """
-
     # HTTP status code 304 (Not Modified) set noUpdate to True.
     if response.status_code == 304:
         demisto.debug('No new indicators fetched, createIndicators will be executed with noUpdate=True.')
@@ -190,12 +192,26 @@ def get_no_update_value(response: requests.Response, feed_name: str) -> bool:
         'etag': etag
     }
     demisto.setLastRun(last_run)
-
+    demisto.debug(f'JSON: The new last run is: {last_run}')
     demisto.debug('New indicators fetched - the Last-Modified value has been updated,'
                   ' createIndicators will be executed with noUpdate=False.')
     return False
 
 
+def get_formatted_feed_name(feed_name: str):
+    """support for AWS Feed config name, that contains $$ in the name.
+        example: AMAZON$$CIDR
+    Args:
+        feed_name (str): The feed config name
+    """
+    prefix_feed_name = ''
+    if '$$' in feed_name:
+        prefix_feed_name = feed_name.split('$$')[0]
+        return prefix_feed_name
+
+    return feed_name
+
+
 def test_module(client: Client, limit) -> str:
     for feed_name, feed in client.feed_name_to_config.items():
         custom_build_iterator = feed.get('custom_build_iterator')
@@ -241,6 +257,7 @@ def fetch_indicators_command(client: Client, indicator_type: str, feedTags: list
         mapping_function = feed_config.get('mapping_function', indicator_mapping)
         handle_indicator_function = feed_config.get('handle_indicator_function', handle_indicator)
         create_relationships_function = feed_config.get('create_relations_function')
+        service_name = get_formatted_feed_name(service_name)
 
         for item in items:
             if isinstance(item, str):

diff --git a/Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule_test.py b/Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule_test.py
@@ -29,7 +29,7 @@ def test_json_feed_no_config():
 CONFIG_PARAMETERS = [
     (
         {
-            'AMAZON': {
+            'AMAZON$$CIDR': {
                 'url': 'https://ip-ranges.amazonaws.com/ip-ranges.json',
                 'extractor': "prefixes[?service=='AMAZON']",
                 'indicator': 'ip_prefix',
@@ -42,13 +42,20 @@ def test_json_feed_no_config():
     ),
     (
         {
-            'AMAZON': {
+            'AMAZON$$CIDR': {
                 'url': 'https://ip-ranges.amazonaws.com/ip-ranges.json',
                 'extractor': "prefixes[?service=='AMAZON']",
                 'indicator': 'ip_prefix',
                 'indicator_type': FeedIndicatorType.CIDR,
                 'fields': ['region', 'service']
             },
+            'AMAZON$$IPV6': {
+                'url': 'https://ip-ranges.amazonaws.com/ip-ranges.json',
+                'extractor': "ipv6_prefixes[?service=='AMAZON']",
+                'indicator': 'ipv6_prefix',
+                'indicator_type': FeedIndicatorType.IPv6,
+                'fields': ['region', 'service']
+            },
             'CLOUDFRONT': {
                 'url': 'https://ip-ranges.amazonaws.com/ip-ranges.json',
                 'extractor': "prefixes[?service=='CLOUDFRONT']",
@@ -57,7 +64,7 @@ def test_json_feed_no_config():
                 'fields': ['region', 'service']
             }
         },
-        1148,
+        1465,
         36
     )
 ]
@@ -89,7 +96,7 @@ def test_json_feed_with_config_mapping():
         ip_ranges = json.load(ip_ranges_json)
 
     feed_name_to_config = {
-        'AMAZON': {
+        'AMAZON$$CIDR': {
             'url': 'https://ip-ranges.amazonaws.com/ip-ranges.json',
             'extractor': "prefixes[?service=='AMAZON']",
             'indicator': 'ip_prefix',
@@ -350,3 +357,111 @@ def test_fetch_indicators_command_google_ip_ranges(mocker):
     indicators, _ = fetch_indicators_command(client, indicator_type=None, feedTags=[], auto_detect=None, limit=100)
     for indicator in indicators:
         assert indicator.get('value')
+
+
+def test_json_feed_with_config_mapping_with_aws_feed_no_update(mocker):
+    """
+    Given
+    - Feed config from AWS feed, with last_run from the same feed, emulating the first
+      fetch after updating the AWS Feed integration when there is no update to the feed.
+      (the last_run object contains an 'AMAZON' entry)
+
+    When
+    - Running fetch indicators command
+
+    Then
+    - Ensure that the correct message displays in demisto.debug, and the last_run object
+     remained the same, and continue to have the previous AWS feed config name 'AMAZON'.
+     (the last_run object contains an 'AMAZON' entry)
+    """
+    with open('test_data/amazon_ip_ranges.json') as ip_ranges_json:
+        ip_ranges = json.load(ip_ranges_json)
+
+    mocker.patch.object(demisto, 'debug')
+    last_run = mocker.patch.object(demisto, 'setLastRun')
+
+    feed_name_to_config = {
+        'AMAZON$$CIDR': {
+            'url': 'https://ip-ranges.amazonaws.com/ip-ranges.json',
+            'extractor': "prefixes[?service=='AMAZON']",
+            'indicator': 'ip_prefix',
+            'indicator_type': FeedIndicatorType.CIDR,
+            'fields': ['region', 'service'],
+            'mapping': {
+                'region': 'Region'
+            }
+        }
+    }
+    mocker.patch('CommonServerPython.is_demisto_version_ge', return_value=True)
+    mocker.patch('JSONFeedApiModule.is_demisto_version_ge', return_value=True)
+    mock_last_run = {"AMAZON": {"last_modified": '2019-12-17-23-03-10', "etag": "etag"}}
+    mocker.patch.object(demisto, 'getLastRun', return_value=mock_last_run)
+
+    with requests_mock.Mocker() as m:
+        m.get('https://ip-ranges.amazonaws.com/ip-ranges.json', json=ip_ranges, status_code=304,)
+
+        client = Client(
+            url='https://ip-ranges.amazonaws.com/ip-ranges.json',
+            credentials={'username': 'test', 'password': 'test'},
+            feed_name_to_config=feed_name_to_config,
+            insecure=True
+        )
+
+        fetch_indicators_command(client=client, indicator_type='CIDR', feedTags=['test'], auto_detect=False)
+        assert demisto.debug.call_args[0][0] == 'No new indicators fetched, createIndicators will be executed with noUpdate=True.'
+        assert last_run.call_count == 0
+
+
+def test_json_feed_with_config_mapping_with_aws_feed_with_update(mocker):
+    """
+    Given
+    - Feed config from AWS feed, with last_run from the same feed, emulating the first
+      fetch after updating the AWS Feed, when there is an update to the indicators
+      (the last_run object contains an 'AMAZON' entry)
+
+    When
+    - Running fetch indicators command
+
+    Then
+    - Ensure that the correct message displays in demisto.debug, and the last_run object
+      contains the new feed config name 'AMAZON$$CIDR'
+    """
+    with open('test_data/amazon_ip_ranges.json') as ip_ranges_json:
+        ip_ranges = json.load(ip_ranges_json)
+
+    mocker.patch.object(demisto, 'debug')
+    last_run = mocker.patch.object(demisto, 'setLastRun')
+
+    feed_name_to_config = {
+        'AMAZON$$CIDR': {
+            'url': 'https://ip-ranges.amazonaws.com/ip-ranges.json',
+            'extractor': "prefixes[?service=='AMAZON']",
+            'indicator': 'ip_prefix',
+            'indicator_type': FeedIndicatorType.CIDR,
+            'fields': ['region', 'service'],
+            'mapping': {
+                'region': 'Region'
+            }
+        }
+    }
+    mocker.patch('CommonServerPython.is_demisto_version_ge', return_value=True)
+    mocker.patch('JSONFeedApiModule.is_demisto_version_ge', return_value=True)
+    mock_last_run = {"AMAZON": {"last_modified": '2019-12-17-23-03-10', "etag": "etag"}}
+    mocker.patch.object(demisto, 'getLastRun', return_value=mock_last_run)
+
+    with requests_mock.Mocker() as m:
+        m.get('https://ip-ranges.amazonaws.com/ip-ranges.json', json=ip_ranges, status_code=200,
+              headers={'Last-Modified': 'Fri, 30 Jul 2021 00:24:13 GMT',  # guardrails-disable-line
+                       'ETag': 'd309ab6e51ed310cf869dab0dfd0d34b'})  # guardrails-disable-line)
+
+        client = Client(
+            url='https://ip-ranges.amazonaws.com/ip-ranges.json',
+            credentials={'username': 'test', 'password': 'test'},
+            feed_name_to_config=feed_name_to_config,
+            insecure=True
+        )
+
+        fetch_indicators_command(client=client, indicator_type='CIDR', feedTags=['test'], auto_detect=False)
+        assert demisto.debug.call_args[0][0] == 'New indicators fetched - the Last-Modified value has been updated,' \
+               ' createIndicators will be executed with noUpdate=False.'
+        assert "AMAZON$$CIDR" in last_run.call_args[0][0]
diff --git a/Packs/FeedAWS/Integrations/FeedAWS/FeedAWS.py b/Packs/FeedAWS/Integrations/FeedAWS/FeedAWS.py
@@ -1,6 +1,30 @@
 import demistomock as demisto
 from CommonServerPython import *
 
+AVAILABLE_FEEDS = ['AMAZON',
+                   'EC2',
+                   'ROUTE53',
+                   'ROUTE53_HEALTHCHECKS',
+                   'CLOUDFRONT',
+                   'S3',
+                   'AMAZON_APPFLOW',
+                   'AMAZON_CONNECT',
+                   'API_GATEWAY',
+                   'CHIME_MEETINGS',
+                   'CHIME_VOICECONNECTOR',
+                   'CLOUD9',
+                   'CLOUDFRONT_ORIGIN_FACING',
+                   'CODEBUILD',
+                   'DYNAMODB',
+                   'EBS',
+                   'EC2_INSTANCE_CONNECT',
+                   'GLOBALACCELERATOR',
+                   'KINESIS_VIDEO_STREAMS',
+                   'ROUTE53_HEALTHCHECKS_PUBLISHING',
+                   'ROUTE53_RESOLVER',
+                   'WORKSPACES_GATEWAYS',
+                   ]
+
 
 def get_feed_config(services: list, regions: list):
     """
@@ -12,39 +36,18 @@ def get_feed_config(services: list, regions: list):
     Returns:
         The feed configuration.
     """
-    available_feeds = {
-        'AMAZON',
-        'EC2',
-        'ROUTE53',
-        'ROUTE53_HEALTHCHECKS',
-        'CLOUDFRONT',
-        'S3',
-        'AMAZON_APPFLOW',
-        'AMAZON_CONNECT',
-        'API_GATEWAY',
-        'CHIME_MEETINGS',
-        'CHIME_VOICECONNECTOR',
-        'CLOUD9',
-        'CLOUDFRONT_ORIGIN_FACING',
-        'CODEBUILD',
-        'DYNAMODB',
-        'EBS',
-        'EC2_INSTANCE_CONNECT',
-        'GLOBALACCELERATOR',
-        'KINESIS_VIDEO_STREAMS',
-        'ROUTE53_HEALTHCHECKS_PUBLISHING',
-        'ROUTE53_RESOLVER',
-        'WORKSPACES_GATEWAYS',
-    }
 
     region_path = ''
-    if regions:
+    if regions and 'All' not in regions:
         region_path = f" && contains({regions}, region)"
 
+    if 'All' in services or not services:
+        services = AVAILABLE_FEEDS
+
     feed_name_to_config = {}
 
-    for feed in available_feeds:
-        feed_name_to_config[feed] = {
+    for feed in services:
+        feed_name_to_config[f'{feed}$$CIDR'] = {
             'url': 'https://ip-ranges.amazonaws.com/ip-ranges.json',
             'extractor': f"prefixes[?service=='{feed}'{region_path}]",
             'indicator': 'ip_prefix',
@@ -56,16 +59,28 @@ def get_feed_config(services: list, regions: list):
             }
         }
 
-    return {feed_name: feed_name_to_config.get(feed_name) for feed_name in services}
+        feed_name_to_config[f'{feed}$$IPv6'] = {
+            'url': 'https://ip-ranges.amazonaws.com/ip-ranges.json',
+            'extractor': f"ipv6_prefixes[?service=='{feed}'{region_path}]",
+            'indicator': 'ipv6_prefix',
+            'indicator_type': FeedIndicatorType.IPv6,
+            'fields': ['region', 'service'],
+            'mapping': {
+                'region': 'region',
+                'service': 'service'
+            }
+        }
+
+    return feed_name_to_config
 
 
 from JSONFeedApiModule import *  # noqa: E402
 
 
 def main():
     params = {k: v for k, v in demisto.params().items() if v is not None}
-    params['feed_name_to_config'] = get_feed_config(params.get('services', ['AMAZON']),
-                                                    argToList(params.get('regions', [])))
+    params['feed_name_to_config'] = get_feed_config(params.get('services', ['All']),
+                                                    argToList(params.get('regions', ['All'])))
     feed_main(params, 'AWS Feed', 'aws')
 
 

diff --git a/Packs/FeedAWS/Integrations/FeedAWS/FeedAWS.yml b/Packs/FeedAWS/Integrations/FeedAWS/FeedAWS.yml
@@ -28,9 +28,12 @@ configuration:
   - ROUTE53_HEALTHCHECKS_PUBLISHING
   - ROUTE53_RESOLVER
   - WORKSPACES_GATEWAYS
+  - All
   required: true
   type: 16
-- additionalinfo: The AWS Regions to fetch indicators by. If empty, all regions will be included.
+  additionalinfo: The services to fetch indicators from. Default value is 'All'. If empty, all services will be included.
+  defaultvalue: 'All'
+- additionalinfo: The AWS Regions to fetch indicators by. Default value is 'All'. If empty, all regions will be included.
   display: Regions
   name: regions
   options:
@@ -67,8 +70,10 @@ configuration:
   - us-west-1
   - us-west-2
   - GLOBAL
+  - All
   required: false
   type: 16
+  defaultvalue: 'All'
 - display: Fetch indicators
   name: feed
   required: false
@@ -163,7 +168,7 @@ script:
     description: Fetches indicators from the feed.
     execution: false
     name: aws-get-indicators
-  dockerimage: demisto/py3-tools:1.0.0.61931
+  dockerimage: demisto/py3-tools:1.0.0.63020
   feed: true
   isfetch: false
   longRunning: false

diff --git a/Packs/FeedAWS/ReleaseNotes/1_1_35.md b/Packs/FeedAWS/ReleaseNotes/1_1_35.md
@@ -0,0 +1,8 @@
+#### Integrations
+
+##### AWS Feed
+
+- Updated the Docker image to: *demisto/py3-tools:1.0.0.63020*.
+- Added support for fetching IPv6 indicators from the feed.
+- Enhanced the ***JSONFeedApiModule*** to support fetching multiple indicator types from the feed.
+- Updated the default value of the *services* integration parameter to *All*.