AWS Feed - added support for IPv6

Packs/ApiModules/ReleaseNotes/2_2_17.md

@@ -0,0 +1,6 @@
+
+#### Scripts
+
+##### JSONFeedApiModule
+
+- Added support for ***AWS Feed*** multiple indicator type fetch.

Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule.py

@@ -111,12 +111,16 @@ def build_iterator(self, feed: dict, feed_name: str, **kwargs) -> Tuple[List, bo
         url = feed.get('url', self.url)
 
         if is_demisto_version_ge('6.5.0'):
+            prefix_feed_name = ''
+            if '-_-' in feed_name:
+                prefix_feed_name = feed_name.split('-_-')[0]  # Support for AWS feed
             # Set the If-None-Match and If-Modified-Since headers
             # if we have etag or last_modified values in the context, with server version higher than 6.5.0.
             last_run = demisto.getLastRun()
-            etag = demisto.get(last_run, f'{feed_name}.etag')
-            last_modified = demisto.get(last_run, f'{feed_name}.last_modified')
+            etag = last_run.get(prefix_feed_name, {}).get('etag') or last_run.get(feed_name, {}).get('etag')
+            last_modified = last_run.get(prefix_feed_name, {}).get('last_modified') or last_run.get(feed_name, {}).get('last_modified')
 
+            demisto.debug(f'AWS: the last run before fetch: {last_run}')
             if etag:
                 self.headers['If-None-Match'] = etag
 
@@ -147,6 +151,7 @@ def build_iterator(self, feed: dict, feed_name: str, **kwargs) -> Tuple[List, bo
         try:
             r.raise_for_status()
             if r.content:
+                demisto.debug(f'AWS: found content for {feed_name}')
                 data = r.json()
                 result = jmespath.search(expression=feed.get('extractor'), data=data)
 
@@ -170,7 +175,6 @@ def get_no_update_value(response: requests.Response, feed_name: str) -> bool:
         boolean with the value for noUpdate argument.
         The value should be False if the response was modified.
     """
-
     # HTTP status code 304 (Not Modified) set noUpdate to True.
     if response.status_code == 304:
         demisto.debug('No new indicators fetched, createIndicators will be executed with noUpdate=True.')
@@ -190,7 +194,7 @@ def get_no_update_value(response: requests.Response, feed_name: str) -> bool:
         'etag': etag
     }
     demisto.setLastRun(last_run)
-
+    demisto.debug(f'AWS: The new last run is: {last_run}')
     demisto.debug('New indicators fetched - the Last-Modified value has been updated,'
                   ' createIndicators will be executed with noUpdate=False.')
     return False
@@ -242,6 +246,9 @@ def fetch_indicators_command(client: Client, indicator_type: str, feedTags: list
         handle_indicator_function = feed_config.get('handle_indicator_function', handle_indicator)
         create_relationships_function = feed_config.get('create_relations_function')
 
+        if '-_-' in service_name:
+            service_name = service_name.split('-_-')[0]  # Support for AWS feed
+
         for item in items:
             if isinstance(item, str):
                 item = {indicator_field: item}

Packs/ApiModules/Scripts/JSONFeedApiModule/JSONFeedApiModule_test.py

@@ -29,7 +29,7 @@ def test_json_feed_no_config():
 CONFIG_PARAMETERS = [
     (
         {
-            'AMAZON': {
+            'AMAZON-_-CIDR': {
                 'url': 'https://ip-ranges.amazonaws.com/ip-ranges.json',
                 'extractor': "prefixes[?service=='AMAZON']",
                 'indicator': 'ip_prefix',
@@ -42,13 +42,20 @@ def test_json_feed_no_config():
     ),
     (
         {
-            'AMAZON': {
+            'AMAZON-_-CIDR': {
                 'url': 'https://ip-ranges.amazonaws.com/ip-ranges.json',
                 'extractor': "prefixes[?service=='AMAZON']",
                 'indicator': 'ip_prefix',
                 'indicator_type': FeedIndicatorType.CIDR,
                 'fields': ['region', 'service']
             },
+            'AMAZON-_-IPV6': {
+                'url': 'https://ip-ranges.amazonaws.com/ip-ranges.json',
+                'extractor': "ipv6_prefixes[?service=='AMAZON']",
+                'indicator': 'ipv6_prefix',
+                'indicator_type': FeedIndicatorType.IPv6,
+                'fields': ['region', 'service']
+            },
             'CLOUDFRONT': {
                 'url': 'https://ip-ranges.amazonaws.com/ip-ranges.json',
                 'extractor': "prefixes[?service=='CLOUDFRONT']",
@@ -57,7 +64,7 @@ def test_json_feed_no_config():
                 'fields': ['region', 'service']
             }
         },
-        1148,
+        1465,
         36
     )
 ]
@@ -89,7 +96,7 @@ def test_json_feed_with_config_mapping():
         ip_ranges = json.load(ip_ranges_json)
 
     feed_name_to_config = {
-        'AMAZON': {
+        'AMAZON-_-CIDR': {
             'url': 'https://ip-ranges.amazonaws.com/ip-ranges.json',
             'extractor': "prefixes[?service=='AMAZON']",
             'indicator': 'ip_prefix',
@@ -350,3 +357,108 @@ def test_fetch_indicators_command_google_ip_ranges(mocker):
     indicators, _ = fetch_indicators_command(client, indicator_type=None, feedTags=[], auto_detect=None, limit=100)
     for indicator in indicators:
         assert indicator.get('value')
+
+
+def test_json_feed_with_config_mapping_with_aws_feed_no_update(mocker):
+    """
+    Given
+    - Feed config from AWS feed, with last_run from the same feed, emulating the first
+      fetch after updating when there is no update to the indicators.
+
+    When
+    - Running fetch indicators command
+
+    Then
+    - Ensure that the correct message displays in demisto.debug, and the last_run object
+      is did not update, and continue to have the previous AWS feed name 'AMAZON'
+    """
+    with open('test_data/amazon_ip_ranges.json') as ip_ranges_json:
+        ip_ranges = json.load(ip_ranges_json)
+
+    mocker.patch.object(demisto, 'debug')
+    last_run = mocker.patch.object(demisto, 'setLastRun')
+
+    feed_name_to_config = {
+        'AMAZON-_-CIDR': {
+            'url': 'https://ip-ranges.amazonaws.com/ip-ranges.json',
+            'extractor': "prefixes[?service=='AMAZON']",
+            'indicator': 'ip_prefix',
+            'indicator_type': FeedIndicatorType.CIDR,
+            'fields': ['region', 'service'],
+            'mapping': {
+                'region': 'Region'
+            }
+        }
+    }
+    mocker.patch('CommonServerPython.is_demisto_version_ge', return_value=True)
+    mocker.patch('JSONFeedApiModule.is_demisto_version_ge', return_value=True)
+    mock_last_run = {"AMAZON": {"last_modified": '2019-12-17-23-03-10', "etag": "etag"}}
+    mocker.patch.object(demisto, 'getLastRun', return_value=mock_last_run)
+
+    with requests_mock.Mocker() as m:
+        m.get('https://ip-ranges.amazonaws.com/ip-ranges.json', json=ip_ranges, status_code=304,)
+
+        client = Client(
+            url='https://ip-ranges.amazonaws.com/ip-ranges.json',
+            credentials={'username': 'test', 'password': 'test'},
+            feed_name_to_config=feed_name_to_config,
+            insecure=True
+        )
+
+        fetch_indicators_command(client=client, indicator_type='CIDR', feedTags=['test'], auto_detect=False)
+        assert demisto.debug.call_args[0][0] == 'No new indicators fetched, createIndicators will be executed with noUpdate=True.'
+        assert last_run.call_count == 0
+
+
+def test_json_feed_with_config_mapping_with_aws_feed_with_update(mocker):
+    """
+    Given
+    - Feed config from AWS feed, with last_run from the same feed, emulating the first
+      fetch after updating when there is an update to the indicators, (the last_run object contains an 'AMAZON' entry)
+
+    When
+    - Running fetch indicators command
+
+    Then
+    - Ensure that the correct message displays in demisto.debug, and the last_run object
+      is updated with the new feed name 'AMAZON-_-CIDR'
+    """
+    with open('test_data/amazon_ip_ranges.json') as ip_ranges_json:
+        ip_ranges = json.load(ip_ranges_json)
+
+    mocker.patch.object(demisto, 'debug')
+    last_run = mocker.patch.object(demisto, 'setLastRun')
+
+    feed_name_to_config = {
+        'AMAZON-_-CIDR': {
+            'url': 'https://ip-ranges.amazonaws.com/ip-ranges.json',
+            'extractor': "prefixes[?service=='AMAZON']",
+            'indicator': 'ip_prefix',
+            'indicator_type': FeedIndicatorType.CIDR,
+            'fields': ['region', 'service'],
+            'mapping': {
+                'region': 'Region'
+            }
+        }
+    }
+    mocker.patch('CommonServerPython.is_demisto_version_ge', return_value=True)
+    mocker.patch('JSONFeedApiModule.is_demisto_version_ge', return_value=True)
+    mock_last_run = {"AMAZON": {"last_modified": '2019-12-17-23-03-10', "etag": "etag"}}
+    mocker.patch.object(demisto, 'getLastRun', return_value=mock_last_run)
+
+    with requests_mock.Mocker() as m:
+        m.get('https://ip-ranges.amazonaws.com/ip-ranges.json', json=ip_ranges, status_code=200,
+              headers={'Last-Modified': 'Fri, 30 Jul 2021 00:24:13 GMT',  # guardrails-disable-line
+                       'ETag': 'd309ab6e51ed310cf869dab0dfd0d34b'})  # guardrails-disable-line)
+
+        client = Client(
+            url='https://ip-ranges.amazonaws.com/ip-ranges.json',
+            credentials={'username': 'test', 'password': 'test'},
+            feed_name_to_config=feed_name_to_config,
+            insecure=True
+        )
+
+        fetch_indicators_command(client=client, indicator_type='CIDR', feedTags=['test'], auto_detect=False)
+        assert demisto.debug.call_args[0][0] == 'New indicators fetched - the Last-Modified value has been updated,' \
+               ' createIndicators will be executed with noUpdate=False.'
+        assert "AMAZON-_-CIDR" in last_run.call_args[0][0]

Packs/ApiModules/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "ApiModules",
     "description": "API Modules",
     "support": "xsoar",
-    "currentVersion": "2.2.16",
+    "currentVersion": "2.2.17",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",

Packs/FeedAWS/Integrations/FeedAWS/FeedAWS.py

@@ -1,6 +1,30 @@
 import demistomock as demisto
 from CommonServerPython import *
 
+AVAILABLE_FEEDS = ['AMAZON',
+                   'EC2',
+                   'ROUTE53',
+                   'ROUTE53_HEALTHCHECKS',
+                   'CLOUDFRONT',
+                   'S3',
+                   'AMAZON_APPFLOW',
+                   'AMAZON_CONNECT',
+                   'API_GATEWAY',
+                   'CHIME_MEETINGS',
+                   'CHIME_VOICECONNECTOR',
+                   'CLOUD9',
+                   'CLOUDFRONT_ORIGIN_FACING',
+                   'CODEBUILD',
+                   'DYNAMODB',
+                   'EBS',
+                   'EC2_INSTANCE_CONNECT',
+                   'GLOBALACCELERATOR',
+                   'KINESIS_VIDEO_STREAMS',
+                   'ROUTE53_HEALTHCHECKS_PUBLISHING',
+                   'ROUTE53_RESOLVER',
+                   'WORKSPACES_GATEWAYS',
+                   ]
+
 
 def get_feed_config(services: list, regions: list):
     """
@@ -12,39 +36,18 @@ def get_feed_config(services: list, regions: list):
     Returns:
         The feed configuration.
     """
-    available_feeds = {
-        'AMAZON',
-        'EC2',
-        'ROUTE53',
-        'ROUTE53_HEALTHCHECKS',
-        'CLOUDFRONT',
-        'S3',
-        'AMAZON_APPFLOW',
-        'AMAZON_CONNECT',
-        'API_GATEWAY',
-        'CHIME_MEETINGS',
-        'CHIME_VOICECONNECTOR',
-        'CLOUD9',
-        'CLOUDFRONT_ORIGIN_FACING',
-        'CODEBUILD',
-        'DYNAMODB',
-        'EBS',
-        'EC2_INSTANCE_CONNECT',
-        'GLOBALACCELERATOR',
-        'KINESIS_VIDEO_STREAMS',
-        'ROUTE53_HEALTHCHECKS_PUBLISHING',
-        'ROUTE53_RESOLVER',
-        'WORKSPACES_GATEWAYS',
-    }
 
     region_path = ''
     if regions:
         region_path = f" && contains({regions}, region)"
 
+    if 'All' in services or not services:
+        services = AVAILABLE_FEEDS
+
     feed_name_to_config = {}
 
-    for feed in available_feeds:
-        feed_name_to_config[feed] = {
+    for feed in services:
+        feed_name_to_config[f'{feed}-_-CIDR'] = {
             'url': 'https://ip-ranges.amazonaws.com/ip-ranges.json',
             'extractor': f"prefixes[?service=='{feed}'{region_path}]",
             'indicator': 'ip_prefix',
@@ -56,15 +59,27 @@ def get_feed_config(services: list, regions: list):
             }
         }
 
-    return {feed_name: feed_name_to_config.get(feed_name) for feed_name in services}
+        feed_name_to_config[f'{feed}-_-IPv6'] = {
+            'url': 'https://ip-ranges.amazonaws.com/ip-ranges.json',
+            'extractor': f"ipv6_prefixes[?service=='{feed}'{region_path}]",
+            'indicator': 'ipv6_prefix',
+            'indicator_type': FeedIndicatorType.IPv6,
+            'fields': ['region', 'service'],
+            'mapping': {
+                'region': 'region',
+                'service': 'service'
+            }
+        }
+
+    return feed_name_to_config
 
 
 from JSONFeedApiModule import *  # noqa: E402
 
 
 def main():
     params = {k: v for k, v in demisto.params().items() if v is not None}
-    params['feed_name_to_config'] = get_feed_config(params.get('services', ['AMAZON']),
+    params['feed_name_to_config'] = get_feed_config(params.get('services', ['All']),
                                                     argToList(params.get('regions', [])))
     feed_main(params, 'AWS Feed', 'aws')
 

Packs/FeedAWS/Integrations/FeedAWS/FeedAWS.yml

@@ -28,8 +28,11 @@ configuration:
   - ROUTE53_HEALTHCHECKS_PUBLISHING
   - ROUTE53_RESOLVER
   - WORKSPACES_GATEWAYS
+  - All
   required: true
   type: 16
+  additionalinfo: The services to fetch indicators from. Default value is all. If empty, all services will be included.
+  defaultvalue: 'All'
 - additionalinfo: The AWS Regions to fetch indicators by. If empty, all regions will be included.
   display: Regions
   name: regions
@@ -164,7 +167,7 @@ script:
     description: Fetches indicators from the feed.
     execution: false
     name: aws-get-indicators
-  dockerimage: demisto/py3-tools:1.0.0.59406
+  dockerimage: demisto/py3-tools:1.0.0.61931
   feed: true
   isfetch: false
   longRunning: false

Packs/FeedAWS/ReleaseNotes/1_1_34.md

@@ -0,0 +1,8 @@
+
+#### Integrations
+
+##### AWS Feed
+
+- Added support for fetching IPv6 indicators from feed.
+- Added a new default value for the **services** integration parameter.
+- Updated the Docker image to: *demisto/py3-tools:1.0.0.61931*.

Packs/FeedAWS/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "AWS Feed",
     "description": "Indicators feed from AWS",
     "support": "xsoar",
-    "currentVersion": "1.1.33",
+    "currentVersion": "1.1.34",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",