CrowdStrike Falcon Intel Feed Actors: add offset to last run (#26243)

* add offset to last run

* Update Packs/FeedCrowdstrikeFalconIntel/Integrations/FeedCrowdstrikeFalconIntel/FeedCrowdstrikeFalconIntel.py

Co-authored-by: Dean Arbel <darbel@paloaltonetworks.com>

---------

Co-authored-by: Dean Arbel <darbel@paloaltonetworks.com>

Packs/FeedCrowdstrikeFalconIntel/Integrations/FeedCrowdstrikeFalconIntel/FeedCrowdstrikeFalconIntel.py

@@ -137,57 +137,37 @@ def build_url_suffix(self, params, actors_filter):
             url_suffix = url_suffix + '?filter=' + params
         return url_suffix
 
-    def build_iterator(self, feed_tags: List, tlp_color: Optional[str], limit=None, offset=None, target_countries=None,
-                       target_industries=None, custom_filter=None):
-        """Builds a list of indicators.
+    def get_indicators(self, feed_tags: List, tlp_color: Optional[str], limit=None, offset=None, target_countries=None,
+                       target_industries=None, custom_filter=None, time_filter=None, sort=None):
+        """Get a list of indicators.
         Returns:
             list. A list of JSON objects representing indicators fetched from a feed.
         """
-        actors_filter = self.build_actors_filter(target_countries, target_industries, custom_filter)
 
-        # for fetch_indicator_command only
         params = {}
-        if not limit:
-            params = self.get_last_modified_time()
-
-        url_suffix_to_filter_by = self.build_url_suffix(params, actors_filter)
         if limit:
-            response = self.http_request('GET', url_suffix_to_filter_by, params={'limit': limit})
-        else:
-            response = self.http_request('GET', url_suffix_to_filter_by)
+            params['limit'] = limit
+        if offset:
+            params['offset'] = offset
+        if sort:
+            params['sort'] = sort
 
-        parsed_indicators = self.create_indicators_from_response(response, feed_tags,
-                                                                 tlp_color)  # list of dict of indicators
+        actors_filter = self.build_actors_filter(target_countries, target_industries, custom_filter)
+        url_suffix_to_filter_by = self.build_url_suffix(time_filter, actors_filter)
 
-        # for get_indicator_command only
-        if limit:
-            parsed_indicators = parsed_indicators[int(offset): int(offset) + int(limit)]
-        return parsed_indicators
+        response = self.http_request('GET', url_suffix_to_filter_by, params=params)
 
-    def set_last_modified_time(self):
-        current_time = datetime.now()
-        current_timestamp = datetime.timestamp(current_time)
-        timestamp = str(int(current_timestamp))
-        integration_context_to_set = {'last_modified_time': timestamp}
-        set_feed_last_run(integration_context_to_set)
-
-    def get_last_modified_time(self):
-        integration_context = get_feed_last_run()
-        if not integration_context:
-            params = ''
-            self.set_last_modified_time()
-        else:
-            last_modified_time = get_feed_last_run()
-            relevant_time = int(last_modified_time['last_modified_time'])
-            params = f"last_modified_date%3A%3E{relevant_time}"
-            self.set_last_modified_time()
-        return params
+        return self.create_indicators_from_response(
+            response,
+            feed_tags,
+            tlp_color
+        )
 
 
 def test_module(client: Client, args: dict, feed_tags: list, tlp_color: Optional[str]):
     try:
         tags = argToList(demisto.params().get('feedTags'))
-        client.build_iterator(tags, tlp_color, limit=1, offset=0)
+        client.get_indicators(tags, tlp_color, limit=1, offset=0)
     except Exception:
         raise Exception("Could not fetch CrowdStrike Feed\n"
                         "\nCheck your API key and your connection to CrowdStrike.")
@@ -215,8 +195,13 @@ def get_indicators_command(client: Client, args: dict, feed_tags: list, tlp_colo
     custom_filter = args.get('custom_filter') if args.get('custom_filter') \
         else demisto.params().get('custom_filter')
 
-    indicators = fetch_indicators(client, feed_tags, tlp_color, limit, offset, target_countries,
-                                  target_industries, custom_filter)
+    indicators = client.get_indicators(
+        feed_tags, tlp_color,
+        limit, offset,
+        target_countries,
+        target_industries,
+        custom_filter
+    )
 
     hr_indicators = []
     for indicator in indicators:
@@ -233,26 +218,53 @@ def get_indicators_command(client: Client, args: dict, feed_tags: list, tlp_colo
     return human_readable, {}, indicators
 
 
-def fetch_indicators(client: Client, feed_tags: List, tlp_color: Optional[str], limit=None, offset=None,
-                     target_countries=None, target_industries=None, custom_filter=None) -> list:
+def fetch_indicators(client: Client, feed_tags: List, tlp_color: Optional[str], limit: int,
+                     target_countries=None, target_industries=None, custom_filter=None) -> tuple:
     """Fetch-indicators command from CrowdStrike Feeds
 
     Args:
         client(Client): CrowdStrike Feed client.
         feed_tags: The indicator tags.
         tlp_color (str): Traffic Light Protocol color.
         limit: limit the amount of indicators fetched.
-        offset: the index of the first index to fetch.
         target_industries: the actor's target_industries.
         target_countries: the actor's target_countries.
         custom_filter: user actor's filter.
     Returns:
-        list. List of indicators.
+        tuple. (List of indicators, last_run data).
     """
-    indicators = client.build_iterator(feed_tags, tlp_color, limit, offset, target_countries,
-                                       target_industries, custom_filter)
+    last_run = demisto.getLastRun() or {}
+    offset = int(last_run.get('offset', '0'))
+
+    last_modified_time = last_run.get('last_modified_time')
+    time_filter = f"last_modified_date%3A%3E{last_modified_time}" if last_modified_time else None
+
+    indicators = client.get_indicators(
+        feed_tags, tlp_color,
+        limit, offset,
+        target_countries,
+        target_industries,
+        custom_filter, time_filter=time_filter,
+        sort='last_modified_date'
+    )
+
+    if len(indicators) >= limit:
+        # we need to store the offset and the same last modified time for the next run
+        last_run = {
+            'last_modified_time': last_modified_time,
+            'offset': offset + limit
+        }
+    elif len(indicators) > 0:
+        # we need to store the latest updateddate from the indictators for the next run
+        latest_modified_time = max(map(lambda indicator: indicator['fields']['updateddate'], indicators))
+        new_last_modified_time = int(latest_modified_time) + 1  # + 1 to avoid get the same
+        last_run = {'last_modified_time': new_last_modified_time}
+    else:
+        # we get 0 new indicators - store the current time
+        current_timestamp = datetime.timestamp(datetime.now())
+        last_run = {'last_modified_time': int(current_timestamp)}
 
-    return indicators
+    return indicators, last_run
 
 
 def main():
@@ -262,7 +274,7 @@ def main():
     target_countries = params.get('target_countries')
     target_industries = params.get('target_industries')
     custom_filter = params.get('custom_filter')
-    fetch_limit = params.get('limit', '200')
+    fetch_limit = int(params.get('limit', '200'))
     client = Client(params)
 
     command = demisto.command()
@@ -274,12 +286,17 @@ def main():
     }
     try:
         if demisto.command() == 'fetch-indicators':
-            indicators = fetch_indicators(client, feed_tags, tlp_color, target_countries=target_countries,
-                                          target_industries=target_industries, custom_filter=custom_filter,
-                                          limit=fetch_limit, offset='0')
+
+            indicators, last_run_data = fetch_indicators(
+                client, feed_tags, tlp_color, target_countries=target_countries,
+                target_industries=target_industries, custom_filter=custom_filter,
+                limit=fetch_limit
+            )
             # we submit the indicators in batches
             for b in batch(indicators, batch_size=2000):
                 demisto.createIndicators(b)
+
+            demisto.setLastRun(last_run_data)
         else:
             readable_output, outputs, raw_response = commands[command](client, demisto.args(),
                                                                        feed_tags, tlp_color)  # type: ignore

Packs/FeedCrowdstrikeFalconIntel/Integrations/FeedCrowdstrikeFalconIntel/FeedCrowdstrikeFalconIntel.yml

@@ -185,7 +185,7 @@ script:
     description: Gets indicators from CrowdStrike Falcon Intel Feed.
     execution: false
     name: crowdstrike-falcon-intel-get-indicators
-  dockerimage: demisto/python3:3.10.11.54132
+  dockerimage: demisto/python3:3.10.11.56082
   feed: true
   isfetch: false
   longRunning: false

Packs/FeedCrowdstrikeFalconIntel/Integrations/FeedCrowdstrikeFalconIntel/FeedCrowdstrikeFalconIntel_test.py

@@ -61,3 +61,32 @@ def test_actor_type(mocker, server_ge_620, expected_actor_type):
 
     # validate
     assert all(indicator['type'] == expected_actor_type for indicator in res)
+
+
+def test_fetch_indicators_with_limit(mocker, requests_mock):
+    """
+    Given:
+        - Limit param are 2
+
+    When:
+        - fetch_indicators
+
+    Then:
+        -
+        Validate there is offset in the last_run for the next run
+    """
+    import re
+    import demistomock as demisto
+    from FeedCrowdstrikeFalconIntel import main
+    mocker.patch.object(Client, '_get_access_token', return_value='test_token')
+    mocker.patch.object(demisto, 'command', return_value='fetch-indicators')
+    mocker.patch.object(demisto, 'params', return_value={'limit': '2'})
+    mocker.patch.object(demisto, 'setLastRun')
+    requests_mock.get(re.compile('.*api.crowdstrike.com.*'),
+                      json=indicators['list_data_cs'])
+
+    main()
+
+    last_run_call_args = demisto.setLastRun.call_args[0][0]
+    assert 'last_modified_time' in last_run_call_args
+    assert last_run_call_args['offset'] == 2

Packs/FeedCrowdstrikeFalconIntel/ReleaseNotes/2_1_5.md

@@ -0,0 +1,7 @@
+
+#### Integrations
+
+##### CrowdStrike Falcon Intel Feed Actors
+
+- Fixed an issue where the **fetch-indicators** fetched the same indicators.
+- Updated the Docker image to: *demisto/python3:3.10.11.56082*.

Packs/FeedCrowdstrikeFalconIntel/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Crowdstrike Falcon Intel Feed",
     "description": "Tracks the activities of threat actor groups and advanced persistent threats (APTs) to understand as much as possible about their known aliases, targets, methods, and more.",
     "support": "xsoar",
-    "currentVersion": "2.1.4",
+    "currentVersion": "2.1.5",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",