[Netskope Event Collector] Add support for event type selection (#32300)

* Add Event Types To Fetch integration parameter

* Update NetskopeEventCollector.py

Updated logic to handle fetching specific events

* UTs update and ruff

* sdk format

* Updated RN

* Add UT; Minor logic updates

* CR comments
Minor updates to logic and UT

* Implement nextFetch feature
Add nextTrigger key-value pair to lastRun dictionary

* Pre commit
docker image; release notes

Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector.py

@@ -1,9 +1,10 @@
-import demistomock as demisto
-from CommonServerPython import *  # noqa # pylint: disable=unused-wildcard-import
-from CommonServerUserPython import *  # noqa
+from typing import Any
 
 import urllib3
-from typing import Dict, Any, Tuple
+from CommonServerUserPython import *  # noqa
+
+import demistomock as demisto
+from CommonServerPython import *  # noqa # pylint: disable=unused-wildcard-import
 
 # Disable insecure warnings
 urllib3.disable_warnings()  # pylint: disable=no-member
@@ -35,8 +36,9 @@ class Client(BaseClient):
         proxy (bool): Specifies if to use XSOAR proxy settings.
     """
 
-    def __init__(self, base_url: str, token: str, validate_certificate: bool, proxy: bool):
-        self.fetch_status: dict = {event_type: False for event_type in ALL_SUPPORTED_EVENT_TYPES}
+    def __init__(self, base_url: str, token: str, validate_certificate: bool, proxy: bool, event_types_to_fetch: list[str]):
+        self.fetch_status: dict = {event_type: False for event_type in event_types_to_fetch}
+        self.event_types_to_fetch: list[str] = event_types_to_fetch
 
         headers = {'Netskope-Api-Token': token}
         super().__init__(base_url, verify=validate_certificate, proxy=proxy, headers=headers)
@@ -77,7 +79,7 @@ def honor_rate_limiting(headers, endpoint):
                     time.sleep(1)
 
     except ValueError as ve:
-        logging.error("Value error when honoring the rate limiting wait time {} {}".format(headers, str(ve)))
+        logging.error(f"Value error when honoring the rate limiting wait time {headers} {str(ve)}")
 
 
 def populate_parsing_rule_fields(event: dict, event_type: str):
@@ -150,7 +152,7 @@ def is_execution_time_exceeded(start_time: datetime) -> bool:
     return secs_from_beginning > EXECUTION_TIMEOUT_SECONDS
 
 
-def setup_last_run(last_run_dict: dict) -> dict:
+def setup_last_run(last_run_dict: dict, event_types_to_fetch: list[str]) -> dict:
     """
     Setting the last_tun object with the right operation to be used throughout the integration run.
 
@@ -161,7 +163,7 @@ def setup_last_run(last_run_dict: dict) -> dict:
         dict: the modified last run dictionary with the needed operation
     """
     first_fetch = int(arg_to_datetime('now').timestamp())  # type: ignore[union-attr]
-    for event_type in ALL_SUPPORTED_EVENT_TYPES:
+    for event_type in event_types_to_fetch:
         if not last_run_dict.get(event_type, {}).get('operation'):
             last_run_dict[event_type] = {'operation': first_fetch}
 
@@ -234,7 +236,7 @@ def handle_data_export_single_event_type(client: Client, event_type: str, operat
     return events, False
 
 
-def get_all_events(client: Client, last_run: dict, limit: int = MAX_EVENTS_PAGE_SIZE) -> Tuple[list, dict]:
+def get_all_events(client: Client, last_run: dict, limit: int = MAX_EVENTS_PAGE_SIZE) -> tuple[list, dict]:
     """
     Iterates over all supported event types and call the handle data export logic. Once each event type is done the operation for
     next run is set to 'next'.
@@ -251,7 +253,7 @@ def get_all_events(client: Client, last_run: dict, limit: int = MAX_EVENTS_PAGE_
 
     all_types_events_result = []
     execution_start_time = datetime.utcnow()
-    for event_type in ALL_SUPPORTED_EVENT_TYPES:
+    for event_type in client.event_types_to_fetch:
         event_type_operation = last_run.get(event_type, {}).get('operation')
 
         events, time_out = handle_data_export_single_event_type(client=client, event_type=event_type,
@@ -271,11 +273,11 @@ def get_all_events(client: Client, last_run: dict, limit: int = MAX_EVENTS_PAGE_
 
 
 def test_module(client: Client, last_run: dict, max_fetch: int) -> str:
-    get_all_events(client, last_run, limit=max_fetch)
+    get_all_events(client, last_run, limit=max_fetch, )
     return 'ok'
 
 
-def get_events_command(client: Client, args: Dict[str, Any], last_run: dict) -> Tuple[CommandResults, list]:
+def get_events_command(client: Client, args: dict[str, Any], last_run: dict) -> tuple[CommandResults, list]:
     limit = arg_to_number(args.get('limit')) or MAX_EVENTS_PAGE_SIZE
     events, _ = get_all_events(client=client, last_run=last_run, limit=limit)
 
@@ -296,6 +298,16 @@ def get_events_command(client: Client, args: Dict[str, Any], last_run: dict) ->
     return results, events
 
 
+def handle_event_types_to_fetch(event_types_to_fetch) -> list[str]:
+    """ Handle event_types_to_fetch parameter.
+        Transform the event_types_to_fetch parameter into a pythonic list with lowercase values.
+    """
+    return argToList(
+        arg=event_types_to_fetch if event_types_to_fetch else ALL_SUPPORTED_EVENT_TYPES,
+        transform=lambda x: x.lower(),
+    )
+
+
 ''' MAIN FUNCTION '''
 
 
@@ -310,11 +322,13 @@ def main() -> None:  # pragma: no cover
         proxy = params.get('proxy', False)
         max_fetch: int = arg_to_number(params.get('max_fetch')) or 10000
         vendor, product = params.get('vendor', 'netskope'), params.get('product', 'netskope')
+        event_types_to_fetch = handle_event_types_to_fetch(params.get('event_types_to_fetch'))
+        demisto.debug(f'Event types that will be fetched in this instance: {event_types_to_fetch}')
         command_name = demisto.command()
         demisto.debug(f'Command being called is {command_name}')
 
-        client = Client(base_url, token, verify_certificate, proxy)
-        last_run = setup_last_run(demisto.getLastRun())
+        client = Client(base_url, token, verify_certificate, proxy, event_types_to_fetch)
+        last_run = setup_last_run(demisto.getLastRun(), event_types_to_fetch)
         demisto.debug(f'Running with the following last_run - {last_run}')
 
         events: list[dict] = []
@@ -347,6 +361,11 @@ def main() -> None:  # pragma: no cover
 
                 end = datetime.utcnow()
                 demisto.debug(f'Handled {len(events)} total events in {(end - start).seconds} seconds')
+
+                # set nextTrigger key in the lastRun dictionary to 0 (seconds) - this will trigger the next
+                # fetch-events to start immediately after the current fetch-events ends (CRTX-89345)
+                new_last_run['nextTrigger'] = '0'
+
                 demisto.setLastRun(new_last_run)
 
     # Log exceptions and return errors

Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector.yml

@@ -24,6 +24,19 @@ configuration:
   section: Collect
   type: 0
   required: false
+- display: Event Types To Fetch
+  section: Collect
+  name: event_types_to_fetch
+  type: 16
+  required: false
+  additionalinfo: 'Event types to fetch. Default is all available types: Application, Alert, Page, Audit, Network.'
+  defaultvalue: Application,Alert,Page,Audit,Network
+  options:
+  - Application
+  - Alert
+  - Page
+  - Audit
+  - Network
 - display: Trust any certificate (not secure)
   section: Connect
   name: insecure
@@ -53,7 +66,7 @@ script:
       defaultValue: 10000
     description: Returns events extracted from SaaS traffic and or logs.
     name: netskope-get-events
-  dockerimage: demisto/python3:3.10.13.84405
+  dockerimage: demisto/python3:3.10.13.87159
   runonce: false
   script: '-'
   subtype: python3

Packs/Netskope/Integrations/NetskopeEventCollector/NetskopeEventCollector_test.py

@@ -1,16 +1,14 @@
-import io
 import json
 import re
 import time
 
 import dateparser
 import pytest
-
-from NetskopeEventCollector import Client, ALL_SUPPORTED_EVENT_TYPES, RATE_LIMIT_REMAINING, RATE_LIMIT_RESET
+from NetskopeEventCollector import ALL_SUPPORTED_EVENT_TYPES, RATE_LIMIT_REMAINING, RATE_LIMIT_RESET, Client
 
 
 def util_load_json(path):
-    with io.open(path, mode='r', encoding='utf-8') as f:
+    with open(path, encoding='utf-8') as f:
         return json.loads(f.read())
 
 
@@ -32,7 +30,7 @@ def test_test_module(mocker):
         - Verify that 'ok' is returned.
     """
     from NetskopeEventCollector import test_module
-    client = Client(BASE_URL, 'dummy_token', False, False)
+    client = Client(BASE_URL, 'dummy_token', False, False, event_types_to_fetch=ALL_SUPPORTED_EVENT_TYPES)
     mocker.patch.object(client, 'perform_data_export', return_value=EVENTS_RAW)
     results = test_module(client, last_run=FIRST_LAST_RUN, max_fetch=1)
     assert results == 'ok'
@@ -72,14 +70,15 @@ def json_callback(request, _):
         return EVENTS_PAGE_RAW[endpoint]
 
     from NetskopeEventCollector import get_all_events
-    client = Client(BASE_URL, 'netskope_token', validate_certificate=False, proxy=False)
+    client = Client(BASE_URL, 'netskope_token', validate_certificate=False,
+                    proxy=False, event_types_to_fetch=ALL_SUPPORTED_EVENT_TYPES)
     url_matcher = re.compile('https://netskope[.]example[.]com/events/dataexport/events')
     requests_mock.get(url_matcher, json=json_callback)
     events, new_last_run = get_all_events(client, FIRST_LAST_RUN)
     assert len(events) == 25
     assert events[0].get('event_id') == '1'
     assert events[0].get('_time') == '2023-05-22T10:30:16.000Z'
-    assert all([new_last_run[event_type]['operation'] == 'next' for event_type in ALL_SUPPORTED_EVENT_TYPES])
+    assert all(new_last_run[event_type]['operation'] == 'next' for event_type in ALL_SUPPORTED_EVENT_TYPES)
 
 
 def test_get_events_command(mocker):
@@ -94,7 +93,7 @@ def test_get_events_command(mocker):
         - Make sure the outputs are set correctly.
     """
     from NetskopeEventCollector import get_events_command
-    client = Client(BASE_URL, 'dummy_token', False, False)
+    client = Client(BASE_URL, 'dummy_token', False, False, event_types_to_fetch=ALL_SUPPORTED_EVENT_TYPES)
     mocker.patch('NetskopeEventCollector.get_all_events', return_value=[MOCK_ENTRY, {}])
     mocker.patch.object(time, "sleep")
     results, events = get_events_command(client, args={}, last_run=FIRST_LAST_RUN)
@@ -161,5 +160,32 @@ def test_setup_last_run(mocker, last_run_dict, expected_operation_value):
     from NetskopeEventCollector import setup_last_run
     first_fetch = dateparser.parse('2023-01-01T10:00:00Z')
     mocker.patch.object(dateparser, "parse", return_value=first_fetch)
-    last_run = setup_last_run(last_run_dict)
-    assert all([val.get('operation') == expected_operation_value for key, val in last_run.items()])
+    last_run = setup_last_run(last_run_dict, ALL_SUPPORTED_EVENT_TYPES)
+    assert all(val.get('operation') == expected_operation_value for key, val in last_run.items())
+
+
+@pytest.mark.parametrize('event_types_to_fetch_param, expected_value', [
+    ('Application', ['application']),
+    ('Alert, Page, Audit', ['alert', 'page', 'audit']),
+    (['Application', 'Audit', 'Network'], ['application', 'audit', 'network']),
+    (None, ALL_SUPPORTED_EVENT_TYPES),
+])
+def test_event_types_to_fetch_parameter_handling(event_types_to_fetch_param, expected_value):
+    """
+    Given:
+        Case a: event_types_to_fetch parameter has a single value
+        Case b: event_types_to_fetch parameter has multiple values
+        Case c: event_types_to_fetch parameter is a pythonic list
+        Case d: event_types_to_fetch parameter is None
+
+    When:
+        Handling the event_types_to_fetch parameter
+
+    Then:
+        - Make sure the parameter converts into a valid pythonic list
+        - The values are lowercase
+        - In the case event_types_to_fetch in None, default ALL_SUPPORTED_EVENT_TYPES is used as parameter
+
+    """
+    from NetskopeEventCollector import handle_event_types_to_fetch
+    assert handle_event_types_to_fetch(event_types_to_fetch_param) == expected_value

Packs/Netskope/ReleaseNotes/3_3_5.md

@@ -0,0 +1,8 @@
+
+#### Integrations
+
+##### Netskope Event Collector
+- Updated the Docker image to: *demisto/python3:3.10.13.87159*.
+
+- Added the *Event Types To Fetch* parameter, allowing to select which event types are being fetched for each instance.
+

Packs/Netskope/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Netskope",
     "description": "Cloud access security broker that enables to find, understand, and secure cloud apps.",
     "support": "xsoar",
-    "currentVersion": "3.3.4",
+    "currentVersion": "3.3.5",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",