feat: improve logging and reports in VD E2E tests

deps/wazuh_testing/wazuh_testing/end_to_end/indexer_api.py

@@ -35,6 +35,7 @@ def get_indexer_values(host_manager: HostManager, credentials: dict = {'user': '
     Returns:
         str: The response text from the indexer API.
     """
+    print('Getting values from the Indexer API')
 
     url = f"https://{host_manager.get_master_ip()}:9200/{index}/_search"
     headers = {
@@ -46,20 +47,35 @@ def get_indexer_values(host_manager: HostManager, credentials: dict = {'user': '
             "match_all": {}
         }
     }
+
     if greater_than_timestamp:
-        data['query'].update(
+        query = {
+                "bool": {
+                    "must": [
+                        {"match_all": {}},
+                        {"range": {"@timestamp": {"gte": f"{greater_than_timestamp}"}}}
+                    ]
+                }
+        }
+
+        sort = [
             {
-                'range': {
-                    "@timestamp": {
-                        "gte": greater_than_timestamp
-                    }
+                "@timestamp": {
+                    "order": "desc"
                 }
-            })
+            }
+        ]
+
+        data['query'] = query
+        data['sort'] = sort
+
     param = {
         'pretty': 'true',
         'size': 10000,
     }
 
+    print(data)
+
     response = requests.get(url=url, params=param, verify=False,
                             auth=requests.auth.HTTPBasicAuth(credentials['user'], credentials['password']), headers=headers,
                             json=data)

deps/wazuh_testing/wazuh_testing/end_to_end/monitoring.py

@@ -21,21 +21,23 @@
 from time import sleep
 from typing import Dict, List
 from multiprocessing.pool import ThreadPool
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
 from wazuh_testing.end_to_end import logs_filepath_os
 from wazuh_testing.end_to_end.regex import get_event_regex
 from wazuh_testing.tools.system import HostManager
 
 
-def monitoring_events_multihost(host_manager: HostManager, monitoring_data: Dict) -> None:
+def monitoring_events_multihost(host_manager: HostManager, monitoring_data: Dict, ignore_error=False) -> Dict:
     """
     Monitor events on multiple hosts concurrently.
 
     Args:
         host_manager: An instance of the HostManager class containing information about hosts.
         monitoring_data: A dictionary containing monitoring data for each host.
     """
-    def monitoring_event(host_manager: HostManager, host: str, monitoring_elements: List[Dict], scan_interval: int = 5):
+    def monitoring_event(host_manager: HostManager, host: str, monitoring_elements: List[Dict], scan_interval: int = 5,
+                         ignore_error=False):
         """
         Monitor the specified elements on a host.
 
@@ -47,6 +49,7 @@ def monitoring_event(host_manager: HostManager, host: str, monitoring_elements:
         Raises:
             TimeoutError: If no match is found within the specified timeout.
         """
+        elements_not_found = []
 
         for element in monitoring_elements:
             regex, timeout, monitoring_file = element['regex'], element['timeout'], element['file']
@@ -63,11 +66,29 @@ def monitoring_event(host_manager: HostManager, host: str, monitoring_elements:
                 current_timeout += 5
 
             if not regex_match:
-                raise TimeoutError("No match found within the specified timeout.")
+                elements_not_found.append(element)
+                if not ignore_error:
+                    raise TimeoutError(f"Element not found: {element}")
 
-    with ThreadPool() as pool:
-        # Use the pool to map the function to the list of hosts
-        pool.starmap(monitoring_event, [(host_manager, host, data) for host, data in monitoring_data.items()])
+        host_elements_not_found = {}
+        host_elements_not_found[host] = elements_not_found
+
+        return host_elements_not_found
+
+    with ThreadPoolExecutor() as executor:
+        futures = []
+        for host, data in monitoring_data.items():
+            futures.append(executor.submit(monitoring_event, host_manager, host, data, ignore_error))
+
+        results = {}
+        for future in as_completed(futures):
+            try:
+                result = future.result()
+                results.update(result)
+            except Exception as e:
+                print(f"An error occurred: {e}")
+
+        return results
 
 
 def generate_monitoring_logs_all_agent(host_manager: HostManager, regex_list: list, timeout_list: list) -> dict:

deps/wazuh_testing/wazuh_testing/end_to_end/vulnerability_detector.py

@@ -47,31 +47,33 @@ def check_vuln_state_index(host_manager: HostManager, vulnerability_data: Dict[s
     assert len(expected_alerts_not_found) == 0, f"Expected alerts were not found {expected_alerts_not_found}"
 
 
-def detect_alerts_by_agent(alerts, regex, current_datetime=None):
+def get_alerts_by_agent(alerts, regex):
+    """
+    Get specific alerts by agent.
+
+    Args:
+        alerts (list): List of alerts.
+        regex (str): Regular expression to match the alerts.
+
+    Returns:
+        dict: Dictionary containing the alerts by agent.
+    """
     alerts_vuln_by_agent = {}
+
     for alert in alerts:
-        valid_timestamp = True
-        if current_datetime:
-            dt = datetime.strptime(alert['_source']['timestamp'], "%Y-%m-%dT%H:%M:%S.%f%z")
-
-            # Convert datetime to Unix timestamp (integer)
-            timestamp = int(dt.timestamp())
-            if timestamp < current_datetime:
-                valid_timestamp = False
-
-        if valid_timestamp:
-            if re.match(regex, alert['_source']['rule']['description']):
-                if 'agent' in alert['_source']:
-                    agent = alert['_source']['agent']['name']
-                    if agent not in alerts_vuln_by_agent:
-                        alerts_vuln_by_agent[agent] = []
-                    else:
-                        alerts_vuln_by_agent[agent].append(alert)
+        if re.match(regex, alert['_source']['rule']['description']):
+            if 'agent' in alert['_source']:
+                agent = alert['_source']['agent']['name']
+                if agent not in alerts_vuln_by_agent:
+                    alerts_vuln_by_agent[agent] = []
+                else:
+                    alerts_vuln_by_agent[agent].append(alert)
 
     return alerts_vuln_by_agent
 
 
-def check_vuln_alert_indexer(host_manager: HostManager, vulnerability_data: Dict[str, Dict], current_datetime: str = None):
+def check_vuln_alert_indexer(host_manager: HostManager, vulnerability_data: Dict[str, Dict],
+                             current_datetime: str = ''):
     """
     Check vulnerability alerts in the indexer for a host.
 
@@ -82,14 +84,15 @@ def check_vuln_alert_indexer(host_manager: HostManager, vulnerability_data: Dict
     Returns:
         list: List of vulnerability alerts.
     """
-    regex_cve_affects = f"CVE.* affects .*"
-    regex_solved_vuln = f"The .* that affected .* was solved due to a package removal"
-
+    regex_cve_affects = "CVE.* affects .*"
+    regex_solved_vuln = "The .* that affected .* was solved due to a package removal"
 
     indexer_alerts = get_indexer_values(host_manager, greater_than_timestamp=current_datetime)['hits']['hits']
+
     # Get CVE affects alerts for all agents
-    detected_vuln_alerts_by_agent = detect_alerts_by_agent(indexer_alerts, regex_cve_affects, current_datetime)
-    solved_alerts_by_agent = detect_alerts_by_agent(indexer_alerts, regex_solved_vuln, current_datetime)
+    detected_vuln_alerts_by_agent = get_alerts_by_agent(indexer_alerts, regex_cve_affects)
+    solved_alerts_by_agent = get_alerts_by_agent(indexer_alerts, regex_solved_vuln)
+
     triggered_alerts = detected_vuln_alerts_by_agent
     expected_alerts_not_found = []
 
@@ -114,4 +117,4 @@ def check_vuln_alert_indexer(host_manager: HostManager, vulnerability_data: Dict
                 if not found:
                     expected_alerts_not_found.append(vulnerability)
 
-    assert len(expected_alerts_not_found) == 0, f"Expected alerts were not found {expected_alerts_not_found}"
+    assert len(expected_alerts_not_found) == 0, f"Expected alerts were not found {expected_alerts_not_found}"

deps/wazuh_testing/wazuh_testing/end_to_end/waiters.py

@@ -35,10 +35,10 @@ def wait_until_vd_is_updated(host_manager: HostManager) -> None:
 
     for manager in host_manager.get_group_hosts('manager'):
         monitoring_data = generate_monitoring_logs_manager(
-            host_manager, manager, 'Message processed', 1000
+            host_manager, manager, "INFO: Action for 'vulnerability_feed_manager' finished", 1000
         )
 
-    monitoring_events_multihost(host_manager, monitoring_data)
+        monitoring_events_multihost(host_manager, monitoring_data)
 
 
 def wait_until_vuln_scan_agents_finished(host_manager: HostManager) -> None:

deps/wazuh_testing/wazuh_testing/modules/syscollector/__init__.py

@@ -1 +1 @@
-TIMEOUT_SYSCOLLECTOR_SCAN = 200
+TIMEOUT_SYSCOLLECTOR_SCAN = 360

deps/wazuh_testing/wazuh_testing/tools/system.py

@@ -4,6 +4,7 @@
 
 import json
 import tempfile
+import logging
 import xml.dom.minidom as minidom
 from typing import Union
 import testinfra
@@ -15,6 +16,11 @@
 from ansible.parsing.dataloader import DataLoader
 from ansible.vars.manager import VariableManager
 
+
+logger = logging.getLogger('testinfra')
+logger.setLevel(logging.CRITICAL)
+
+
 class HostManager:
     """This class is an extensible remote host management interface. Within this we have multiple functions to modify
     the remote hosts depending on what our tests need.
@@ -300,7 +306,7 @@ def get_api_token(self, host, user='wazuh', password='wazuh', auth_context=None,
         """Return an API token for the specified user.
 
         Args:
-            host (str): Hostname.
+            host (str): HostName in inventory.
             user (str, optional): API username. Default `wazuh`
             password (str, optional): API password. Default `wazuh`
             auth_context (dict, optional): Authorization context body. Default `None`
@@ -512,6 +518,25 @@ def get_master_ip(self):
 
         return master_ip
 
+    def get_master(self):
+        """
+        Retrieves the master node from the inventory.
+
+        Returns:
+            str: The master node, or None if not found.
+        """
+        master_node = None
+
+        for manager in self.get_group_hosts('manager'):
+            if 'type' in self.get_host_variables(manager) and \
+                         self.get_host_variables(manager)['type'] == 'master':
+                master_node = manager
+                break
+        if master_node is None:
+            raise ValueError('Master node not found in inventory')
+
+        return master_node
+
     def remove_package(self, host, package_name, system):
         """
         Removes a package from the specified host.
@@ -588,6 +613,47 @@ def control_environment(self, operation, group_list):
             for host in self.get_group_hosts(group):
                 self.handle_wazuh_services(host, operation)
 
+    def get_agents_ids(self):
+        """
+        Retrieves the ID of the agents from the API.
+
+        Args:
+            agent_name (str): The name of the agent.
+
+        Returns:
+            str: The ID of the agent.
+        """
+        token = self.get_api_token(self.get_master())
+        agents = self.make_api_call(self.get_master(), endpoint='/agents/', token=token)['json']['data']
+
+        agents_ids = []
+
+        for agent in agents['affected_items']:
+            if agent['id'] != '000':
+                agents_ids.append(agent['id'])
+
+        return agents_ids
+
+    def remove_agents(self):
+        """
+        Removes all the agents from the API.
+
+        Args:
+            host (str): The target host from which to remove the agent.
+
+        Example:
+            host_manager.remove_agent('my_host', 'my_agent_id')
+        """
+        token = self.get_api_token(self.get_master())
+        agents_ids = self.get_agents_ids()
+        result = self.make_api_call(
+            host=self.get_master(),
+            method='DELETE',
+            endpoint=f'/agents?agents_list={",".join(agents_ids)}&status=all&older_than=0s',
+            token=token,
+        )
+
+
 
 def clean_environment(host_manager, target_files):
     """Clears a series of files on target hosts managed by a host manager

tests/end_to_end/conftest.py

@@ -346,3 +346,5 @@ def pytest_addoption(parser):
         type=str,
         help='Ansible roles path.',
     )
+
+

tests/end_to_end/pytest.ini

@@ -1,5 +1,10 @@
 [pytest]
 log_cli = 1
+
 log_cli_level = ERROR
 log_cli_format = %(asctime)s %(message)s (%(filename)s:%(lineno)s)
 log_cli_date_format=%Y-%m-%d %H:%M:%S
+
+log_file_level = ERROR
+log_file_format = %(asctime)s %(message)s (%(filename)s:%(lineno)s)
+log_file_date_format = %Y-%m-%d %H:%M:%S

tests/end_to_end/test_vulnerability_detector/configurations/agent.yaml

@@ -21,4 +21,4 @@
         - disabled:
             value: 'no'
         - interval:
-            value: '1m'
+            value: '6m'