Fim integration tests tags

[crd1985]

This PR closes #148

It adds the following tests:

• Use of common tags
• Use of non ASCII characters
• Use of long tags

Furthermore, wazuh_testing.fim.regular_file_cud function have been improved to be much more generic. Now, some custom validators can be added. I.e.:

def tag_validator(event):
    assert(defined_tags == event['data']['tags'])

    regular_file_cud(folder, wazuh_log_monitor,
                     min_timeout=3,
                     time_travel=get_configuration['metadata']['fim_mode'] == 'scheduled',
                     validators_after_cud=[tag_validator]
                     )

Tests

The current status of integration tests remain as follows:

[root@wazuh-master vagrant]# python3 -m pytest /vagrant/shared/test_wazuh/test_fim
================================================ test session starts =================================================
platform linux -- Python 3.6.8, pytest-5.2.1, py-1.8.0, pluggy-0.13.0
rootdir: /vagrant/shared/test_wazuh, inifile: pytest.ini
collected 767 items                                                                                                  

../../vagrant/shared/test_wazuh/test_fim/test_audit/test_audit.py ..F..                                        [  0%]
../../vagrant/shared/test_wazuh/test_fim/test_basic_usage/test_basic_usage_changes.py ..ssssss..ssssss..       [  2%]
../../vagrant/shared/test_wazuh/test_fim/test_basic_usage/test_basic_usage_create.py ............sssssssssssss [  6%]
sssssssssss............ssssssssssss............                                                                [ 12%]
../../vagrant/shared/test_wazuh/test_fim/test_benchmark/test_benchmark.py .....FFF                             [ 13%]
../../vagrant/shared/test_wazuh/test_fim/test_ignore/test_ignore_invalid.py .                                  [ 13%]
../../vagrant/shared/test_wazuh/test_fim/test_ignore/test_ignore_valid.py sssssssssssssssssss......s......ssss [ 18%]
.sssss....s......ssss.sssss....s......ssss.sssss................s.sss..................sss..................ss [ 32%]
s....s......s.s..s.sss....s......s.s..ss.sssssssssssssssssssss......s......ssss.sssss....s......ssss.sssss.... [ 46%]
s......ssss.sssss................s.sss..................sss..................sss....s......s.s..s.sss....s.... [ 61%]
..s.s..ss.sssssssssssssssssssss......s......ssss.sssss....s......ssss.sssss....s......ssss.sssss.............. [ 75%]
..s.sss..................sss..................sss....s......s.s..s.sss....s......s.s..ss.ss                    [ 87%]
../../vagrant/shared/test_wazuh/test_fim/test_recursion_level/test_recursion_level.py ......................FF [ 90%]
                                                                                                               [ 90%]
../../vagrant/shared/test_wazuh/test_fim/test_tags/test_tags.py .............................................. [ 96%]
..                                                                                                             [ 96%]
../../vagrant/shared/test_wazuh/test_fim/test_whodata/test_whodata.py ........................                 [100%]

====================================================== FAILURES ======================================================
_______________________________ test_readded_rules[get_configuration0-tags_to_apply0] ________________________________

tags_to_apply = {'all'}
get_configuration = {'apply_to_modules': ['test_audit'], 'elements': [{'disabled': {'value': 'no'}}, {'directories': {'attributes': [{'che...: 'yes'}, {'whodata': 'yes'}], 'value': '/testdir1,/testdir2,/testdir3'}}], 'metadata': {}, 'section': 'syscheck', ...}
configure_environment = None, restart_wazuh = None

    @pytest.mark.parametrize('tags_to_apply', [
        ({'all'})
    ])
    def test_readded_rules(tags_to_apply, get_configuration,
                           configure_environment, restart_wazuh):
        """Checks if the removed rules are added to Audit rules list."""
        check_apply_test(tags_to_apply, get_configuration['tags'])
    
        # Remove added rules
        for dir in (testdir1, testdir2, testdir3):
            os.system("auditctl -W {0} -p wa -k wazuh_fim".format(dir))
    
            wazuh_log_monitor.start(timeout=20,
                                    callback=callback_audit_rules_manipulation)
    
            events = wazuh_log_monitor.start(timeout=10,
                                             callback=callback_audit_loaded_rule).result()
    
>           assert (dir in events)
E           AssertionError: assert '/testdir1' in '/testdir2'

/vagrant/shared/test_wazuh/test_fim/test_audit/test_audit.py:91: AssertionError
________________ test_benchmark_regular_files[get_configuration1-100-/testdir1-False-tags_to_apply1] _________________

n_regular = 100, folder = '/testdir1', is_scheduled = False, tags_to_apply = {'realtime', 'whodata'}
get_configuration = {'apply_to_modules': ['test_benchmark'], 'elements': [{'disabled': {'value': 'no'}}, {'directories': {'attributes': [{...: 'yes'}, {'whodata': 'yes'}], 'value': '/testdir1,/testdir2,/noexists'}}], 'metadata': {}, 'section': 'syscheck', ...}
configure_environment = None, restart_wazuh = None, wait_for_initial_scan = None

    @pytest.mark.benchmark
    @pytest.mark.parametrize('n_regular, folder, is_scheduled, tags_to_apply', [
        (10, testdir1, False, {'realtime', 'whodata'}),
        (100, testdir1, False, {'realtime', 'whodata'}),
        (1000, testdir1, False, {'realtime', 'whodata'}),
        (10000, testdir1, False, {'realtime', 'whodata'})
    ])
    def test_benchmark_regular_files(n_regular, folder, is_scheduled,
                                     tags_to_apply, get_configuration,
                                     configure_environment, restart_wazuh,
                                     wait_for_initial_scan):
        """Checks syscheckd detects a minimum volume of file changes (add, modify, delete)"""
        check_apply_test(tags_to_apply, get_configuration['tags'])
        min_timeout = 30
    
        regular_file_cud(folder, wazuh_log_monitor, time_travel=is_scheduled,
>                        n_regular=n_regular, min_timeout=min_timeout)

/vagrant/shared/test_wazuh/test_fim/test_benchmark/test_benchmark.py:54: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/usr/local/lib/python3.6/site-packages/wazuh_testing/fim.py:415: in regular_file_cud
    events = fetch_events()
/usr/local/lib/python3.6/site-packages/wazuh_testing/fim.py:383: in fetch_events
    accum_results=n_regular
/usr/local/lib/python3.6/site-packages/wazuh_testing/tools.py:310: in start
    self._monitor(callback=callback, accum_results=accum_results)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <wazuh_testing.tools.FileMonitor object at 0x7f5dcb73f438>
callback = <function callback_detect_event at 0x7f5dcbc6ef28>, accum_results = 100

    def _monitor(self, callback=_callback_default, accum_results=1):
        """Wait for new lines to be appended to the file.
    
        A callback function will be called every time a new line is detected. This function must receive two
        positional parameters: a references to the FileMonitor object and the line detected.
        """
        self._result = [] if accum_results > 1 else None
        with open(self.file_path) as f:
            f.seek(self._position)
            while self._continue:
                if self._abort:
                    self.stop()
>                   raise TimeoutError()
E                   TimeoutError

/usr/local/lib/python3.6/site-packages/wazuh_testing/tools.py:282: TimeoutError
________________ test_benchmark_regular_files[get_configuration1-1000-/testdir1-False-tags_to_apply2] ________________

n_regular = 1000, folder = '/testdir1', is_scheduled = False, tags_to_apply = {'realtime', 'whodata'}
get_configuration = {'apply_to_modules': ['test_benchmark'], 'elements': [{'disabled': {'value': 'no'}}, {'directories': {'attributes': [{...: 'yes'}, {'whodata': 'yes'}], 'value': '/testdir1,/testdir2,/noexists'}}], 'metadata': {}, 'section': 'syscheck', ...}
configure_environment = None, restart_wazuh = None, wait_for_initial_scan = None

    @pytest.mark.benchmark
    @pytest.mark.parametrize('n_regular, folder, is_scheduled, tags_to_apply', [
        (10, testdir1, False, {'realtime', 'whodata'}),
        (100, testdir1, False, {'realtime', 'whodata'}),
        (1000, testdir1, False, {'realtime', 'whodata'}),
        (10000, testdir1, False, {'realtime', 'whodata'})
    ])
    def test_benchmark_regular_files(n_regular, folder, is_scheduled,
                                     tags_to_apply, get_configuration,
                                     configure_environment, restart_wazuh,
                                     wait_for_initial_scan):
        """Checks syscheckd detects a minimum volume of file changes (add, modify, delete)"""
        check_apply_test(tags_to_apply, get_configuration['tags'])
        min_timeout = 30
    
        regular_file_cud(folder, wazuh_log_monitor, time_travel=is_scheduled,
>                        n_regular=n_regular, min_timeout=min_timeout)

/vagrant/shared/test_wazuh/test_fim/test_benchmark/test_benchmark.py:54: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/usr/local/lib/python3.6/site-packages/wazuh_testing/fim.py:415: in regular_file_cud
    events = fetch_events()
/usr/local/lib/python3.6/site-packages/wazuh_testing/fim.py:383: in fetch_events
    accum_results=n_regular
/usr/local/lib/python3.6/site-packages/wazuh_testing/tools.py:310: in start
    self._monitor(callback=callback, accum_results=accum_results)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <wazuh_testing.tools.FileMonitor object at 0x7f5dcb73f438>
callback = <function callback_detect_event at 0x7f5dcbc6ef28>, accum_results = 1000

    def _monitor(self, callback=_callback_default, accum_results=1):
        """Wait for new lines to be appended to the file.
    
        A callback function will be called every time a new line is detected. This function must receive two
        positional parameters: a references to the FileMonitor object and the line detected.
        """
        self._result = [] if accum_results > 1 else None
        with open(self.file_path) as f:
            f.seek(self._position)
            while self._continue:
                if self._abort:
                    self.stop()
>                   raise TimeoutError()
E                   TimeoutError

/usr/local/lib/python3.6/site-packages/wazuh_testing/tools.py:282: TimeoutError
_______________ test_benchmark_regular_files[get_configuration1-10000-/testdir1-False-tags_to_apply3] ________________

n_regular = 10000, folder = '/testdir1', is_scheduled = False, tags_to_apply = {'realtime', 'whodata'}
get_configuration = {'apply_to_modules': ['test_benchmark'], 'elements': [{'disabled': {'value': 'no'}}, {'directories': {'attributes': [{...: 'yes'}, {'whodata': 'yes'}], 'value': '/testdir1,/testdir2,/noexists'}}], 'metadata': {}, 'section': 'syscheck', ...}
configure_environment = None, restart_wazuh = None, wait_for_initial_scan = None

    @pytest.mark.benchmark
    @pytest.mark.parametrize('n_regular, folder, is_scheduled, tags_to_apply', [
        (10, testdir1, False, {'realtime', 'whodata'}),
        (100, testdir1, False, {'realtime', 'whodata'}),
        (1000, testdir1, False, {'realtime', 'whodata'}),
        (10000, testdir1, False, {'realtime', 'whodata'})
    ])
    def test_benchmark_regular_files(n_regular, folder, is_scheduled,
                                     tags_to_apply, get_configuration,
                                     configure_environment, restart_wazuh,
                                     wait_for_initial_scan):
        """Checks syscheckd detects a minimum volume of file changes (add, modify, delete)"""
        check_apply_test(tags_to_apply, get_configuration['tags'])
        min_timeout = 30
    
        regular_file_cud(folder, wazuh_log_monitor, time_travel=is_scheduled,
>                        n_regular=n_regular, min_timeout=min_timeout)

/vagrant/shared/test_wazuh/test_fim/test_benchmark/test_benchmark.py:54: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/usr/local/lib/python3.6/site-packages/wazuh_testing/fim.py:415: in regular_file_cud
    events = fetch_events()
/usr/local/lib/python3.6/site-packages/wazuh_testing/fim.py:383: in fetch_events
    accum_results=n_regular
/usr/local/lib/python3.6/site-packages/wazuh_testing/tools.py:310: in start
    self._monitor(callback=callback, accum_results=accum_results)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <wazuh_testing.tools.FileMonitor object at 0x7f5dcb73f438>
callback = <function callback_detect_event at 0x7f5dcbc6ef28>, accum_results = 10000

    def _monitor(self, callback=_callback_default, accum_results=1):
        """Wait for new lines to be appended to the file.
    
        A callback function will be called every time a new line is detected. This function must receive two
        positional parameters: a references to the FileMonitor object and the line detected.
        """
        self._result = [] if accum_results > 1 else None
        with open(self.file_path) as f:
            f.seek(self._position)
            while self._continue:
                if self._abort:
                    self.stop()
>                   raise TimeoutError()
E                   TimeoutError

/usr/local/lib/python3.6/site-packages/wazuh_testing/tools.py:282: TimeoutError
______________________ test_recursion_level[get_configuration2-/test_recursion_320-subdir-318] _______________________

dirname = '/test_recursion_320', subdirname = 'subdir', recursion_level = 318
get_configuration = {'apply_to_modules': ['test_recursion_level'], 'elements': [{'disabled': {'value': 'no'}}, {'directories': {'attribute... {'whodata': 'yes'}], 'value': '/test_recursion_5'}}, ...], 'metadata': {'fim_mode': 'whodata'}, 'section': 'syscheck'}
configure_environment = None, restart_wazuh = None, wait_for_initial_scan = None

    @pytest.mark.parametrize('dirname, subdirname, recursion_level', [
        (dir_no_recursion, subdir, 0),
        (dir_no_recursion_space, subdir_space, 0),
        (dir_recursion_1, subdir, 1),
        (dir_recursion_1_space, subdir_space, 1),
        (dir_recursion_5, subdir, 5),
        (dir_recursion_5_space, subdir_space, 5),
        (dir_recursion_320, subdir, 318),
        (dir_recursion_320_space, subdir_space, 318)
    ])
    def test_recursion_level(dirname, subdirname, recursion_level,
                             get_configuration, configure_environment,
                             restart_wazuh, wait_for_initial_scan):
        """Checks if files are correctly detected by syscheck with recursion level using scheduled, realtime and whodata monitoring
    
        This test is intended to be used with valid ignore configurations. It applies RegEx to match the name
        of the configuration file where the test applies. If the configuration file does not match the test
        is skipped.
    
        :param dirname string The path being monitored by syscheck (indicated in the .conf file)
        :param subdirname string The name of the subdirectories that will be created during the execution for testing purpouses.
        :param recursion_level int Recursion level. Also used as the number of subdirectories to be created and checked for the current test.
        """
        if get_configuration['metadata']['fim_mode'] == 'scheduled':
            recursion_test(dirname, subdirname, recursion_level, is_scheduled=True)
        else:
>           recursion_test(dirname, subdirname, recursion_level, timeout=2)

/vagrant/shared/test_wazuh/test_fim/test_recursion_level/test_recursion_level.py:136: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/vagrant/shared/test_wazuh/test_fim/test_recursion_level/test_recursion_level.py:92: in recursion_test
    n_regular=num_files, min_timeout=timeout)
/usr/local/lib/python3.6/site-packages/wazuh_testing/fim.py:415: in regular_file_cud
    events = fetch_events()
/usr/local/lib/python3.6/site-packages/wazuh_testing/fim.py:383: in fetch_events
    accum_results=n_regular
/usr/local/lib/python3.6/site-packages/wazuh_testing/tools.py:310: in start
    self._monitor(callback=callback, accum_results=accum_results)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <wazuh_testing.tools.FileMonitor object at 0x7f5dcbacb278>
callback = <function callback_detect_event at 0x7f5dcbc6ef28>, accum_results = 1

    def _monitor(self, callback=_callback_default, accum_results=1):
        """Wait for new lines to be appended to the file.
    
        A callback function will be called every time a new line is detected. This function must receive two
        positional parameters: a references to the FileMonitor object and the line detected.
        """
        self._result = [] if accum_results > 1 else None
        with open(self.file_path) as f:
            f.seek(self._position)
            while self._continue:
                if self._abort:
                    self.stop()
>                   raise TimeoutError()
E                   TimeoutError

/usr/local/lib/python3.6/site-packages/wazuh_testing/tools.py:282: TimeoutError
_____________________ test_recursion_level[get_configuration2-/test recursion 320-sub dir -318] ______________________

dirname = '/test recursion 320', subdirname = 'sub dir ', recursion_level = 318
get_configuration = {'apply_to_modules': ['test_recursion_level'], 'elements': [{'disabled': {'value': 'no'}}, {'directories': {'attribute... {'whodata': 'yes'}], 'value': '/test_recursion_5'}}, ...], 'metadata': {'fim_mode': 'whodata'}, 'section': 'syscheck'}
configure_environment = None, restart_wazuh = None, wait_for_initial_scan = None

    @pytest.mark.parametrize('dirname, subdirname, recursion_level', [
        (dir_no_recursion, subdir, 0),
        (dir_no_recursion_space, subdir_space, 0),
        (dir_recursion_1, subdir, 1),
        (dir_recursion_1_space, subdir_space, 1),
        (dir_recursion_5, subdir, 5),
        (dir_recursion_5_space, subdir_space, 5),
        (dir_recursion_320, subdir, 318),
        (dir_recursion_320_space, subdir_space, 318)
    ])
    def test_recursion_level(dirname, subdirname, recursion_level,
                             get_configuration, configure_environment,
                             restart_wazuh, wait_for_initial_scan):
        """Checks if files are correctly detected by syscheck with recursion level using scheduled, realtime and whodata monitoring
    
        This test is intended to be used with valid ignore configurations. It applies RegEx to match the name
        of the configuration file where the test applies. If the configuration file does not match the test
        is skipped.
    
        :param dirname string The path being monitored by syscheck (indicated in the .conf file)
        :param subdirname string The name of the subdirectories that will be created during the execution for testing purpouses.
        :param recursion_level int Recursion level. Also used as the number of subdirectories to be created and checked for the current test.
        """
        if get_configuration['metadata']['fim_mode'] == 'scheduled':
            recursion_test(dirname, subdirname, recursion_level, is_scheduled=True)
        else:
>           recursion_test(dirname, subdirname, recursion_level, timeout=2)

/vagrant/shared/test_wazuh/test_fim/test_recursion_level/test_recursion_level.py:136: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/vagrant/shared/test_wazuh/test_fim/test_recursion_level/test_recursion_level.py:92: in recursion_test
    n_regular=num_files, min_timeout=timeout)
/usr/local/lib/python3.6/site-packages/wazuh_testing/fim.py:415: in regular_file_cud
    events = fetch_events()
/usr/local/lib/python3.6/site-packages/wazuh_testing/fim.py:383: in fetch_events
    accum_results=n_regular
/usr/local/lib/python3.6/site-packages/wazuh_testing/tools.py:310: in start
    self._monitor(callback=callback, accum_results=accum_results)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <wazuh_testing.tools.FileMonitor object at 0x7f5dcbacb278>
callback = <function callback_detect_event at 0x7f5dcbc6ef28>, accum_results = 1

    def _monitor(self, callback=_callback_default, accum_results=1):
        """Wait for new lines to be appended to the file.
    
        A callback function will be called every time a new line is detected. This function must receive two
        positional parameters: a references to the FileMonitor object and the line detected.
        """
        self._result = [] if accum_results > 1 else None
        with open(self.file_path) as f:
            f.seek(self._position)
            while self._continue:
                if self._abort:
                    self.stop()
>                   raise TimeoutError()
E                   TimeoutError

/usr/local/lib/python3.6/site-packages/wazuh_testing/tools.py:282: TimeoutError
======================= 6 failed, 494 passed, 267 skipped in 12732699.93s (147 days, 8:51:39) ========================

• test_audit.py fails depending on the machine. @albertomn86 is fixing that.
• test_benchmark.py fails due to missing events for a high volume in whodata config.
• test_recursion_level.py fails due to max length of recursion level for whodata. Fix pending.