Add test code, etc.

CHANGELOG.md

@@ -9,6 +9,20 @@
 ## Other changes
 - None
 
+
+# 2.x.x
+
+## Breaking changes
+- None
+
+## New features
+- None
+
+## Other changes
+- [Tests] Add test code. Changed ubuntu version of Dockerfile-test from latest to 21.10. - [#354](https://github.com/jertel/elastalert2/pull/354) - @nsano-rururu
+- Remove Python 2.x compatibility code - [#354](https://github.com/jertel/elastalert2/pull/354) - @nsano-rururu
+
+
 # 2.1.2
 ## Breaking changes
 - None

elastalert/util.py

@@ -256,13 +256,10 @@ def seconds(td):
 
 
 def total_seconds(dt):
-    # For python 2.6 compatability
     if dt is None:
         return 0
-    elif hasattr(dt, 'total_seconds'):
-        return dt.total_seconds()
     else:
-        return (dt.microseconds + (dt.seconds + dt.days * 24 * 3600) * 10**6) / 10**6
+        return dt.total_seconds()
 
 
 def dt_to_int(dt):

tests/Dockerfile-test

@@ -1,4 +1,4 @@
-FROM ubuntu:latest
+FROM ubuntu:21.10
 
 RUN apt update && apt upgrade -y
 RUN apt install software-properties-common -y

tests/alerters/thehive_test.py

@@ -263,3 +263,42 @@ def test_thehive_alerter2():
     del actual_data['sourceRef']
 
     assert expected_data == actual_data
+
+
+@pytest.mark.parametrize('tags, expect', [
+    ({'tags': ['a', 'b']}, {'a', 'b'}),
+    (['a', 'b'],           {'a', 'b'}),
+    ([],                   set()),
+    ('a',                  {'a'})
+])
+def test_load_tags(tags, expect):
+    rule = {'alert': [],
+            'alert_text': '',
+            'alert_text_type': 'alert_text_only',
+            'description': 'test',
+            'hive_alert_config': {'customFields': [{'name': 'test',
+                                                    'type': 'string',
+                                                    'value': 2}],
+                                  'follow': True,
+                                  'severity': 2,
+                                  'source': 'elastalert',
+                                  'status': 'New',
+                                  'tags': ['test.port'],
+                                  'tlp': 3,
+                                  'type': 'external'},
+            'hive_connection': {'hive_apikey': '',
+                                'hive_host': 'https://localhost',
+                                'hive_port': 9000},
+            'hive_observable_data_mapping': [{'ip': 'test.ip', 'autonomous-system': 'test.as_number'}],
+            'name': 'test-thehive',
+            'tags': ['a', 'b'],
+            'type': 'any'}
+    rules_loader = FileRulesLoader({})
+    rules_loader.load_modules(rule)
+    alert = HiveAlerter(rule)
+    match = {
+        '@timestamp': '2021-01-01T00:00:00',
+        'somefield': 'a'
+    }
+    actual = alert.load_tags(tags, match)
+    assert expect == actual

tests/base_test.py

@@ -12,6 +12,7 @@
 
 from elastalert.enhancements import BaseEnhancement
 from elastalert.enhancements import DropMatchException
+from elastalert.enhancements import TimeEnhancement
 from elastalert.kibana import dashboard_temp
 from elastalert.util import dt_to_ts
 from elastalert.util import dt_to_unix
@@ -1443,3 +1444,24 @@ def test_get_elasticsearch_client_different_rule(ea):
     y = ea.get_elasticsearch_client(y_rule)
 
     assert x is not y, 'Should return unique client for each rule'
+
+
+def test_base_enhancement_process_error(ea):
+    try:
+        be = BaseEnhancement(ea.rules[0])
+        be.process('')
+        assert False
+    except NotImplementedError:
+        assert True
+
+
+def test_time_enhancement(ea):
+    be = BaseEnhancement(ea.rules[0])
+    te = TimeEnhancement(be)
+    match = {
+        '@timestamp': '2021-01-01T00:00:00',
+        'somefield': 'foobarbaz'
+    }
+    te.process(match)
+    excepted = '2021-01-01 00:00 +0000'
+    assert match['@timestamp'] == excepted

tests/config_test.py

@@ -1,9 +1,13 @@
 # -*- coding: utf-8 -*-
-import os
-from unittest import mock
 import datetime
+import logging
+import os
+import pytest
 
 from elastalert.config import load_conf
+from elastalert.util import EAException
+
+from unittest import mock
 
 
 def test_config_loads():
@@ -31,3 +35,165 @@ def test_config_loads():
     assert conf['writeback_index'] == 'elastalert_status'
 
     assert conf['alert_time_limit'] == datetime.timedelta(days=2)
+
+
+def test_config_loads_ea_execption():
+    with pytest.raises(EAException) as ea:
+        os.environ['ELASTIC_PASS'] = 'password_from_env'
+
+        test_args = mock.Mock()
+        test_args.config = ''
+        test_args.rule = None
+        test_args.debug = False
+        test_args.es_debug_trace = None
+
+        load_conf(test_args)
+
+    assert 'No --config or config.yaml found' in str(ea)
+
+
+@pytest.mark.parametrize('config, expected', [
+    ('/example.config.type_error.run_every.yaml', 'Invalid time format used: '),
+    ('/example.config.type_error.buffer_time.yaml', 'Invalid time format used: ')
+])
+def test_config_loads_type_error(config, expected):
+    with pytest.raises(EAException) as ea:
+        os.environ['ELASTIC_PASS'] = 'password_from_env'
+        dir_path = os.path.dirname(os.path.realpath(__file__))
+
+        test_args = mock.Mock()
+        test_args.config = dir_path + config
+        test_args.rule = None
+        test_args.debug = False
+        test_args.es_debug_trace = None
+
+        load_conf(test_args)
+
+    assert expected in str(ea)
+
+
+@pytest.mark.parametrize('config, expected', [
+    ('/example.config.not_found.run_every.yaml', 'must contain '),
+    ('/example.config.not_found.es_host.yaml', 'must contain '),
+    ('/example.config.not_found.es_port.yaml', 'must contain '),
+    ('/example.config.not_found.writeback_index.yaml', 'must contain '),
+    ('/example.config.not_found.buffer_time.yaml', 'must contain ')
+])
+def test_config_loads_required_globals_error(config, expected):
+    with pytest.raises(EAException) as ea:
+        os.environ['ELASTIC_PASS'] = 'password_from_env'
+        dir_path = os.path.dirname(os.path.realpath(__file__))
+
+        test_args = mock.Mock()
+        test_args.config = dir_path + config
+        test_args.rule = None
+        test_args.debug = False
+        test_args.verbose = None
+        test_args.es_debug_trace = None
+
+        load_conf(test_args)
+
+    assert expected in str(ea)
+
+
+def test_config_loads_debug(caplog):
+    caplog.set_level(logging.INFO)
+    os.environ['ELASTIC_PASS'] = 'password_from_env'
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+
+    test_args = mock.Mock()
+    test_args.config = dir_path + '/example.config.yaml'
+    test_args.rule = None
+    test_args.debug = True
+    test_args.verbose = None
+    test_args.es_debug_trace = None
+
+    load_conf(test_args)
+
+    expected_msg = 'Note: In debug mode, alerts will be logged to console but NOT actually sent.\n'
+    expected_msg += '            To send them but remain verbose, use --verbose instead.'
+    assert ('elastalert', logging.INFO, expected_msg) == caplog.record_tuples[0]
+
+
+def test_config_loads_debug_and_verbose(caplog):
+    caplog.set_level(logging.INFO)
+    os.environ['ELASTIC_PASS'] = 'password_from_env'
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+
+    test_args = mock.Mock()
+    test_args.config = dir_path + '/example.config.yaml'
+    test_args.rule = None
+    test_args.debug = True
+    test_args.verbose = True
+    test_args.es_debug_trace = None
+
+    load_conf(test_args)
+
+    expected_msg = 'Note: --debug and --verbose flags are set. --debug takes precedent.'
+    assert ('elastalert', logging.INFO, expected_msg) == caplog.record_tuples[0]
+
+
+def test_config_loads_old_query_limit():
+    os.environ['ELASTIC_PASS'] = 'password_from_env'
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+
+    test_args = mock.Mock()
+    test_args.config = dir_path + '/example.config.old_query_limit.yaml'
+    test_args.rule = None
+    test_args.debug = False
+    test_args.es_debug_trace = None
+
+    conf = load_conf(test_args)
+
+    assert conf['rules_folder'] == '/opt/elastalert/rules'
+    assert conf['run_every'] == datetime.timedelta(seconds=10)
+    assert conf['buffer_time'] == datetime.timedelta(minutes=15)
+    assert conf['es_host'] == 'elasticsearch'
+    assert conf['es_port'] == 9200
+    assert conf['es_username'] == 'elastic'
+    assert conf['es_password'] == 'password_from_env'
+    assert conf['writeback_index'] == 'elastalert_status'
+    assert conf['alert_time_limit'] == datetime.timedelta(days=2)
+    assert conf['old_query_limit'] == datetime.timedelta(days=3)
+
+
+def test_config_loads_logging(capfd):
+    os.environ['ELASTIC_PASS'] = 'password_from_env'
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+
+    test_args = mock.Mock()
+    test_args.config = dir_path + '/example.config.logging.yaml'
+    test_args.rule = None
+    test_args.debug = True
+    test_args.verbose = True
+    test_args.es_debug_trace = None
+
+    load_conf(test_args)
+
+    expected1 = 'Note: --debug and --verbose flags are set. --debug takes precedent.'
+    expected2 = 'Note: In debug mode, alerts will be logged to console but NOT actually sent.\n'
+    expected3 = '           To send them but remain verbose, use --verbose instead.\n'
+    out, err = capfd.readouterr()
+    assert expected1 in err
+    assert expected2 in err
+    assert expected3 in err
+
+
+def test_config_loads_logging2(caplog):
+    os.environ['ELASTIC_PASS'] = 'password_from_env'
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+
+    test_args = mock.Mock()
+    test_args.config = dir_path + '/example.config.yaml'
+    test_args.rule = None
+    test_args.debug = True
+    test_args.verbose = False
+    test_args.es_debug_trace = None
+
+    load_conf(test_args)
+
+    expected1 = 'Note: In debug mode, alerts will be logged to console but NOT actually sent.'
+    expected2 = '            To send them but remain verbose, use --verbose instead.'
+    user, level, message = caplog.record_tuples[0]
+    assert expected1 in message
+    assert expected2 in message

tests/create_index_test.py

@@ -51,3 +51,96 @@ def test_read_es_6_index_mappings():
     mappings = elastalert.create_index.read_es_index_mappings(6)
     assert len(mappings) == len(es_mappings)
     print((json.dumps(mappings, indent=2)))
+
+
+@pytest.mark.parametrize('es_version, expected', [
+    ('5.6.0', False),
+    ('6.0.0', True),
+    ('6.1.0', True),
+    ('6.2.0', True),
+    ('6.3.0', True),
+    ('6.4.0', True),
+    ('6.5.0', True),
+    ('6.6.0', True),
+    ('6.7.0', True),
+    ('6.8.0', True),
+    ('7.0.0', True),
+    ('7.1.0', True),
+    ('7.2.0', True),
+    ('7.3.0', True),
+    ('7.4.0', True),
+    ('7.5.0', True),
+    ('7.6.0', True),
+    ('7.7.0', True),
+    ('7.8.0', True),
+    ('7.9.0', True),
+    ('7.10.0', True),
+    ('7.11.0', True),
+    ('7.12.0', True),
+    ('7.13.0', True)
+])
+def test_is_atleastsix(es_version, expected):
+    result = elastalert.create_index.is_atleastsix(es_version)
+    assert result == expected
+
+
+@pytest.mark.parametrize('es_version, expected', [
+    ('5.6.0', False),
+    ('6.0.0', False),
+    ('6.1.0', False),
+    ('6.2.0', True),
+    ('6.3.0', True),
+    ('6.4.0', True),
+    ('6.5.0', True),
+    ('6.6.0', True),
+    ('6.7.0', True),
+    ('6.8.0', True),
+    ('7.0.0', True),
+    ('7.1.0', True),
+    ('7.2.0', True),
+    ('7.3.0', True),
+    ('7.4.0', True),
+    ('7.5.0', True),
+    ('7.6.0', True),
+    ('7.7.0', True),
+    ('7.8.0', True),
+    ('7.9.0', True),
+    ('7.10.0', True),
+    ('7.11.0', True),
+    ('7.12.0', True),
+    ('7.13.0', True)
+])
+def test_is_atleastsixtwo(es_version, expected):
+    result = elastalert.create_index.is_atleastsixtwo(es_version)
+    assert result == expected
+
+
+@pytest.mark.parametrize('es_version, expected', [
+    ('5.6.0', False),
+    ('6.0.0', False),
+    ('6.1.0', False),
+    ('6.2.0', False),
+    ('6.3.0', False),
+    ('6.4.0', False),
+    ('6.5.0', False),
+    ('6.6.0', False),
+    ('6.7.0', False),
+    ('6.8.0', False),
+    ('7.0.0', True),
+    ('7.1.0', True),
+    ('7.2.0', True),
+    ('7.3.0', True),
+    ('7.4.0', True),
+    ('7.5.0', True),
+    ('7.6.0', True),
+    ('7.7.0', True),
+    ('7.8.0', True),
+    ('7.9.0', True),
+    ('7.10.0', True),
+    ('7.11.0', True),
+    ('7.12.0', True),
+    ('7.13.0', True)
+])
+def test_is_atleastseven(es_version, expected):
+    result = elastalert.create_index.is_atleastseven(es_version)
+    assert result == expected