From 6022d01698e1a2f1c4aa78b3df4cd5958c6b8b21 Mon Sep 17 00:00:00 2001 From: Constantin Date: Wed, 5 Oct 2011 17:22:08 +0200 Subject: [PATCH] modified and finished vmware normaliser, also modified snare, test_log_sample and test_normalizer --- normalizers/VMWare_ESX4-ESXi4.xml | 70 ++++++++++++++++++------------- normalizers/snare.xml | 4 +- tests/test_log_samples.py | 65 +++++++++++++++++++++++++--- tests/test_normalizer.py | 5 +++ 4 files changed, 108 insertions(+), 36 deletions(-) diff --git a/normalizers/VMWare_ESX4-ESXi4.xml b/normalizers/VMWare_ESX4-ESXi4.xml index bbf5d76..af59245 100644 --- a/normalizers/VMWare_ESX4-ESXi4.xml +++ b/normalizers/VMWare_ESX4-ESXi4.xml @@ -32,12 +32,12 @@ version="0.99" unicode="yes" ignorecase="yes" - matchtype= "search" + matchtype= "match" appliedTo= "raw"> - This normalizer parses VMware ESX 4.x and ESXi 4.x logs. - Ce normaliser analyse les logs de VMware ESX 4.x et ESXi 4.x. + This normalizer parses VMware ESX 4.x and ESXi 4.x logs that are not handled by the Syslog normalizer. + Ce normaliseur analyse les logs de VMware ESX 4.x et ESXi 4.x. qui ne sont pas gérés pas le normaliseur Syslog. clo@wallix.com @@ -45,53 +45,65 @@ - - + Expression matching a date in the format yyyy-mm-dd hh:mm:ss. + Expression correspondant à une date au format yyyy-mm-dd hh:mm:ss. \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} - - + Expression matching a hexadecimal number. + Expression correspondant à un nombre héxadécimal. [A-F0-9]{8} - + - - + Expression matching the 'alpha' field, words between '. + Expression correspondant au champ 'alpha', qui contient les mots entre '. [^']+(?: [^']+)* - - + Expression matching the 'level' field. + Expression correspondant au champ 'level'. [^\s]+ - - date = str(value) - date = date[:10] + 'T' + date[11:] - log["date"] = date +reg = re.compile('(?P<year>\d{4})-(?P<month>\d{2})-(?P<day>\d{2}) (?P<hours>\d{2}):(?P<minutes>\d{2}):(?P<seconds>\d{2})') + +date = reg.search(value) + +date = date.groupdict() + +year= int(date.get('year')) +month = int(date.get('month')) +day = int(date.get('day')) +hours = int(date.get('hours')) +minutes = int(date.get('minutes')) +seconds = int(date.get('seconds')) + +newdate = datetime(year, month, day, hours, minutes, seconds) + +log["date"] = newdate - Hostd.log and vpxa.log logs' pattern. (Both agent) - + Logs contained in hostd.log file. + Logs contenus dans le fichier hostd.log. \[DATE NUMERIC LEVEL 'ALPHA'[^\]]*\] BODY - the time at which the request was issued - please note that the timezone information is not carried over. - la date à laquelle la requête a été émise. Veuillez noter que l'information de fuseau horaire n'est pas prise en compte. + The time at which the request was issued - please note that the timezone information is not carried over. + La date à laquelle la requête a été émise. Veuillez noter que l'information de fuseau horaire n'est pas prise en compte. DATE @@ -107,12 +119,12 @@ - - + The level is the type of the log. + Le level correspond au type du log. LEVEL - + @@ -121,8 +133,8 @@ - the actual event message - le message décrivant l'événement + The actual event message. + Le message décrivant l'événement. BODY @@ -150,15 +162,15 @@ - Log from sysboot.log file. - + Logs contained in sysboot.log file. + Log contenu dans le fichier sysboot.log. sysboot: EVENT - the actual event message - le message décrivant l'événement + The actual event message. + Le message décrivant l'événement. EVENT diff --git a/normalizers/snare.xml b/normalizers/snare.xml index 5f51cb4..963f95d 100644 --- a/normalizers/snare.xml +++ b/normalizers/snare.xml @@ -109,8 +109,8 @@ log['technet_link'] = url + str(value) - - + This is the Snare log format. + Description du format des logs Snare. SNARE_EVENT_LOG_TYPE\s+CRITICALITY\s+SOURCE_NAME\s+SNARE_EVENT_COUNTER\s+[a-zA-Z]{3}. \w+ [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [0-9]{3}\s+EVENT_ID\s+EXPANDED_SOURCENAME\sUSER_NAME\s+SID_TYPE\s+EVENT_LOGTYPE\s+COMPUTER_NAME\s+CATEGORY_STRING\s+DATA_STRING(?:\s+MD5_CHECKSUM)? diff --git a/tests/test_log_samples.py b/tests/test_log_samples.py index 433d69a..6cc04a0 100644 --- a/tests/test_log_samples.py +++ b/tests/test_log_samples.py @@ -515,14 +515,69 @@ def test_Snare(self): def test_vmwareESX4_ESXi4(self): """Test VMware ESX 4.x and VMware ESXi 4.x log normalization""" self.aS("""[2011-09-05 16:06:30.016 F4CD1B90 verbose 'Locale' opID=996867CC-000002A6] Default resource used for 'host.SystemIdentificationInfo.IdentifierType.ServiceTag.summary' expected in module 'enum'.""", - {'date': '2011-09-05T16:06:30.016', - 'numeric': 'F4CD1B90', - 'level': 'verbose', - 'alpha': 'Locale', - 'body': 'Default resource used for \'host.SystemIdentificationInfo.IdentifierType.ServiceTag.summary\' expected in module \'enum\'.'}) + {'date': datetime(2011, 9, 5, 16, 6, 30), + 'numeric': 'F4CD1B90', + 'level': 'verbose', + 'alpha': 'Locale', + 'body': 'Default resource used for \'host.SystemIdentificationInfo.IdentifierType.ServiceTag.summary\' expected in module \'enum\'.'}) self.aS("""sysboot: Executing 'kill -TERM 314'""", {'body': 'Executing \'kill -TERM 314\''}) + def test_mysql(self): + """Test mysql log normalization""" + self.aS("""110923 11:04:58 36 Query show databases""", + {'date': datetime(2011, 9, 23, 11, 4, 58), + 'id': '36', + 'type': 'Query', + 'event': 'show databases'}) + + self.aS("""110923 10:09:11 [Note] Plugin 'FEDERATED' is disabled.""", + {'date': datetime(2011, 9, 23, 10, 9, 11), + 'component': 'Note', + 'event': 'Plugin \'FEDERATED\' is disabled.'}) + + def test_IIS(self): + """Test IIS log normalization""" + self.aS("""172.16.255.255, anonymous, 03/20/01, 23:58:11, MSFTPSVC, SALES1, 172.16.255.255, 60, 275, 0, 0, 0, PASS, /Intro.htm, -,""", + {'source_ip': '172.16.255.255', + 'user': 'anonymous', + 'date': datetime(2001, 3, 20, 23, 58, 11), + 'service': 'MSFTPSVC', + 'dest_host': 'SALES1', + 'dest_ip': '172.16.255.255', + 'time_taken': 0.06, + 'sent_bytes_number': '275', + 'returned_bytes_number': '0', + 'status': '0', + 'windows_status_code': '0', + 'method': 'PASS', + 'target': '/Intro.htm', + 'script_parameters': '-'}) + + self.aS("""2011-09-26 13:57:48 W3SVC1 127.0.0.1 GET /tapage.asp - 80 - 127.0.0.1 Mozilla/4.0+(compatible;MSIE+6.0;+windows+NT5.2;+SV1;+.NET+CLR+1.1.4322) 404 0 2""", + {'date': datetime(2011, 9, 26, 13, 57, 48), + 'service': 'W3SVC1', + 'dest_ip': '127.0.0.1', + 'method': 'GET', + 'target': '/tapage.asp', + 'query': '-', + 'port': '80', + 'user': '-', + 'source_ip': '127.0.0.1', + 'browser': 'Mozilla/4.0+(compatible;MSIE+6.0;+windows+NT5.2;+SV1;+.NET+CLR+1.1.4322)', + 'status': '404', + 'substatus': '0', + 'win_status': '2'}) + if __name__ == "__main__": unittest.main() + + + + + + + + + diff --git a/tests/test_normalizer.py b/tests/test_normalizer.py index a7acdc4..5850536 100644 --- a/tests/test_normalizer.py +++ b/tests/test_normalizer.py @@ -94,6 +94,11 @@ def test_normalize_samples_016_snare(self): def test_normalize_samples_017_vmware(self): self.normalize_samples('VMWare_ESX4-ESXi4.xml', 'VMWare_ESX4-ESXi4', 0.99) + def test_normalize_samples_018_mysql(self): + self.normalize_samples('mysql.xml', 'mysql', 0.99) + + def test_normalize_samples_019_IIS(self): + self.normalize_samples('IIS.xml', 'IIS', 0.99) class TestCSVPattern(unittest.TestCase): """Test CSVPattern behaviour"""