Navigation Menu

Skip to content

Commit

Permalink
modified and finished vmware normaliser, also modified snare, test_lo…
Browse files Browse the repository at this point in the history 
…g_sample and test_normalizer
  • Loading branch information
Constantin committed Oct 5, 2011
1 parent 1e63d52 commit 6022d01
Show file tree
Hide file tree
Showing 4 changed files with 108 additions and 36 deletions.
70 changes: 41 additions & 29 deletions normalizers/VMWare_ESX4-ESXi4.xml
View file
Expand Up @@ -32,66 +32,78 @@
version="0.99"
unicode="yes"
ignorecase="yes"
matchtype= "search"
matchtype= "match"
appliedTo= "raw">

<description>
<localized_desc language="en">This normalizer parses VMware ESX 4.x and ESXi 4.x logs.</localized_desc>
<localized_desc language="fr">Ce normaliser analyse les logs de VMware ESX 4.x et ESXi 4.x.</localized_desc>
<localized_desc language="en">This normalizer parses VMware ESX 4.x and ESXi 4.x logs that are not handled by the Syslog normalizer.</localized_desc>
<localized_desc language="fr">Ce normaliseur analyse les logs de VMware ESX 4.x et ESXi 4.x. qui ne sont pas gérés pas le normaliseur Syslog.</localized_desc>
</description>
<authors>
<author>clo@wallix.com</author>
</authors>
<tagTypes>
<tagType name="date" type="datetime">
<description>
<localized_desc language="en"></localized_desc>
<localized_desc language="fr"></localized_desc>
<localized_desc language="en">Expression matching a date in the format yyyy-mm-dd hh:mm:ss.</localized_desc>
<localized_desc language="fr">Expression correspondant à une date au format yyyy-mm-dd hh:mm:ss.</localized_desc>
</description>
<regexp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}</regexp>
</tagType>
<tagType name="hexa_num" type="integer">
<description>
<localized_desc language="en"></localized_desc>
<localized_desc language="fr"></localized_desc>
<localized_desc language="en">Expression matching a hexadecimal number.</localized_desc>
<localized_desc language="fr">Expression correspondant à un nombre héxadécimal.</localized_desc>
</description>
<regexp>[A-F0-9]{8}</regexp>
</tagType>
<tagType name="words" type="basestring">
<tagType name="alpha_words" type="basestring">
<description>
<localized_desc language="en"></localized_desc>
<localized_desc language="fr"></localized_desc>
<localized_desc language="en">Expression matching the 'alpha' field, words between '.</localized_desc>
<localized_desc language="fr">Expression correspondant au champ 'alpha', qui contient les mots entre '.</localized_desc>
</description>
<regexp>[^']+(?: [^']+)*</regexp>
</tagType>
<tagType name="level_word" type="basestring">
<description>
<localized_desc language="en"></localized_desc>
<localized_desc language="fr"></localized_desc>
<localized_desc language="en">Expression matching the 'level' field.</localized_desc>
<localized_desc language="fr">Expression correspondant au champ 'level'.</localized_desc>
</description>
<regexp>[^\s]+</regexp>
</tagType>

</tagTypes>
<callbacks>
<callback name="modify_date">
date = str(value)
date = date[:10] + 'T' + date[11:]
log["date"] = date
reg = re.compile('(?P&lt;year&gt;\d{4})-(?P&lt;month&gt;\d{2})-(?P&lt;day&gt;\d{2}) (?P&lt;hours&gt;\d{2}):(?P&lt;minutes&gt;\d{2}):(?P&lt;seconds&gt;\d{2})')

date = reg.search(value)

date = date.groupdict()

year= int(date.get('year'))
month = int(date.get('month'))
day = int(date.get('day'))
hours = int(date.get('hours'))
minutes = int(date.get('minutes'))
seconds = int(date.get('seconds'))

newdate = datetime(year, month, day, hours, minutes, seconds)

log["date"] = newdate
</callback>
</callbacks>
<patterns>
<pattern name="VMWare-001">
<description>
<localized_desc language="en">Hostd.log and vpxa.log logs' pattern. (Both agent)</localized_desc>
<localized_desc language="fr"></localized_desc>
<localized_desc language="en">Logs contained in hostd.log file.</localized_desc>
<localized_desc language="fr">Logs contenus dans le fichier hostd.log.</localized_desc>
</description>
<text>\[DATE NUMERIC LEVEL 'ALPHA'[^\]]*\] BODY</text>
<tags>
<tag name="__date" tagType="date">
<description>
<localized_desc language="en">the time at which the request was issued - please note that the timezone information is not carried over.</localized_desc>
<localized_desc language="fr">la date à laquelle la requête a été émise. Veuillez noter que l'information de fuseau horaire n'est pas prise en compte.</localized_desc>
<localized_desc language="en">The time at which the request was issued - please note that the timezone information is not carried over.</localized_desc>
<localized_desc language="fr">La date à laquelle la requête a été émise. Veuillez noter que l'information de fuseau horaire n'est pas prise en compte.</localized_desc>
</description>
<substitute>DATE</substitute>
<callbacks>
Expand All @@ -107,12 +119,12 @@
</tag>
<tag name="level" tagType="level_word">
<description>
<localized_desc language="en"></localized_desc>
<localized_desc language="fr"></localized_desc>
<localized_desc language="en">The level is the type of the log.</localized_desc>
<localized_desc language="fr">Le level correspond au type du log.</localized_desc>
</description>
<substitute>LEVEL</substitute>
</tag>
<tag name="alpha" tagType="words">
<tag name="alpha" tagType="alpha_words">
<description>
<localized_desc language="en"></localized_desc>
<localized_desc language="fr"></localized_desc>
Expand All @@ -121,8 +133,8 @@
</tag>
<tag name="body" tagType="Anything">
<description>
<localized_desc language="en">the actual event message</localized_desc>
<localized_desc language="fr">le message décrivant l'événement</localized_desc>
<localized_desc language="en">The actual event message.</localized_desc>
<localized_desc language="fr">Le message décrivant l'événement.</localized_desc>
</description>
<substitute>BODY</substitute>
</tag>
Expand Down Expand Up @@ -150,15 +162,15 @@
</pattern>
<pattern name="VMWare-002">
<description>
<localized_desc language="en">Log from sysboot.log file.</localized_desc>
<localized_desc language="fr"></localized_desc>
<localized_desc language="en">Logs contained in sysboot.log file.</localized_desc>
<localized_desc language="fr">Log contenu dans le fichier sysboot.log.</localized_desc>
</description>
<text>sysboot: EVENT</text>
<tags>
<tag name="body" tagType="Anything">
<description>
<localized_desc language="en">the actual event message</localized_desc>
<localized_desc language="fr">le message décrivant l'événement</localized_desc>
<localized_desc language="en">The actual event message.</localized_desc>
<localized_desc language="fr">Le message décrivant l'événement.</localized_desc>
</description>
<substitute>EVENT</substitute>
</tag>
Expand Down
4 changes: 2 additions & 2 deletions normalizers/snare.xml
View file
Expand Up @@ -109,8 +109,8 @@ log['technet_link'] = url + str(value)
<patterns>
<pattern name="Snare-001">
<description>
<localized_desc language="en"></localized_desc>
<localized_desc language="fr"></localized_desc>
<localized_desc language="en">This is the Snare log format.</localized_desc>
<localized_desc language="fr">Description du format des logs Snare.</localized_desc>
</description>
<text>SNARE_EVENT_LOG_TYPE\s+CRITICALITY\s+SOURCE_NAME\s+SNARE_EVENT_COUNTER\s+[a-zA-Z]{3}. \w+ [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [0-9]{3}\s+EVENT_ID\s+EXPANDED_SOURCENAME\sUSER_NAME\s+SID_TYPE\s+EVENT_LOGTYPE\s+COMPUTER_NAME\s+CATEGORY_STRING\s+DATA_STRING(?:\s+MD5_CHECKSUM)?</text>
<tags>
Expand Down
65 changes: 60 additions & 5 deletions tests/test_log_samples.py
View file
Expand Up @@ -515,14 +515,69 @@ def test_Snare(self):
def test_vmwareESX4_ESXi4(self):
"""Test VMware ESX 4.x and VMware ESXi 4.x log normalization"""
self.aS("""[2011-09-05 16:06:30.016 F4CD1B90 verbose 'Locale' opID=996867CC-000002A6] Default resource used for 'host.SystemIdentificationInfo.IdentifierType.ServiceTag.summary' expected in module 'enum'.""",
{'date': '2011-09-05T16:06:30.016',
'numeric': 'F4CD1B90',
'level': 'verbose',
'alpha': 'Locale',
'body': 'Default resource used for \'host.SystemIdentificationInfo.IdentifierType.ServiceTag.summary\' expected in module \'enum\'.'})
{'date': datetime(2011, 9, 5, 16, 6, 30),
'numeric': 'F4CD1B90',
'level': 'verbose',
'alpha': 'Locale',
'body': 'Default resource used for \'host.SystemIdentificationInfo.IdentifierType.ServiceTag.summary\' expected in module \'enum\'.'})

self.aS("""sysboot: Executing 'kill -TERM 314'""",
{'body': 'Executing \'kill -TERM 314\''})

def test_mysql(self):
"""Test mysql log normalization"""
self.aS("""110923 11:04:58 36 Query show databases""",
{'date': datetime(2011, 9, 23, 11, 4, 58),
'id': '36',
'type': 'Query',
'event': 'show databases'})

self.aS("""110923 10:09:11 [Note] Plugin 'FEDERATED' is disabled.""",
{'date': datetime(2011, 9, 23, 10, 9, 11),
'component': 'Note',
'event': 'Plugin \'FEDERATED\' is disabled.'})

def test_IIS(self):
"""Test IIS log normalization"""
self.aS("""172.16.255.255, anonymous, 03/20/01, 23:58:11, MSFTPSVC, SALES1, 172.16.255.255, 60, 275, 0, 0, 0, PASS, /Intro.htm, -,""",
{'source_ip': '172.16.255.255',
'user': 'anonymous',
'date': datetime(2001, 3, 20, 23, 58, 11),
'service': 'MSFTPSVC',
'dest_host': 'SALES1',
'dest_ip': '172.16.255.255',
'time_taken': 0.06,
'sent_bytes_number': '275',
'returned_bytes_number': '0',
'status': '0',
'windows_status_code': '0',
'method': 'PASS',
'target': '/Intro.htm',
'script_parameters': '-'})

self.aS("""2011-09-26 13:57:48 W3SVC1 127.0.0.1 GET /tapage.asp - 80 - 127.0.0.1 Mozilla/4.0+(compatible;MSIE+6.0;+windows+NT5.2;+SV1;+.NET+CLR+1.1.4322) 404 0 2""",
{'date': datetime(2011, 9, 26, 13, 57, 48),
'service': 'W3SVC1',
'dest_ip': '127.0.0.1',
'method': 'GET',
'target': '/tapage.asp',
'query': '-',
'port': '80',
'user': '-',
'source_ip': '127.0.0.1',
'browser': 'Mozilla/4.0+(compatible;MSIE+6.0;+windows+NT5.2;+SV1;+.NET+CLR+1.1.4322)',
'status': '404',
'substatus': '0',
'win_status': '2'})

if __name__ == "__main__":
unittest.main()









5 changes: 5 additions & 0 deletions tests/test_normalizer.py
View file
Expand Up @@ -94,6 +94,11 @@ def test_normalize_samples_016_snare(self):
def test_normalize_samples_017_vmware(self):
self.normalize_samples('VMWare_ESX4-ESXi4.xml', 'VMWare_ESX4-ESXi4', 0.99)

def test_normalize_samples_018_mysql(self):
self.normalize_samples('mysql.xml', 'mysql', 0.99)

def test_normalize_samples_019_IIS(self):
self.normalize_samples('IIS.xml', 'IIS', 0.99)

class TestCSVPattern(unittest.TestCase):
"""Test CSVPattern behaviour"""
Expand Down

0 comments on commit 6022d01

Please sign in to comment.