diff --git a/intelmq/bots/parsers/shadowserver/config.py b/intelmq/bots/parsers/shadowserver/config.py index 16bf04689..bda7fe85a 100644 --- a/intelmq/bots/parsers/shadowserver/config.py +++ b/intelmq/bots/parsers/shadowserver/config.py @@ -166,8 +166,6 @@ def validate_ip(value): 'classification.type': 'vulnerable service', 'classification.taxonomy': 'Vulnerable', 'protocol.application': 'mdns', - 'feed.code': 'shadowserver-openmdns', - 'feed.name': 'shadowserver', 'classification.identifier': 'openmdns', }, } @@ -197,8 +195,6 @@ def validate_ip(value): 'classification.type': 'vulnerable service', 'classification.taxonomy': 'Vulnerable', 'classification.identifier': 'openchargen', - 'feed.code': 'shadowserver-openchargen', - 'feed.name': 'shadowserver', 'protocol.application': 'chargen', }, } @@ -231,8 +227,6 @@ def validate_ip(value): 'classification.type': 'vulnerable service', 'classification.taxonomy': 'Vulnerable', 'classification.identifier': 'opentftp', - 'feed.code': 'shadowserver-opentftp', - 'feed.name': 'shadowserver', 'protocol.application': 'tftp', }, } @@ -274,8 +268,6 @@ def validate_ip(value): 'classification.type': 'botnet drone', 'classification.taxonomy': 'Malicious Code', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-sinkhole-http-drone', - 'feed.name': 'shadowserver', }, } @@ -316,8 +308,6 @@ def validate_ip(value): 'protocol.application': 'http', 'classification.taxonomy': 'Malicious Code', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-microsoft-sinkhole', - 'feed.name': 'shadowserver', }, } @@ -358,8 +348,6 @@ def validate_ip(value): 'classification.type': 'vulnerable service', 'classification.taxonomy': 'Vulnerable', 'classification.identifier': 'openredis', - 'feed.code': 'shadowserver-openredis', - 'feed.name': 'shadowserver', 'protocol.application': 'redis', }, } @@ -391,8 +379,6 @@ def validate_ip(value): 'classification.type': 'vulnerable service', 'classification.taxonomy': 'Vulnerable', 'classification.identifier': 'openportmapper', - 'feed.code': 'shadowserver-openportmapper', - 'feed.name': 'shadowserver', 'protocol.application': 'portmapper', }, } @@ -437,8 +423,6 @@ def validate_ip(value): 'classification.type': 'vulnerable service', 'classification.taxonomy': 'Vulnerable', 'classification.identifier': 'openipmi', - 'feed.code': 'shadowserver-openipmi', - 'feed.name': 'shadowserver', 'protocol.application': 'ipmi', 'protocol.transport': 'udp', }, @@ -470,8 +454,6 @@ def validate_ip(value): 'classification.type': 'vulnerable service', 'classification.taxonomy': 'Vulnerable', 'classification.identifier': 'openqotd', - 'feed.code': 'shadowserver-openqotd', - 'feed.name': 'shadowserver', 'protocol.application': 'qotd', }, } @@ -511,8 +493,6 @@ def validate_ip(value): 'classification.type': 'vulnerable service', 'classification.taxonomy': 'Vulnerable', 'classification.identifier': 'openssdp', - 'feed.code': 'shadowserver-openssdp', - 'feed.name': 'shadowserver', 'protocol.application': 'ssdp', }, } @@ -544,8 +524,6 @@ def validate_ip(value): 'classification.taxonomy': 'Vulnerable', 'protocol.application': 'snmp', 'classification.identifier': 'opensnmp', - 'feed.code': 'shadowserver-opensnmp', - 'feed.name': 'shadowserver', }, } @@ -579,8 +557,6 @@ def validate_ip(value): 'classification.type': 'vulnerable service', 'classification.taxonomy': 'Vulnerable', 'classification.identifier': 'openmssql', - 'feed.code': 'shadowserver-openmssql', - 'feed.name': 'shadowserver', 'protocol.application': 'mssql', }, } @@ -619,8 +595,6 @@ def validate_ip(value): 'classification.type': 'vulnerable service', 'classification.taxonomy': 'Vulnerable', 'classification.identifier': 'openmongodb', - 'feed.code': 'shadowserver-openmongodb', - 'feed.name': 'shadowserver', 'protocol.application': 'mongodb', }, } @@ -650,8 +624,6 @@ def validate_ip(value): 'classification.type': 'vulnerable service', 'classification.taxonomy': 'Vulnerable', 'classification.identifier': 'opennetbios', - 'feed.code': 'shadowserver-opennetbios', - 'feed.name': 'shadowserver', 'protocol.application': 'netbios', }, } @@ -690,8 +662,6 @@ def validate_ip(value): 'classification.type': 'vulnerable service', 'classification.taxonomy': 'Vulnerable', 'classification.identifier': 'openelasticsearch', - 'feed.code': 'shadowserver-openelasticsearch', - 'feed.name': 'shadowserver', 'protocol.application': 'elasticsearch', }, } @@ -720,8 +690,6 @@ def validate_ip(value): 'classification.type': 'vulnerable service', 'classification.taxonomy': 'Vulnerable', 'classification.identifier': 'opendns', - 'feed.code': 'shadowserver-opendns', - 'feed.name': 'shadowserver', 'protocol.application': 'dns', }, } @@ -745,8 +713,6 @@ def validate_ip(value): 'classification.type': 'vulnerable service', 'classification.taxonomy': 'Vulnerable', 'classification.identifier': 'openntp', - 'feed.code': 'shadowserver-openntp', - 'feed.name': 'shadowserver', 'protocol.application': 'ntp', }, } @@ -769,8 +735,6 @@ def validate_ip(value): 'classification.type': 'vulnerable service', 'classification.taxonomy': 'Vulnerable', 'classification.identifier': 'SSL-FREAK', - 'feed.code': 'shadowserver-ssl-freak-scan', - 'feed.name': 'shadowserver', 'protocol.application': 'https', }, } @@ -794,8 +758,6 @@ def validate_ip(value): 'classification.type': 'vulnerable service', 'classification.taxonomy': 'Vulnerable', 'classification.identifier': 'SSL-Poodle', - 'feed.code': 'shadowserver-ssl-scan', - 'feed.name': 'shadowserver', 'protocol.application': 'https', }, } @@ -819,8 +781,6 @@ def validate_ip(value): 'classification.type': 'vulnerable service', 'classification.taxonomy': 'Vulnerable', 'classification.identifier': 'openmemcached', - 'feed.code': 'shadowserver-openmemcached', - 'feed.name': 'shadowserver', 'protocol.application': 'memcached', }, } @@ -859,8 +819,6 @@ def validate_ip(value): 'classification.type': 'botnet drone', 'classification.taxonomy': 'Malicious Code', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-botnet-drone-hadoop', - 'feed.name': 'shadowserver', }, } @@ -889,9 +847,7 @@ def validate_ip(value): 'classification.type': 'vulnerable service', 'classification.taxonomy': 'Vulnerable', 'protocol.application': 'xdmcp', - 'feed.code': 'shadowserver-openxdmcp', - 'feed.name': 'shadowserver', - 'feed.url': 'https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-XDMCP', + # 'feed.url': 'https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-XDMCP', 'classification.identifier': 'openxdmcp', }, } @@ -922,9 +878,7 @@ def validate_ip(value): 'classification.type': 'vulnerable service', 'classification.taxonomy': 'Vulnerable', 'protocol.application': 'nat-pmp', - 'feed.code': 'shadowserver-opennatpmp', - 'feed.name': 'shadowserver', - 'feed.url': 'https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-NATPMP', + # 'feed.url': 'https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-NATPMP', 'classification.identifier': 'opennatpmp', }, } diff --git a/intelmq/tests/bots/parsers/shadowserver/test_chargen.py b/intelmq/tests/bots/parsers/shadowserver/test_chargen.py index 603d388f8..95a46e89c 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_chargen.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_chargen.py @@ -30,13 +30,12 @@ "time.observation": "2015-01-01T00:00:00+00:00", } EVENTS = [{'__type': 'Event', + 'feed.name': 'ShadowServer Chargen', 'classification.type': 'vulnerable service', 'classification.identifier': 'openchargen', 'classification.taxonomy': 'Vulnerable', 'extra': '{"naics": 123456, "response_size": 116, "sic": 654321, ' '"tag": "chargen"}', - 'feed.code': 'shadowserver-openchargen', - 'feed.name': 'shadowserver', 'protocol.application': 'chargen', 'protocol.transport': 'udp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], @@ -50,12 +49,11 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-03-16T04:15:19+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Chargen', 'classification.type': 'vulnerable service', 'classification.identifier': 'openchargen', 'classification.taxonomy': 'Vulnerable', 'extra': '{"response_size": 116, "tag": "chargen"}', - 'feed.code': 'shadowserver-openchargen', - 'feed.name': 'shadowserver', 'protocol.application': 'chargen', 'protocol.transport': 'udp', 'raw': utils.base64_encode( @@ -72,12 +70,11 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-03-16T04:15:19+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Chargen', 'classification.type': 'vulnerable service', 'classification.identifier': 'openchargen', 'classification.taxonomy': 'Vulnerable', 'extra': '{"response_size": 116, "tag": "chargen"}', - 'feed.code': 'shadowserver-openchargen', - 'feed.name': 'shadowserver', 'protocol.application': 'chargen', 'protocol.transport': 'udp', 'raw': utils.base64_encode( @@ -96,12 +93,11 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-03-16T04:15:19+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Chargen', 'classification.type': 'vulnerable service', 'classification.identifier': 'openchargen', 'classification.taxonomy': 'Vulnerable', 'extra': '{"response_size": 116, "tag": "chargen"}', - 'feed.code': 'shadowserver-openchargen', - 'feed.name': 'shadowserver', 'protocol.application': 'chargen', 'protocol.transport': 'udp', 'raw': utils.base64_encode( @@ -118,12 +114,11 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-03-16T04:15:19+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Chargen', 'classification.type': 'vulnerable service', 'classification.identifier': 'openchargen', 'classification.taxonomy': 'Vulnerable', 'extra': '{"response_size": 116, "tag": "chargen"}', - 'feed.code': 'shadowserver-openchargen', - 'feed.name': 'shadowserver', 'protocol.application': 'chargen', 'protocol.transport': 'udp', 'raw': utils.base64_encode( @@ -141,12 +136,11 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-03-16T04:15:19+00:00'}] EVENT_SHORT = {'__type': 'Event', + 'feed.name': 'ShadowServer Chargen', 'classification.type': 'vulnerable service', 'classification.identifier': 'openchargen', 'classification.taxonomy': 'Vulnerable', 'extra': '{"tag": "chargen"}', - 'feed.code': 'shadowserver-openchargen', - 'feed.name': 'shadowserver', 'protocol.application': 'chargen', 'protocol.transport': 'udp', 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINE_SHORT[0], diff --git a/intelmq/tests/bots/parsers/shadowserver/test_drone_hadoop.py b/intelmq/tests/bots/parsers/shadowserver/test_drone_hadoop.py index c38f8e196..90587ec42 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_drone_hadoop.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_drone_hadoop.py @@ -22,16 +22,15 @@ "time.observation": "2015-01-01T00:00:00+00:00", } EVENTS = [{'__type': 'Event', + 'feed.name': 'ShadowServer Botnet Drone Hadoop', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-botnet-drone-hadoop', 'destination.asn': 8560, 'destination.geolocation.cc': 'US', 'destination.ip': '74.208.164.166', 'destination.port': 80, 'extra': '{"connection_count": 1, "os.name": "Windows", "os.version": "2000 SP4, XP SP1+"}', - 'feed.name': 'shadowserver', 'malware.name': 'sinkhole', 'protocol.transport': 'tcp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], @@ -45,16 +44,15 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2011-04-23T00:00:05+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Botnet Drone Hadoop', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-botnet-drone-hadoop', 'destination.asn': 16265, 'destination.fqdn': '015.maxided.com', 'destination.geolocation.cc': 'NL', 'destination.ip': '94.75.228.147', 'extra': '{"connection_count": 1, "os.name": "WINXP"}', - 'feed.name': 'shadowserver', 'malware.name': 'spyeye', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], RECONSTRUCTED_LINES[2], ''])), @@ -67,16 +65,15 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2011-04-23T00:00:08+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Botnet Drone Hadoop', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-botnet-drone-hadoop', 'destination.asn': 8560, 'destination.geolocation.cc': 'DE', 'destination.ip': '87.106.24.200', 'destination.port': 80, 'extra': '{"os.name": "Windows", "os.version": "XP SP1+, 2000 SP3 (2)"}', - 'feed.name': 'shadowserver', 'malware.name': 'sinkhole', 'protocol.transport': 'tcp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], @@ -90,16 +87,15 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2011-04-23T00:00:10+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Botnet Drone Hadoop', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-botnet-drone-hadoop', 'destination.asn': 8560, 'destination.geolocation.cc': 'DE', 'destination.ip': '87.106.24.200', 'destination.port': 443, 'extra': '{"connection_count": 1, "os.name": "Windows", "os.version": "2000 SP4, XP SP1+"}', - 'feed.name': 'shadowserver', 'malware.name': 'sinkhole', 'protocol.transport': 'tcp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], @@ -113,16 +109,15 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2011-04-23T00:00:15+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Botnet Drone Hadoop', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-botnet-drone-hadoop', 'destination.asn': 8560, 'destination.geolocation.cc': 'US', 'destination.ip': '74.208.164.166', 'destination.port': 443, 'extra': '{"connection_count": 1, "os.name": "Windows", "os.version": "2000 SP4, XP SP1+"}', - 'feed.name': 'shadowserver', 'malware.name': 'sinkhole', 'protocol.transport': 'tcp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], @@ -136,16 +131,15 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2011-04-23T00:00:26+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Botnet Drone Hadoop', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-botnet-drone-hadoop', 'destination.asn': 8560, 'destination.geolocation.cc': 'DE', 'destination.ip': '87.106.24.200', 'destination.port': 443, 'extra': '{"os.name": "Windows", "os.version": "2000 SP4, XP SP1+"}', - 'feed.name': 'shadowserver', 'malware.name': 'sinkhole', 'protocol.transport': 'tcp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], @@ -159,16 +153,15 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2011-04-23T00:00:28+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Botnet Drone Hadoop', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-botnet-drone-hadoop', 'destination.asn': 8560, 'destination.geolocation.cc': 'DE', 'destination.ip': '87.106.24.200', 'destination.port': 443, 'extra': '{"connection_count": 1, "os.name": "Windows", "os.version": "XP/2000 (RFC1323+, w+, tstamp+)"}', - 'feed.name': 'shadowserver', 'malware.name': 'sinkhole', 'protocol.transport': 'tcp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], @@ -182,16 +175,15 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2011-04-23T00:00:29+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Botnet Drone Hadoop', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-botnet-drone-hadoop', 'destination.asn': 8560, 'destination.geolocation.cc': 'US', 'destination.ip': '74.208.164.166', 'destination.port': 80, 'extra': '{"connection_count": 1, "os.name": "Windows", "os.version": "XP SP1+, 2000 SP3 (2)"}', - 'feed.name': 'shadowserver', 'malware.name': 'sinkhole', 'protocol.transport': 'tcp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], @@ -205,16 +197,15 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2011-04-23T00:00:33+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Botnet Drone Hadoop', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-botnet-drone-hadoop', 'destination.asn': 8560, 'destination.geolocation.cc': 'US', 'destination.ip': '74.208.164.166', 'destination.port': 443, 'extra': '{"connection_count": 1, "os.name": "Windows", "os.version": "2000 SP4, XP SP1+"}', - 'feed.name': 'shadowserver', 'malware.name': 'sinkhole', 'protocol.transport': 'tcp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], @@ -228,16 +219,15 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2011-04-23T00:00:36+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Botnet Drone Hadoop', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-botnet-drone-hadoop', 'destination.asn': 8560, 'destination.geolocation.cc': 'DE', 'destination.ip': '87.106.24.200', 'destination.port': 443, 'extra': '{"connection_count": 1, "os.name": "Windows", "os.version": "2000 SP4, XP SP1+"}', - 'feed.name': 'shadowserver', 'malware.name': 'sinkhole', 'protocol.transport': 'tcp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], diff --git a/intelmq/tests/bots/parsers/shadowserver/test_microsoft_sinkhole.py b/intelmq/tests/bots/parsers/shadowserver/test_microsoft_sinkhole.py index 33719dd47..c7b2ef684 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_microsoft_sinkhole.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_microsoft_sinkhole.py @@ -17,21 +17,20 @@ RECONSTRUCTED_FILE = handle.read() RECONSTRUCTED_LINES = RECONSTRUCTED_FILE.splitlines() -EXAMPLE_REPORT = {"feed.name": "ShadowServer QOTD", +EXAMPLE_REPORT = {"feed.name": "ShadowServer Microsoft-Sinkhole", "raw": utils.base64_encode(EXAMPLE_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", } EVENTS = [{'__type': 'Event', + 'feed.name': 'ShadowServer Microsoft-Sinkhole', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-microsoft-sinkhole', 'destination.asn': 8075, 'destination.geolocation.cc': 'SG', 'destination.ip': '168.63.184.224', 'destination.port': 16470, - 'feed.name': 'shadowserver', 'malware.name': 'b68-zeroaccess-1-64bit', 'protocol.application': 'http', 'protocol.transport': 'tcp', @@ -44,15 +43,14 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-09-12T00:00:00+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Microsoft-Sinkhole', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-microsoft-sinkhole', 'destination.asn': 8075, 'destination.geolocation.cc': 'HK', 'destination.ip': '168.63.202.23', 'destination.port': 16470, - 'feed.name': 'shadowserver', 'malware.name': 'b68-zeroaccess-1-64bit', 'protocol.application': 'http', 'protocol.transport': 'tcp', @@ -65,15 +63,14 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-09-12T00:00:00+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Microsoft-Sinkhole', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-microsoft-sinkhole', 'destination.asn': 16265, 'destination.geolocation.cc': 'NL', 'destination.ip': '82.192.70.219', 'destination.port': 16471, - 'feed.name': 'shadowserver', 'malware.name': 'b68-zeroaccess-1-32bit', 'protocol.application': 'http', 'protocol.transport': 'tcp', @@ -86,15 +83,14 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-09-12T00:00:00+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Microsoft-Sinkhole', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-microsoft-sinkhole', 'destination.asn': 8075, 'destination.geolocation.cc': 'SG', 'destination.ip': '168.63.184.224', 'destination.port': 16470, - 'feed.name': 'shadowserver', 'malware.name': 'b68-zeroaccess-1-64bit', 'protocol.application': 'http', 'protocol.transport': 'tcp', @@ -107,15 +103,14 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-09-12T00:00:00+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Microsoft-Sinkhole', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-microsoft-sinkhole', 'destination.asn': 8075, 'destination.geolocation.cc': 'HK', 'destination.ip': '207.46.138.117', 'destination.port': 16464, - 'feed.name': 'shadowserver', 'malware.name': 'b68-zeroaccess-2-32bit', 'protocol.application': 'http', 'protocol.transport': 'tcp', @@ -128,15 +123,14 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-09-12T00:00:00+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Microsoft-Sinkhole', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-microsoft-sinkhole', 'destination.asn': 8075, 'destination.geolocation.cc': 'SG', 'destination.ip': '168.63.240.164', 'destination.port': 16464, - 'feed.name': 'shadowserver', 'malware.name': 'b68-zeroaccess-2-32bit', 'protocol.application': 'http', 'protocol.transport': 'tcp', @@ -149,17 +143,16 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-09-12T00:00:00+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Microsoft-Sinkhole', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-microsoft-sinkhole', 'destination.asn': 8075, 'destination.geolocation.cc': 'US', 'destination.ip': '204.95.99.205', 'destination.port': 443, 'destination.url': 'http://204.95.99.205/index.php', 'extra': '{"http_host": "204.95.99.205", "user_agent": "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.8077)"}', - 'feed.name': 'shadowserver', 'malware.name': 'caphaw', 'protocol.application': 'http', 'protocol.transport': 'tcp', @@ -172,10 +165,10 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-09-12T00:00:00+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Microsoft-Sinkhole', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-microsoft-sinkhole', 'destination.asn': 8075, 'destination.geolocation.cc': 'US', 'destination.ip': '204.95.99.204', @@ -183,7 +176,6 @@ 'destination.port': 443, 'destination.url': 'http://xf5wau9lcpf5.oonucoog.cc/ping.html', 'extra': '{"user_agent": "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.7357)"}', - 'feed.name': 'shadowserver', 'malware.name': 'caphaw', 'protocol.application': 'http', 'protocol.transport': 'tcp', @@ -196,10 +188,10 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-09-12T00:00:00+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Microsoft-Sinkhole', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-microsoft-sinkhole', 'destination.asn': 8075, 'destination.geolocation.cc': 'US', 'destination.ip': '204.95.99.204', @@ -207,7 +199,6 @@ 'destination.port': 443, 'destination.url': 'http://3k3kwrnj.rgk.cc/index.php', 'extra': '{"user_agent": "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.9121)"}', - 'feed.name': 'shadowserver', 'malware.name': 'caphaw', 'protocol.application': 'http', 'protocol.transport': 'tcp', @@ -220,10 +211,10 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-09-12T00:00:00+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Microsoft-Sinkhole', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-microsoft-sinkhole', 'destination.asn': 3598, 'destination.geolocation.cc': 'US', 'destination.ip': '199.2.137.201', @@ -231,7 +222,6 @@ 'destination.port': 80, 'destination.url': 'http://ultimaresource.com/wild/live/file.php', 'extra': '{"user_agent": "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; BRI/1)"}', - 'feed.name': 'shadowserver', 'malware.name': 'citadel-b54', 'protocol.application': 'http', 'protocol.transport': 'tcp', @@ -244,17 +234,16 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-09-12T00:00:01+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Microsoft-Sinkhole', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-microsoft-sinkhole', 'destination.asn': 3598, 'destination.geolocation.cc': 'US', 'destination.ip': '199.2.137.202', 'destination.port': 80, 'destination.url': 'http://199.2.137.202/file-b29d40.php', 'extra': '{"http_host": "199.2.137.202", "user_agent": "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; WOW64; .NET CLR 3.5.21022)"}', - 'feed.name': 'shadowserver', 'malware.name': 'citadel-b54', 'protocol.application': 'http', 'protocol.transport': 'tcp', @@ -267,10 +256,10 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-09-12T00:00:01+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Microsoft-Sinkhole', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-microsoft-sinkhole', 'destination.asn': 3598, 'destination.geolocation.cc': 'US', 'destination.ip': '199.2.137.201', @@ -278,7 +267,6 @@ 'destination.port': 80, 'destination.url': 'http://prohomemain.com/367601b6737825deb58a244576e4f098/file.php', 'extra': '{"user_agent": "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; AskTB5.6)"}', - 'feed.name': 'shadowserver', 'malware.name': 'citadel-b54', 'protocol.application': 'http', 'protocol.transport': 'tcp', @@ -291,10 +279,10 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-09-12T00:00:01+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Microsoft-Sinkhole', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-microsoft-sinkhole', 'destination.asn': 3598, 'destination.geolocation.cc': 'US', 'destination.ip': '199.2.137.202', @@ -302,7 +290,6 @@ 'destination.port': 80, 'destination.url': 'http://ronapri.com/view/file.php', 'extra': '{"user_agent": "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; AskTbFWV5/5.11.3.15590)"}', - 'feed.name': 'shadowserver', 'malware.name': 'citadel-b54', 'protocol.application': 'http', 'protocol.transport': 'tcp', @@ -315,10 +302,10 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-09-12T00:00:01+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer Microsoft-Sinkhole', 'classification.taxonomy': 'Malicious Code', 'classification.type': 'botnet drone', 'classification.identifier': 'botnet', - 'feed.code': 'shadowserver-microsoft-sinkhole', 'destination.asn': 3598, 'destination.geolocation.cc': 'US', 'destination.ip': '199.2.137.201', @@ -326,7 +313,6 @@ 'destination.port': 80, 'destination.url': 'http://9A5BB34EEDE4B85B9E81F40D530B68FF.co.cc/message.php', 'extra': '{"user_agent": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; .NET4.0C)"}', - 'feed.name': 'shadowserver', 'malware.name': 'bamital-b58', 'protocol.application': 'http', 'protocol.transport': 'tcp', diff --git a/intelmq/tests/bots/parsers/shadowserver/test_qotd.py b/intelmq/tests/bots/parsers/shadowserver/test_qotd.py index faca0291b..ff58919ed 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_qotd.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_qotd.py @@ -22,12 +22,11 @@ "time.observation": "2015-01-01T00:00:00+00:00", } EVENTS = [{'__type': 'Event', + 'feed.name': 'ShadowServer QOTD', 'classification.identifier': 'openqotd', 'classification.taxonomy': 'Vulnerable', 'classification.type': 'vulnerable service', 'extra': '{"naics": 123456, "quote": "N?s matamos o tempo, mas ele enterra-nos.?? (Machado de Assis)??", "sic": 654321, "tag": "qotd"}', - 'feed.code': 'shadowserver-openqotd', - 'feed.name': 'shadowserver', 'protocol.application': 'qotd', 'protocol.transport': 'udp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], @@ -42,12 +41,11 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-03-16T08:12:33+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer QOTD', 'classification.identifier': 'openqotd', 'classification.taxonomy': 'Vulnerable', 'classification.type': 'vulnerable service', 'extra': '{"quote": "When a stupid man is doing something he is ashamed of, he always declares?? that it is his duty. George Bernard Shaw (1856-1950)??", "tag": "qotd"}', - 'feed.code': 'shadowserver-openqotd', - 'feed.name': 'shadowserver', 'protocol.application': 'qotd', 'protocol.transport': 'udp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], @@ -61,12 +59,11 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-03-16T08:12:33+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer QOTD', 'classification.identifier': 'openqotd', 'classification.taxonomy': 'Vulnerable', 'classification.type': 'vulnerable service', 'extra': '{"quote": "_The secret of being miserable is to have leisure to bother about whether?? you are happy or not. The cure for it is occupation._?? George Bernard Shaw (1856-1950)??", "tag": "qotd"}', - 'feed.code': 'shadowserver-openqotd', - 'feed.name': 'shadowserver', 'protocol.application': 'qotd', 'protocol.transport': 'udp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], @@ -81,12 +78,11 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-03-16T08:12:34+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer QOTD', 'classification.identifier': 'openqotd', 'classification.taxonomy': 'Vulnerable', 'classification.type': 'vulnerable service', 'extra': '{"quote": "_We have no more right to consume happiness without producing it than to?? consume wealth without producing it._ George Bernard Shaw (1856-1950)??", "tag": "qotd"}', - 'feed.code': 'shadowserver-openqotd', - 'feed.name': 'shadowserver', 'protocol.application': 'qotd', 'protocol.transport': 'udp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], diff --git a/intelmq/tests/bots/parsers/shadowserver/test_snmp.py b/intelmq/tests/bots/parsers/shadowserver/test_snmp.py index 91e9e6757..9935427b6 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_snmp.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_snmp.py @@ -22,12 +22,11 @@ "time.observation": "2015-01-01T00:00:00+00:00", } EVENTS = [{'__type': 'Event', + 'feed.name': 'ShadowServer SNMP', 'classification.identifier': 'opensnmp', 'classification.taxonomy': 'Vulnerable', 'classification.type': 'vulnerable service', 'extra': '{"sysdesc": "Hardware: x86 Family 6 Model 8 Stepping 6 AT/AT COMPATIBLE - Software: Windows 2000 Version 5.0 (Build 2195 Uniprocessor Free)", "sysname": "ORSONKA", "version": 2}', - 'feed.code': 'shadowserver-opensnmp', - 'feed.name': 'shadowserver', 'protocol.application': 'snmp', 'protocol.transport': 'udp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], @@ -42,12 +41,11 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-03-16T03:45:50+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer SNMP', 'classification.identifier': 'opensnmp', 'classification.taxonomy': 'Vulnerable', 'classification.type': 'vulnerable service', 'extra': '{"sysdesc": "ADSL Modem", "sysname": "tc", "version": 2}', - 'feed.code': 'shadowserver-opensnmp', - 'feed.name': 'shadowserver', 'protocol.application': 'snmp', 'protocol.transport': 'udp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], @@ -62,12 +60,11 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-03-16T03:45:51+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer SNMP', 'classification.identifier': 'opensnmp', 'classification.taxonomy': 'Vulnerable', 'classification.type': 'vulnerable service', 'extra': '{"version": 2}', - 'feed.code': 'shadowserver-opensnmp', - 'feed.name': 'shadowserver', 'protocol.application': 'snmp', 'protocol.transport': 'udp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], @@ -82,12 +79,11 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-03-16T03:45:51+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer SNMP', 'classification.identifier': 'opensnmp', 'classification.taxonomy': 'Vulnerable', 'classification.type': 'vulnerable service', 'extra': '{"sysdesc": "Linux ADSL2PlusRouter 2.6.19 #7 Tue Apr 9 17:06:16 CST 2013 mips", "sysname": "TD5130", "version": 2}', - 'feed.code': 'shadowserver-opensnmp', - 'feed.name': 'shadowserver', 'protocol.application': 'snmp', 'protocol.transport': 'udp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], @@ -102,12 +98,11 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-03-16T03:45:51+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer SNMP', 'classification.identifier': 'opensnmp', 'classification.taxonomy': 'Vulnerable', 'classification.type': 'vulnerable service', 'extra': '{"sysdesc": "Linux R6100 2.6.31 #1 Tue Jun 4 06:50:58 EDT 2013 mips MIB=01a01", "sysname": "Unknow", "version": 2}', - 'feed.code': 'shadowserver-opensnmp', - 'feed.name': 'shadowserver', 'protocol.application': 'snmp', 'protocol.transport': 'udp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], @@ -122,12 +117,11 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-03-16T03:45:51+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer SNMP', 'classification.identifier': 'opensnmp', 'classification.taxonomy': 'Vulnerable', 'classification.type': 'vulnerable service', 'extra': '{"sysdesc": "110TC1", "sysname": "Beetel", "version": 2}', - 'feed.code': 'shadowserver-opensnmp', - 'feed.name': 'shadowserver', 'protocol.application': 'snmp', 'protocol.transport': 'udp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], @@ -142,12 +136,11 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-03-16T03:45:51+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer SNMP', 'classification.identifier': 'opensnmp', 'classification.taxonomy': 'Vulnerable', 'classification.type': 'vulnerable service', 'extra': '{"sysdesc": "BCW710J <>", "sysname": "CableHome", "version": 2}', - 'feed.code': 'shadowserver-opensnmp', - 'feed.name': 'shadowserver', 'protocol.application': 'snmp', 'protocol.transport': 'udp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], @@ -162,12 +155,11 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-03-16T03:45:51+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer SNMP', 'classification.identifier': 'opensnmp', 'classification.taxonomy': 'Vulnerable', 'classification.type': 'vulnerable service', 'extra': '{"sysdesc": "Linux WNR1000v2 2.6.15 #199 Thu Jan 28 09:49:57 CST 2010 mips MIB=01a01", "sysname": "Unknow", "version": 2}', - 'feed.code': 'shadowserver-opensnmp', - 'feed.name': 'shadowserver', 'protocol.application': 'snmp', 'protocol.transport': 'udp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], @@ -182,12 +174,11 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-03-16T03:45:51+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer SNMP', 'classification.identifier': 'opensnmp', 'classification.taxonomy': 'Vulnerable', 'classification.type': 'vulnerable service', 'extra': '{"version": 2}', - 'feed.code': 'shadowserver-opensnmp', - 'feed.name': 'shadowserver', 'protocol.application': 'snmp', 'protocol.transport': 'udp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], @@ -201,12 +192,11 @@ 'time.observation': '2015-01-01T00:00:00+00:00', 'time.source': '2014-03-16T03:45:51+00:00'}, {'__type': 'Event', + 'feed.name': 'ShadowServer SNMP', 'classification.identifier': 'opensnmp', 'classification.taxonomy': 'Vulnerable', 'classification.type': 'vulnerable service', 'extra': '{"sysdesc": "D-Link Wireless Voice Gateway <>", "sysname": "CableHome", "version": 2}', - 'feed.code': 'shadowserver-opensnmp', - 'feed.name': 'shadowserver', 'protocol.application': 'snmp', 'protocol.transport': 'udp', 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], diff --git a/intelmq/tests/bots/parsers/shadowserver/test_xdmcp.py b/intelmq/tests/bots/parsers/shadowserver/test_xdmcp.py new file mode 100644 index 000000000..cfda8875c --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/test_xdmcp.py @@ -0,0 +1,207 @@ +# -*- coding: utf-8 -*- + +import os +import unittest + +import intelmq.lib.test as test +import intelmq.lib.utils as utils +import pprint + +from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot + +with open(os.path.join(os.path.dirname(__file__), 'xdmcp.csv')) as handle: + EXAMPLE_FILE = handle.read() +EXAMPLE_LINES = EXAMPLE_FILE.splitlines() + +with open(os.path.join(os.path.dirname(__file__), + 'xdmcp_RECONSTRUCTED.csv')) as handle: + RECONSTRUCTED_FILE = handle.read() +RECONSTRUCTED_LINES = RECONSTRUCTED_FILE.splitlines() + +pprint.pprint(RECONSTRUCTED_LINES) + +EXAMPLE_REPORT = {"feed.name": "ShadowServer XDMCP", + "raw": utils.base64_encode(EXAMPLE_FILE), + "__type": "Report", + "time.observation": "2015-01-01T00:00:00+00:00", + } +EVENTS = [{'__type': 'Event', + 'feed.name': 'ShadowServer XDMCP', + 'classification.identifier': 'openxdmcp', + 'classification.taxonomy': 'Vulnerable', + 'classification.type': 'vulnerable service', + 'extra': '{"opcode": "Willing", "reported_hostname": "netmanage", "size": "50", "status": "Linux 2.6.32-573.3.1.el6.i686", "tag": "xdmcp"}', + 'protocol.application': 'xdmcp', + 'protocol.transport': 'udp', + 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], + RECONSTRUCTED_LINES[1], ''])), + 'source.asn': 4812, + 'source.geolocation.cc': 'CN', + 'source.geolocation.city': 'SHANGHAI', + 'source.geolocation.region': 'SHANGHAI', + 'source.ip': '61.152.122.54', + 'source.port': 177, + 'time.observation': '2016-05-17T19:04:55+00:00', + 'time.source': '2016-05-17T19:04:55+00:00'}, + {'__type': 'Event', + 'feed.name': 'ShadowServer XDMCP', + 'classification.identifier': 'openxdmcp', + 'classification.taxonomy': 'Vulnerable', + 'classification.type': 'vulnerable service', + 'extra': '{"opcode": "Willing", "reported_hostname": "bimsdev1", "size": "48", "status": "0 users load: 0.0, 0.0, 0.0", "tag": "xdmcp"}', + 'protocol.application': 'xdmcp', + 'protocol.transport': 'udp', + 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], + RECONSTRUCTED_LINES[2], ''])), + 'source.asn': 4837, + 'source.geolocation.cc': 'CN', + 'source.geolocation.city': 'TIANJIN', + 'source.geolocation.region': 'TIANJIN', + 'source.ip': '218.68.63.240', + 'source.port': 177, + 'time.observation': '2015-01-01T00:00:00+00:00', + 'time.source': '2016-05-17T19:04:56+00:00'}, + {'__type': 'Event', + 'feed.name': 'ShadowServer XDMCP', + 'classification.identifier': 'openxdmcp', + 'classification.taxonomy': 'Vulnerable', + 'classification.type': 'vulnerable service', + 'extra': '{"opcode": "Willing", "reported_hostname": "zyite01", "size": "50", "status": "4 users load: 28.2, 28.6, 28.8", "tag": "xdmcp"}', + 'protocol.application': 'xdmcp', + 'protocol.transport': 'udp', + 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], + RECONSTRUCTED_LINES[3], ''])), + 'source.asn': 9808, + 'source.geolocation.cc': 'CN', + 'source.geolocation.city': 'HARBIN', + 'source.geolocation.region': 'HEILONGJIANG', + 'source.ip': '211.137.249.158', + 'source.port': 177, + 'time.source': '2016-05-17T19:04:56+00:00'}, + {'__type': 'Event', + 'feed.name': 'ShadowServer XDMCP', + 'classification.identifier': 'openxdmcp', + 'classification.taxonomy': 'Vulnerable', + 'classification.type': 'vulnerable service', + 'extra': '{"opcode": "Willing", "reported_hostname": "PAGOS", "size": "44", "status": "Linux 3.12.55-52.42-default", "tag": "xdmcp"}', + 'protocol.application': 'xdmcp', + 'protocol.transport': 'udp', + 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], + RECONSTRUCTED_LINES[4], ''])), + 'source.asn': 8151, + 'source.geolocation.cc': 'MX', + 'source.geolocation.city': 'MEDELLIN DE BRAVO', + 'source.geolocation.region': 'VERACRUZ', + 'source.reverse_dns': 'customer-187-174-250-38.uninet-ide.com.mx', + 'source.ip': '187.174.250.38', + 'source.port': 177, + 'time.source': '2016-05-17T19:04:57+00:00'}, + {'__type': 'Event', + 'feed.name': 'ShadowServer XDMCP', + 'classification.identifier': 'openxdmcp', + 'classification.taxonomy': 'Vulnerable', + 'classification.type': 'vulnerable service', + 'extra': '{"opcode": "Willing", "reported_hostname": "linux-ws15", "size": "52", "status": "0 user, load: 0.00, 0.00, 0.00", "tag": "xdmcp"}', + 'protocol.application': 'xdmcp', + 'protocol.transport': 'udp', + 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], + RECONSTRUCTED_LINES[5], ''])), + 'source.asn': 3549, + 'source.geolocation.cc': 'CO', + 'source.geolocation.city': 'SANTIAGO DE CALI', + 'source.geolocation.region': 'VALLE DEL CAUCA', + 'source.ip': '152.231.30.35', + 'source.port': 177, + 'time.source': '2016-05-17T19:04:57+00:00'}, + {'__type': 'Event', + 'feed.name': 'ShadowServer XDMCP', + 'classification.identifier': 'openxdmcp', + 'classification.taxonomy': 'Vulnerable', + 'classification.type': 'vulnerable service', + 'extra': '{"opcode": "Unwilling", "reported_hostname": "mvodtown", "size": "51", "status": "!Display not authorized to connect", "tag": "xdmcp"}', + 'protocol.application': 'xdmcp', + 'protocol.transport': 'udp', + 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], + RECONSTRUCTED_LINES[6], ''])), + 'source.asn': 9318, + 'source.geolocation.cc': 'KR', + 'source.geolocation.city': 'SEOUL', + 'source.geolocation.region': 'SEOUL TEUGBYEOLSI', + 'source.ip': '218.39.178.182', + 'source.port': 177, + 'time.source': '2016-05-17T19:04:57+00:00' }, + {'__type': 'Event', + 'feed.name': 'ShadowServer XDMCP', + 'classification.identifier': 'openxdmcp', + 'classification.taxonomy': 'Vulnerable', + 'classification.type': 'vulnerable service', + 'protocol.application': 'xdmcp', + 'protocol.transport': 'udp', + 'extra': '{"opcode": "Willing", "reported_hostname": "WASWP", "size": "45", "status": "0 users load: 0.1, 0.2, 0.2", "tag": "xdmcp"}', + 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], + RECONSTRUCTED_LINES[7], ''])), + 'source.asn': 38661, + 'source.geolocation.cc': 'KR', + 'source.geolocation.city': 'GURO-DONG', + 'source.geolocation.region': 'SEOUL TEUGBYEOLSI', + 'source.ip': '121.0.141.75', + 'source.port': 177, + 'time.source': '2016-05-17T19:04:57+00:00'}, + {'__type': 'Event', + 'feed.name': 'ShadowServer XDMCP', + 'classification.identifier': 'openxdmcp', + 'classification.taxonomy': 'Vulnerable', + 'classification.type': 'vulnerable service', + 'extra': '{"opcode": "Willing", "reported_hostname": "VENDITTI.localdomain.net", "size": "58", "status": "Linux 2.6.32-64GB-i686", "tag": "xdmcp"}', + 'protocol.application': 'xdmcp', + 'protocol.transport': 'udp', + 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], + RECONSTRUCTED_LINES[8], ''])), + 'source.asn': 12874, + 'source.geolocation.cc': 'IT', + 'source.geolocation.city': 'CASORIA', + 'source.geolocation.region': 'NAPOLI', + 'source.ip': '89.97.0.73', + 'source.port': 177, + 'source.reverse_dns': '89-97-0-73.ip2.fastwebnet.it', + 'time.source': '2016-05-17T19:04:58+00:00'}, + {'__type': 'Event', + 'feed.name': 'ShadowServer XDMCP', + 'classification.identifier': 'openxdmcp', + 'classification.taxonomy': 'Vulnerable', + 'classification.type': 'vulnerable service', + 'extra': '{"opcode": "Willing", "reported_hostname": "kasei", "size": "45", "status": "0 users load: 11., 11., 11.", "tag": "xdmcp"}', + 'protocol.application': 'xdmcp', + 'protocol.transport': 'udp', + 'raw': utils.base64_encode('\n'.join([RECONSTRUCTED_LINES[0], + RECONSTRUCTED_LINES[9], ''])), + 'source.asn': 11105, + 'source.geolocation.cc': 'CA', + 'source.geolocation.city': 'BURNABY', + 'source.geolocation.region': 'BRITISH COLUMBIA', + 'source.ip': '209.87.31.2', + 'source.port': 177, + 'source.reverse_dns': 'kasei.cecm.sfu.ca', + 'time.source': '2016-05-17T19:04:58+00:00'}] + + +class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): + """ + A TestCase for a ShadowserverParserBot. + """ + + @classmethod + def set_bot(cls): + cls.bot_reference = ShadowserverParserBot + cls.default_input_message = EXAMPLE_REPORT + cls.sysconfig = {'feedname': 'Open-XDMCP'} + + def test_event(self): + """ Test if correct Event has been produced. """ + self.run_bot() + for i, EVENT in enumerate(EVENTS): + self.assertMessageEqual(i, EVENT) + + +if __name__ == '__main__': + unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/xdmcp.csv b/intelmq/tests/bots/parsers/shadowserver/xdmcp.csv new file mode 100644 index 000000000..03172d233 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/xdmcp.csv @@ -0,0 +1,10 @@ +"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","opcode","reported_hostname","status","size" +"2016-05-17 19:04:55","61.152.122.54","udp",177,,"xdmcp",4812,"CN","SHANGHAI","SHANGHAI",0,0,"Willing","netmanage","Linux 2.6.32-573.3.1.el6.i686",50 +"2016-05-17 19:04:56","218.68.63.240","udp",177,,"xdmcp",4837,"CN","TIANJIN","TIANJIN",0,0,"Willing","bimsdev1","0 users load: 0.0, 0.0, 0.0",48 +"2016-05-17 19:04:56","211.137.249.158","udp",177,,"xdmcp",9808,"CN","HEILONGJIANG","HARBIN",0,0,"Willing","zyite01","4 users load: 28.2, 28.6, 28.8",50 +"2016-05-17 19:04:57","187.174.250.38","udp",177,"customer-187-174-250-38.uninet-ide.com.mx","xdmcp",8151,"MX","VERACRUZ","MEDELLIN DE BRAVO",0,0,"Willing","PAGOS","Linux 3.12.55-52.42-default",44 +"2016-05-17 19:04:57","152.231.30.35","udp",177,,"xdmcp",3549,"CO","VALLE DEL CAUCA","SANTIAGO DE CALI",0,0,"Willing","linux-ws15","0 user, load: 0.00, 0.00, 0.00",52 +"2016-05-17 19:04:57","218.39.178.182","udp",177,,"xdmcp",9318,"KR","SEOUL TEUGBYEOLSI","SEOUL",0,0,"Unwilling","mvodtown","!Display not authorized to connect",51 +"2016-05-17 19:04:57","121.0.141.75","udp",177,,"xdmcp",38661,"KR","SEOUL TEUGBYEOLSI","GURO-DONG",0,0,"Willing","WASWP","0 users load: 0.1, 0.2, 0.2",45 +"2016-05-17 19:04:58","89.97.0.73","udp",177,"89-97-0-73.ip2.fastwebnet.it","xdmcp",12874,"IT","NAPOLI","CASORIA",0,0,"Willing","VENDITTI.localdomain.net","Linux 2.6.32-64GB-i686",58 +"2016-05-17 19:04:58","209.87.31.2","udp",177,"kasei.cecm.sfu.ca","xdmcp",11105,"CA","BRITISH COLUMBIA","BURNABY",0,0,"Willing","kasei","0 users load: 11., 11., 11.",45 diff --git a/intelmq/tests/bots/parsers/shadowserver/xdmcp_RECONSTRUCTED.csv b/intelmq/tests/bots/parsers/shadowserver/xdmcp_RECONSTRUCTED.csv new file mode 100644 index 000000000..a30ff3648 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/xdmcp_RECONSTRUCTED.csv @@ -0,0 +1,10 @@ +"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","opcode","reported_hostname","status","size" +"2016-05-17 19:04:55","61.152.122.54","udp","177","","xdmcp","4812","CN","SHANGHAI","SHANGHAI","0","0","Willing","netmanage","Linux 2.6.32-573.3.1.el6.i686","50" +"2016-05-17 19:04:56","218.68.63.240","udp","177","","xdmcp","4837","CN","TIANJIN","TIANJIN","0","0","Willing","bimsdev1","0 users load: 0.0, 0.0, 0.0","48" +"2016-05-17 19:04:56","211.137.249.158","udp","177","","xdmcp","9808","CN","HEILONGJIANG","HARBIN","0","0","Willing","zyite01","4 users load: 28.2, 28.6, 28.8","50" +"2016-05-17 19:04:57","187.174.250.38","udp","177","customer-187-174-250-38.uninet-ide.com.mx","xdmcp","8151","MX","VERACRUZ","MEDELLIN DE BRAVO","0","0","Willing","PAGOS","Linux 3.12.55-52.42-default","44" +"2016-05-17 19:04:57","152.231.30.35","udp","177","","xdmcp","3549","CO","VALLE DEL CAUCA","SANTIAGO DE CALI","0","0","Willing","linux-ws15","0 user, load: 0.00, 0.00, 0.00","52" +"2016-05-17 19:04:57","218.39.178.182","udp","177","","xdmcp","9318","KR","SEOUL TEUGBYEOLSI","SEOUL","0","0","Unwilling","mvodtown","!Display not authorized to connect","51" +"2016-05-17 19:04:57","121.0.141.75","udp","177","","xdmcp","38661","KR","SEOUL TEUGBYEOLSI","GURO-DONG","0","0","Willing","WASWP","0 users load: 0.1, 0.2, 0.2","45" +"2016-05-17 19:04:58","89.97.0.73","udp","177","89-97-0-73.ip2.fastwebnet.it","xdmcp","12874","IT","NAPOLI","CASORIA","0","0","Willing","VENDITTI.localdomain.net","Linux 2.6.32-64GB-i686","58" +"2016-05-17 19:04:58","209.87.31.2","udp","177","kasei.cecm.sfu.ca","xdmcp","11105","CA","BRITISH COLUMBIA","BURNABY","0","0","Willing","kasei","0 users load: 11., 11., 11.","45"