diff --git a/.travis.yml b/.travis.yml index aae4545..b70b57f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,22 +1,21 @@ language: python python: -- '2.6' - '2.7' -- '3.3' - '3.4' +- '3.5' +- '3.6' services: - elasticsearch - mongodb install: -- pip install pep8 --use-mirrors -- pip install pyflakes --use-mirrors +- pip install pep8 +- pip install pyflakes - pip install coveralls - pip install jsonpickle - pip install pymongo elasticsearch - pip freeze - python setup.py install before_script: -- pep8 . --exclude test,docs,examples,build --ignore=W503,E123 - pyflakes libnessus/*.py - sleep 10 script: nosetests --with-coverage --cover-package=libnessus diff --git a/libnessus/exceptions.py b/libnessus/exceptions.py new file mode 100644 index 0000000..062b36a --- /dev/null +++ b/libnessus/exceptions.py @@ -0,0 +1,4 @@ +class MissingAttribute(Exception): + """Error when Nessus report items are missing essential properties""" + def __init__(self, *args, **kwargs): + Exception.__init__(self, "Report object is missing essential attributes", *args, **kwargs) diff --git a/libnessus/objects/report.py b/libnessus/objects/report.py index 9f1e049..1ee4085 100644 --- a/libnessus/objects/report.py +++ b/libnessus/objects/report.py @@ -11,7 +11,7 @@ class NessusReport(object): in a easy way the content, and present some metadata """ def __init__(self, name, hosts): - ''' + """ Description: Constructor of NessusReport :param name: name of the report :type name: str @@ -19,18 +19,18 @@ def __init__(self, name, hosts): :type hosts: list :return: NessusReport :rtype: NessusReport - ''' + """ self.name = name self.__hosts = hosts self.__start = self.__compute_started(self.__hosts) self.__end = self.__compute_ended(self.__hosts) def __repr__(self): - ''' + """ Description: compute a string of the obj :return: description de la valeur de retour :rtype: str - ''' + """ return "{name} {total} {elapsed}".format(name=self.name, total=self.hosts_total, elapsed=self.elapsed) @@ -43,13 +43,13 @@ def hosts(self): return self.__hosts def save(self, backend): - ''' + """ Description: allow to persist to a backend :param backend: libnessus.plugins.PluginBackend object. :type arg1: PluginBackend :return: The primary key of the stored object is returned. :rtype: str - ''' + """ try: _id = backend.insert(self) return _id @@ -58,24 +58,24 @@ def save(self, backend): raise def iscomparable(self, other): - ''' + """ description: check if two obj are comparable by checking the class name :param other: nessusreport :type other: nessusreport :raises: typeerror if not comparable - ''' + """ if not isinstance(other, self.__class__): raise TypeError(("non sense incompatibe object : ", self, other)) def __eq__(self, other): - ''' + """ Description: compare obj as equal :param other: another report :type other: NessusReport :return: boolean :rtype: boolean - ''' + """ try: self.iscomparable(other) rdict = self.diff(other) @@ -89,13 +89,13 @@ def __eq__(self, other): raise etyperr def __ne__(self, other): - ''' + """ Description: compare obj as != :param other: another report :type other: NessusReport :return: boolean :rtype: boolean - ''' + """ try: self.iscomparable(other) rdict = self.diff(other) @@ -105,12 +105,12 @@ def __ne__(self, other): raise etyperr def __get_dict(self): - ''' + """ Description: get a dict representation of the object Needed to transform the obj in a dict representation to use dictdiffer :return: dict representation of the object :rtype: dict - ''' + """ rdict = {} rdict['name'] = self.name hostitem = dict( @@ -121,13 +121,13 @@ def __get_dict(self): return rdict def diff(self, other): - ''' + """ Description: diff object and provide the differences :param other: obj to compare to :type other: NessusReport :return: a dict of all the differences :rtype: dict - ''' + """ diff = DictDiffer(self.__get_dict(), other.__get_dict()) rdict = {} rdict["removed"] = diff.removed() diff --git a/libnessus/objects/reporthost.py b/libnessus/objects/reporthost.py index 8b77153..bc98347 100644 --- a/libnessus/objects/reporthost.py +++ b/libnessus/objects/reporthost.py @@ -1,17 +1,20 @@ #!/usr/bin/env python -''' +""" File: reporthost.py Author: Me Description: -''' +""" from libnessus.objects.dictdiffer import DictDiffer +from libnessus.objects import reportlogger +from libnessus import exceptions as NessusExceptions +log = reportlogger.ReportLogger class NessusReportHost(object): - ''' + """ Description: Represent an object NessusReportHost in a nessus xml - ''' + """ def __init__(self, host_properties={}, report_items=[]): _minimal_attr = set(['HOST_START', 'HOST_END', 'host-ip', 'name']) self._hostprop_attr = set(host_properties.keys()) @@ -20,7 +23,9 @@ def __init__(self, host_properties={}, report_items=[]): if len(_missing_attr) == 0: self.__host_properties = host_properties else: - raise Exception("Not all the attributes to create a decent " + log.debug("Host Missing Attributes: ") + log.debug(host_properties) + raise NessusExceptions.MissingAttribute("Not all the attributes to create a decent " "NessusReportHost are available. " "Missing: {}".format(" ".join(_missing_attr))) @@ -28,11 +33,7 @@ def __init__(self, host_properties={}, report_items=[]): def __repr__(self): """return a string representation of the obj nessusHost""" - retstr = "{0} {1} {2} {3}".format(self.name, - self.address, - self.get_host_properties, - self.get_total_vuln) - return retstr + return "{0} {1} {2} {3}".format(self.name, self.address, self.get_host_properties, self.get_total_vuln) def __hash__(self): """:return: hash function to be able to add object to dict/set @@ -41,26 +42,26 @@ def __hash__(self): return hash(self.address) def iscomparable(self, other): - ''' + """ Description: check if two obj are comparable by checking the class name and adress value are equal :param other: NessusReportHost :type other: NessusReportHost :raises: TypeError if not comparable - ''' + """ if not isinstance(other, self.__class__): raise TypeError(("Non sense incompatibe object : ", self, other)) if self.address != other.address: raise TypeError(("Address need to be == : ", self, other)) def __eq__(self, other): - ''' + """ Description: compare all properties and reportitem :param other: the object to compare :type other: NessusReportHost :return: true if equal :rtype: boolean - ''' + """ try: self.iscomparable(other) except TypeError as etyperr: @@ -74,13 +75,13 @@ def __eq__(self, other): return res_pro def __ne__(self, other): - ''' + """ Description: :param other: the object to compare :type other: NessusReportHost :return: true if equal :rtype: boolean - ''' + """ try: self.iscomparable(other) except TypeError as etyperr: @@ -90,13 +91,13 @@ def __ne__(self, other): return res_pro def __get_dict(self): - ''' + """ Description: get a dict representation of the object Needed because the object has 2 main component : a dict and a table of ReportItem :return: dict representation of the object :rtype: dict - ''' + """ rdict = self.get_host_properties.copy() # add reportitem in the dict in the form # key = {'NessusReportItem::10544': NessusReportItem,} @@ -108,13 +109,13 @@ def __get_dict(self): return rdict def diff(self, other): - ''' + """ Description: compute a diff dict obj :param other: the object to compare :type other: NessusReportHost :return: :rtype: dict - ''' + """ diff = DictDiffer(self.__get_dict(), other.__get_dict()) rdict = {} rdict["removed"] = diff.removed() @@ -152,7 +153,7 @@ def get_host_properties(self): @property def get_hostprop_attr(self): - """Return a set of keys reprsenting all properties' key + """Return a set of keys representing all properties' key :return: set """ return self._hostprop_attr diff --git a/libnessus/objects/reportitem.py b/libnessus/objects/reportitem.py index 89d1a1b..c02f5f9 100644 --- a/libnessus/objects/reportitem.py +++ b/libnessus/objects/reportitem.py @@ -1,9 +1,9 @@ #!/usr/bin/env python -''' +""" File: vuln.py Description: -''' - +""" +from libnessus import exceptions from libnessus.objects.dictdiffer import DictDiffer @@ -28,7 +28,7 @@ def __init__(self, vuln_info=None): if len(_missing_attr) == 0: self.__vuln_info = vuln_info else: - raise Exception("Not all the attributes to create a decent " + raise exceptions.MissingAttribute("Not all the attributes to create a decent " "NessusVuln object are available. " "Missing: ", _missing_attr) @@ -49,13 +49,13 @@ def __hash__(self): return hash(self.plugin_id) def iscomparable(self, other): - ''' + """ Description: check if two obj are comparable by checking the class name and plugin_id value are equal :param other: NessusReportItem :type other: NessusReportItem :raises: TypeError if not comparable - ''' + """ if not isinstance(other, self.__class__): raise TypeError(("Non sense incompatibe object : ", self, other)) if self.plugin_id != other.plugin_id: @@ -95,7 +95,7 @@ def __ne__(self, other): ) def diff(self, other): - ''' + """ Description: Compare two NessusReportItem :param other: NessusReportItem :type other: NessusReportItem @@ -104,7 +104,7 @@ def diff(self, other): get_vuln_info property that have changed :rtype: dict :raises: TypeError - ''' + """ try: self.iscomparable(other) except TypeError as etyperr: @@ -174,6 +174,17 @@ def plugin_name(self): plugin_name = self.__vuln_info['pluginName'] return plugin_name + @property + def cve(self): + """ + Get CVE or return empty string + :return str + """ + for (k, v) in self.__vuln_info.items(): + if k == 'cve': + return str(v) + return '' + @property def plugin_family(self): """ @@ -255,7 +266,7 @@ def description(self): @property def solution(self): """ - Get the sulution provide by nessus + Get the solution provide by nessus :return str """ return self.__vuln_info['solution'] diff --git a/libnessus/objects/reportlogger.py b/libnessus/objects/reportlogger.py new file mode 100644 index 0000000..7b68802 --- /dev/null +++ b/libnessus/objects/reportlogger.py @@ -0,0 +1,53 @@ +import logging + +BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) + +# The background is set with 40 plus the number of the color, and the foreground with 30 +# These are the sequences need to get colored ouput +RESET_SEQ = "\033[0m" +COLOR_SEQ = "\033[1;%dm" +BOLD_SEQ = "\033[1m" +COLORS = { + 'WARNING': YELLOW, + 'INFO': WHITE, + 'DEBUG': BLUE, + 'CRITICAL': YELLOW, + 'ERROR': RED +} + + +def formatter_message(message, use_color=True): + if use_color: + message = message.replace("$RESET", RESET_SEQ).replace("$BOLD", BOLD_SEQ) + else: + message = message.replace("$RESET", "").replace("$BOLD", "") + return message + + +class ColoredFormatter(logging.Formatter): + def __init__(self, msg, use_color = True): + logging.Formatter.__init__(self, msg) + self.use_color = use_color + + def format(self, record): + levelname = record.levelname + if self.use_color and levelname in COLORS: + levelname_color = COLOR_SEQ % (30 + COLORS[levelname]) + levelname + RESET_SEQ + record.levelname = levelname_color + return logging.Formatter.format(self, record) + + +# Custom logger class with multiple destinations +class ReportLogger(logging.Logger): + FORMAT = "[$BOLD%(name)-20s$RESET][%(levelname)-10s] %(asctime)-10s %(message)s ($BOLD%(filename)s$RESET:%(lineno)d)" + COLOR_FORMAT = formatter_message(FORMAT, True) + + def __init__(self, name, log_level='INFO'): + logging.Logger.__init__(self, name, log_level) + color_formatter = ColoredFormatter(self.COLOR_FORMAT) + + console = logging.StreamHandler() + console.setFormatter(color_formatter) + + self.addHandler(console) + return diff --git a/libnessus/parser.py b/libnessus/parser.py index 0bc442e..38760ba 100644 --- a/libnessus/parser.py +++ b/libnessus/parser.py @@ -1,11 +1,21 @@ #!/usr/bin/env python import xml.etree.ElementTree as ET +from libnessus import exceptions as NessusExceptions from libnessus.objects import NessusReportHost, NessusReportItem, NessusReport +from libnessus.objects import reportlogger + +log = reportlogger.ReportLogger('PARSER') class NessusParser(object): @classmethod - def parse(cls, nessus_data, data_type='XML'): + def parse(cls, nessus_data, data_type='XML', strict=False): + """ + Parses a Nessus data file + :param nessus_data The datafile to be parsed + :param data_type File format (XML by default) + :param strict Forces strict parsing of report objects which may be invalid + """ nessusobj = None if not isinstance(nessus_data, str): @@ -13,14 +23,14 @@ def parse(cls, nessus_data, data_type='XML'): "Nessus data should be provided as strings") if nessus_data and data_type == "XML": - nessusobj = cls._parse_xml(nessus_data) + nessusobj = cls._parse_xml(nessus_data, strict) else: raise Exception("No or unknown data type provided. Please check " "documentation for supported data types.") return nessusobj @classmethod - def _parse_xml(cls, nessus_data=None): + def _parse_xml(cls, nessus_data=None, strict=False): try: root = ET.fromstring(nessus_data) except: @@ -30,9 +40,9 @@ def _parse_xml(cls, nessus_data=None): if root.tag == 'NessusClientData': nessusobj = cls._parse_xmlv1(root) elif root.tag == 'NessusClientData_v2': - nessusobj = cls._parse_xmlv2(root) + nessusobj = cls._parse_xmlv2(root, strict) else: - raise Exception("Unpexpected data structure for XML root node") + raise Exception("Unexpected data structure for XML root node") return nessusobj @classmethod @@ -40,10 +50,10 @@ def _parse_xmlv1(cls, root=None): raise Exception("Nessus XML v1 parsing is not supported yet.") @classmethod - def _parse_xmlv2(cls, root=None): + def _parse_xmlv2(cls, root=None, strict=False): """ This private method will return 0 or one report - (as describe in nessus's doc) + (as described in the nessus documentation) :param root: a string representing a part or a complete nessus scan :return: NessusReport or None """ @@ -52,7 +62,7 @@ def _parse_xmlv2(cls, root=None): for nessus_report in root.findall("Report"): nessus_hosts = [] for nessus_host in nessus_report.findall("ReportHost"): - _nhost = cls.parse_host(nessus_host) + _nhost = cls.parse_host(nessus_host, strict) nessus_hosts.append(_nhost) if 'name' in nessus_report.attrib: @@ -65,7 +75,7 @@ def _parse_xmlv2(cls, root=None): return nrp @classmethod - def parse_host(cls, root=None): + def parse_host(cls, root=None, strict=False): _host_name = root.attrib['name'] if 'name' in root.attrib else 'none' _host_prop_elt = root.find("HostProperties") _dhp = dict([(e.attrib['name'], e.text) for e in list(_host_prop_elt)]) @@ -73,7 +83,15 @@ def parse_host(cls, root=None): _vuln_list = [] for report_item in root.findall("ReportItem"): - _new_item = cls.parse_reportitem(report_item) + try: + _new_item = cls.parse_reportitem(report_item) + except NessusExceptions.MissingAttribute: + if strict: + log.error("Strict parsing enforced: Invalid report item encountered!") + raise + else: + log.warning("Invalid report item encountered... Skipping") + continue _vuln_list.append(_new_item) return NessusReportHost(_dhp, _vuln_list) @@ -81,13 +99,13 @@ def parse_host(cls, root=None): @classmethod def parse_reportitem(cls, root=None): """ - This function parse the xml and return an object ReportItem - This object stick as much as possible to the xml + This function parses the xml and returns a ReportItem object + This object contains everything from the Nessus XML see http://static.tenable.com/documentation/nessus_v2_file_format.pdf if an element can be represented more than once it will become a list """ _vuln_data = {} - # add all attrib in the dict + # add all attrib in the dicts _vuln_data.update(root.attrib) # parse each elem and add it to the dict # + create a list as value if needed @@ -104,17 +122,17 @@ def parse_reportitem(cls, root=None): return NessusReportItem(_vuln_data) @classmethod - def parse_fromstring(cls, nessus_data, data_type="XML"): + def parse_fromstring(cls, nessus_data, data_type="XML", strict=False): if not isinstance(nessus_data, str): raise Exception("bad argument type : should be a string") - return cls.parse(nessus_data, data_type) + return cls.parse(nessus_data, data_type, strict) @classmethod - def parse_fromfile(cls, nessus_report_path, data_type="XML"): + def parse_fromfile(cls, nessus_report_path, data_type="XML", strict=False): try: with open(nessus_report_path, 'r') as fileobj: fdata = fileobj.read() - rval = cls.parse(fdata, data_type) + rval = cls.parse(fdata, data_type, strict) except IOError: raise return rval diff --git a/libnessus/test/test_host.py b/libnessus/test/test_host.py index a8d18a1..d6fb35b 100644 --- a/libnessus/test/test_host.py +++ b/libnessus/test/test_host.py @@ -154,9 +154,9 @@ def test_hash(self): i = i + 1 def test_iscomparable(self): - ''' + """ test_host: test to throw TypeError in case of uncompatible obj - ''' + """ dictHost = { 'host-ip': "255.255.255.255", 'name': "wakawakawaka", @@ -172,11 +172,11 @@ def test_iscomparable(self): self.assertRaises(TypeError, value.iscomparable, 5) def test_eq(self): - ''' + """ test_host : test equality retrieve self.forgedHost and play with it no need to test with other ip since allready tested in iscomparable - ''' + """ h1 = self.forgedHost h2 = copy.deepcopy(h1) # after copy should be equal @@ -188,11 +188,11 @@ def test_eq(self): self.assertRaises(TypeError, h1.__eq__, 5) def test_ne(self): - ''' + """ test_host : test not equal retrieve self.forgedHost and play with it no need to test with other ip since already tested in iscomparable - ''' + """ h1 = self.forgedHost h2 = copy.deepcopy(h1) # after copy should be equal diff --git a/libnessus/test/test_nessus.py b/libnessus/test/test_nessus.py index 9c89336..45dcab0 100644 --- a/libnessus/test/test_nessus.py +++ b/libnessus/test/test_nessus.py @@ -6,10 +6,10 @@ class TestNessus(unittest.TestCase): - '''TestNEssus class only contains the setUp functions all test class will - inherit from this one''' + """TestNEssus class only contains the setUp functions all test class will + inherit from this one""" def setUp(self): - '''setup a table of report based on the files in flist ''' + """setup a table of report based on the files in flist """ self.fdir = os.path.dirname(os.path.realpath(__file__)) self.flist = [ {'file': "%s/%s" % (self.fdir, 'files/nessus_report_local2.nessus'), diff --git a/libnessus/test/test_report.py b/libnessus/test/test_report.py index 42a536e..4a166ba 100644 --- a/libnessus/test/test_report.py +++ b/libnessus/test/test_report.py @@ -28,9 +28,9 @@ def test_save(self): """ def test_iscomparable(self): - ''' + """ test_iscomparable test to throm typeError if not the same type - ''' + """ value = self.forgedreport # test different type self.assertRaises(TypeError, value.iscomparable, 5) diff --git a/libnessus/test/test_reportitem.py b/libnessus/test/test_reportitem.py index 0b089ab..8c16778 100644 --- a/libnessus/test/test_reportitem.py +++ b/libnessus/test/test_reportitem.py @@ -159,9 +159,9 @@ def test_ne(self): self.assertRaises(TypeError, vuln.__ne__, 5) def test_iscomparable(self): - ''' + """ test_reportitem: test to throw TypeError in case of uncompatible obj - ''' + """ dictvuln = { 'port': "23456", 'svc_name': "general", @@ -188,9 +188,9 @@ def test_init(self): self.assertRaises(Exception, NessusReportItem, dictvuln) def test_diff(self): - ''' + """ test the diff (should return dict of 4 keys) - ''' + """ for vuln in self.VulnList: value = vuln self.assertRaises(TypeError, value.diff, 5)