Bug 774419 - New package: MozTest (or similar) for Mozbase;r=jgriffin

moztest/README.md

@@ -0,0 +1,16 @@
+# Moztest
+
+Package for handling Mozilla test results.
+
+
+## Usage example
+
+This shows how you can create an xUnit representation of python unittest results.
+
+    from results import TestResultCollection
+    from output import XUnitOutput
+
+    collection = TestResultCollection.from_unittest_results(results)
+    out = XUnitOutput()
+    with open('out.xml', 'w') as f:
+        out.serialize(collection, f)

moztest/moztest/output.py

@@ -0,0 +1,105 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+from abc import abstractmethod
+from contextlib import closing
+from StringIO import StringIO
+import xml.dom.minidom as dom
+
+
+class Output(object):
+    """ Abstract base class for outputting test results """
+
+    @abstractmethod
+    def serialize(self, results_collection, file_obj):
+        """ Writes the string representation of the results collection
+        to the given file object"""
+
+    def dump_string(self, results_collection):
+        """ Returns the string representation of the results collection """
+        with closing(StringIO()) as s:
+            self.serialize(results_collection, s)
+            return s.getvalue()
+
+
+class XUnitOutput(Output):
+    """ Class for writing xUnit formatted test results in an XML file """
+
+    def serialize(self, results_collection, file_obj):
+        """ Writes the xUnit formatted results to the given file object """
+
+        def _extract_xml(test_result, text='', result='Pass'):
+            if not isinstance(text, basestring):
+                text = '\n'.join(text)
+
+            cls_name = test_result.test_class
+
+            # if the test class is not already created, create it
+            if cls_name not in classes:
+                cls = doc.createElement('class')
+                cls.setAttribute('name', cls_name)
+                assembly.appendChild(cls)
+                classes[cls_name] = cls
+
+            t = doc.createElement('test')
+            t.setAttribute('name', test_result.name)
+            t.setAttribute('result', result)
+
+            if result == 'Fail':
+                f = doc.createElement('failure')
+                st = doc.createElement('stack-trace')
+                st.appendChild(doc.createTextNode(text))
+
+                f.appendChild(st)
+                t.appendChild(f)
+
+            elif result == 'Skip':
+                r = doc.createElement('reason')
+                msg = doc.createElement('message')
+                msg.appendChild(doc.createTextNode(text))
+
+                r.appendChild(msg)
+                t.appendChild(f)
+
+            cls = classes[cls_name]
+            cls.appendChild(t)
+
+        doc = dom.Document()
+
+        assembly = doc.createElement('assembly')
+        assembly.setAttribute('name', results_collection.suite_name)
+        assembly.setAttribute('time', str(results_collection.time_taken))
+        assembly.setAttribute('total', str(len(results_collection)))
+        assembly.setAttribute('passed',
+                              str(len(list(results_collection.tests_with_result('PASS')))))
+        assembly.setAttribute('failed', str(len(list(results_collection.unsuccessful))))
+        assembly.setAttribute('skipped',
+                              str(len(list(results_collection.tests_with_result('SKIPPED')))))
+
+        classes = {}  # str -> xml class element
+
+        for tr in results_collection.tests_with_result('ERROR'):
+            _extract_xml(tr, text=tr.output, result='Fail')
+
+        for tr in results_collection.tests_with_result('UNEXPECTED-FAIL'):
+            _extract_xml(tr, text=tr.output, result='Fail')
+
+        for tr in results_collection.tests_with_result('UNEXPECTED-PASS'):
+            _extract_xml(tr, text='TEST-UNEXPECTED-PASS', result='Fail')
+
+        for tr in results_collection.tests_with_result('SKIPPED'):
+            _extract_xml(tr, text=tr.output, result='Skip')
+
+        for tr in results_collection.tests_with_result('KNOWN-FAIL'):
+            _extract_xml(tr, text=tr.output, result='Skip')
+
+        for tr in results_collection.tests_with_result('PASS'):
+            _extract_xml(tr, result='Pass')
+
+        for cls in classes.itervalues():
+            assembly.appendChild(cls)
+
+        doc.appendChild(assembly)
+        file_obj.write(doc.toxml(encoding='utf-8'))

moztest/moztest/results.py

@@ -0,0 +1,223 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import time
+import os
+import mozinfo
+
+
+class TestContext(object):
+    """ Stores context data about the test """
+
+    def __init__(self, hostname='localhost'):
+        self.hostname = hostname
+        self.arch = mozinfo.processor
+        self.env = os.environ.copy()
+        self.os = mozinfo.os
+        self.os_version = mozinfo.version
+
+    def __str__(self):
+        return '%s (%s, %s)' % (self.hostname, self.os, self.arch)
+
+    def __repr__(self):
+        return '<%s>' % self.__str__()
+
+
+class TestResult(object):
+    """ Stores test result data """
+
+    POSSIBLE_RESULTS = [
+        'START',
+        'PASS',
+        'UNEXPECTED-PASS',
+        'UNEXPECTED-FAIL',
+        'KNOWN-FAIL',
+        'SKIPPED',
+        'ERROR',
+    ]
+
+    def __init__(self, name, test_class='', time_start=None, context=None,
+                 result_expected='PASS'):
+        """ Create a TestResult instance.
+        name = name of the test that is running
+        test_class = the class that the test belongs to
+        time_start = timestamp (seconds since UNIX epoch) of when the test started
+                     running; if not provided, defaults to the current time
+        context = TestContext instance; can be None
+        result_expected = string representing the expected outcome of the test"""
+
+        assert isinstance(name, basestring), "name has to be a string"
+        assert result_expected in self.POSSIBLE_RESULTS, "Result '%s' not in \
+                    possible results: %s" %\
+                    (result_expected, ', '.join(self.POSSIBLE_RESULTS))
+
+        self.name = name
+        self.test_class = test_class
+        self.context = context
+        self.time_start = time_start or time.time()
+        self.time_end = None
+        self.result_expected = result_expected
+        self.result_actual = None
+        self.filename = None
+        self.description = None
+        self.output = []
+        self.reason = None
+
+    def __str__(self):
+        return '%s | %s (%s) | %s' % (self.result_actual or 'PENDING',
+                                      self.name, self.test_class, self.reason)
+
+    def __repr__(self):
+        return '<%s>' % self.__str__()
+
+    def finish(self, result, time_end=None, output=None, reason=None):
+        """ Marks the test as finished, storing its end time and status """
+        assert result in TestResult.POSSIBLE_RESULTS
+
+        # use lists instead of multiline strings
+        if isinstance(output, basestring):
+            output = output.splitlines()
+
+        self.time_end = time_end or time.time()
+        self.output = output or self.output
+        self.result_actual = result
+        self.reason = reason
+
+    @property
+    def finished(self):
+        """ Boolean saying if the test is finished or not """
+        return self.result_actual is not None
+
+    @property
+    def duration(self):
+        """ Returns the time it took for the test to finish. If the test is
+        not finished, returns the elapsed time so far """
+        if self.result_actual is not None:
+            return self.time_end - self.time_start
+        else:
+            # returns the elapsed time
+            return time.time() - self.time_start
+
+    @property
+    def successful(self):
+        """ Boolean saying if the test was successful or not. None in
+        case the test is not finished """
+        if self.result_actual is not None:
+            return self.result_expected == self.result_actual
+        else:
+            return None
+
+
+class TestResultCollection(list):
+    """ Container class that stores test results """
+
+    def __init__(self, suite_name, time_taken=0):
+        list.__init__(self)
+        self.suite_name = suite_name
+        self.time_taken = time_taken
+
+    def __str__(self):
+        return "%s (%.2fs)\n%s" % (self.suite_name, self.time_taken,
+                                list.__str__(self))
+
+    @property
+    def contexts(self):  # don't mind test contexts yet
+        """ List of unique contexts for the test results contained """
+        cs = [tr.context for tr in self]
+        return list(set(cs))
+
+    def filter(self, predicate):
+        """ Returns a generator of TestResults that satisfy a given predicate """
+        return (tr for tr in self if predicate(tr))
+
+    def tests_with_result(self, result):
+        """ Returns a generator of TestResults with the given result """
+        return self.filter(lambda t: t.result_actual == result)
+
+    @property
+    def successful(self):
+        """ Returns a generator of successful TestResults"""
+        return self.filter(lambda t: t.successful)
+
+    @property
+    def unsuccessful(self):
+        """ Returns a generator of unsuccessful TestResults"""
+        return self.filter(lambda t: not t.successful)
+
+    @property
+    def tests(self):
+        """ Generator of all tests in the collection """
+        return (t for t in self)
+
+    def add_unittest_result(self, result, context=None):
+        """ Adds the python unittest result provided to the collection"""
+
+        def get_class(test):
+            return test.__class__.__module__ + '.' + test.__class__.__name__
+
+        def add_test_result(test, result_expected='PASS',
+                            result_actual='PASS', output=''):
+            t = TestResult(name=str(test).split()[0], test_class=get_class(test),
+                           time_start=0, result_expected=result_expected,
+                           context=context)
+            t.finish(result_actual, time_end=0, reason=relevant_line(output),
+                     output=output)
+            self.append(t)
+
+        self.time_taken += result.time_taken
+
+        for test, output in result.errors:
+            add_test_result(test, result_actual='ERROR', output=output)
+
+        for test, output in result.failures:
+            add_test_result(test, result_actual='UNEXPECTED-FAIL',
+                            output=output)
+
+        for test in result.unexpectedSuccesses:
+            add_test_result(test, result_expected='KNOWN-FAIL',
+                            result_actual='UNEXPECTED-PASS')
+
+        for test, output in result.skipped:
+            add_test_result(test, result_expected='SKIPPED',
+                            result_actual='SKIPPED', output=output)
+
+        for test, output in result.expectedFailures:
+            add_test_result(test, result_expected='KNOWN-FAIL',
+                            result_actual='KNOWN-FAIL', output=output)
+
+        # unittest does not store these by default
+        if hasattr(result, 'tests_passed'):
+            for test in result.tests_passed:
+                add_test_result(test)
+
+    @classmethod
+    def from_unittest_results(cls, results_list):
+        """ Creates a TestResultCollection containing the given python
+        unittest results """
+
+        # so we can pass in a single unittest result instance as well
+        if type(results_list) is not list:
+            results_list = list(results_list)
+
+        # all the TestResult instances share the same context
+        context = TestContext()
+
+        collection = cls('from %s' % results_list[0].__class__.__name__)
+
+        for result in results_list:
+            collection.add_unittest_result(result, context)
+
+        return collection
+
+
+# used to get exceptions/errors from tracebacks
+def relevant_line(s):
+    KEYWORDS = ('Error:', 'Exception:', 'error:', 'exception:')
+    lines = s.splitlines()
+    for line in lines:
+        for keyword in KEYWORDS:
+            if keyword in line:
+                return line
+    return 'N/A'

moztest/setup.py

@@ -0,0 +1,39 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import os
+from setuptools import setup, find_packages
+
+PACKAGE_VERSION = '0.1'
+
+# get documentation from the README
+try:
+    here = os.path.dirname(os.path.abspath(__file__))
+    description = file(os.path.join(here, 'README.md')).read()
+except (OSError, IOError):
+    description = ''
+
+# dependencies
+deps = ['mozinfo']
+try:
+    import json
+except ImportError:
+    deps.append('simplejson')
+
+setup(name='moztest',
+      version=PACKAGE_VERSION,
+      description="Package for storing and outputting Mozilla test results",
+      long_description=description,
+      classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
+      keywords='mozilla',
+      author='Mihnea Dobrescu-Balaur',
+      author_email='mbalaur@mozilla.com',
+      url='https://wiki.mozilla.org/Auto-tools',
+      license='MPL',
+      packages=find_packages(exclude=['legacy']),
+      include_package_data=True,
+      zip_safe=False,
+      install_requires=deps,
+      )