Merge pull request #21 from mulawamichal/fea/failedlogsdir

easy access to logs from failed tests

CHANGELOG

@@ -2,6 +2,7 @@ next
 -----------------------------------
 - class with testcases is represented with additional directory instead of '.'
   separating class name and testcase name
+- add the "split logs by outcome" functionality
 
 0.4.0
 -----------------------------------

docs/usage.rst

@@ -135,6 +135,48 @@ It has structure following pytest test item's `nodeid`_.
         └── test_cat
             └── proc
 
+
+.. _`split logs by outcome`:
+
+Split logs by outcome
+---------------------------------------
+It is possible to split the logs by test outcome. If chosen to do so (by calling below method):
+
+::
+
+    # content of conftest.py
+    def pytest_logger_config(logger_config):
+        logger_config.split_by_outcome()
+
+Will result in below directory structure:
+
+::
+
+    logs/
+    ├── classtests
+    │   └── test_y.py
+    │       └── TestClass
+    │           ├── test_class
+    │           |   ├── daemon
+    │           |   └── setup
+    |           └── test_that_failed_two
+    |               └── somelogfile
+    ├── by_outcome
+    |   └── failed
+    |       ├── classtests
+    │       |   └── test_y.py
+    │       |       └── TestClass
+    |       |           └── test_that_failed_two -> ../../../../../../classtests/test_y.py/TestClass/test_that_failed_two
+    |       └── test_p.py
+    |           └── test_that_failed_one -> ../../../../test_p.py/test_that_failed_one
+    └── test_p.py
+        ├── test_cat
+        |   └── proc
+        └── test_that_failed_one
+            └── somelog
+
+You can change the default `by_outcome` dirname to something else, as well as add more "per-outcome" subdirectories by passing proper arguments to the `split_by_outcome` method.
+
 .. _`link to logs dir`:
 
 Link to logs directory
@@ -184,7 +226,8 @@ API reference
 .. autoclass:: LoggerConfig()
     :members: add_loggers,
               set_log_option_default,
-              set_formatter_class
+              set_formatter_class,
+              split_by_outcome
 
 .. _`conftest.py`: http://docs.pytest.org/en/latest/writing_plugins.html#conftest-py
 .. _`unwanted message`: https://docs.python.org/2/howto/logging.html#what-happens-if-no-configuration-is-provided

pytest_logger/plugin.py

@@ -63,6 +63,8 @@ def __init__(self, config, logcfg):
         self._loggers = _loggers_from_logcfg(logcfg, config.getoption('loggers')) if logcfg._enabled else None
         self._formatter_class = logcfg._formatter_class or DefaultFormatter
         self._logsdir = None
+        self._split_by_outcome_subdir = logcfg._split_by_outcome_subdir
+        self._split_by_outcome_outcomes = logcfg._split_by_outcome_outcomes
 
     def logsdir(self):
         ldir = self._logsdir
@@ -97,9 +99,20 @@ def pytest_runtest_teardown(self, item, nextitem):
         if logger:
             logger.on_teardown()
 
+    @pytest.mark.hookwrapper
     def pytest_runtest_makereport(self, item, call):
+        outcome = yield
+        tr = outcome.get_result()
         logger = getattr(item, '_logger', None)
         if logger:
+            if self._logsdir and self._split_by_outcome_subdir and tr.outcome in self._split_by_outcome_outcomes:
+                split_by_outcome_logdir = self._logsdir.join(self._split_by_outcome_subdir, tr.outcome)
+                nodeid = _sanitize_nodeid(item.nodeid)
+                nodepath = os.path.dirname(nodeid)
+                split_by_outcome_logdir.join(nodepath).ensure(dir=1)
+                destdir_relpath = os.path.relpath(str(self._logsdir.join(nodeid)),
+                                                  str(split_by_outcome_logdir.join(nodepath)))
+                _refresh_link(destdir_relpath, str(split_by_outcome_logdir.join(nodeid)))
             if call.when == 'teardown':
                 logger.on_makereport()
 
@@ -159,6 +172,8 @@ def __init__(self):
         self._loggers = []
         self._formatter_class = None
         self._log_option_default = ''
+        self._split_by_outcome_subdir = None
+        self._split_by_outcome_outcomes = []
 
     def add_loggers(self, loggers, stdout_level=logging.NOTSET, file_level=logging.NOTSET):
         """Adds loggers for stdout/filesystem handling.
@@ -195,6 +210,23 @@ def set_log_option_default(self, value):
         """ Sets default value of `log` option."""
         self._log_option_default = value
 
+    def split_by_outcome(self, outcomes=None, subdir='by_outcome'):
+        """Makes a directory inside main logdir where logs are further split by test outcome
+
+        :param subdir: name for the subdirectory in main log directory
+        :param outcomes: list of test outcomes to be handled (failed/passed/skipped)
+        """
+        if outcomes is not None:
+            allowed_outcomes = ['passed', 'failed', 'skipped']
+            unexpected_outcomes = set(outcomes) - set(allowed_outcomes)
+            if unexpected_outcomes:
+                raise ValueError('got unexpected_outcomes: <' + str(list(unexpected_outcomes)) + '>')
+            self._split_by_outcome_outcomes = outcomes
+        else:
+            self._split_by_outcome_outcomes = ['failed']
+
+        self._split_by_outcome_subdir = subdir
+
 
 class LoggerHookspec(object):
     def pytest_logger_config(self, logger_config):

tests/test_pytest_logger.py

@@ -1,11 +1,13 @@
+import os
 import sys
 import pytest
 import platform
 from py.code import Source
 from _pytest.pytester import LineMatcher
 
-win32py2 = sys.platform == 'win32' and sys.version_info[0] == 2
-win32pypy = sys.platform == 'win32' and platform.python_implementation() == 'PyPy'
+win32 = sys.platform == 'win32'
+win32py2 = win32 and sys.version_info[0] == 2
+win32pypy = win32 and platform.python_implementation() == 'PyPy'
 
 
 def makefile(testdir, path, content):
@@ -215,6 +217,58 @@ def test_file_handlers(testdir, conftest_py, test_case_py):
     ])
 
 
+@pytest.mark.skipif(win32py2, reason="python 2 on windows doesn't have symlink feature")
+def test_split_logs_by_outcome(testdir):
+    makefile(testdir, ['conftest.py'], """
+        import logging
+        def pytest_logger_fileloggers(item):
+            return [
+                'foo',
+                ('bar', logging.ERROR),
+            ]
+
+        def pytest_logger_config(logger_config):
+            logger_config.split_by_outcome(outcomes=['passed', 'failed'])
+    """)
+    makefile(testdir, ['test_case.py'], """
+            import pytest
+            import logging
+            def test_case_that_fails():
+                for lgr in (logging.getLogger(name) for name in ['foo', 'bar', 'baz']):
+                    lgr.error('this is error')
+                    lgr.warning('this is warning')
+                pytest.fail('just checking')
+
+            def test_case_that_passes():
+                for lgr in (logging.getLogger(name) for name in ['foo', 'bar', 'baz']):
+                    lgr.error('this is error')
+                    lgr.warning('this is warning')
+        """)
+    result = testdir.runpytest('-s')
+    assert result.ret != 0
+    result.stdout.fnmatch_lines([
+        '',
+        'test_case.py F.',
+        '',
+    ])
+
+    assert 'by_outcome' in ls(basetemp(testdir).join('logs'))
+
+    assert 'failed' in ls(basetemp(testdir).join('logs', 'by_outcome'))
+    failedlogpath = basetemp(testdir).join('logs', 'by_outcome', 'failed', 'test_case.py', 'test_case_that_fails')
+    assert failedlogpath.islink()
+    failedlogdest = os.path.join(
+        os.path.pardir, os.path.pardir, os.path.pardir, 'test_case.py', 'test_case_that_fails')
+    assert os.readlink(str(failedlogpath)) == failedlogdest
+
+    assert 'passed' in ls(basetemp(testdir).join('logs', 'by_outcome'))
+    passedlogpath = basetemp(testdir).join('logs', 'by_outcome', 'passed', 'test_case.py', 'test_case_that_passes')
+    assert passedlogpath.islink()
+    passedlogdest = os.path.join(
+        os.path.pardir, os.path.pardir, os.path.pardir, 'test_case.py', 'test_case_that_passes')
+    assert os.readlink(str(passedlogpath)) == passedlogdest
+
+
 def test_file_handlers_root(testdir):
     makefile(testdir, ['conftest.py'], """
         import logging

tests/test_units.py

@@ -96,3 +96,9 @@ def test_loggers_from_logcfg():
     assert loggers.stdout == [('b', logging.FATAL), ('d', 10)]
     assert loggers.file == [('a', logging.WARN), ('b', logging.WARN), ('c', logging.WARN), ('d', 0)]
     assert loggers
+
+
+def test_split_by_outcome_wrong_config():
+    logcfg = plugin.LoggerConfig()
+    with pytest.raises(ValueError, match="got unexpected_outcomes: <\\['sthelese'\\]>"):
+        logcfg.split_by_outcome(outcomes=['failed', 'sthelese'])