forked from pytest-dev/pytest
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Showing
10 changed files
with
231 additions
and
21 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
New ``--report-log=FILE`` option that writes *report logs* into a file as the test session executes. | ||
|
||
Each line of the report log contains a self contained JSON object corresponding to a testing event, | ||
such as a collection or a test result report. The file is guaranteed to be flushed after writing | ||
each line, so systems can read and process events in real-time. | ||
|
||
This option is meant to replace ``--resultlog``, which is deprecated and meant to be removed | ||
in a future release. If you use ``--resultlog``, please try out ``--report-log`` and | ||
provide feedback. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -27,6 +27,7 @@ Full pytest documentation | |
unittest | ||
nose | ||
xunit_setup | ||
report_log | ||
plugins | ||
writing_plugins | ||
logging | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,70 @@ | ||
.. _report_log: | ||
|
||
Report files | ||
============ | ||
|
||
.. versionadded:: 5.3 | ||
|
||
The ``--report-log=FILE`` option writes *report logs* into a file as the test session executes. | ||
|
||
Each line of the report log contains a self contained JSON object corresponding to a testing event, | ||
such as a collection or a test result report. The file is guaranteed to be flushed after writing | ||
each line, so systems can read and process events in real-time. | ||
|
||
Each JSON object contains a special key ``$report_type``, which contains a unique identifier for | ||
that kind of report object. For future compatibility, consumers of the file should ignore reports | ||
they don't recognize, as well as ignore unknown properties/keys in JSON objects that they do know, | ||
as future pytest versions might enrich the objects with more properties/keys. | ||
|
||
.. note:: | ||
This option is meant to the replace ``--resultlog``, which is deprecated and meant to be removed | ||
in a future release. If you use ``--resultlog``, please try out ``--report-log`` and | ||
provide feedback. | ||
|
||
Example | ||
------- | ||
|
||
Consider this file: | ||
|
||
.. code-block:: python | ||
# content of test_report_example.py | ||
def test_ok(): | ||
assert 5 + 5 == 10 | ||
def test_fail(): | ||
assert 4 + 4 == 1 | ||
.. code-block:: pytest | ||
$ pytest test_report_example.py -q --report-log=log.json | ||
.F [100%] | ||
================================= FAILURES ================================= | ||
________________________________ test_fail _________________________________ | ||
def test_fail(): | ||
> assert 4 + 4 == 1 | ||
E assert (4 + 4) == 1 | ||
test_report_example.py:8: AssertionError | ||
------------------- generated report log file: log.json -------------------- | ||
1 failed, 1 passed in 0.12s | ||
The generated ``log.json`` will contain a JSON object per line: | ||
|
||
:: | ||
|
||
$ cat log.json | ||
{"pytest_version": "5.2.3.dev90+gd1129cf96.d20191026", "$report_type": "Header"} | ||
{"nodeid": "", "outcome": "passed", "longrepr": null, "result": null, "sections": [], "$report_type": "CollectReport"} | ||
{"nodeid": "test_report_example.py", "outcome": "passed", "longrepr": null, "result": null, "sections": [], "$report_type": "CollectReport"} | ||
{"nodeid": "test_report_example.py::test_ok", "location": ["test_report_example.py", 2, "test_ok"], "keywords": {"report_log.rst-39": 1, "test_report_example.py": 1, "test_ok": 1}, "outcome": "passed", "longrepr": null, "when": "setup", "user_properties": [], "sections": [], "duration": 0.00021314620971679688, "$report_type": "TestReport"} | ||
{"nodeid": "test_report_example.py::test_ok", "location": ["test_report_example.py", 2, "test_ok"], "keywords": {"report_log.rst-39": 1, "test_report_example.py": 1, "test_ok": 1}, "outcome": "passed", "longrepr": null, "when": "call", "user_properties": [], "sections": [], "duration": 0.00014543533325195312, "$report_type": "TestReport"} | ||
{"nodeid": "test_report_example.py::test_ok", "location": ["test_report_example.py", 2, "test_ok"], "keywords": {"report_log.rst-39": 1, "test_report_example.py": 1, "test_ok": 1}, "outcome": "passed", "longrepr": null, "when": "teardown", "user_properties": [], "sections": [], "duration": 0.00016427040100097656, "$report_type": "TestReport"} | ||
{"nodeid": "test_report_example.py::test_fail", "location": ["test_report_example.py", 6, "test_fail"], "keywords": {"test_fail": 1, "test_report_example.py": 1, "report_log.rst-39": 1}, "outcome": "passed", "longrepr": null, "when": "setup", "user_properties": [], "sections": [], "duration": 0.00013589859008789062, "$report_type": "TestReport"} | ||
{"nodeid": "test_report_example.py::test_fail", "location": ["test_report_example.py", 6, "test_fail"], "keywords": {"test_fail": 1, "test_report_example.py": 1, "report_log.rst-39": 1}, "outcome": "failed", "longrepr": {"reprcrash": {"path": "$REGENDOC_TMPDIR/test_report_example.py", "lineno": 8, "message": "assert (4 + 4) == 1"}, "reprtraceback": {"reprentries": [{"type": "ReprEntry", "data": {"lines": [" def test_fail():", "> assert 4 + 4 == 1", "E assert (4 + 4) == 1"], "reprfuncargs": {"args": []}, "reprlocals": null, "reprfileloc": {"path": "test_report_example.py", "lineno": 8, "message": "AssertionError"}, "style": "long"}}], "extraline": null, "style": "long"}, "sections": [], "chain": [[{"reprentries": [{"type": "ReprEntry", "data": {"lines": [" def test_fail():", "> assert 4 + 4 == 1", "E assert (4 + 4) == 1"], "reprfuncargs": {"args": []}, "reprlocals": null, "reprfileloc": {"path": "test_report_example.py", "lineno": 8, "message": "AssertionError"}, "style": "long"}}], "extraline": null, "style": "long"}, {"path": "$REGENDOC_TMPDIR/test_report_example.py", "lineno": 8, "message": "assert (4 + 4) == 1"}, null]]}, "when": "call", "user_properties": [], "sections": [], "duration": 0.00027489662170410156, "$report_type": "TestReport"} | ||
{"nodeid": "test_report_example.py::test_fail", "location": ["test_report_example.py", 6, "test_fail"], "keywords": {"test_fail": 1, "test_report_example.py": 1, "report_log.rst-39": 1}, "outcome": "passed", "longrepr": null, "when": "teardown", "user_properties": [], "sections": [], "duration": 0.00016689300537109375, "$report_type": "TestReport"} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,72 @@ | ||
import json | ||
from pathlib import Path | ||
|
||
import pytest | ||
|
||
|
||
def pytest_addoption(parser): | ||
group = parser.getgroup("terminal reporting", "report-log plugin options") | ||
group.addoption( | ||
"--report-log", | ||
action="store", | ||
metavar="path", | ||
default=None, | ||
help="Path to line-based json objects of test session events.", | ||
) | ||
|
||
|
||
def pytest_configure(config): | ||
report_log = config.option.report_log | ||
if report_log and not hasattr(config, "slaveinput"): | ||
config._report_log_plugin = ReportLogPlugin(config, Path(report_log)) | ||
config.pluginmanager.register(config._report_log_plugin) | ||
|
||
|
||
def pytest_unconfigure(config): | ||
report_log_plugin = getattr(config, "_report_log_plugin", None) | ||
if report_log_plugin: | ||
report_log_plugin.close() | ||
del config._report_log_plugin | ||
|
||
|
||
class ReportLogPlugin: | ||
def __init__(self, config, log_path: Path): | ||
self._config = config | ||
self._log_path = log_path | ||
|
||
log_path.parent.mkdir(parents=True, exist_ok=True) | ||
self._file = log_path.open("w", buffering=1, encoding="UTF-8") | ||
|
||
def close(self): | ||
if self._file is not None: | ||
self._file.close() | ||
self._file = None | ||
|
||
def _write_json_data(self, data): | ||
self._file.write(json.dumps(data) + "\n") | ||
self._file.flush() | ||
|
||
def pytest_sessionstart(self): | ||
data = {"pytest_version": pytest.__version__, "$report_type": "SessionStart"} | ||
self._write_json_data(data) | ||
|
||
def pytest_sessionfinish(self, exitstatus): | ||
data = {"exitstatus": exitstatus, "$report_type": "SessionFinish"} | ||
self._write_json_data(data) | ||
|
||
def pytest_runtest_logreport(self, report): | ||
data = self._config.hook.pytest_report_to_serializable( | ||
config=self._config, report=report | ||
) | ||
self._write_json_data(data) | ||
|
||
def pytest_collectreport(self, report): | ||
data = self._config.hook.pytest_report_to_serializable( | ||
config=self._config, report=report | ||
) | ||
self._write_json_data(data) | ||
|
||
def pytest_terminal_summary(self, terminalreporter): | ||
terminalreporter.write_sep( | ||
"-", "generated report log file: {}".format(self._log_path) | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,54 @@ | ||
import json | ||
|
||
import pytest | ||
from _pytest.reports import BaseReport | ||
|
||
|
||
def test_basics(testdir, tmp_path, pytestconfig): | ||
"""Basic testing of the report log functionality. | ||
We don't test the test reports extensively because they have been | ||
tested already in ``test_reports``. | ||
""" | ||
testdir.makepyfile( | ||
""" | ||
def test_ok(): | ||
pass | ||
def test_fail(): | ||
assert 0 | ||
""" | ||
) | ||
|
||
log_file = tmp_path / "log.json" | ||
|
||
result = testdir.runpytest("--report-log", str(log_file)) | ||
assert result.ret == pytest.ExitCode.TESTS_FAILED | ||
result.stdout.fnmatch_lines(["* generated report log file: {}*".format(log_file)]) | ||
|
||
json_objs = [json.loads(x) for x in log_file.read_text().splitlines()] | ||
assert len(json_objs) == 10 | ||
|
||
# first line should be the session_start | ||
session_start = json_objs[0] | ||
assert session_start == { | ||
"pytest_version": pytest.__version__, | ||
"$report_type": "SessionStart", | ||
} | ||
|
||
# last line should be the session_finish | ||
session_start = json_objs[-1] | ||
assert session_start == { | ||
"exitstatus": pytest.ExitCode.TESTS_FAILED, | ||
"$report_type": "SessionFinish", | ||
} | ||
|
||
# rest of the json objects should be unserialized into report objects; we don't test | ||
# the actual report object extensively because it has been tested in ``test_reports`` | ||
# already. | ||
pm = pytestconfig.pluginmanager | ||
for json_obj in json_objs[1:-1]: | ||
rep = pm.hook.pytest_report_from_serializable( | ||
config=pytestconfig, data=json_obj | ||
) | ||
assert isinstance(rep, BaseReport) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters