From 43c87c5a042128b8de64e14a1724c6e3559dc125 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Fri, 12 Jul 2019 15:14:04 -0300 Subject: [PATCH] Validate xunit2 files against the schema Fix #5095 --- changelog/5095.trivial.rst | 2 + setup.py | 5 +- testing/example_scripts/junit-10.xsd | 147 ++++++++++++++ testing/test_junitxml.py | 291 +++++++++++++++++---------- 4 files changed, 331 insertions(+), 114 deletions(-) create mode 100644 changelog/5095.trivial.rst create mode 100644 testing/example_scripts/junit-10.xsd diff --git a/changelog/5095.trivial.rst b/changelog/5095.trivial.rst new file mode 100644 index 00000000000..2256cf9f4cd --- /dev/null +++ b/changelog/5095.trivial.rst @@ -0,0 +1,2 @@ +XML files of the ``xunit2`` family are now validated against the schema by pytest's own test suite +to avoid future regressions. diff --git a/setup.py b/setup.py index e8bc8e895ca..adbafb557a9 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,6 @@ def main(): use_scm_version={"write_to": "src/_pytest/_version.py"}, setup_requires=["setuptools-scm", "setuptools>=40.0"], package_dir={"": "src"}, - # fmt: off extras_require={ "testing": [ "argcomplete", @@ -29,9 +28,9 @@ def main(): "mock", "nose", "requests", - ], + "xmlschema", + ] }, - # fmt: on install_requires=INSTALL_REQUIRES, ) diff --git a/testing/example_scripts/junit-10.xsd b/testing/example_scripts/junit-10.xsd new file mode 100644 index 00000000000..286fbf7c87b --- /dev/null +++ b/testing/example_scripts/junit-10.xsd @@ -0,0 +1,147 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/testing/test_junitxml.py b/testing/test_junitxml.py index 3196f0ebd8d..5f2d73c78aa 100644 --- a/testing/test_junitxml.py +++ b/testing/test_junitxml.py @@ -1,18 +1,50 @@ import os +from pathlib import Path from xml.dom import minidom import py +import xmlschema import pytest from _pytest.junitxml import LogXML from _pytest.reports import BaseReport -def runandparse(testdir, *args): - resultpath = testdir.tmpdir.join("junit.xml") - result = testdir.runpytest("--junitxml=%s" % resultpath, *args) - xmldoc = minidom.parse(str(resultpath)) - return result, DomNode(xmldoc) +@pytest.fixture(scope="session") +def schema(): + """Returns a xmlschema.XMLSchema object for the junit-10.xsd file""" + fn = Path(__file__).parent / "example_scripts/junit-10.xsd" + with fn.open() as f: + return xmlschema.XMLSchema(f) + + +@pytest.fixture(name="run_and_parse") +def run_and_parse_(testdir, schema): + """ + Fixture that returns a function that can be used to execute pytest and return + the parsed ``DomNode`` of the root xml node. It accepts a ``validate`` argument, + in which case it will configure pytest to write with the xunit2 family and validate + against the schema. + """ + + def run(*args, validate=False): + if validate and not testdir.tmpdir.join("tox.ini").isfile(): + testdir.makeini( + """ + [pytest] + junit_family=xunit2 + """ + ) + + resultpath = testdir.tmpdir.join("junit.xml") + result = testdir.runpytest("--junitxml=%s" % resultpath, *args) + if validate: + with resultpath.open() as f: + schema.validate(f) + xmldoc = minidom.parse(str(resultpath)) + return result, DomNode(xmldoc) + + return run def assert_attr(node, **kwargs): @@ -90,7 +122,8 @@ def next_sibling(self): class TestPython: - def test_summing_simple(self, testdir): + @pytest.mark.parametrize("validate", [True, False]) + def test_summing_simple(self, testdir, run_and_parse, validate): testdir.makepyfile( """ import pytest @@ -108,12 +141,13 @@ def test_xpass(): assert 1 """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) assert result.ret node = dom.find_first_by_tag("testsuite") node.assert_attr(name="pytest", errors=0, failures=1, skipped=2, tests=5) - def test_summing_simple_with_errors(self, testdir): + @pytest.mark.parametrize("validate", [True, False]) + def test_summing_simple_with_errors(self, testdir, run_and_parse, validate): testdir.makepyfile( """ import pytest @@ -134,12 +168,12 @@ def test_xpass(): assert True """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) assert result.ret node = dom.find_first_by_tag("testsuite") node.assert_attr(name="pytest", errors=1, failures=2, skipped=1, tests=5) - def test_timing_function(self, testdir): + def test_timing_function(self, testdir, run_and_parse): testdir.makepyfile( """ import time, pytest @@ -151,14 +185,16 @@ def test_sleep(): time.sleep(0.01) """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse() node = dom.find_first_by_tag("testsuite") tnode = node.find_first_by_tag("testcase") val = tnode["time"] assert round(float(val), 2) >= 0.03 @pytest.mark.parametrize("duration_report", ["call", "total"]) - def test_junit_duration_report(self, testdir, monkeypatch, duration_report): + def test_junit_duration_report( + self, testdir, monkeypatch, duration_report, run_and_parse + ): # mock LogXML.node_reporter so it always sets a known duration to each test report object original_node_reporter = LogXML.node_reporter @@ -176,8 +212,8 @@ def test_foo(): pass """ ) - result, dom = runandparse( - testdir, "-o", "junit_duration_report={}".format(duration_report) + result, dom = run_and_parse( + "-o", "junit_duration_report={}".format(duration_report) ) node = dom.find_first_by_tag("testsuite") tnode = node.find_first_by_tag("testcase") @@ -188,7 +224,8 @@ def test_foo(): assert duration_report == "call" assert val == 1.0 - def test_setup_error(self, testdir): + @pytest.mark.parametrize("validate", [True, False]) + def test_setup_error(self, testdir, run_and_parse, validate): testdir.makepyfile( """ import pytest @@ -200,7 +237,7 @@ def test_function(arg): pass """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) assert result.ret node = dom.find_first_by_tag("testsuite") node.assert_attr(errors=1, tests=1) @@ -210,7 +247,8 @@ def test_function(arg): fnode.assert_attr(message="test setup failure") assert "ValueError" in fnode.toxml() - def test_teardown_error(self, testdir): + @pytest.mark.parametrize("validate", [True, False]) + def test_teardown_error(self, testdir, run_and_parse, validate): testdir.makepyfile( """ import pytest @@ -223,7 +261,7 @@ def test_function(arg): pass """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) assert result.ret node = dom.find_first_by_tag("testsuite") tnode = node.find_first_by_tag("testcase") @@ -232,7 +270,8 @@ def test_function(arg): fnode.assert_attr(message="test teardown failure") assert "ValueError" in fnode.toxml() - def test_call_failure_teardown_error(self, testdir): + @pytest.mark.parametrize("validate", [True, False]) + def test_call_failure_teardown_error(self, testdir, run_and_parse, validate): testdir.makepyfile( """ import pytest @@ -245,7 +284,7 @@ def test_function(arg): raise Exception("Call Exception") """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) assert result.ret node = dom.find_first_by_tag("testsuite") node.assert_attr(errors=1, failures=1, tests=1) @@ -257,7 +296,8 @@ def test_function(arg): snode = second.find_first_by_tag("error") snode.assert_attr(message="test teardown failure") - def test_skip_contains_name_reason(self, testdir): + @pytest.mark.parametrize("validate", [True, False]) + def test_skip_contains_name_reason(self, testdir, run_and_parse, validate): testdir.makepyfile( """ import pytest @@ -265,7 +305,7 @@ def test_skip(): pytest.skip("hello23") """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) assert result.ret == 0 node = dom.find_first_by_tag("testsuite") node.assert_attr(skipped=1) @@ -274,7 +314,8 @@ def test_skip(): snode = tnode.find_first_by_tag("skipped") snode.assert_attr(type="pytest.skip", message="hello23") - def test_mark_skip_contains_name_reason(self, testdir): + @pytest.mark.parametrize("validate", [True, False]) + def test_mark_skip_contains_name_reason(self, testdir, run_and_parse, validate): testdir.makepyfile( """ import pytest @@ -283,7 +324,7 @@ def test_skip(): assert True """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) assert result.ret == 0 node = dom.find_first_by_tag("testsuite") node.assert_attr(skipped=1) @@ -294,7 +335,8 @@ def test_skip(): snode = tnode.find_first_by_tag("skipped") snode.assert_attr(type="pytest.skip", message="hello24") - def test_mark_skipif_contains_name_reason(self, testdir): + @pytest.mark.parametrize("validate", [True, False]) + def test_mark_skipif_contains_name_reason(self, testdir, run_and_parse, validate): testdir.makepyfile( """ import pytest @@ -304,7 +346,7 @@ def test_skip(): assert True """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) assert result.ret == 0 node = dom.find_first_by_tag("testsuite") node.assert_attr(skipped=1) @@ -315,7 +357,8 @@ def test_skip(): snode = tnode.find_first_by_tag("skipped") snode.assert_attr(type="pytest.skip", message="hello25") - def test_mark_skip_doesnt_capture_output(self, testdir): + @pytest.mark.parametrize("validate", [True, False]) + def test_mark_skip_doesnt_capture_output(self, testdir, run_and_parse, validate): testdir.makepyfile( """ import pytest @@ -324,12 +367,13 @@ def test_skip(): print("bar!") """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) assert result.ret == 0 node_xml = dom.find_first_by_tag("testsuite").toxml() assert "bar!" not in node_xml - def test_classname_instance(self, testdir): + @pytest.mark.parametrize("validate", [True, False]) + def test_classname_instance(self, testdir, run_and_parse, validate): testdir.makepyfile( """ class TestClass(object): @@ -337,7 +381,7 @@ def test_method(self): assert 0 """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) assert result.ret node = dom.find_first_by_tag("testsuite") node.assert_attr(failures=1) @@ -346,20 +390,22 @@ def test_method(self): classname="test_classname_instance.TestClass", name="test_method" ) - def test_classname_nested_dir(self, testdir): + @pytest.mark.parametrize("validate", [True, False]) + def test_classname_nested_dir(self, testdir, run_and_parse, validate): p = testdir.tmpdir.ensure("sub", "test_hello.py") p.write("def test_func(): 0/0") - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) assert result.ret node = dom.find_first_by_tag("testsuite") node.assert_attr(failures=1) tnode = node.find_first_by_tag("testcase") tnode.assert_attr(classname="sub.test_hello", name="test_func") - def test_internal_error(self, testdir): + @pytest.mark.parametrize("validate", [True, False]) + def test_internal_error(self, testdir, run_and_parse, validate): testdir.makeconftest("def pytest_runtest_protocol(): 0 / 0") testdir.makepyfile("def test_function(): pass") - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) assert result.ret node = dom.find_first_by_tag("testsuite") node.assert_attr(errors=1, tests=1) @@ -370,7 +416,8 @@ def test_internal_error(self, testdir): assert "Division" in fnode.toxml() @pytest.mark.parametrize("junit_logging", ["no", "system-out", "system-err"]) - def test_failure_function(self, testdir, junit_logging): + @pytest.mark.parametrize("validate", [True, False]) + def test_failure_function(self, testdir, junit_logging, run_and_parse, validate): testdir.makepyfile( """ import logging @@ -385,7 +432,9 @@ def test_fail(): """ ) - result, dom = runandparse(testdir, "-o", "junit_logging=%s" % junit_logging) + result, dom = run_and_parse( + "-o", "junit_logging=%s" % junit_logging, validate=validate + ) assert result.ret node = dom.find_first_by_tag("testsuite") node.assert_attr(failures=1, tests=1) @@ -413,7 +462,8 @@ def test_fail(): assert "warning msg" not in systemout.toxml() assert "warning msg" not in systemerr.toxml() - def test_failure_verbose_message(self, testdir): + @pytest.mark.parametrize("validate", [True, False]) + def test_failure_verbose_message(self, testdir, run_and_parse, validate): testdir.makepyfile( """ import sys @@ -421,14 +471,14 @@ def test_fail(): assert 0, "An error" """ ) - - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) node = dom.find_first_by_tag("testsuite") tnode = node.find_first_by_tag("testcase") fnode = tnode.find_first_by_tag("failure") fnode.assert_attr(message="AssertionError: An error assert 0") - def test_failure_escape(self, testdir): + @pytest.mark.parametrize("validate", [True, False]) + def test_failure_escape(self, testdir, run_and_parse, validate): testdir.makepyfile( """ import pytest @@ -438,7 +488,7 @@ def test_func(arg1): assert 0 """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) assert result.ret node = dom.find_first_by_tag("testsuite") node.assert_attr(failures=3, tests=3) @@ -453,7 +503,8 @@ def test_func(arg1): text = sysout.text assert text == "%s\n" % char - def test_junit_prefixing(self, testdir): + @pytest.mark.parametrize("validate", [True, False]) + def test_junit_prefixing(self, testdir, run_and_parse, validate): testdir.makepyfile( """ def test_func(): @@ -463,7 +514,7 @@ def test_hello(self): pass """ ) - result, dom = runandparse(testdir, "--junitprefix=xyz") + result, dom = run_and_parse("--junitprefix=xyz", validate=validate) assert result.ret node = dom.find_first_by_tag("testsuite") node.assert_attr(failures=1, tests=2) @@ -474,7 +525,8 @@ def test_hello(self): classname="xyz.test_junit_prefixing.TestHello", name="test_hello" ) - def test_xfailure_function(self, testdir): + @pytest.mark.parametrize("validate", [True, False]) + def test_xfailure_function(self, testdir, run_and_parse, validate): testdir.makepyfile( """ import pytest @@ -482,7 +534,7 @@ def test_xfail(): pytest.xfail("42") """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) assert not result.ret node = dom.find_first_by_tag("testsuite") node.assert_attr(skipped=1, tests=1) @@ -490,9 +542,9 @@ def test_xfail(): tnode.assert_attr(classname="test_xfailure_function", name="test_xfail") fnode = tnode.find_first_by_tag("skipped") fnode.assert_attr(type="pytest.xfail", message="42") - # assert "ValueError" in fnode.toxml() - def test_xfailure_marker(self, testdir): + @pytest.mark.parametrize("validate", [True, False]) + def test_xfailure_marker(self, testdir, run_and_parse, validate): testdir.makepyfile( """ import pytest @@ -501,7 +553,7 @@ def test_xfail(): assert False """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) assert not result.ret node = dom.find_first_by_tag("testsuite") node.assert_attr(skipped=1, tests=1) @@ -510,7 +562,7 @@ def test_xfail(): fnode = tnode.find_first_by_tag("skipped") fnode.assert_attr(type="pytest.xfail", message="42") - def test_xfail_captures_output_once(self, testdir): + def test_xfail_captures_output_once(self, testdir, run_and_parse): testdir.makepyfile( """ import sys @@ -523,13 +575,14 @@ def test_fail(): assert 0 """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse() node = dom.find_first_by_tag("testsuite") tnode = node.find_first_by_tag("testcase") assert len(tnode.find_by_tag("system-err")) == 1 assert len(tnode.find_by_tag("system-out")) == 1 - def test_xfailure_xpass(self, testdir): + @pytest.mark.parametrize("validate", [True, False]) + def test_xfailure_xpass(self, testdir, run_and_parse, validate): testdir.makepyfile( """ import pytest @@ -538,14 +591,15 @@ def test_xpass(): pass """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) # assert result.ret node = dom.find_first_by_tag("testsuite") node.assert_attr(skipped=0, tests=1) tnode = node.find_first_by_tag("testcase") tnode.assert_attr(classname="test_xfailure_xpass", name="test_xpass") - def test_xfailure_xpass_strict(self, testdir): + @pytest.mark.parametrize("validate", [True, False]) + def test_xfailure_xpass_strict(self, testdir, run_and_parse, validate): testdir.makepyfile( """ import pytest @@ -554,7 +608,7 @@ def test_xpass(): pass """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) # assert result.ret node = dom.find_first_by_tag("testsuite") node.assert_attr(skipped=0, tests=1) @@ -563,9 +617,10 @@ def test_xpass(): fnode = tnode.find_first_by_tag("failure") fnode.assert_attr(message="[XPASS(strict)] This needs to fail!") - def test_collect_error(self, testdir): + @pytest.mark.parametrize("validate", [True, False]) + def test_collect_error(self, testdir, run_and_parse, validate): testdir.makepyfile("syntax error") - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) assert result.ret node = dom.find_first_by_tag("testsuite") node.assert_attr(errors=1, tests=1) @@ -574,7 +629,7 @@ def test_collect_error(self, testdir): fnode.assert_attr(message="collection failure") assert "SyntaxError" in fnode.toxml() - def test_unicode(self, testdir): + def test_unicode(self, testdir, run_and_parse): value = "hx\xc4\x85\xc4\x87\n" testdir.makepyfile( """\ @@ -585,13 +640,13 @@ def test_hello(): """ % value ) - result, dom = runandparse(testdir) + result, dom = run_and_parse() assert result.ret == 1 tnode = dom.find_first_by_tag("testcase") fnode = tnode.find_first_by_tag("failure") assert "hx" in fnode.toxml() - def test_assertion_binchars(self, testdir): + def test_assertion_binchars(self, testdir, run_and_parse): """this test did fail when the escaping wasn't strict""" testdir.makepyfile( """ @@ -603,23 +658,23 @@ def test_str_compare(): assert M1 == M2 """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse() print(dom.toxml()) - def test_pass_captures_stdout(self, testdir): + def test_pass_captures_stdout(self, testdir, run_and_parse): testdir.makepyfile( """ def test_pass(): print('hello-stdout') """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse() node = dom.find_first_by_tag("testsuite") pnode = node.find_first_by_tag("testcase") systemout = pnode.find_first_by_tag("system-out") assert "hello-stdout" in systemout.toxml() - def test_pass_captures_stderr(self, testdir): + def test_pass_captures_stderr(self, testdir, run_and_parse): testdir.makepyfile( """ import sys @@ -627,13 +682,13 @@ def test_pass(): sys.stderr.write('hello-stderr') """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse() node = dom.find_first_by_tag("testsuite") pnode = node.find_first_by_tag("testcase") systemout = pnode.find_first_by_tag("system-err") assert "hello-stderr" in systemout.toxml() - def test_setup_error_captures_stdout(self, testdir): + def test_setup_error_captures_stdout(self, testdir, run_and_parse): testdir.makepyfile( """ import pytest @@ -646,13 +701,13 @@ def test_function(arg): pass """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse() node = dom.find_first_by_tag("testsuite") pnode = node.find_first_by_tag("testcase") systemout = pnode.find_first_by_tag("system-out") assert "hello-stdout" in systemout.toxml() - def test_setup_error_captures_stderr(self, testdir): + def test_setup_error_captures_stderr(self, testdir, run_and_parse): testdir.makepyfile( """ import sys @@ -666,13 +721,13 @@ def test_function(arg): pass """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse() node = dom.find_first_by_tag("testsuite") pnode = node.find_first_by_tag("testcase") systemout = pnode.find_first_by_tag("system-err") assert "hello-stderr" in systemout.toxml() - def test_avoid_double_stdout(self, testdir): + def test_avoid_double_stdout(self, testdir, run_and_parse): testdir.makepyfile( """ import sys @@ -687,7 +742,7 @@ def test_function(arg): sys.stdout.write('hello-stdout call') """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse() node = dom.find_first_by_tag("testsuite") pnode = node.find_first_by_tag("testcase") systemout = pnode.find_first_by_tag("system-out") @@ -730,7 +785,8 @@ def getini(self, name): class TestNonPython: - def test_summing_simple(self, testdir): + @pytest.mark.parametrize("validate", [True, False]) + def test_summing_simple(self, testdir, run_and_parse, validate): testdir.makeconftest( """ import pytest @@ -748,7 +804,7 @@ def repr_failure(self, excinfo): """ ) testdir.tmpdir.join("myfile.xyz").write("hello") - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) assert result.ret node = dom.find_first_by_tag("testsuite") node.assert_attr(errors=0, failures=1, skipped=0, tests=1) @@ -796,8 +852,8 @@ def test_print_nullbyte(): def test_invalid_xml_escape(): # Test some more invalid xml chars, the full range should be - # tested really but let's just thest the edges of the ranges - # intead. + # tested really but let's just test the edges of the ranges + # instead. # XXX This only tests low unicode character points for now as # there are some issues with the testing infrastructure for # the higher ones. @@ -881,7 +937,7 @@ def test_logxml_check_isdir(testdir): result.stderr.fnmatch_lines(["*--junitxml must be a filename*"]) -def test_escaped_parametrized_names_xml(testdir): +def test_escaped_parametrized_names_xml(testdir, run_and_parse): testdir.makepyfile( """\ import pytest @@ -890,13 +946,13 @@ def test_func(char): assert char """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse() assert result.ret == 0 node = dom.find_first_by_tag("testcase") node.assert_attr(name="test_func[\\x00]") -def test_double_colon_split_function_issue469(testdir): +def test_double_colon_split_function_issue469(testdir, run_and_parse): testdir.makepyfile( """ import pytest @@ -905,14 +961,14 @@ def test_func(param): pass """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse() assert result.ret == 0 node = dom.find_first_by_tag("testcase") node.assert_attr(classname="test_double_colon_split_function_issue469") node.assert_attr(name="test_func[double::colon]") -def test_double_colon_split_method_issue469(testdir): +def test_double_colon_split_method_issue469(testdir, run_and_parse): testdir.makepyfile( """ import pytest @@ -922,7 +978,7 @@ def test_func(self, param): pass """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse() assert result.ret == 0 node = dom.find_first_by_tag("testcase") node.assert_attr(classname="test_double_colon_split_method_issue469.TestClass") @@ -958,7 +1014,7 @@ class Report(BaseReport): log.pytest_sessionfinish() -def test_record_property(testdir): +def test_record_property(testdir, run_and_parse): testdir.makepyfile( """ import pytest @@ -970,7 +1026,7 @@ def test_record(record_property, other): record_property("foo", "<1"); """ ) - result, dom = runandparse(testdir, "-rwv") + result, dom = run_and_parse("-rwv") node = dom.find_first_by_tag("testsuite") tnode = node.find_first_by_tag("testcase") psnode = tnode.find_first_by_tag("properties") @@ -979,7 +1035,7 @@ def test_record(record_property, other): pnodes[1].assert_attr(name="foo", value="<1") -def test_record_property_same_name(testdir): +def test_record_property_same_name(testdir, run_and_parse): testdir.makepyfile( """ def test_record_with_same_name(record_property): @@ -987,7 +1043,7 @@ def test_record_with_same_name(record_property): record_property("foo", "baz") """ ) - result, dom = runandparse(testdir, "-rw") + result, dom = run_and_parse("-rw") node = dom.find_first_by_tag("testsuite") tnode = node.find_first_by_tag("testcase") psnode = tnode.find_first_by_tag("properties") @@ -1011,7 +1067,7 @@ def test_record({fixture_name}): @pytest.mark.filterwarnings("default") -def test_record_attribute(testdir): +def test_record_attribute(testdir, run_and_parse): testdir.makeini( """ [pytest] @@ -1029,7 +1085,7 @@ def test_record(record_xml_attribute, other): record_xml_attribute("foo", "<1"); """ ) - result, dom = runandparse(testdir, "-rw") + result, dom = run_and_parse("-rw") node = dom.find_first_by_tag("testsuite") tnode = node.find_first_by_tag("testcase") tnode.assert_attr(bar="1") @@ -1041,7 +1097,7 @@ def test_record(record_xml_attribute, other): @pytest.mark.filterwarnings("default") @pytest.mark.parametrize("fixture_name", ["record_xml_attribute", "record_property"]) -def test_record_fixtures_xunit2(testdir, fixture_name): +def test_record_fixtures_xunit2(testdir, fixture_name, run_and_parse): """Ensure record_xml_attribute and record_property drop values when outside of legacy family """ testdir.makeini( @@ -1064,7 +1120,7 @@ def test_record({fixture_name}, other): ) ) - result, dom = runandparse(testdir, "-rw") + result, dom = run_and_parse("-rw", validate=fixture_name != "record_property") expected_lines = [] if fixture_name == "record_xml_attribute": expected_lines.append( @@ -1079,7 +1135,7 @@ def test_record({fixture_name}, other): result.stdout.fnmatch_lines(expected_lines) -def test_random_report_log_xdist(testdir, monkeypatch): +def test_random_report_log_xdist(testdir, monkeypatch, run_and_parse): """xdist calls pytest_runtest_logreport as they are executed by the slaves, with nodes from several nodes overlapping, so junitxml must cope with that to produce correct reports. #1064 @@ -1094,7 +1150,7 @@ def test_x(i): assert i != 22 """ ) - _, dom = runandparse(testdir, "-n2") + _, dom = run_and_parse("-n2") suite_node = dom.find_first_by_tag("testsuite") failed = [] for case_node in suite_node.find_by_tag("testcase"): @@ -1104,21 +1160,22 @@ def test_x(i): assert failed == ["test_x[22]"] -def test_root_testsuites_tag(testdir): +@pytest.mark.parametrize("validate", [True, False]) +def test_root_testsuites_tag(testdir, run_and_parse, validate): testdir.makepyfile( """ def test_x(): pass """ ) - _, dom = runandparse(testdir) + _, dom = run_and_parse(validate=validate) root = dom.get_unique_child assert root.tag == "testsuites" suite_node = root.get_unique_child assert suite_node.tag == "testsuite" -def test_runs_twice(testdir): +def test_runs_twice(testdir, run_and_parse): f = testdir.makepyfile( """ def test_pass(): @@ -1126,14 +1183,14 @@ def test_pass(): """ ) - result, dom = runandparse(testdir, f, f) + result, dom = run_and_parse(f, f) assert "INTERNALERROR" not in result.stdout.str() first, second = [x["classname"] for x in dom.find_by_tag("testcase")] assert first == second @pytest.mark.xfail(reason="hangs", run=False) -def test_runs_twice_xdist(testdir): +def test_runs_twice_xdist(testdir, run_and_parse): pytest.importorskip("xdist") f = testdir.makepyfile( """ @@ -1142,13 +1199,13 @@ def test_pass(): """ ) - result, dom = runandparse(testdir, f, "--dist", "each", "--tx", "2*popen") + result, dom = run_and_parse(f, "--dist", "each", "--tx", "2*popen") assert "INTERNALERROR" not in result.stdout.str() first, second = [x["classname"] for x in dom.find_by_tag("testcase")] assert first == second -def test_fancy_items_regression(testdir): +def test_fancy_items_regression(testdir, run_and_parse): # issue 1259 testdir.makeconftest( """ @@ -1181,7 +1238,7 @@ def test_pass(): """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse() assert "INTERNALERROR" not in result.stdout.str() @@ -1200,9 +1257,10 @@ def test_pass(): ] -def test_global_properties(testdir): +@pytest.mark.parametrize("validate", [True, False]) +def test_global_properties(testdir, validate): path = testdir.tmpdir.join("test_global_properties.xml") - log = LogXML(str(path), None) + log = LogXML(str(path), None, family=("xunit2" if validate else "xunit1")) class Report(BaseReport): sections = [] @@ -1260,7 +1318,8 @@ class Report(BaseReport): ), "The URL did not get written to the xml" -def test_record_testsuite_property(testdir): +@pytest.mark.parametrize("validate", [True, False]) +def test_record_testsuite_property(testdir, run_and_parse, validate): testdir.makepyfile( """ def test_func1(record_testsuite_property): @@ -1270,7 +1329,7 @@ def test_func2(record_testsuite_property): record_testsuite_property("stats", 10) """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) assert result.ret == 0 node = dom.find_first_by_tag("testsuite") properties_node = node.find_first_by_tag("properties") @@ -1308,14 +1367,17 @@ def test_func1(record_testsuite_property): @pytest.mark.parametrize("suite_name", ["my_suite", ""]) -def test_set_suite_name(testdir, suite_name): +@pytest.mark.parametrize("validate", [True, False]) +def test_set_suite_name(testdir, suite_name, run_and_parse, validate): + family = "xunit2" if validate else "xunit1" if suite_name: testdir.makeini( """ [pytest] - junit_suite_name={} + junit_suite_name={suite_name} + junit_family={family} """.format( - suite_name + suite_name=suite_name, family=family ) ) expected = suite_name @@ -1329,13 +1391,13 @@ def test_func(): pass """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) assert result.ret == 0 node = dom.find_first_by_tag("testsuite") node.assert_attr(name=expected) -def test_escaped_skipreason_issue3533(testdir): +def test_escaped_skipreason_issue3533(testdir, run_and_parse): testdir.makepyfile( """ import pytest @@ -1344,20 +1406,27 @@ def test_skip(): pass """ ) - _, dom = runandparse(testdir) + _, dom = run_and_parse() node = dom.find_first_by_tag("testcase") snode = node.find_first_by_tag("skipped") assert "1 <> 2" in snode.text snode.assert_attr(message="1 <> 2") -def test_logging_passing_tests_disabled_does_not_log_test_output(testdir): +@pytest.mark.parametrize("validate", [True, False]) +def test_logging_passing_tests_disabled_does_not_log_test_output( + testdir, run_and_parse, validate +): + family = "xunit2" if validate else "xunit1" testdir.makeini( """ [pytest] junit_log_passing_tests=False junit_logging=system-out - """ + junit_family={family} + """.format( + family=family + ) ) testdir.makepyfile( """ @@ -1371,7 +1440,7 @@ def test_func(): logging.warning('hello') """ ) - result, dom = runandparse(testdir) + result, dom = run_and_parse(validate=validate) assert result.ret == 0 node = dom.find_first_by_tag("testcase") assert len(node.find_by_tag("system-err")) == 0