Pytester runexamples

changelog/3623.feature.rst

@@ -0,0 +1 @@
+introduce ``pytester.copy_example`` as helper to do acceptance tests against examples from the project

doc/en/writing_plugins.rst

@@ -386,11 +386,52 @@ return a result object, with which we can assert the tests' outcomes.
         result.assert_outcomes(passed=4)
 
 
+additionally it is possible to copy examples for a example folder before running pytest on it
+
+.. code:: ini
+
+  # content of pytest.ini
+  [pytest]
+  pytester_example_dir = .
+
+
+.. code:: python
+
+    # content of test_example.py
+
+
+    def test_plugin(testdir):
+      testdir.copy_example("test_example.py")
+      testdir.runpytest("-k", "test_example")
+
+    def test_example():
+      pass
+
+.. code::
+
+    $ pytest
+    =========================== test session starts ============================
+    platform linux -- Python 3.x.y, pytest-3.x.y, py-1.x.y, pluggy-0.x.y
+    rootdir: $REGENDOC_TMPDIR, inifile: pytest.ini
+    collected 2 items
+
+    test_example.py ..                                                   [100%]
+
+    ============================= warnings summary =============================
+    test_example.py::test_plugin
+      $REGENDOC_TMPDIR/test_example.py:4: PytestExerimentalApiWarning: testdir.copy_example is an experimental api that may change over time
+        testdir.copy_example("test_example.py")
+
+    -- Docs: http://doc.pytest.org/en/latest/warnings.html
+    =================== 2 passed, 1 warnings in 0.12 seconds ===================
+
 For more information about the result object that ``runpytest()`` returns, and
 the methods that it provides please check out the :py:class:`RunResult
 <_pytest.pytester.RunResult>` documentation.
 
 
+
+
 .. _`writinghooks`:
 
 Writing hook functions

src/_pytest/experiments.py

@@ -0,0 +1,13 @@
+class PytestExerimentalApiWarning(FutureWarning):
+    "warning category used to denote experiments in pytest"
+
+    @classmethod
+    def simple(cls, apiname):
+        return cls(
+            "{apiname} is an experimental api that may change over time".format(
+                apiname=apiname
+            )
+        )
+
+
+PYTESTER_COPY_EXAMPLE = PytestExerimentalApiWarning.simple("testdir.copy_example")

src/_pytest/pytester.py

@@ -53,6 +53,10 @@ def pytest_addoption(parser):
         ),
     )
 
+    parser.addini(
+        "pytester_example_dir", help="directory to take the pytester example files from"
+    )
+
 
 def pytest_configure(config):
     if config.getvalue("lsof"):
@@ -628,6 +632,22 @@ def mkpydir(self, name):
         p.ensure("__init__.py")
         return p
 
+    def copy_example(self, name):
+        from . import experiments
+        import warnings
+
+        warnings.warn(experiments.PYTESTER_COPY_EXAMPLE, stacklevel=2)
+        example_dir = self.request.config.getini("pytester_example_dir")
+        if example_dir is None:
+            raise ValueError("pytester_example_dir is unset, can't copy examples")
+        example_path = self.request.config.rootdir.join(example_dir, name)
+        if example_path.isdir() and not example_path.join("__init__.py").isfile():
+            example_path.copy(self.tmpdir)
+        elif example_path.isfile():
+            example_path.copy(self.tmpdir.join(example_path.basename))
+        else:
+            raise LookupError("example is not found as a file or directory")
+
     Session = Session
 
     def getnode(self, config, arg):

testing/acceptance_test.py

@@ -14,13 +14,7 @@
 
 class TestGeneralUsage(object):
     def test_config_error(self, testdir):
-        testdir.makeconftest(
-            """
-            def pytest_configure(config):
-                import pytest
-                raise pytest.UsageError("hello")
-        """
-        )
+        testdir.copy_example("conftest_usageerror/conftest.py")
         result = testdir.runpytest(testdir.tmpdir)
         assert result.ret != 0
         result.stderr.fnmatch_lines(["*ERROR: hello"])
@@ -170,18 +164,7 @@ def pytest_collect_directory():
         result.stdout.fnmatch_lines(["*1 skip*"])
 
     def test_issue88_initial_file_multinodes(self, testdir):
-        testdir.makeconftest(
-            """
-            import pytest
-            class MyFile(pytest.File):
-                def collect(self):
-                    return [MyItem("hello", parent=self)]
-            def pytest_collect_file(path, parent):
-                return MyFile(path, parent)
-            class MyItem(pytest.Item):
-                pass
-        """
-        )
+        testdir.copy_example("issue88_initial_file_multinodes")
         p = testdir.makepyfile("def test_hello(): pass")
         result = testdir.runpytest(p, "--collect-only")
         result.stdout.fnmatch_lines(["*MyFile*test_issue88*", "*Module*test_issue88*"])

testing/example_scripts/conftest_usageerror/conftest.py

@@ -0,0 +1,4 @@
+def pytest_configure(config):
+    import pytest
+
+    raise pytest.UsageError("hello")

testing/example_scripts/issue88_initial_file_multinodes/conftest.py

@@ -0,0 +1,14 @@
+import pytest
+
+
+class MyFile(pytest.File):
+    def collect(self):
+        return [MyItem("hello", parent=self)]
+
+
+def pytest_collect_file(path, parent):
+    return MyFile(path, parent)
+
+
+class MyItem(pytest.Item):
+    pass

testing/example_scripts/issue88_initial_file_multinodes/test_hello.py

@@ -0,0 +1,2 @@
+def test_hello():
+    pass

testing/examples/test_issue519.py

@@ -0,0 +1,3 @@
+def test_510(testdir):
+    testdir.copy_example("issue_519.py")
+    testdir.runpytest("issue_519.py")

tox.ini

@@ -204,7 +204,9 @@ filterwarnings =
     ignore:.*type argument to addoption.*:DeprecationWarning
     # produced by python >=3.5 on execnet (pytest-xdist)
     ignore:.*inspect.getargspec.*deprecated, use inspect.signature.*:DeprecationWarning
-
+    #pytests own futurewarnings
+    ignore::_pytest.experiments.PytestExerimentalApiWarning
+pytester_example_dir = testing/example_scripts
 [flake8]
 max-line-length = 120
 ignore = E203,W503