Decorate potentially flaky tests with @pytest.mark.flaky from pytest-rerunfailures

changelog/2483.trivial.rst

@@ -0,0 +1,2 @@
+Added ``pytest-rerunfailures`` to the ``tests`` set of dependencies,
+and applied it to a flaky test.

docs/contributing/testing_guide.rst

@@ -471,6 +471,22 @@ balanced with each other rather than absolute principles.
   failing. If a test depends on random numbers, use the same random
   seed for each automated test run.
 
+  .. tip::
+
+     Tests that fail intermittently can be decorated with the
+     :py:`@pytest.mark.flaky` decorator from `pytest-rerunfailures`_
+     to indicate that the test should be rerun in case of failures:
+
+     .. code-block:: python
+
+        @pytest.mark.flaky(reruns=5)  # see issue 1548
+        def test_optical_density_histogram(): ...
+
+     Each usage of this decorator should have a comment that either
+     indicates why the test occasionally fails (for example, if the
+     test must download data from an external source) or refers to an
+     issue describing the intermittent failures.
+
 * **Avoid testing implementation details.** Fine-grained tests help us
   find and fix bugs. However, tests that are too fine-grained become
   brittle and lose resistance to refactoring. Avoid testing
@@ -833,6 +849,7 @@ popular IDEs:
 .. _PyCharm: https://www.jetbrains.com/pycharm
 .. _pytest: https://docs.pytest.org
 .. _`pytest-cov`: https://pytest-cov.readthedocs.io
+.. _`pytest-rerunfailures`: https://github.com/pytest-dev/pytest-rerunfailures
 .. _`Python debugger`: https://docs.python.org/3/library/pdb.html
 .. _refactoring: https://refactoring.guru/refactoring/techniques
 .. _`test discovery conventions`: https://docs.pytest.org/en/latest/goodpractices.html#conventions-for-python-test-discovery

plasmapy/utils/data/tests/test_downloader.py

@@ -19,11 +19,9 @@
 
 
 @pytest.mark.parametrize(("filename", "expected"), test_files)
+@pytest.mark.flaky(reruns=5)  # in case of intermittent connection to World Wide Web™
 def test_get_file(filename, expected, tmp_path) -> None:
-    """
-    Test the get_file function
-
-    """
+    """Test the get_file function."""
     # Delete file if it already exists, so the test always downloads it
     dl_path = tmp_path / filename
     if dl_path.exists():
@@ -41,11 +39,9 @@ def test_get_file(filename, expected, tmp_path) -> None:
         downloader.get_file(filename, directory=tmp_path)
 
 
+@pytest.mark.flaky(reruns=5)  # in case of intermittent connection to World Wide Web™
 def test_get_file_NIST_PSTAR_datafile(tmp_path) -> None:
-    """
-    Test the get_file function on a NIST PSTAR datafile
-
-    """
+    """Test the get_file function on a NIST PSTAR datafile."""
     filename = "NIST_PSTAR_aluminum.txt"
 
     # Delete file if it already exists, so the test always downloads it

pyproject.toml

@@ -90,6 +90,7 @@ tests = [
   "pre-commit >= 3.6.0",
   "pytest >= 8.0.2",
   "pytest-regressions >= 2.5.0",
+  "pytest-rerunfailures >= 13.0",
   "pytest-xdist >= 3.5.0",
   "tomli >= 2.0.1",
   "tox >= 4.12.1",

requirements.txt

@@ -292,6 +292,7 @@ pytest==8.0.2
 pytest-datadir==1.5.0
     # via pytest-regressions
 pytest-regressions==2.5.0
+pytest-rerunfailures==13.0
 pytest-xdist==3.5.0
 python-dateutil==2.9.0.post0
     # via