diff --git a/doc/development/setup.rst b/doc/development/setup.rst
index 83aeb70988e..486f486fe97 100644
--- a/doc/development/setup.rst
+++ b/doc/development/setup.rst
@@ -96,9 +96,6 @@ Before you check in your code into git, always run the static checkers and unit
           package ``pytest-xdist`` using ``pip install pytest-xdist`` and then run ``py.test -n NUM`` with
           ``NUM`` being the number of threads you want to use.
 
-.. note:: We currently have some flaky tests that we haven't yet been able to debug. If you just run
-          ``py.test --rerun 5`` they'll be rerun and only reported as failures if they fail repeatedly.
-
 It is a good idea to put this command into your git hook ``.git/hooks/pre-commit``,
 for example::
 
diff --git a/src/pytest.ini b/src/pytest.ini
index 929815d9757..7a7ce31252d 100644
--- a/src/pytest.ini
+++ b/src/pytest.ini
@@ -1,2 +1,3 @@
 [pytest]
 DJANGO_SETTINGS_MODULE=tests.settings
+addopts =--rerun 3 -rw
diff --git a/src/requirements/dev.txt b/src/requirements/dev.txt
index 82b9bf5449c..6567a61d38b 100644
--- a/src/requirements/dev.txt
+++ b/src/requirements/dev.txt
@@ -12,3 +12,4 @@ pytest-django
 isort
 pytest-mock
 pytest-rerunfailures
+pytest-warnings