Remove tests/requirements.txt and refactor extra deps (#6272)

Co-authored-by: lucasbelo777 <lucas.belo@cialdnb.com>

conftest.py

@@ -1,10 +1,6 @@
-import platform
-import sys
 from pathlib import Path
 
 import pytest
-from twisted import version as twisted_version
-from twisted.python.versions import Version
 from twisted.web.http import H2_ENABLED
 
 from scrapy.utils.reactor import install_reactor
@@ -85,14 +81,12 @@ def only_not_asyncio(request, reactor_pytest):
 def requires_uvloop(request):
     if not request.node.get_closest_marker("requires_uvloop"):
         return
-    if sys.implementation.name == "pypy":
-        pytest.skip("uvloop does not support pypy properly")
-    if platform.system() == "Windows":
-        pytest.skip("uvloop does not support Windows")
-    if twisted_version == Version("twisted", 21, 2, 0):
-        pytest.skip("https://twistedmatrix.com/trac/ticket/10106")
-    if sys.version_info >= (3, 12):
-        pytest.skip("uvloop doesn't support Python 3.12 yet")
+    try:
+        import uvloop
+
+        del uvloop
+    except ImportError:
+        pytest.skip("uvloop is not installed")
 
 
 def pytest_configure(config):

tox.ini

@@ -7,9 +7,23 @@
 envlist = pre-commit,pylint,typing,py
 minversion = 1.7.0
 
+[test-requirements]
+deps =
+    attrs
+    pexpect >= 4.8.0
+    pyftpdlib >= 1.5.8
+    pygments
+    pytest
+    pytest-cov==4.0.0
+    pytest-xdist
+    sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
+    testfixtures
+    pywin32; sys_platform == "win32"
+
 [testenv]
 deps =
-    -rtests/requirements.txt
+    {[test-requirements]deps}
+
     # mitmproxy does not support PyPy
     mitmproxy; implementation_name != 'pypy'
 passenv =
@@ -81,7 +95,7 @@ deps =
     w3lib==1.17.0
     zope.interface==5.1.0
     lxml==4.4.1
-    -rtests/requirements.txt
+    {[test-requirements]deps}
 
     # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies
     # above, hence we do not install it in pinned environments at the moment
@@ -124,8 +138,12 @@ deps =
     robotexclusionrulesparser
     Pillow
     Twisted[http2]
-    brotli
-    zstandard
+    uvloop; platform_system != "Windows"
+    bpython  # optional for shell wrapper tests
+    brotli; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
+    brotlicffi; implementation_name == 'pypy'  # optional for HTTP compress downloader middleware tests
+    zstandard; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
+    ipython
 
 [testenv:extra-deps-pinned]
 basepython = python3.8
@@ -136,6 +154,12 @@ deps =
     Pillow==7.1.0
     robotexclusionrulesparser==1.6.2
     brotlipy
+    uvloop==0.14.0; platform_system != "Windows"
+    bpython==0.7.1
+    zstandard==0.1; implementation_name != 'pypy'
+    ipython==2.0.0
+    brotli==0.5.2; implementation_name != 'pypy'
+    brotlicffi==0.8.0; implementation_name == 'pypy'
 install_command = {[pinned]install_command}
 setenv =
     {[pinned]setenv}