Merge remote-tracking branch 'Gallaecio/no-callback' into no-callback

.github/workflows/checks.yml

@@ -8,12 +8,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: "3.11"
-          env:
-            TOXENV: security
-        - python-version: "3.11"
-          env:
-            TOXENV: flake8
         - python-version: "3.11"
           env:
             TOXENV: pylint
@@ -26,9 +20,6 @@ jobs:
         - python-version: "3.11"
           env:
             TOXENV: twinecheck
-        - python-version: "3.11"
-          env:
-            TOXENV: black
 
     steps:
     - uses: actions/checkout@v3
@@ -43,3 +34,9 @@ jobs:
       run: |
         pip install -U tox
         tox
+
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: pre-commit/action@v3.0.0

.isort.cfg

@@ -0,0 +1,2 @@
+[settings]
+profile = black

.pre-commit-config.yaml

@@ -8,12 +8,11 @@ repos:
   rev: 6.0.0
   hooks:
   - id: flake8
-- repo: https://github.com/PyCQA/pylint
-  rev: v2.15.6
-  hooks:
-  - id: pylint
-    args: [conftest.py, docs, extras, scrapy, setup.py, tests]
 - repo: https://github.com/psf/black.git
   rev: 22.12.0
   hooks:
   - id: black
+- repo: https://github.com/pycqa/isort
+  rev: 5.12.0
+  hooks:
+  - id: isort

conftest.py

@@ -4,7 +4,6 @@
 from twisted.web.http import H2_ENABLED
 
 from scrapy.utils.reactor import install_reactor
-
 from tests.keys import generate_keys
 
 

docs/_ext/scrapydocs.py

@@ -1,7 +1,8 @@
 from operator import itemgetter
-from docutils.parsers.rst.roles import set_classes
+
 from docutils import nodes
 from docutils.parsers.rst import Directive
+from docutils.parsers.rst.roles import set_classes
 from sphinx.util.nodes import make_refnode
 
 

extras/qps-bench-server.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python
-from time import time
 from collections import deque
-from twisted.web.server import Site, NOT_DONE_YET
-from twisted.web.resource import Resource
+from time import time
+
 from twisted.internet import reactor
+from twisted.web.resource import Resource
+from twisted.web.server import NOT_DONE_YET, Site
 
 
 class Root(Resource):

extras/qpsclient.py

@@ -8,8 +8,8 @@
 
 """
 
-from scrapy.spiders import Spider
 from scrapy.http import Request
+from scrapy.spiders import Spider
 
 
 class QPSSpider(Spider):

scrapy/__init__.py

@@ -9,11 +9,10 @@
 from twisted import version as _txv
 
 # Declare top-level shortcuts
-from scrapy.spiders import Spider
-from scrapy.http import Request, FormRequest
+from scrapy.http import FormRequest, Request
+from scrapy.item import Field, Item
 from scrapy.selector import Selector
-from scrapy.item import Item, Field
-
+from scrapy.spiders import Spider
 
 __all__ = [
     "__version__",

scrapy/cmdline.py

@@ -1,16 +1,17 @@
-import sys
-import os
 import argparse
 import cProfile
 import inspect
+import os
+import sys
+
 import pkg_resources
 
 import scrapy
+from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
 from scrapy.crawler import CrawlerProcess
-from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, BaseRunSpiderCommand
 from scrapy.exceptions import UsageError
 from scrapy.utils.misc import walk_modules
-from scrapy.utils.project import inside_project, get_project_settings
+from scrapy.utils.project import get_project_settings, inside_project
 from scrapy.utils.python import garbage_collect
 
 

scrapy/commands/__init__.py

@@ -1,16 +1,16 @@
 """
 Base class for Scrapy commands
 """
-import os
 import argparse
+import os
 from pathlib import Path
 from typing import Any, Dict, Optional
 
 from twisted.python import failure
-from scrapy.crawler import CrawlerProcess
 
-from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
+from scrapy.crawler import CrawlerProcess
 from scrapy.exceptions import UsageError
+from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
 
 
 class ScrapyCommand:

scrapy/commands/bench.py

@@ -1,6 +1,6 @@
+import subprocess
 import sys
 import time
-import subprocess
 from urllib.parse import urlencode
 
 import scrapy

scrapy/commands/check.py

@@ -1,11 +1,12 @@
 import time
 from collections import defaultdict
-from unittest import TextTestRunner, TextTestResult as _TextTestResult
+from unittest import TextTestResult as _TextTestResult
+from unittest import TextTestRunner
 
 from scrapy.commands import ScrapyCommand
 from scrapy.contracts import ContractsManager
-from scrapy.utils.misc import load_object, set_environ
 from scrapy.utils.conf import build_component_list
+from scrapy.utils.misc import load_object, set_environ
 
 
 class TextTestResult(_TextTestResult):

scrapy/commands/edit.py

@@ -1,5 +1,5 @@
-import sys
 import os
+import sys
 
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError

scrapy/commands/fetch.py

@@ -1,11 +1,12 @@
 import sys
+
 from w3lib.url import is_url
 
 from scrapy.commands import ScrapyCommand
-from scrapy.http import Request
 from scrapy.exceptions import UsageError
+from scrapy.http import Request
 from scrapy.utils.datatypes import SequenceExclude
-from scrapy.utils.spider import spidercls_for_request, DefaultSpider
+from scrapy.utils.spider import DefaultSpider, spidercls_for_request
 
 
 class Command(ScrapyCommand):

scrapy/commands/genspider.py

@@ -1,16 +1,15 @@
 import os
 import shutil
 import string
-
-from pathlib import Path
 from importlib import import_module
+from pathlib import Path
 from typing import Optional, cast
 from urllib.parse import urlparse
 
 import scrapy
 from scrapy.commands import ScrapyCommand
-from scrapy.utils.template import render_templatefile, string_camelcase
 from scrapy.exceptions import UsageError
+from scrapy.utils.template import render_templatefile, string_camelcase
 
 
 def sanitize_module_name(module_name):

scrapy/commands/parse.py

@@ -2,15 +2,15 @@
 import logging
 from typing import Dict
 
-from itemadapter import is_item, ItemAdapter
+from itemadapter import ItemAdapter, is_item
+from twisted.internet.defer import maybeDeferred
 from w3lib.url import is_url
 
-from twisted.internet.defer import maybeDeferred
 from scrapy.commands import BaseRunSpiderCommand
+from scrapy.exceptions import UsageError
 from scrapy.http import Request
 from scrapy.utils import display
 from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
-from scrapy.exceptions import UsageError
 
 logger = logging.getLogger(__name__)
 

scrapy/commands/runspider.py

@@ -1,13 +1,13 @@
 import sys
+from importlib import import_module
 from os import PathLike
 from pathlib import Path
-from importlib import import_module
 from types import ModuleType
 from typing import Union
 
-from scrapy.utils.spider import iter_spider_classes
-from scrapy.exceptions import UsageError
 from scrapy.commands import BaseRunSpiderCommand
+from scrapy.exceptions import UsageError
+from scrapy.utils.spider import iter_spider_classes
 
 
 def _import_file(filepath: Union[str, PathLike]) -> ModuleType:

scrapy/commands/shell.py

@@ -8,7 +8,7 @@
 from scrapy.commands import ScrapyCommand
 from scrapy.http import Request
 from scrapy.shell import Shell
-from scrapy.utils.spider import spidercls_for_request, DefaultSpider
+from scrapy.utils.spider import DefaultSpider, spidercls_for_request
 from scrapy.utils.url import guess_scheme
 
 

scrapy/commands/startproject.py

@@ -1,16 +1,15 @@
-import re
 import os
+import re
 import string
 from importlib.util import find_spec
 from pathlib import Path
-from shutil import ignore_patterns, move, copy2, copystat
+from shutil import copy2, copystat, ignore_patterns, move
 from stat import S_IWUSR as OWNER_WRITE_PERMISSION
 
 import scrapy
 from scrapy.commands import ScrapyCommand
-from scrapy.utils.template import render_templatefile, string_camelcase
 from scrapy.exceptions import UsageError
-
+from scrapy.utils.template import render_templatefile, string_camelcase
 
 TEMPLATES_TO_RENDER = (
     ("scrapy.cfg",),

scrapy/commands/view.py

@@ -1,4 +1,5 @@
 import argparse
+
 from scrapy.commands import fetch
 from scrapy.utils.response import open_in_browser
 

scrapy/contracts/default.py

@@ -1,6 +1,6 @@
 import json
 
-from itemadapter import is_item, ItemAdapter
+from itemadapter import ItemAdapter, is_item
 
 from scrapy.contracts import Contract
 from scrapy.exceptions import ContractFail

scrapy/core/downloader/__init__.py

@@ -1,16 +1,16 @@
 import random
-from time import time
-from datetime import datetime
 from collections import deque
+from datetime import datetime
+from time import time
 
 from twisted.internet import defer, task
 
-from scrapy.utils.defer import mustbe_deferred
-from scrapy.utils.httpobj import urlparse_cached
-from scrapy.resolver import dnscache
 from scrapy import signals
-from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
 from scrapy.core.downloader.handlers import DownloadHandlers
+from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
+from scrapy.resolver import dnscache
+from scrapy.utils.defer import mustbe_deferred
+from scrapy.utils.httpobj import urlparse_cached
 
 
 class Slot:

scrapy/core/downloader/contextfactory.py

@@ -3,10 +3,10 @@
 from OpenSSL import SSL
 from twisted.internet._sslverify import _setAcceptableProtocols
 from twisted.internet.ssl import (
-    optionsForClientTLS,
+    AcceptableCiphers,
     CertificateOptions,
+    optionsForClientTLS,
     platformTrust,
-    AcceptableCiphers,
 )
 from twisted.web.client import BrowserLikePolicyForHTTPS
 from twisted.web.iweb import IPolicyForHTTPS
@@ -15,8 +15,8 @@
 
 from scrapy.core.downloader.tls import (
     DEFAULT_CIPHERS,
-    openssl_methods,
     ScrapyClientTLSOptions,
+    openssl_methods,
 )
 from scrapy.utils.misc import create_instance, load_object
 

scrapy/core/downloader/handlers/__init__.py

@@ -10,7 +10,6 @@
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.python import without_none_values
 
-
 logger = logging.getLogger(__name__)
 
 

scrapy/core/downloader/handlers/http11.py

@@ -13,15 +13,15 @@
 from twisted.internet.error import TimeoutError
 from twisted.python.failure import Failure
 from twisted.web.client import (
+    URI,
     Agent,
     HTTPConnectionPool,
     ResponseDone,
     ResponseFailed,
-    URI,
 )
-from twisted.web.http import _DataLoss, PotentialDataLoss
+from twisted.web.http import PotentialDataLoss, _DataLoss
 from twisted.web.http_headers import Headers as TxHeaders
-from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
+from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer
 from zope.interface import implementer
 
 from scrapy import signals

scrapy/core/downloader/handlers/http2.py

@@ -16,7 +16,6 @@
 from scrapy.spiders import Spider
 from scrapy.utils.python import to_bytes
 
-
 H2DownloadHandlerOrSubclass = TypeVar(
     "H2DownloadHandlerOrSubclass", bound="H2DownloadHandler"
 )

scrapy/core/downloader/middleware.py

@@ -12,8 +12,8 @@
 from scrapy.exceptions import _InvalidOutput
 from scrapy.http import Request, Response
 from scrapy.middleware import MiddlewareManager
-from scrapy.utils.defer import mustbe_deferred, deferred_from_coro
 from scrapy.utils.conf import build_component_list
+from scrapy.utils.defer import deferred_from_coro, mustbe_deferred
 
 
 class DownloaderMiddlewareManager(MiddlewareManager):