Use pylint

.travis.yml

@@ -7,6 +7,8 @@ branches:
     - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
 matrix:
   include:
+    - env: TOXENV=static
+      python: 3.7
     - env: TOXENV=py27
       python: 2.7
     - env: TOXENV=py27-pinned

docs/utils/linkfix.py

@@ -14,50 +14,57 @@
 
 import re
 
-# Used for remembering the file (and its contents)
-# so we don't have to open the same file again.
-_filename = None
-_contents = None
-
-# A regex that matches standard linkcheck output lines
-line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
-
-# Read lines from the linkcheck output file
-try:
-    with open("build/linkcheck/output.txt") as out:
-        output_lines = out.readlines()
-except IOError:
-    print("linkcheck output not found; please run linkcheck first.")
-    exit(1)
-
-# For every line, fix the respective file
-for line in output_lines:
-    match = re.match(line_re, line)
-
-    if match:
-        newfilename = match.group(1)
-        errortype = match.group(2)
-
-        # Broken links can't be fixed and
-        # I am not sure what do with the local ones.
-        if errortype.lower() in ["broken", "local"]:
-            print("Not Fixed: " + line)
-        else:
-            # If this is a new file
-            if newfilename != _filename:
 
-                # Update the previous file
-                if _filename:
-                    with open(_filename, "w") as _file:
-                        _file.write(_contents)
+def main():
+
+    # Used for remembering the file (and its contents)
+    # so we don't have to open the same file again.
+    _filename = None
+    _contents = None
+
+    # A regex that matches standard linkcheck output lines
+    line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
+
+    # Read lines from the linkcheck output file
+    try:
+        with open("build/linkcheck/output.txt") as out:
+            output_lines = out.readlines()
+    except IOError:
+        print("linkcheck output not found; please run linkcheck first.")
+        exit(1)
+
+    # For every line, fix the respective file
+    for line in output_lines:
+        match = re.match(line_re, line)
+
+        if match:
+            newfilename = match.group(1)
+            errortype = match.group(2)
 
-                _filename = newfilename
+            # Broken links can't be fixed and
+            # I am not sure what do with the local ones.
+            if errortype.lower() in ["broken", "local"]:
+                print("Not Fixed: " + line)
+            else:
+                # If this is a new file
+                if newfilename != _filename:
+
+                    # Update the previous file
+                    if _filename:
+                        with open(_filename, "w") as _file:
+                            _file.write(_contents)
+
+                    _filename = newfilename
+
+                    # Read the new file to memory
+                    with open(_filename) as _file:
+                        _contents = _file.read()
+
+                _contents = _contents.replace(match.group(3), match.group(4))
+        else:
+            # We don't understand what the current line means!
+            print("Not Understood: " + line)
 
-                # Read the new file to memory
-                with open(_filename) as _file:
-                    _contents = _file.read()
 
-            _contents = _contents.replace(match.group(3), match.group(4))
-    else:
-        # We don't understand what the current line means!
-        print("Not Understood: " + line)
+if __name__ == '__main__':
+    main()

pylintrc

@@ -0,0 +1,7 @@
+[MASTER]
+persistent=no
+jobs=1  # >1 hides results
+
+[MESSAGES CONTROL]
+disable=duplicate-code,  # https://github.com/PyCQA/pylint/issues/214
+        useless-object-inheritance  # Required for Python 2 support

scrapy/__init__.py

@@ -1,3 +1,5 @@
+# pylint: disable=invalid-name,wrong-import-order,wrong-import-position
+
 """
 Scrapy - a web crawling and web scraping framework written for Python
 """

scrapy/__main__.py

@@ -1,3 +1,5 @@
+# pylint: disable=missing-docstring
+
 from scrapy.cmdline import execute
 
 if __name__ == '__main__':

scrapy/_monkeypatches.py

@@ -1,3 +1,5 @@
+# pylint: disable=import-error,missing-docstring,trailing-whitespace,unused-import,wrong-import-position
+
 import six
 from six.moves import copyreg
 

scrapy/cmdline.py

@@ -1,3 +1,5 @@
+# pylint: disable=deprecated-module,fixme,inconsistent-return-statements,invalid-name,line-too-long,missing-docstring,multiple-imports,multiple-statements,unused-argument
+
 from __future__ import print_function
 import sys
 import os

scrapy/commands/__init__.py

@@ -1,3 +1,5 @@
+# pylint: disable=attribute-defined-outside-init,bad-continuation,deprecated-module,invalid-name,missing-docstring,no-self-use,unused-argument
+
 """
 Base class for Scrapy commands
 """

scrapy/commands/bench.py

@@ -1,3 +1,5 @@
+# pylint: disable=attribute-defined-outside-init,missing-docstring
+
 import sys
 import time
 import subprocess

scrapy/commands/check.py

@@ -1,3 +1,5 @@
+# pylint: disable=invalid-name,missing-docstring,too-many-locals,trailing-newlines,unused-import
+
 from __future__ import print_function
 import time
 import sys

scrapy/commands/crawl.py

@@ -1,3 +1,5 @@
+# pylint: disable=len-as-condition,line-too-long,missing-docstring,no-else-raise
+
 import os
 from scrapy.commands import ScrapyCommand
 from scrapy.utils.conf import arglist_to_dict

scrapy/commands/edit.py

@@ -1,3 +1,5 @@
+# pylint: disable=inconsistent-return-statements,missing-docstring,unsubscriptable-object
+
 import sys
 import os
 

scrapy/commands/fetch.py

@@ -1,3 +1,5 @@
+# pylint: disable=bad-continuation,invalid-name,missing-docstring,multiple-imports,no-self-use
+
 from __future__ import print_function
 import sys, six
 from w3lib.url import is_url

scrapy/commands/genspider.py

@@ -1,3 +1,5 @@
+# pylint: disable=bad-continuation,inconsistent-return-statements,invalid-name,missing-docstring,too-many-arguments,ungrouped-imports,unsubscriptable-object
+
 from __future__ import print_function
 import os
 import shutil

scrapy/commands/list.py

@@ -1,3 +1,5 @@
+# pylint: disable=invalid-name,missing-docstring
+
 from __future__ import print_function
 from scrapy.commands import ScrapyCommand
 

scrapy/commands/parse.py

@@ -1,3 +1,5 @@
+# pylint: disable=attribute-defined-outside-init,bad-continuation,inconsistent-return-statements,invalid-name,line-too-long,missing-docstring,no-else-raise,no-self-use,too-many-branches
+
 from __future__ import print_function
 import json
 import logging

scrapy/commands/runspider.py

@@ -1,3 +1,5 @@
+# pylint: disable=invalid-name,missing-docstring,redefined-builtin
+
 import sys
 import os
 from importlib import import_module

scrapy/commands/settings.py

@@ -1,3 +1,5 @@
+# pylint: disable=bad-continuation,invalid-name,missing-docstring
+
 from __future__ import print_function
 import json
 

scrapy/commands/shell.py

@@ -1,3 +1,5 @@
+# pylint: disable=bad-continuation,invalid-name,missing-docstring,protected-access,redefined-builtin,unnecessary-pass
+
 """
 Scrapy Shell
 

scrapy/commands/startproject.py

@@ -1,3 +1,5 @@
+# pylint: disable=bad-continuation,missing-docstring,no-self-use,trailing-newlines,ungrouped-imports,unsubscriptable-object
+
 from __future__ import print_function
 import re
 import os

scrapy/commands/version.py

@@ -1,3 +1,5 @@
+# pylint: disable=bad-continuation,missing-docstring,trailing-newlines
+
 from __future__ import print_function
 
 import scrapy

scrapy/commands/view.py

@@ -1,3 +1,5 @@
+# pylint: disable=missing-docstring,unused-import
+
 from scrapy.commands import fetch, ScrapyCommand
 from scrapy.utils.response import open_in_browser
 

scrapy/contracts/__init__.py

@@ -1,3 +1,5 @@
+# pylint: disable=broad-except,inconsistent-return-statements,invalid-name,lost-exception,missing-docstring,no-member,no-self-argument,no-self-use
+
 import sys
 import re
 from functools import wraps

scrapy/contracts/default.py

@@ -1,3 +1,5 @@
+# pylint: disable=invalid-name,missing-docstring
+
 from scrapy.item import BaseItem
 from scrapy.http import Request
 from scrapy.exceptions import ContractFail

scrapy/core/downloader/__init__.py

@@ -1,3 +1,5 @@
+# pylint: disable=no-member,missing-docstring,too-many-instance-attributes,unused-argument
+
 from __future__ import absolute_import
 import random
 import warnings

scrapy/core/downloader/contextfactory.py

@@ -1,3 +1,5 @@
+# pylint: disable=arguments-differ,bad-continuation,invalid-name,keyword-arg-before-vararg,missing-docstring,too-few-public-methods,ungrouped-imports,unused-argument
+
 from OpenSSL import SSL
 from twisted.internet.ssl import ClientContextFactory
 

scrapy/core/downloader/handlers/__init__.py

@@ -1,3 +1,5 @@
+# pylint: disable=broad-except,invalid-name,missing-docstring,too-few-public-methods
+
 """Download handlers for different schemes"""
 
 import logging

scrapy/core/downloader/handlers/datauri.py

@@ -1,3 +1,5 @@
+# pylint: disable=missing-docstring,no-self-use,too-few-public-methods,unused-argument
+
 from w3lib.url import parse_data_uri
 
 from scrapy.http import TextResponse

scrapy/core/downloader/handlers/file.py

@@ -1,3 +1,5 @@
+# pylint: disable=invalid-name,missing-docstring,no-self-use,too-few-public-methods,unused-argument
+
 from w3lib.url import file_uri_to_path
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.decorators import defers

scrapy/core/downloader/handlers/ftp.py

@@ -1,3 +1,5 @@
+# pylint: disable=anomalous-backslash-in-string,attribute-defined-outside-init,bad-continuation,expression-not-assigned,invalid-name,line-too-long,missing-docstring,trailing-newlines,unused-argument
+
 """
 An asynchronous FTP file download handler for scrapy which somehow emulates an http response.
 

scrapy/core/downloader/handlers/http.py

@@ -1,3 +1,5 @@
+# pylint: disable=missing-docstring,too-few-public-methods,unused-import
+
 from __future__ import absolute_import
 from .http10 import HTTP10DownloadHandler
 from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler

scrapy/core/downloader/handlers/http10.py

@@ -1,3 +1,5 @@
+# pylint: disable=invalid-name,line-too-long,missing-docstring,no-else-return,no-member,too-few-public-methods,unused-argument
+
 """Download handlers for http and https schemes
 """
 from twisted.internet import reactor

scrapy/core/downloader/handlers/http11.py

@@ -1,3 +1,5 @@
+# pylint: disable=anomalous-backslash-in-string,arguments-differ,attribute-defined-outside-init,bad-continuation,bad-whitespace,deprecated-method,invalid-name,line-too-long,missing-docstring,no-else-return,no-member,no-self-use,protected-access,redefined-builtin,redefined-outer-name,signature-differs,too-few-public-methods,too-many-arguments,too-many-function-args,too-many-instance-attributes,unused-argument,unused-import
+
 """Download handlers for http and https schemes"""
 
 import re

scrapy/core/downloader/handlers/s3.py

@@ -1,3 +1,5 @@
+# pylint: disable=arguments-differ,bad-continuation,import-error,invalid-name,missing-docstring,no-name-in-module,no-self-use,too-few-public-methods,unused-argument,unused-import
+
 from six.moves.urllib.parse import unquote
 
 from scrapy.exceptions import NotConfigured

scrapy/core/downloader/middleware.py

@@ -1,3 +1,5 @@
+# pylint: disable=line-too-long,missing-docstring
+
 """
 Downloader Middleware manager
 

scrapy/core/downloader/tls.py

@@ -1,3 +1,5 @@
+# pylint: disable=catching-non-exception,fixme,invalid-name,line-too-long,logging-format-interpolation,missing-docstring,no-name-in-module,protected-access,too-few-public-methods,ungrouped-imports
+
 import logging
 from OpenSSL import SSL
 

scrapy/core/downloader/webclient.py

@@ -1,3 +1,5 @@
+# pylint: disable=arguments-differ,attribute-defined-outside-init,invalid-name,missing-docstring,no-member,super-init-not-called,too-many-instance-attributes,trailing-newlines
+
 from time import time
 from six.moves.urllib.parse import urlparse, urlunparse, urldefrag
 

scrapy/core/engine.py

@@ -1,3 +1,5 @@
+# pylint: disable=attribute-defined-outside-init,bad-continuation,broad-except,inconsistent-return-statements,invalid-name,missing-docstring,no-else-return,too-many-instance-attributes,unused-argument
+
 """
 This is the Scrapy engine which controls the Scheduler, Downloader and Spiders.
 

scrapy/core/scheduler.py

@@ -1,3 +1,5 @@
+# pylint: disable=attribute-defined-outside-init,inconsistent-return-statements,invalid-name,missing-docstring,no-self-use,too-many-arguments,too-many-instance-attributes,ungrouped-imports,unused-argument
+
 import os
 import json
 import logging

scrapy/core/scraper.py

@@ -1,3 +1,5 @@
+# pylint: disable=bad-continuation,inconsistent-return-statements,invalid-name,missing-docstring,no-else-return,no-self-use,trailing-newlines
+
 """This module implements the Scraper component which parses responses and
 extracts information from them"""
 

scrapy/core/spidermw.py

@@ -1,3 +1,5 @@
+# pylint: disable=bad-continuation,bad-whitespace,broad-except,invalid-name,line-too-long,missing-docstring,no-else-return
+
 """
 Spider Middleware manager
 

scrapy/crawler.py

@@ -1,3 +1,5 @@
+# pylint: disable=attribute-defined-outside-init,invalid-name,missing-docstring,no-member,no-self-use,too-many-instance-attributes,wrong-import-order
+
 import six
 import signal
 import logging

scrapy/downloadermiddlewares/ajaxcrawl.py

@@ -1,4 +1,6 @@
 # -*- coding: utf-8 -*-
+# pylint: disable=fixme,invalid-name,line-too-long,missing-docstring
+
 from __future__ import absolute_import
 import re
 import logging

scrapy/downloadermiddlewares/chunked.py

@@ -1,3 +1,5 @@
+# pylint: disable=missing-docstring,no-self-use,too-few-public-methods,unused-argument
+
 import warnings
 
 from scrapy.exceptions import ScrapyDeprecationWarning

scrapy/downloadermiddlewares/cookies.py

@@ -1,3 +1,5 @@
+# pylint: disable=invalid-name,missing-docstring,no-self-use,unused-import,wrong-import-order
+
 import os
 import six
 import logging

scrapy/downloadermiddlewares/decompression.py

@@ -1,3 +1,5 @@
+# pylint: disable=inconsistent-return-statements,invalid-name,missing-docstring,no-self-use,too-few-public-methods,unused-argument
+
 """ This module implements the DecompressionMiddleware which tries to recognise
 and extract the potentially compressed responses that may arrive.
 """

scrapy/downloadermiddlewares/defaultheaders.py

@@ -1,3 +1,5 @@
+# pylint: disable=invalid-name,missing-docstring,unused-argument
+
 """
 DefaultHeaders downloader middleware
 

scrapy/downloadermiddlewares/downloadtimeout.py

@@ -1,3 +1,5 @@
+# pylint: disable=invalid-name,missing-docstring,unused-argument
+
 """
 Download timeout middleware
 

scrapy/downloadermiddlewares/httpauth.py

@@ -1,3 +1,5 @@
+# pylint: disable=attribute-defined-outside-init,invalid-name,missing-docstring,unused-argument
+
 """
 HTTP basic auth downloader middleware