Merge pull request #8524 from gsnedders/flake8-wptrunner

Run flake8 on wptrunner, and run the same flakes everywhere

.travis.yml

@@ -87,7 +87,7 @@ matrix:
         apt:
           packages:
             - libnss3-tools
-      env: JOB=wpt_integration TOXENV=py27 SCRIPT=tools/ci/ci_wpt.sh
+      env: JOB=wpt_integration TOXENV=py27,py27-flake8 SCRIPT=tools/ci/ci_wpt.sh
   exclude:
     - env:  # exclude empty env from the top-level above
   allow_failures:

tools/ci/check_stability.py

@@ -132,11 +132,6 @@ def get_sha1():
     return git("rev-parse", "HEAD").strip()
 
 
-def install_wptrunner():
-    """Install wptrunner."""
-    call("pip", "install", wptrunner_root)
-
-
 def deepen_checkout(user):
     """Convert from a shallow checkout to a full one"""
     fetch_args = [user, "+refs/heads/*:refs/remotes/origin/*"]

tools/ci/ci_tools_unittest.sh

@@ -16,6 +16,7 @@ fi
 
 if [[ $(./wpt test-jobs --includes wptrunner_unittest; echo $?) -eq 0 ]]; then
     if [ $TOXENV == "py27" ] || [ $TOXENV == "pypy" ]; then
+        TOXENV="$TOXENV,py27-flake8"
         cd tools/wptrunner
         tox
     fi

tools/manifest/XMLParser.py

@@ -70,8 +70,8 @@ def _data(self, text):
     def _end(self, tag):
         return self._target.end(_fixname(tag))
 
-    def _external(self, context, base, systemId, publicId):
-        if publicId in {
+    def _external(self, context, base, system_id, public_id):
+        if public_id in {
                 "-//W3C//DTD XHTML 1.0 Transitional//EN",
                 "-//W3C//DTD XHTML 1.1//EN",
                 "-//W3C//DTD XHTML 1.0 Strict//EN",

tools/manifest/manifest.py

@@ -2,7 +2,7 @@
 import os
 import re
 from collections import defaultdict
-from six import iteritems, itervalues, viewkeys
+from six import iteritems, itervalues, viewkeys, string_types
 
 from .item import ManualTest, WebdriverSpecTest, Stub, RefTestNode, RefTest, TestharnessTest, SupportFile, ConformanceCheckerTest, VisualTest
 from .log import get_logger
@@ -20,14 +20,6 @@ class ManifestVersionMismatch(ManifestError):
     pass
 
 
-def sourcefile_items(args):
-    tests_root, url_base, rel_path, status = args
-    source_file = SourceFile(tests_root,
-                             rel_path,
-                             url_base)
-    return rel_path, source_file.manifest_items()
-
-
 class Manifest(object):
     def __init__(self, url_base="/"):
         assert url_base is not None
@@ -221,7 +213,7 @@ def load(tests_root, manifest):
     logger = get_logger()
 
     # "manifest" is a path or file-like object.
-    if isinstance(manifest, basestring):
+    if isinstance(manifest, string_types):
         if os.path.exists(manifest):
             logger.debug("Opening manifest at %s" % manifest)
         else:

tools/runner/report.py

@@ -1,3 +1,5 @@
+# flake8: noqa
+
 from __future__ import print_function
 
 import argparse

tools/sslutils/base.py

@@ -1,9 +1,3 @@
-def get_logger(name="ssl"):
-    logger = structured.get_default_logger(name)
-    if logger is None:
-        logger = structured.structuredlog.StructuredLogger(name)
-    return logger
-
 class NoSSLEnvironment(object):
     ssl_enabled = False
 

tools/tox.ini

@@ -3,8 +3,12 @@ envlist = py27,py36,pypy
 skipsdist=True
 
 [testenv]
+# flake8 versions should be kept in sync across tools/tox.ini, tools/wpt/tox.ini, and tools/wptrunner/tox.ini
 deps =
-  flake8
+  flake8==3.5.0
+  pycodestyle==2.3.1
+  pyflakes==1.6.0
+  pep8-naming==0.4.1
   pytest
   pytest-cov
   mock
@@ -19,6 +23,28 @@ passenv =
   HYPOTHESIS_PROFILE
 
 [flake8]
-ignore = E128,E129,E221,E226,E231,E251,E265,E302,E303,E305,E402,E901,F401,F821,F841
+# flake8 config should be kept in sync across tools/tox.ini, tools/wpt/tox.ini, and tools/wptrunner/tox.ini
+select = E,W,F,N
+# E128: continuation line under-indented for visual indent
+# E129: visually indented line with same indent as next logical line
+# E221: multiple spaces before operator
+# E226: missing whitespace around arithmetic operator
+# E231: missing whitespace after ‘,’, ‘;’, or ‘:’
+# E251: unexpected spaces around keyword / parameter equals
+# E265: block comment should start with ‘# ‘
+# E302: expected 2 blank lines, found 0
+# E303: too many blank lines (3)
+# E305: expected 2 blank lines after end of function or class
+# E402: module level import not at top of file
+# E731: do not assign a lambda expression, use a def
+# E901: SyntaxError or IndentationError
+# W601: .has_key() is deprecated, use ‘in’
+# F401: module imported but unused
+# F403: ‘from module import *’ used; unable to detect undefined names
+# F405: name may be undefined, or defined from star imports: module
+# F841: local variable name is assigned to but never used
+# N801: class names should use CapWords convention
+# N802: function name should be lowercase
+ignore = E128,E129,E221,E226,E231,E251,E265,E302,E303,E305,E402,E731,E901,W601,F401,F403,F405,F841,N801,N802
 max-line-length = 141
 exclude = .tox,html5lib,third_party/py,third_party/pytest,third_party/funcsigs,third_party/attrs,third_party/pluggy/,pywebsocket,six,_venv,webencodings,wptserve/docs,wptserve/tests/functional/docroot/,wpt,wptrunner

tools/webdriver/webdriver/client.py

@@ -4,6 +4,8 @@
 import protocol
 import transport
 
+from six import string_types
+
 from mozlog import get_default_logger
 
 logger = get_default_logger()
@@ -313,7 +315,7 @@ def __setitem__(self, name, value):
         cookie = {"name": name,
                   "value": None}
 
-        if isinstance(name, (str, unicode)):
+        if isinstance(name, string_types):
             cookie["value"] = value
         elif hasattr(value, "value"):
             cookie["value"] = value.value

tools/webdriver/webdriver/protocol.py

@@ -16,7 +16,7 @@ def default(self, obj):
             return [self.default(x) for x in obj]
         elif isinstance(obj, webdriver.Element):
             return {webdriver.Element.identifier: obj.id}
-        return super(ProtocolEncoder, self).default(obj)
+        return super(Encoder, self).default(obj)
 
 
 class Decoder(json.JSONDecoder):

tools/webdriver/webdriver/transport.py

@@ -4,6 +4,7 @@
 
 import error
 
+from six import text_type
 
 """Implements HTTP transport for the WebDriver wire protocol."""
 
@@ -135,7 +136,7 @@ def send(self,
         except ValueError:
             raise ValueError("Failed to encode request body as JSON:\n"
                 "%s" % json.dumps(body, indent=2))
-        if isinstance(payload, unicode):
+        if isinstance(payload, text_type):
             payload = body.encode("utf-8")
 
         if headers is None:

tools/wpt/browser.py

@@ -4,9 +4,12 @@
 import re
 import shutil
 import stat
+import subprocess
+import sys
 from abc import ABCMeta, abstractmethod
 from ConfigParser import RawConfigParser
 from distutils.spawn import find_executable
+from io import BytesIO
 
 from utils import call, get, untar, unzip
 

tools/wpt/install.py

@@ -39,8 +39,3 @@ def install(name, component, destination):
     subclass = getattr(browser, name.title())
     sys.stdout.write('Now installing %s %s...\n' % (name, component))
     getattr(subclass(), method)(dest=destination)
-
-
-if __name__ == '__main__':
-    args = parser.parse_args()
-    run(None, **vars(args))

tools/wpt/markdown.py

@@ -1,3 +1,5 @@
+from .stability import is_inconsistent
+
 def format_comment_title(product):
     """Produce a Markdown-formatted string based on a given "product"--a string
     containing a browser identifier optionally followed by a colon and a

tools/wpt/tox.ini

@@ -1,10 +1,9 @@
 [tox]
-envlist = py27
+envlist = py27,py27-flake8
 skipsdist=True
 
 [testenv]
 deps =
-  flake8
   pytest
   pytest-cov
   hypothesis
@@ -14,8 +13,40 @@ deps =
 
 commands =
   pytest --cov
-  flake8
+
+[testenv:py27-flake8]
+# flake8 versions should be kept in sync across tools/tox.ini, tools/wpt/tox.ini, and tools/wptrunner/tox.ini
+deps =
+     flake8==3.5.0
+     pycodestyle==2.3.1
+     pyflakes==1.6.0
+     pep8-naming==0.4.1
+
+commands =
+     flake8
 
 [flake8]
-ignore = E128,E129,E221,E226,E231,E251,E265,E302,E303,E305,E402,E901,F401,F821,F841
+# flake8 config should be kept in sync across tools/tox.ini, tools/wpt/tox.ini, and tools/wptrunner/tox.ini
+select = E,W,F,N
+# E128: continuation line under-indented for visual indent
+# E129: visually indented line with same indent as next logical line
+# E221: multiple spaces before operator
+# E226: missing whitespace around arithmetic operator
+# E231: missing whitespace after ‘,’, ‘;’, or ‘:’
+# E251: unexpected spaces around keyword / parameter equals
+# E265: block comment should start with ‘# ‘
+# E302: expected 2 blank lines, found 0
+# E303: too many blank lines (3)
+# E305: expected 2 blank lines after end of function or class
+# E402: module level import not at top of file
+# E731: do not assign a lambda expression, use a def
+# E901: SyntaxError or IndentationError
+# W601: .has_key() is deprecated, use ‘in’
+# F401: module imported but unused
+# F403: ‘from module import *’ used; unable to detect undefined names
+# F405: name may be undefined, or defined from star imports: module
+# F841: local variable name is assigned to but never used
+# N801: class names should use CapWords convention
+# N802: function name should be lowercase
+ignore = E128,E129,E221,E226,E231,E251,E265,E302,E303,E305,E402,E731,E901,W601,F401,F403,F405,F841,N801,N802
 max-line-length = 141

tools/wpt/utils.py

@@ -27,7 +27,7 @@ def set_if_none(self,
                 value = value()
             if not value:
                 if err_fn is not None:
-                    return err_fn(kwargs, "Failed to find %s" % desc)
+                    return err_fn(self, "Failed to find %s" % desc)
                 else:
                     return
             self[name] = value

tools/wpt/virtualenv.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 import sys
 import logging
 from distutils.spawn import find_executable

tools/wptrunner/docs/conf.py

@@ -186,22 +186,22 @@
 # -- Options for LaTeX output ---------------------------------------------
 
 latex_elements = {
-# The paper size ('letterpaper' or 'a4paper').
-#'papersize': 'letterpaper',
+    # The paper size ('letterpaper' or 'a4paper').
+    #'papersize': 'letterpaper',
 
-# The font size ('10pt', '11pt' or '12pt').
-#'pointsize': '10pt',
+    # The font size ('10pt', '11pt' or '12pt').
+    #'pointsize': '10pt',
 
-# Additional stuff for the LaTeX preamble.
-#'preamble': '',
+    # Additional stuff for the LaTeX preamble.
+    #'preamble': '',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-  ('index', 'wptrunner.tex', u'wptrunner Documentation',
-   u'James Graham', 'manual'),
+    ('index', 'wptrunner.tex', u'wptrunner Documentation',
+     u'James Graham', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -244,9 +244,9 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'wptrunner', u'wptrunner Documentation',
-   u'James Graham', 'wptrunner', 'One line description of project.',
-   'Miscellaneous'),
+    ('index', 'wptrunner', u'wptrunner Documentation',
+     u'James Graham', 'wptrunner', 'One line description of project.',
+     'Miscellaneous'),
 ]
 
 # Documents to append as an appendix to all manuals.

tools/wptrunner/setup.py

@@ -59,7 +59,7 @@
       include_package_data=True,
       data_files=[("requirements", requirements_files)],
       install_requires=deps
-     )
+      )
 
 if "install" in sys.argv:
     path = os.path.relpath(os.path.join(sys.prefix, "requirements"), os.curdir)

tools/wptrunner/test/test.py

@@ -156,7 +156,8 @@ def main():
         run(config, args)
     except Exception:
         if args.pdb:
-            import pdb, traceback
+            import pdb
+            import traceback
             print traceback.format_exc()
             pdb.post_mortem()
         else: