Skip to content
Fetching contributors…
Cannot retrieve contributors at this time
executable file 200 lines (166 sloc) 6.33 KB
#!/usr/bin/env python
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (c) 2012, AT&T Labs, Yun Mao <>
# All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""pylint error checking"""
import cStringIO as StringIO
import json
import re
import sys
from pylint import lint
from pylint.reporters import text
# Note(maoy): E1103 is error code related to partial type inference
ignore_codes = ["E1103"]
# Note(maoy): the error message is the pattern of E0202. It should be ignored
# for nova.tests modules
ignore_messages = ["An attribute affected in nova.tests"]
# Note(maoy): we ignore all errors in openstack.common because it should be
# checked elsewhere. We also ignore nova.tests for now due to high false
# positive rate.
ignore_modules = ["nova/openstack/common/", "nova/tests/"]
KNOWN_PYLINT_EXCEPTIONS_FILE = "tools/pylint_exceptions"
class LintOutput(object):
_cached_filename = None
_cached_content = None
def __init__(self, filename, lineno, line_content, code, message,
self.filename = filename
self.lineno = lineno
self.line_content = line_content
self.code = code
self.message = message
self.lintoutput = lintoutput
def from_line(cls, line):
m ="(\S+):(\d+): \[(\S+)(, \S+)?] (.*)", line)
matched = m.groups()
filename, lineno, code, message = (matched[0], int(matched[1]),
matched[2], matched[-1])
if cls._cached_filename != filename:
with open(filename) as f:
cls._cached_content = list(f.readlines())
cls._cached_filename = filename
line_content = cls._cached_content[lineno - 1].rstrip()
return cls(filename, lineno, line_content, code, message,
def from_msg_to_dict(cls, msg):
"""From the output of pylint msg, to a dict, where each key
is a unique error identifier, value is a list of LintOutput
result = {}
for line in msg.splitlines():
obj = cls.from_line(line)
if obj.is_ignored():
key = obj.key()
if key not in result:
result[key] = []
return result
def is_ignored(self):
if self.code in ignore_codes:
return True
if any(self.filename.startswith(name) for name in ignore_modules):
return True
if any(msg in self.message for msg in ignore_messages):
return True
return False
def key(self):
if self.code in ["E1101", "E1103"]:
# These two types of errors are like Foo class has no member bar.
# We discard the source code so that the error will be ignored
# next time another is encountered.
return self.message, ""
return self.message, self.line_content.strip()
def json(self):
return json.dumps(self.__dict__)
def review_str(self):
return ("File %(filename)s\nLine %(lineno)d:%(line_content)s\n"
"%(code)s: %(message)s" % self.__dict__)
class ErrorKeys(object):
def print_json(cls, errors, output=sys.stdout):
print >>output, "# automatically generated by tools/"
for i in sorted(errors.keys()):
print >>output, json.dumps(i)
def from_file(cls, filename):
keys = set()
for line in open(filename):
if line and line[0] != "#":
d = json.loads(line)
return keys
def run_pylint():
buff = StringIO.StringIO()
reporter = text.ParseableTextReporter(output=buff)
args = ["--include-ids=y", "-E", "nova"]
lint.Run(args, reporter=reporter, exit=False)
val = buff.getvalue()
return val
def generate_error_keys(msg=None):
if msg is None:
msg = run_pylint()
errors = LintOutput.from_msg_to_dict(msg)
with open(KNOWN_PYLINT_EXCEPTIONS_FILE, "w") as f:
ErrorKeys.print_json(errors, output=f)
def validate(newmsg=None):
known = ErrorKeys.from_file(KNOWN_PYLINT_EXCEPTIONS_FILE)
if newmsg is None:
print "Running pylint. Be patient..."
newmsg = run_pylint()
errors = LintOutput.from_msg_to_dict(newmsg)
print "Unique errors reported by pylint: was %d, now %d." \
% (len(known), len(errors))
passed = True
for err_key, err_list in errors.items():
for err in err_list:
if err_key not in known:
print err.lintoutput
passed = False
if passed:
print "Congrats! pylint check passed."
redundant = known - set(errors.keys())
if redundant:
print "Extra credit: some known pylint exceptions disappeared."
for i in sorted(redundant):
print json.dumps(i)
print "Consider regenerating the exception file if you will."
print ("Please fix the errors above. If you believe they are false"
" positives, run 'tools/ generate' to overwrite.")
def usage():
print """Usage: tools/ [generate|validate]
To generate pylint_exceptions file: tools/ generate
To validate the current commit: tools/
def main():
option = "validate"
if len(sys.argv) > 1:
option = sys.argv[1]
if option == "generate":
elif option == "validate":
if __name__ == "__main__":
Jump to Line
Something went wrong with that request. Please try again.