-
Notifications
You must be signed in to change notification settings - Fork 1.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'pep8fix' of git://github.com/tardyp/buildbot
- Loading branch information
Showing
547 changed files
with
19,114 additions
and
15,290 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,166 @@ | ||
#!/usr/bin/env python | ||
'''Check and sort import statement from a python file ''' | ||
|
||
import re | ||
import sys | ||
|
||
|
||
class FixImports(object): | ||
|
||
''' | ||
I can be used to check and sort import statement of a python file | ||
Please use sortImportGroups() method | ||
''' | ||
|
||
_regexImport = re.compile(r"^import\s+(.*)") | ||
_regexFromImport = re.compile(r"^from\s+([a-zA-Z0-9\._]+)\s+import\s+(.*)$") | ||
_regexFromFutureImport = re.compile(r"^from\s+__future__\s+import\s+(.*)$") | ||
|
||
def printErrorMsg(self, filename, lineNb, errorMessage): | ||
''' I print the error message following pylint convention''' | ||
print ("%(filename)s:%(line_nb)s: %(error_msg)s" % | ||
dict(filename=filename, | ||
line_nb=lineNb, | ||
error_msg=errorMessage)) | ||
|
||
def isImportLine(self, line): | ||
'''I return True is the given line is an import statement, False otherwize''' | ||
return self._regexImport.match(line) or self._regexFromImport.match(line) | ||
|
||
def isBadLineFixable(self, line): | ||
'''I return True is the given line is an import line than I know how to split''' | ||
if self.isImportLine(line) and ',' in line and '(' not in line and '\\' not in line: | ||
return True | ||
return False | ||
|
||
def analyzeLine(self, filename, line, lineNb): | ||
'''I look at the line and print all error I find''' | ||
res = True | ||
if self.isImportLine(line): | ||
if ';' in line: | ||
self.printErrorMsg(filename, lineNb, | ||
"multiple import statement on one line. " | ||
"Put each import on its own line.") | ||
res = False | ||
if ',' in line: | ||
self.printErrorMsg(filename, lineNb, | ||
"multiple module imported on one line. " | ||
"Please import each module on a single line.") | ||
res = False | ||
if '\\' in line: | ||
self.printErrorMsg(filename, lineNb, | ||
"new line character found. " | ||
"Please import each module on a single line") | ||
if '(' in line: | ||
self.printErrorMsg(filename, lineNb, | ||
"parenthesis character found. " | ||
"Please import each module on a single line") | ||
res = False | ||
return res | ||
|
||
def importOrder(self, line): | ||
''' | ||
I define how import lines should be sorted | ||
return a tuple of order criterias sorted be importance | ||
''' | ||
ret = ("__future__" not in line, # always put __future__ import first | ||
self._regexFromImport.match(line) is not None, # import before from import | ||
line, # then lexicographic order | ||
) | ||
return ret | ||
|
||
def sortImportGroups(self, filename, data=None): | ||
''' | ||
I perform the analysis of the given file, print the error I find and try to split and | ||
sort the import statement | ||
''' | ||
lines = data.split("\n") | ||
res = True | ||
for cur_line_nb, line in enumerate(lines): | ||
if not self.analyzeLine(filename, line, cur_line_nb): | ||
if not self.isBadLineFixable(line): | ||
res = False | ||
if not res: | ||
return False, data | ||
|
||
# First split the import we can split | ||
newlines = [] | ||
self.groups = [] | ||
self.group_start = None | ||
|
||
def maybeEndGroup(): | ||
if self.group_start is not None: | ||
self.groups.append((self.group_start, len(newlines))) | ||
self.group_start = None | ||
|
||
for line in lines: | ||
if self.isImportLine(line): | ||
if self.group_start is None: | ||
self.group_start = len(newlines) | ||
|
||
if self.isBadLineFixable(line): | ||
match = self._regexFromImport.match(line) | ||
if match: | ||
module = match.group(1) | ||
imports = [s.strip() for s in match.group(2).split(",")] | ||
for imp in imports: | ||
newlines.append("from %s import %s" % (module, imp)) | ||
continue | ||
else: | ||
maybeEndGroup() | ||
newlines.append(line) | ||
|
||
maybeEndGroup() | ||
|
||
lines = newlines | ||
for start, end in self.groups: | ||
lines[start:end] = sorted(lines[start:end], key=self.importOrder) | ||
|
||
# reiterate line by line to split mixed groups | ||
splitted_groups_lines = [] | ||
prev_import_line_type = "" | ||
for line in lines: | ||
if not line.strip() or not self.isImportLine(line): | ||
splitted_groups_lines.append(line) | ||
prev_import_line_type = "" | ||
else: | ||
import_match = self._regexImport.match(line) | ||
from_match = self._regexFromImport.match(line) | ||
current_line_type = None | ||
if import_match is not None: | ||
module = import_match | ||
current_line_type = "import" | ||
elif from_match is not None: | ||
module = from_match | ||
current_line_type = "from" | ||
assert(current_line_type) | ||
if prev_import_line_type and current_line_type != prev_import_line_type: | ||
splitted_groups_lines.append("") | ||
prev_import_line_type = current_line_type | ||
splitted_groups_lines.append(line) | ||
|
||
return True, "\n".join(splitted_groups_lines) | ||
|
||
|
||
def main(): | ||
'''I am the main method''' | ||
if len(sys.argv) != 2: | ||
print "usage: %s <python file>" % (sys.argv[0]) | ||
sys.exit(1) | ||
|
||
filename = sys.argv[1] | ||
|
||
with open(filename, 'r') as filedesc: | ||
data = filedesc.read() | ||
res, content = FixImports().sortImportGroups(filename, data) | ||
if not res: | ||
sys.exit(1) | ||
|
||
with open(filename, 'w') as filedesc: | ||
filedesc.write(content) | ||
if data != content: | ||
print "import successfully reordered for file: %s" % (filename) | ||
sys.exit(0) | ||
|
||
if __name__ == "__main__": | ||
main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
git diff --name-only HEAD^ | common/style_check_and_fix.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,89 @@ | ||
#!/bin/bash | ||
function status() | ||
{ | ||
_ESC=$'\e' | ||
LTCYAN="$_ESC[1;36m" | ||
NORM="$_ESC[0;0m" | ||
|
||
echo "" | ||
echo "${LTCYAN}-- ${*} --${NORM}" | ||
} | ||
|
||
function newshell() | ||
{ | ||
echo "I will launch a new shell. When you are done, just exit the shell" | ||
echo "and I will continue the process" | ||
bash | ||
echo "ok lets continue" | ||
} | ||
|
||
function unittests() | ||
{ | ||
status run the whole test suite as a double check | ||
find . -name \*.pyc -exec rm {} \; | ||
trial --reporter=text buildslave buildbot | ||
if [[ $? != 0 ]] | ||
then | ||
echo "Oups.. the tests are failing, better resolve them now before the big autopep8 work" | ||
newshell | ||
fi | ||
} | ||
if [ $# -eq 0 ]; then | ||
echo "USAGE: common/merge_and_pep8.sh <refs/to/master>" | ||
echo " This script will merge your branch to master" | ||
echo " and apply pep8" | ||
echo "Run this if you want to contribute a branch based on pre-autopep8 rework" | ||
exit 1 | ||
fi | ||
|
||
MASTER=$1 | ||
PREPEP8=`git log $MASTER --grep "PRE_PEP8_COMMIT" --pretty="format:%H"` | ||
POSTPEP8=`git log $MASTER --grep "POST_PEP8_COMMIT" --pretty="format:%H"` | ||
|
||
status "merging against last commit before autopep8" | ||
|
||
git merge $PREPEP8 | ||
if [[ $? != 0 ]] | ||
then | ||
echo "Please fix the merge conflicts between your branch, and last commit before autopep8!" | ||
newshell | ||
fi | ||
|
||
status "merging against first commit after autopep8 and take our version when there are conflicts" | ||
git merge $POSTPEP8 | ||
# autopep8 takes 1h30 to run on the whole codebase, so let git resolve the obvious merge conflicts. | ||
# using -s recursive -x ours works at chunk level, which proved not to work for nine -> master merge | ||
if [[ $? != 0 ]] | ||
then | ||
status "resolve conflicts by checking out ours file" | ||
git status --porcelain |egrep "^DU" | awk '{print $2}' | xargs git rm | ||
git status --porcelain |egrep "^UU" | awk '{print $2}' | xargs git checkout --ours | ||
git status --porcelain |egrep "^UU" | awk '{print $2}' | xargs git add | ||
git commit --no-edit | ||
fi | ||
|
||
unittests | ||
|
||
status "re-apply autopep8 on the files modified by our branch" | ||
git diff --name-only $POSTPEP8 | | ||
( | ||
# there is no real use of displaying output of autopep8 | ||
# so we just display a simple progress status | ||
FILES=() | ||
while read filename; do | ||
FILES+=($filename) | ||
done | ||
n=0 | ||
for filename in ${FILES[@]}; do | ||
n=$(($n + 1)) | ||
echo -n $(($n * 100 / ${#FILES[@]}))% | ||
echo " processing $filename" | ||
echo "$filename" | bash common/style_check_and_fix.sh >&/dev/null | ||
done | ||
) | ||
git commit -s -a -m "re-auto-pep8" | ||
|
||
unittests | ||
|
||
status "finally merge to latest version of master" | ||
git merge $MASTER |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,90 @@ | ||
#!/bin/bash | ||
|
||
FILES=() | ||
EXIT_CODE=0 | ||
while read filename; do | ||
extension="${filename##*.}" | ||
if [[ $extension == "py" && -f $filename ]]; then | ||
if [[ "$filename" =~ "/contrib/" || "$filename" =~ "master/docs" || "$filename" =~ "setup.py" ]]; then | ||
echo "not processing file: $filename" | ||
else | ||
echo "Will process file: $filename" | ||
FILES+=($filename) | ||
fi | ||
fi | ||
done | ||
|
||
echo "======== Checking Import module convention in modified files ========" | ||
|
||
RES=true | ||
for filename in ${FILES[@]}; do | ||
python common/fiximports.py "$filename" | ||
if [[ $? != 0 ]]; then | ||
echo "cannot fix imports of $filename" | ||
RES=false | ||
fi | ||
done | ||
|
||
if [[ $RES == false ]]; then | ||
echo "======= Some import fix could be done. not enforcing for now ========" | ||
else | ||
echo "========================== No error found ===========================" | ||
fi | ||
|
||
if [[ -z `which autopep8` ]]; then | ||
echo "please 'pip install autopep8' in order to automatically fix your pep8 issues" | ||
else | ||
echo "============================== Auto pep8 ==============================" | ||
|
||
for filename in ${FILES[@]}; do | ||
if [[ -f common/pep8rc ]]; then | ||
LINEWIDTH=$(grep -E "max-line-length" common/pep8rc | sed 's/ //g' | cut -d'=' -f 2) | ||
# even if we dont enforce errors, if they can be fixed automatically, thats better.. | ||
IGNORES=E501,W6 | ||
# ignore is not None for sqlaclhemy code.. | ||
if [[ "$filename" =~ "/db/" ]]; then | ||
IGNORES=$IGNORES,E711,E712 | ||
fi | ||
autopep8 --in-place --max-line-length=$LINEWIDTH --ignore=$IGNORES "$filename" | ||
else | ||
echo "No pep8rc found. Discard" | ||
fi | ||
done | ||
echo "=========================== autopep8 done =============================" | ||
fi | ||
if [[ -z `which pep8` ]]; then | ||
echo "please 'pip install pep8' in order to automatically check your pep8 issues" | ||
else | ||
echo "=============================== Pep8 ==================================" | ||
|
||
if [[ -f common/pep8rc ]]; then | ||
for filename in ${FILES[@]}; do | ||
pep8 --config=common/pep8rc "$filename" | ||
if [[ $? != 0 ]]; then | ||
echo "pep8 issues" | ||
EXIT_CODE=1 | ||
fi | ||
done | ||
else | ||
echo "No pep8rc found. Discard" | ||
fi | ||
echo "============================= Pep8 done ===============================" | ||
fi | ||
if [[ -z `which pylint` ]]; then | ||
echo "please 'pip install pylint' in order to automatically fix your pylint issues" | ||
else | ||
echo "========================== Pylint ==========================" | ||
|
||
if [[ -f common/pylintrc ]]; then | ||
for filename in ${FILES[@]}; do | ||
pylint --rcfile=common/pylintrc --disable=R,line-too-long --enable=W0611 --output-format=text --report=no "$filename" | ||
if [[ $? != 0 ]]; then | ||
echo "pylint issues" | ||
EXIT_CODE=1 | ||
fi | ||
done | ||
else | ||
echo "No pylintrc found. Discard" | ||
fi | ||
fi | ||
exit $EXIT_CODE |
Oops, something went wrong.