From 4d91b5de03394281d12b1450eb08d5d64bc4814b Mon Sep 17 00:00:00 2001 From: dabercro Date: Wed, 1 Aug 2018 11:32:36 -0400 Subject: [PATCH 01/11] Move dependencies explicitly into test --- .travis.yml | 13 ++++++------- cmstoolbox/__init__.py | 2 +- setup.py | 1 - test/requirements.txt | 2 ++ test/requirements26.txt | 3 +++ 5 files changed, 12 insertions(+), 9 deletions(-) create mode 100644 test/requirements.txt create mode 100644 test/requirements26.txt diff --git a/.travis.yml b/.travis.yml index e42e879..529b1bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,22 +4,21 @@ matrix: include: - python: 2.6 env: - - pylinter=old - nodoc=true + - testreq=26 - python: 2.7 - env: nodoc=true + env: + - nodoc=true - dist: trusty sudo: required python: 2.7 install: - - if [ "$pylinter" = "old" ]; - then pip install "astroid<1.3" "pylint<1.4" "testfixtures<6"; - else pip install pylint; - fi + # 2.6 will need a different requirements file for tests + - pip install -r test/requirements${testrequirements}.txt - python setup.py install script: - opsspace-test - - if [ "$pylinter" = "old" ]; + - if [ "$testrequrements" = "26" ]; then pylint --rcfile <(sed 's/load-plugins=/#/g' test/pylint.cfg) cmstoolbox; else pylint --rcfile test/pylint.cfg cmstoolbox; fi diff --git a/cmstoolbox/__init__.py b/cmstoolbox/__init__.py index f82510f..cb04f0a 100644 --- a/cmstoolbox/__init__.py +++ b/cmstoolbox/__init__.py @@ -2,4 +2,4 @@ __all__ = [] -__version__ = '0.9.3' +__version__ = '0.9.4' diff --git a/setup.py b/setup.py index 7ac2057..66be8dd 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,5 @@ author_email='dabercro@mit.edu', description='Tools used by CMS Computing Operations', url='https://github.com/CMSCompOps/OpsSpace', - install_requires=['testfixtures'], scripts=[s for s in glob.glob('bin/*') if not s.endswith('~')] ) diff --git a/test/requirements.txt b/test/requirements.txt new file mode 100644 index 0000000..6c2324f --- /dev/null +++ b/test/requirements.txt @@ -0,0 +1,2 @@ +testfixtures +pylint diff --git a/test/requirements26.txt b/test/requirements26.txt new file mode 100644 index 0000000..b5c3914 --- /dev/null +++ b/test/requirements26.txt @@ -0,0 +1,3 @@ +astroid<1.3 +pylint<1.4 +testfixtures<6 From 5d226fa87aee27e7e08c489f695dbaf475ff528a Mon Sep 17 00:00:00 2001 From: dabercro Date: Wed, 1 Aug 2018 11:36:09 -0400 Subject: [PATCH 02/11] Didn't agree with myself on variable name --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 529b1bf..d38dc30 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,11 +14,11 @@ matrix: python: 2.7 install: # 2.6 will need a different requirements file for tests - - pip install -r test/requirements${testrequirements}.txt + - pip install -r test/requirements${testreq}.txt - python setup.py install script: - opsspace-test - - if [ "$testrequrements" = "26" ]; + - if [ "$testreq" = "26" ]; then pylint --rcfile <(sed 's/load-plugins=/#/g' test/pylint.cfg) cmstoolbox; else pylint --rcfile test/pylint.cfg cmstoolbox; fi From 5894a4ef6aea20f16c25d4f2155e4be4ad18dc5a Mon Sep 17 00:00:00 2001 From: dabercro Date: Wed, 1 Aug 2018 14:31:02 -0400 Subject: [PATCH 03/11] Added automatic pylint tests. --- .travis.yml | 7 - bin/opsspace-test | 440 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 439 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index d38dc30..68df40e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ matrix: - python: 2.6 env: - nodoc=true - - testreq=26 - python: 2.7 env: - nodoc=true @@ -13,12 +12,6 @@ matrix: sudo: required python: 2.7 install: - # 2.6 will need a different requirements file for tests - - pip install -r test/requirements${testreq}.txt - python setup.py install script: - opsspace-test - - if [ "$testreq" = "26" ]; - then pylint --rcfile <(sed 's/load-plugins=/#/g' test/pylint.cfg) cmstoolbox; - else pylint --rcfile test/pylint.cfg cmstoolbox; - fi diff --git a/bin/opsspace-test b/bin/opsspace-test index 986522d..77e5430 100755 --- a/bin/opsspace-test +++ b/bin/opsspace-test @@ -8,12 +8,14 @@ fi errorsfound=0 + + # Function to do the test for us _do_test () { packagetest="$1" # Setup the test to be done - $packagetest # Do the test + eval $packagetest # Do the test errorcode=$? # Get the error code if [ $errorcode -ne 0 ] # If failed tell user, and increase error count @@ -34,6 +36,8 @@ _do_test () { } + + # Check if we should test build Sphinx documentation if [ -f docs/conf.py ] && [ "$nodoc" != "true" ] then @@ -44,9 +48,25 @@ then fi + # Now do the tests inside the test directory cd test +# Get the python version +# 26 for Python 2.6 +# 27 for Python 2.7 +# and so on +pyvers=$(python --version 2>&1 | perl -ne '/\b(\d)\.(\d+)\./ && print "$1$2"') + +# If versioned file not here, fall back to requirements.txt +reqfile=$(test -f requirements${pyvers}.txt && echo requirements${pyvers}.txt || echo requirements.txt) + +# Install requirements, if they exist +test -f $reqfile && pip install -r $reqfile + + + +# Run the test for all files that start with "test_". They should be executable. for packagetest in test_* do @@ -54,6 +74,38 @@ do done + +# Check for pylint in test requirements, and run test if desired +if grep pylint $reqfile +then + + cd .. + # Look for packages from root directory + package=$(dirname $(find * -maxdepth 1 -name __init__.py)) + + # This exits 0 when the pylint version is old, and we have to disable the extensions + if pylint --version 2> /dev/null | perl -ne '/pylint\s(\d+.\d+)/ && exit($1 > 1.3)' + then + + lintfilter='s/load-plugins/#/' + + fi + + # PyLint Config is towards the bottom of this file + # It's used as default when local pylint.cfg does not exist + if [ ! -f test/pylint.cfg ] + then + + perl -0ne '/PyLint Config\n(\-+)\n(.*)\n\1/sm && print $2' $0 > test/pylint.cfg + + fi + + _do_test "pylint --rcfile=<(sed '$lintfilter' test/pylint.cfg) $package" + +fi + + + if [ $errorsfound -eq 1 ] # Setting plural correctly then # looks impressive to some errstr="error" @@ -73,3 +125,389 @@ echo "$errorsfound $errstr found" tput sgr0 2> /dev/null # Reset color exit $errorsfound + +PyLint Config +------------- + +[MASTER] + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins=pylint.extensions.docparams,pylint.extensions.mccabe + +# Use multiple processes to speed up Pylint. +jobs=1 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=bad-option-value,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,no-name-in-module,broad-except,no-member,star-args + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[BASIC] + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + + +[ELIF] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=100 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,future.builtins + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=8 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=20 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=20 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=1 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception + +------------- From a85f41b7f1f100d47bc56237cdf12fbbf687359e Mon Sep 17 00:00:00 2001 From: dabercro Date: Wed, 1 Aug 2018 14:51:54 -0400 Subject: [PATCH 04/11] Test should clean up after self --- bin/opsspace-test | 4 + cmstoolbox/__init__.py | 2 +- test/pylint.cfg | 407 ----------------------------------------- 3 files changed, 5 insertions(+), 408 deletions(-) delete mode 100644 test/pylint.cfg diff --git a/bin/opsspace-test b/bin/opsspace-test index 77e5430..9d7ae02 100755 --- a/bin/opsspace-test +++ b/bin/opsspace-test @@ -97,11 +97,15 @@ then then perl -0ne '/PyLint Config\n(\-+)\n(.*)\n\1/sm && print $2' $0 > test/pylint.cfg + plseraselintcfg=1 fi _do_test "pylint --rcfile=<(sed '$lintfilter' test/pylint.cfg) $package" + # If PyLint file was created, get rid of it unless asked to be kept + test -z $plseraselintcfg || test ! -z $keepcfg || rm test/pylint.cfg + fi diff --git a/cmstoolbox/__init__.py b/cmstoolbox/__init__.py index cb04f0a..1d4e944 100644 --- a/cmstoolbox/__init__.py +++ b/cmstoolbox/__init__.py @@ -2,4 +2,4 @@ __all__ = [] -__version__ = '0.9.4' +__version__ = '0.9.5' diff --git a/test/pylint.cfg b/test/pylint.cfg deleted file mode 100644 index e9b2a4d..0000000 --- a/test/pylint.cfg +++ /dev/null @@ -1,407 +0,0 @@ -[MASTER] - -# Specify a configuration file. -#rcfile= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns= - -# Pickle collected data for later comparisons. -persistent=yes - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins=pylint.extensions.docparams,pylint.extensions.mccabe - -# Use multiple processes to speed up Pylint. -jobs=1 - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= - -# Allow optimization of some AST trees. This will activate a peephole AST -# optimizer, which will apply various small optimizations. For instance, it can -# be used to obtain the result of joining multiple strings with the addition -# operator. Joining a lot of strings can lead to a maximum recursion error in -# Pylint and this flag can prevent that. It has one side effect, the resulting -# AST will be different than the one from reality. This option is deprecated -# and it will be removed in Pylint 2.0. -optimize-ast=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -#enable= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=bad-option-value,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,no-name-in-module,broad-except,no-member,star-args - - -[REPORTS] - -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html. You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". This option is deprecated -# and it will be removed in Pylint 2.0. -files-output=no - -# Tells whether to display a full report or only the messages -reports=yes - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - - -[BASIC] - -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty - -# Regular expression matching correct function names -function-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for function names -function-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct variable names -variable-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for variable names -variable-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct constant names -const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Regular expression matching correct attribute names -attr-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for attribute names -attr-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct argument names -argument-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for argument names -argument-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ - -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression matching correct method names -method-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for method names -method-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - - -[ELIF] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - - -[FORMAT] - -# Maximum number of characters on a single line. -max-line-length=100 - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator - -# Maximum number of lines in a module -max-module-lines=1000 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO - - -[SIMILARITIES] - -# Minimum lines number of a similarity. -min-similarity-lines=4 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - - -[SPELLING] - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[TYPECHECK] - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - - -[VARIABLES] - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,future.builtins - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=8 - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.* - -# Maximum number of locals for function / method body -max-locals=20 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of branch for function / method body -max-branches=12 - -# Maximum number of statements in function / method body -max-statements=50 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of attributes for a class (see R0902). -max-attributes=20 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=1 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 - - -[IMPORTS] - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,TERMIOS,Bastion,rexec - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception From 593b4dd2671657745070badba4d46b4cc23358f3 Mon Sep 17 00:00:00 2001 From: dabercro Date: Wed, 1 Aug 2018 15:08:40 -0400 Subject: [PATCH 05/11] Don't need mock for this package alone. --- docs/conf.py | 10 ---- docs/requirements.txt | 1 - test/test_shell_style.sh | 106 --------------------------------------- 3 files changed, 117 deletions(-) delete mode 100755 test/test_shell_style.sh diff --git a/docs/conf.py b/docs/conf.py index 6a05da2..9df476b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -284,13 +284,3 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'https://docs.python.org/': None} - -from mock import Mock as MagicMock - -class Mock(MagicMock): - @classmethod - def __getattr__(cls, name): - return Mock() - -MOCK_MODULES = ['cjson', 'pycurl'] -sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) diff --git a/docs/requirements.txt b/docs/requirements.txt index 79568f7..a8cfa67 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -3,4 +3,3 @@ sphinx_rtd_theme sphinxcontrib-programoutput sphinxcontrib-autoanysrc customdocs -mock diff --git a/test/test_shell_style.sh b/test/test_shell_style.sh deleted file mode 100755 index 3f277c9..0000000 --- a/test/test_shell_style.sh +++ /dev/null @@ -1,106 +0,0 @@ -#! /bin/bash - -# Check for shellcheck - -if [ "$(which shellcheck 2> /dev/null)" = "" ] -then - - if [ "$TRAVIS" = "true" ] - then - - # Install shellcheck - sudo apt-get install shellcheck - - else - - echo "Please install shellcheck: https://github.com/koalaman/shellcheck" 1>&2 - exit 0 - - fi - -fi - -# Save here, in case user is not in test dir -here=$(pwd) - -# Get the test dir -testdir=${0%%$(basename "$0")} -cd "$testdir" || exit 1 -testdir=$(pwd) - -# Set text output location -outputdir=$testdir"/shellcheck_output" - -test -d "$outputdir" || mkdir "$outputdir" - -cd .. - -ERRORSFOUND=0 - -# Define the function to check each repository -check_package () { - - location=$(pwd) - location=${location##*/} - - # shellcheck disable=SC2046 - shellcheck $(git ls-files | grep "\.sh") > "$outputdir/$location.txt" - - if [ "$(wc -l < "$outputdir/$location.txt")" -eq 0 ] - then - tput setaf 2 2> /dev/null - echo "$outputdir/$location.txt passed the check." - tput sgr0 2> /dev/null - return - fi - - tput setaf 1 2> /dev/null - echo "$outputdir/$location.txt failed the check." - tput sgr0 2> /dev/null - - if [ "$TRAVIS" != "true" ] || [ "$location" = "$MUSTWORK" ] || [ "$location" = "OpsSpace" ] - then - - ERRORSFOUND=$((ERRORSFOUND + 1)) - - fi - - if [ "$TRAVIS" = "true" ] - then - - cat "$outputdir/$location.txt" - - fi - -} - -# Check OpsSpace -check_package - -# Check each installed package -while read -r package -do - - if [ "$package" = "WmAgentScripts" ] - then - - continue - - fi - - if [ -d "$package" ] - then - - cd "$package" || exit 1 - check_package - cd .. - - fi - -done < PackageList.txt - -cd "$here" || exit 1 - -echo "$ERRORSFOUND errors found" - -exit "$ERRORSFOUND" From d0e479542353bf95f93e0024a760d8bad87a9a01 Mon Sep 17 00:00:00 2001 From: dabercro Date: Wed, 1 Aug 2018 15:13:05 -0400 Subject: [PATCH 06/11] Working towards next version --- cmstoolbox/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmstoolbox/__init__.py b/cmstoolbox/__init__.py index 1d4e944..574c93a 100644 --- a/cmstoolbox/__init__.py +++ b/cmstoolbox/__init__.py @@ -2,4 +2,4 @@ __all__ = [] -__version__ = '0.9.5' +__version__ = '0.9.6' From 544d71e1c212153cb30904cbb05a17aa7c0dd519 Mon Sep 17 00:00:00 2001 From: dabercro Date: Wed, 1 Aug 2018 16:07:14 -0400 Subject: [PATCH 07/11] Option to select tests by commands --- bin/opsspace-test | 121 ++++++++++++++++++++++++++++++---------------- 1 file changed, 79 insertions(+), 42 deletions(-) diff --git a/bin/opsspace-test b/bin/opsspace-test index 9d7ae02..bb794c8 100755 --- a/bin/opsspace-test +++ b/bin/opsspace-test @@ -9,6 +9,16 @@ fi errorsfound=0 +# Get the python version +# 26 for Python 2.6 +# 27 for Python 2.7 +# and so on +pyvers=$(python --version 2>&1 | perl -ne '/\b(\d)\.(\d+)\./ && print "$1$2"') + +# If versioned file not here, fall back to requirements.txt +# Get the full path to the test requirements file +reqfile=$(pwd)/test/requirements$(test -f test/requirements${pyvers}.txt && echo ${pyvers}).txt + # Function to do the test for us _do_test () { @@ -37,77 +47,104 @@ _do_test () { } +_doc_test () { -# Check if we should test build Sphinx documentation -if [ -f docs/conf.py ] && [ "$nodoc" != "true" ] -then + # Check if we should test build Sphinx documentation + if [ -f docs/conf.py ] && [ "$nodoc" != "true" ] + then - test -f docs/requirements.txt && pip install -r docs/requirements.txt - _do_test "sphinx-build -W -b html -E docs test/html" + test -f docs/requirements.txt && pip install -r docs/requirements.txt + _do_test "sphinx-build -W -b html -E docs test/html" -fi + fi +} -# Now do the tests inside the test directory -cd test +_unit_test () { -# Get the python version -# 26 for Python 2.6 -# 27 for Python 2.7 -# and so on -pyvers=$(python --version 2>&1 | perl -ne '/\b(\d)\.(\d+)\./ && print "$1$2"') + # Install requirements, if they exist + test -f $reqfile && pip install -r $reqfile -# If versioned file not here, fall back to requirements.txt -reqfile=$(test -f requirements${pyvers}.txt && echo requirements${pyvers}.txt || echo requirements.txt) + # Now do the tests inside the test directory + cd test -# Install requirements, if they exist -test -f $reqfile && pip install -r $reqfile + # Run the test for all files that start with "test_". They should be executable. + for packagetest in test_* + do + _do_test "./$packagetest" + done -# Run the test for all files that start with "test_". They should be executable. -for packagetest in test_* -do + cd - - _do_test "./$packagetest" +} -done +_style_test () { -# Check for pylint in test requirements, and run test if desired -if grep pylint $reqfile -then + # Check for pylint in test requirements, and run test if desired + if grep pylint $reqfile + then - cd .. - # Look for packages from root directory - package=$(dirname $(find * -maxdepth 1 -name __init__.py)) + # Look for packages from root directory + package=$(dirname $(find * -maxdepth 1 -name __init__.py)) - # This exits 0 when the pylint version is old, and we have to disable the extensions - if pylint --version 2> /dev/null | perl -ne '/pylint\s(\d+.\d+)/ && exit($1 > 1.3)' - then + # This exits 0 when the pylint version is old, and we have to disable the extensions + if pylint --version 2> /dev/null | perl -ne '/pylint\s(\d+.\d+)/ && exit($1 > 1.3)' + then - lintfilter='s/load-plugins/#/' + lintfilter='s/load-plugins/#/' - fi + fi - # PyLint Config is towards the bottom of this file - # It's used as default when local pylint.cfg does not exist - if [ ! -f test/pylint.cfg ] - then + # PyLint Config is towards the bottom of this file + # It's used as default when local pylint.cfg does not exist + if [ ! -f test/pylint.cfg ] + then + + perl -0ne '/PyLint Config\n(\-+)\n(.*)\n\1/sm && print $2' $0 > test/pylint.cfg + plseraselintcfg=1 - perl -0ne '/PyLint Config\n(\-+)\n(.*)\n\1/sm && print $2' $0 > test/pylint.cfg - plseraselintcfg=1 + fi + + _do_test "pylint --rcfile=<(sed '$lintfilter' test/pylint.cfg) $package" + + # If PyLint file was created, get rid of it unless asked to be kept + test -z $plseraselintcfg || test ! -z $keepcfg || rm test/pylint.cfg fi - _do_test "pylint --rcfile=<(sed '$lintfilter' test/pylint.cfg) $package" +} - # If PyLint file was created, get rid of it unless asked to be kept - test -z $plseraselintcfg || test ! -z $keepcfg || rm test/pylint.cfg +if [ -z $1 ] +then + actions="docs unit style" +else + actions=$@ fi +for action in $actions +do + + case $action in + "docs") + _doc_test + ;; + "unit") + _unit_test + ;; + "style") + _style_test + ;; + *) + echo "Don't recognize action '$action'" + ;; + esac + +done if [ $errorsfound -eq 1 ] # Setting plural correctly From a23d9d7cdfe5e2c057c908fb1c2354db2855e1e9 Mon Sep 17 00:00:00 2001 From: dabercro Date: Thu, 2 Aug 2018 11:21:35 -0400 Subject: [PATCH 08/11] Added coverage to opsspace-test --- bin/opsspace-test | 31 ++++++++++++++++++++++++++++++- cmstoolbox/__init__.py | 2 +- test/requirements.txt | 1 + 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/bin/opsspace-test b/bin/opsspace-test index bb794c8..f900553 100755 --- a/bin/opsspace-test +++ b/bin/opsspace-test @@ -69,14 +69,43 @@ _unit_test () { # Now do the tests inside the test directory cd test + if grep coverage $reqfile + then + + docoverage=1 + # Remove any old .coverage file + test ! -f .coverage || rm .coverage + + fi + + # Run the test for all files that start with "test_". They should be executable. for packagetest in test_* do - _do_test "./$packagetest" + # Run the coverage on top of the unit tests + if [ ! -z $docoverage ] && [ "${packagetest##*.}" == "py" ] + then + + _do_test "coverage run -a $packagetest" + + else + + _do_test "./$packagetest" + + fi done + + if [ ! -z $docoverage ] + then + + coverage report + + fi + + cd - } diff --git a/cmstoolbox/__init__.py b/cmstoolbox/__init__.py index 574c93a..b621198 100644 --- a/cmstoolbox/__init__.py +++ b/cmstoolbox/__init__.py @@ -2,4 +2,4 @@ __all__ = [] -__version__ = '0.9.6' +__version__ = '0.9.7' diff --git a/test/requirements.txt b/test/requirements.txt index 6c2324f..1b988aa 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,2 +1,3 @@ testfixtures pylint +coverage From 3f69e274c454f19456a7846008b4184bfbbe941a Mon Sep 17 00:00:00 2001 From: dabercro Date: Thu, 2 Aug 2018 15:31:48 -0400 Subject: [PATCH 09/11] Don't generate pylint report that I never look at --- .gitignore | 1 + bin/opsspace-test | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index fe482a2..688d7c2 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ test/*/* test/*.cache.json dist/* cmstoolbox.egg-info/* +test/.coverage diff --git a/bin/opsspace-test b/bin/opsspace-test index f900553..7c09b29 100755 --- a/bin/opsspace-test +++ b/bin/opsspace-test @@ -256,7 +256,7 @@ disable=bad-option-value,old-octal-literal,oct-method,print-statement,unpacking- output-format=text # Tells whether to display a full report or only the messages -reports=yes +reports=no # Python expression which should return a note less than 10 (10 is the highest # note). You have access to the variables errors warning, statement which From 7003fff837f05b0cd81105cbf384d98d31048d80 Mon Sep 17 00:00:00 2001 From: dabercro Date: Thu, 2 Aug 2018 19:36:07 -0400 Subject: [PATCH 10/11] Only run coverage on package being tested. --- bin/opsspace-test | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bin/opsspace-test b/bin/opsspace-test index 7c09b29..9faacf2 100755 --- a/bin/opsspace-test +++ b/bin/opsspace-test @@ -9,6 +9,8 @@ fi errorsfound=0 +package=$(dirname $(find * -maxdepth 1 -name __init__.py)) + # Get the python version # 26 for Python 2.6 # 27 for Python 2.7 @@ -87,7 +89,7 @@ _unit_test () { if [ ! -z $docoverage ] && [ "${packagetest##*.}" == "py" ] then - _do_test "coverage run -a $packagetest" + _do_test "coverage run --source $package -a $packagetest" else @@ -117,9 +119,6 @@ _style_test () { if grep pylint $reqfile then - # Look for packages from root directory - package=$(dirname $(find * -maxdepth 1 -name __init__.py)) - # This exits 0 when the pylint version is old, and we have to disable the extensions if pylint --version 2> /dev/null | perl -ne '/pylint\s(\d+.\d+)/ && exit($1 > 1.3)' then From 1af5c769b72a3ea64be0cb56b83fe8cfc81ca319 Mon Sep 17 00:00:00 2001 From: dabercro Date: Fri, 3 Aug 2018 22:02:27 -0400 Subject: [PATCH 11/11] Was accidentially using the slow version of the unmerged cleaner --- cmstoolbox/__init__.py | 2 +- cmstoolbox/unmergedcleaner/listdeletable.py | 121 ++++++++++++-------- test/test_unmerged_cleaner.py | 9 +- 3 files changed, 78 insertions(+), 54 deletions(-) diff --git a/cmstoolbox/__init__.py b/cmstoolbox/__init__.py index b621198..cf49812 100644 --- a/cmstoolbox/__init__.py +++ b/cmstoolbox/__init__.py @@ -2,4 +2,4 @@ __all__ = [] -__version__ = '0.9.7' +__version__ = '0.9.8' diff --git a/cmstoolbox/unmergedcleaner/listdeletable.py b/cmstoolbox/unmergedcleaner/listdeletable.py index 361a793..fb22599 100644 --- a/cmstoolbox/unmergedcleaner/listdeletable.py +++ b/cmstoolbox/unmergedcleaner/listdeletable.py @@ -67,12 +67,16 @@ import datetime import subprocess import shutil +import logging from bisect import bisect_left from optparse import OptionParser from . import configtools +LOG = logging.getLogger(__name__) + + if __name__ == '__main__': PARSER = OptionParser('Usage: ./%prog [options]\n\n' ' This script can list and delete directories over the course of\n' @@ -93,6 +97,9 @@ (OPTS, ARGS) = PARSER.parse_args() + logging.basicConfig(level=logging.INFO, + format='%(asctime)s:%(levelname)s:%(name)s: %(message)s') + config = None @@ -339,7 +346,7 @@ def get_protected(): result = json.loads(res.read()) protected = result['protected'] except Exception: - print 'Cannot read Protected LFNs. Have to stop...' + LOG.error('Cannot read Protected LFNs. Have to stop...') exit(1) conn.close() @@ -371,7 +378,7 @@ def hadoop_delete(directory, mount_point='/mnt/hadoop'): # than we are expecting at the moment. if os.path.exists(os.path.normpath(os.path.sep.join([mount_point, directory]))): command = 'hdfs dfs -rm -r %s' % directory - print 'Will do:', command + LOG.info('Will do: %s', command) time.sleep(config.SLEEP_TIME) os.system(command) @@ -382,8 +389,8 @@ def dcache_delete(directory): :param str directory: The directory name for dCache to delete """ - print 'Not implimented yet. %s has not been deleted.' % directory - print 'Try posix or editing dcache_delete() in ListDeletable.py' + LOG.error('Not implimented yet. %s has not been deleted.', directory) + LOG.error('Try posix or editing dcache_delete() in ListDeletable.py') def do_delete(): @@ -407,15 +414,15 @@ def do_delete(): """ if not os.path.isfile(config.DELETION_FILE): - print 'Deletion file %s has not been created yet.' % config.DELETION_FILE + LOG.error('Deletion file %s has not been created yet.', config.DELETION_FILE) exit() if config.WHICH_LIST != 'directories': - print '-' * 40 - print 'Deleting individual files.' - print 'Your sleep time is set to %s seconds.' % config.SLEEP_TIME - print 'To change it, edit your config.py.' - print '-' * 40 + LOG.info('-' * 40) + LOG.info('Deleting individual files.') + LOG.info('Your sleep time is set to %s seconds.', config.SLEEP_TIME) + LOG.info('To change it, edit your config.py.') + LOG.info('-' * 40) with open(config.DELETION_FILE, 'r') as deletions: for deleted in deletions.readlines(): @@ -423,10 +430,10 @@ def do_delete(): # Do a check of the directory names. End process if something is wrong. if '/unmerged/' not in deleting: - print 'Something is either wrong with your deletions file or' - print 'ListDetetable.do_delete().' - print 'Your deletions file is at', config.DELETION_FILE - print 'Refusing to continue.' + LOG.error('Something is either wrong with your deletions file or') + LOG.error('ListDetetable.do_delete().') + LOG.error('Your deletions file is at %s', config.DELETION_FILE) + LOG.error('Refusing to continue.') exit() if config.WHICH_LIST == 'directories': @@ -442,13 +449,13 @@ def do_delete(): else: # The default, 'posix', goes here - print 'About to delete %s' % deleting + LOG.warning('About to delete %s', deleting) time.sleep(config.SLEEP_TIME) shutil.rmtree(deleting) else: if os.path.isfile(deleting): - print 'About to delete %s' % deleting + LOG.warning('About to delete %s', deleting) time.sleep(config.SLEEP_TIME) os.remove(deleting) @@ -462,8 +469,8 @@ def get_unmerged_files(): find_cmd = 'find {0} -type f -not -newermt \'-{1} seconds\' -print'.format( config.UNMERGED_DIR_LOCATION, config.MIN_AGE) - print 'About to run:' - print find_cmd + LOG.info('About to run:') + LOG.info(find_cmd) out = subprocess.Popen(find_cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -482,8 +489,8 @@ def get_unmerged_files_hadoop(): hdfs_cmd = "hdfs dfs -ls -R {0} | grep -v '^d' | sed '1d;s/ */ /g' | cut -d\ -f6-8".format( config.LFN_TO_CLEAN) - print 'About to run:' - print hdfs_cmd + LOG.info('About to run:') + LOG.info(hdfs_cmd) out = subprocess.Popen(hdfs_cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -517,39 +524,56 @@ def filter_protected(unmerged_files, protected): or if there is a partial match with a protected LFN """ - print 'Got %i deletion candidates' % len(unmerged_files) - print 'Have %i protected dirs' % len(protected) - print 'Have %i avoided dirs' % len(config.DIRS_TO_AVOID) + LOG.info('Got %i deletion candidates', len(unmerged_files)) + LOG.info('Have %i protected dirs', len(protected)) + LOG.info('Have %i avoided dirs', len(config.DIRS_TO_AVOID)) n_protect = 0 n_delete = 0 output = [] - for unmerged_file in unmerged_files: - protect = False + # Double check this + if not protected: + raise SuspiciousConditions( + '\nNo directories are protected.\n' + 'Check https://cmst2.web.cern.ch/cmst2/unified/listProtectedLFN.txt') + + # Speed up by comparing two sorted lists + protected.sort() + unmerged_files.sort() + iter_protect = iter(protected) + pfn = lfn_to_pfn(iter_protect.next()) # We should never have 0 protected here + + for unmerged_file in unmerged_files: if not unmerged_file.startswith(config.UNMERGED_DIR_LOCATION): raise SuspiciousConditions( '\nFile %s\nis not in your configured unmerged location:\n%s' % (unmerged_file, config.UNMERGED_DIR_LOCATION)) - for lfn in protected: - pfn = lfn_to_pfn(lfn) - if pfn in unmerged_file: + protect = False + while pfn < unmerged_file: + + if unmerged_file.startswith(pfn): protect = True break - elif lfn in unmerged_file: - raise SuspiciousConditions( - '\nFile %s\nhas partial match to LFN %s,\n' - 'but LFN mapped to %s\nCheck your configuration file' % - (unmerged_file, lfn, pfn)) - - for root_dir in config.DIRS_TO_AVOID: - pfn = os.path.join(config.UNMERGED_DIR_LOCATION, root_dir) - if pfn in unmerged_file: - protect = True + try: + pfn = lfn_to_pfn(iter_protect.next()) + except StopIteration: + pfn = 'None' # Paths start with '/', so this never satisfies pfn < unmerged_file break + if not pfn.startswith(config.UNMERGED_DIR_LOCATION): + raise SuspiciousConditions( + '\nDirectory %s\nis not in your configured unmerged location:\n%s' % + (pfn, config.UNMERGED_DIR_LOCATION)) + + if not protect: + for root_dir in config.DIRS_TO_AVOID: + if os.path.join(config.UNMERGED_DIR_LOCATION, root_dir) in unmerged_file: + protect = True + break + if not protect: output.append(unmerged_file) n_delete += 1 @@ -559,7 +583,8 @@ def filter_protected(unmerged_files, protected): with open(config.DELETION_FILE, 'w') as deletions: deletions.write('\n'.join(output)) - print 'Number to delete: %i,\nNumber protected/avoided: %i' % (n_delete, n_protect) + LOG.info('Number to delete: %i', n_delete) + LOG.info('Number protected/avoided: %i', n_protect) def main(): @@ -600,9 +625,9 @@ def main(): PROTECTED_UPPER_DIRS.add(parent) parent = os.path.dirname(parent) - print "Some statistics about what is going to be deleted" - print "# Folders Total Total DiskSize FolderName" - print "# Folders Files [GB] " + LOG.info("Some statistics about what is going to be deleted") + LOG.info("# Folders Total Total DiskSize FolderName") + LOG.info("# Folders Files [GB] ") # Get the location of the PFN and the subdirectories there @@ -636,9 +661,9 @@ def main(): todelete_size += item.size todelete_size /= (1024 * 1024 * 1024) - print " %-8d %-8d %-6d %-9d %-s" \ - % (len(list_to_del), num_todelete_dirs, num_todelete_files, - todelete_size, subdir) + LOG.info(" %-8d %-8d %-6d %-9d %-s", + len(list_to_del), num_todelete_dirs, num_todelete_files, + todelete_size, subdir) tot_upper_dirs += len(list_to_del) tot_dirs += num_todelete_dirs @@ -647,8 +672,8 @@ def main(): dirs_to_delete.extend(list_to_del) - print "-" * 30 - print " %-8d %-8d %-6d %-9d TOTALS" % (tot_upper_dirs, tot_dirs, tot_files, tot_site) + LOG.info("-" * 30) + LOG.info(" %-8d %-8d %-6d %-9d TOTALS", tot_upper_dirs, tot_dirs, tot_files, tot_site) deletion_dir = os.path.dirname(config.DELETION_FILE) if not os.path.exists(deletion_dir): @@ -662,7 +687,7 @@ def main(): ) + '\n') else: - print 'The WHICH_LIST parameter in config.py is not valid.' + LOG.error('The WHICH_LIST parameter in config.py is not valid.') # Generate documentation for the options in the configuration file. diff --git a/test/test_unmerged_cleaner.py b/test/test_unmerged_cleaner.py index 503d169..e4fbd9d 100755 --- a/test/test_unmerged_cleaner.py +++ b/test/test_unmerged_cleaner.py @@ -19,11 +19,11 @@ import unittest import random import uuid +import testfixtures import time import shutil -import testfixtures - +import cmstoolbox._loadtestpath from cmstoolbox.unmergedcleaner import listdeletable @@ -243,10 +243,9 @@ def test_bad_file_start(self): [os.path.join(listdeletable.config.UNMERGED_DIR_LOCATION.replace('/store/', '/disk/store/'), 'example/file/location.root')], []) - def test_partial_match(self): self.assertRaises(listdeletable.SuspiciousConditions, listdeletable.filter_protected, - [os.path.join(listdeletable.config.UNMERGED_DIR_LOCATION, 'store/unmerged/protected/file.root')], - ['/store/unmerged/protected/file.root']) + [os.path.join(listdeletable.config.UNMERGED_DIR_LOCATION.replace('/store/', '/disk/store/'), + 'example/file/location.root')], ['/store/unmerged/protected/']) if __name__ == '__main__':