Merge pull request #73 from databio/dev

0.7.2 release

.gitignore

@@ -1,44 +1,78 @@
-# First, ignore everything.
-*
-# Now, whitelist anything that's a directory
-!*/
+# ignore test results
+tests/test/*
 
+# toy/experimental files
+*.csv
+*.tsv
+*.pkl
 
-# Ignore Python build-related directories.
+# ignore eggs
 .eggs/
+
+# ignore built docs
+doc/build/*
+
+# generic ignore list:
+*.lst
+
+# Compiled source 
+*.com
+*.class
+*.dll
+*.exe
+*.o
+*.so
+*.pyc
+
+# Packages 
+# it's better to unpack these files and commit the raw source
+# git has its own built in compression methods
+*.7z
+*.dmg
+*.gz
+*.iso
+*.jar
+*.rar
+*.tar
+*.zip
+
+# Logs and databases 
+*.log
+*.sql
+*.sqlite
+
+# OS generated files 
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Gedit temporary files 
+*~
+
+# libreoffice lock files:
+.~lock*
+
+# Default-named test output
+microtest/
+open_pipelines/
+
+# IDE-specific items
+.idea/
+
+# pytest-related
+.cache/
+.coverage
+.pytest_cache
+
+# Reserved files for comparison
+*RESERVE*
+
+# Build-related stuff
 build/
 dist/
-
-# Keep everything in data.
-!metadata/*
-results_pipeline/
-results_analysis
-resources/
-pipeline_output/
-
-# And all the file types you're interested in.
-!*.sh
-!*.svg
-!*.R
-!*.pl
-!*.py
-!*.c
-!*.md
-!*.sub
-!*readme.txt*
-!*README.txt*
-!*readme.md*
-!*README.md*
-!*.rst
-!*.xlsx
-!*.in
-
-# Make sure to track .gitignore.
-!.gitignore
-
-# Sphinx documentation - build folder
-doc/build/
-
-# Track requirements files
-!requirements/*.txt
+looper.egg-info/
 

doc/source/changelog.rst

@@ -1,5 +1,14 @@
 Changelog
 ******************************
+- **v0.7.2** (*2018-06-05*):
+
+    - Cleanup files are now relative, so a moved folder could still be cleaned.
+
+    - Fixed a bug that prevented install if pypandoc was not installed
+
+    - Fixed a bug that caused an error in containers where /proc wasn't accessible
+
+
 - **v0.7.1** (*2018-02-27*):
 
     - Package cleanup for Pypi.

pypiper/_version.py

@@ -1 +1 @@
-__version__ = "0.7.1"
+__version__ = "0.7.2"

pypiper/manager.py

@@ -146,6 +146,7 @@ def __init__(
         # this could become customizable if necessary
         self.mem = params['mem'] + "m"
         self.container = None
+        self.clean_initialized = False
 
         # Do some cores math for split processes
         # If a pipeline wants to run a process using half the cores, or 1/4 of the cores,
@@ -1693,18 +1694,28 @@ def clean_add(self, regex, conditional=False, manual=False):
             # Override the user-provided option and force manual cleanup.
             manual = True
 
+        if not self.clean_initialized:
+            # Make cleanup files relative to the cleanup script in case the result folder moves.
+            with open(self.cleanup_file, "a") as myfile:
+                clean_init = 'DIR="$(cd -P -- "$(dirname -- "$0")" && pwd -P)"'
+                myfile.write(clean_init + "\n")
+                myfile.write("cd ${DIR}\n")
+                self.clean_initialized = True
+
         if manual:
             try:
-                files = glob.glob(regex)
-                for file in files:
+                filenames = glob.glob(regex)
+                for filename in filenames:
                     with open(self.cleanup_file, "a") as myfile:
-                        if os.path.isfile(file):
-                            myfile.write("rm " + file + "\n")
-                        elif os.path.isdir(file):
-                            # first, add all files in the directory
-                            myfile.write("rm " + file + "/*\n")
+                        relative_filename = os.path.relpath(filename, self.outfolder) \
+                            if os.path.isabs(filename) else filename
+                        if os.path.isfile(relative_filename):
+                            myfile.write("rm " + relative_filename + "\n")
+                        elif os.path.isdir(relative_filename):
+                            # first, add all filenames in the directory
+                            myfile.write("rm " + relative_filename + "/*\n")
                             # and the directory itself
-                            myfile.write("rmdir " + file + "\n")
+                            myfile.write("rmdir " + relative_filename + "\n")
             except:
                 pass
         elif conditional:
@@ -1830,6 +1841,8 @@ def _memory_usage(self, pid='self', category="hwm", container=None):
         # Thanks Martin Geisler:
         status = None
         result = {'peak': 0, 'rss': 0, 'hwm': 0}
+
+
         try:
             # This will only work on systems with a /proc file system
             # (like Linux).
@@ -1841,6 +1854,9 @@ def _memory_usage(self, pid='self', category="hwm", container=None):
                 key = parts[0][2:-1].lower()
                 if key in result:
                     result[key] = int(parts[1])
+        except:
+            return 0
+
         finally:
             if status is not None:
                 status.close()

pypiper/utils.py

@@ -16,7 +16,7 @@
 # What to export/attach to pypiper package namespace.
 # Conceptually, reserve this for functions expected to be used in other
 # packages, and import from utils within pypiper for other functions.
-__all__ = ["add_pypiper_args", "build_command", "get_parameter"]
+__all__ = ["add_pypiper_args", "build_command", "get_first_value"]
 
 
 
@@ -259,11 +259,10 @@ def flag_name(status):
 
 
 
-def get_parameter(param, param_pools, on_missing=None, error=True):
+def get_first_value(param, param_pools, on_missing=None, error=True):
     """
-    Get the value for a particular parameter.
-
-    Other than the parameter name itself, the other critical
+    Get the value for a particular parameter from the first pool in the provided
+    priority list of parameter pools.
 
     :param str param: Name of parameter for which to determine/fetch value.
     :param Sequence[Mapping[str, object]] param_pools: Ordered (priority)

setup.py

@@ -47,7 +47,7 @@ def read_reqs_file(reqs_name):
 try:
     import pypandoc
     long_description = pypandoc.convert_file('README.md', 'rst')
-except(IOError, ImportError):
+except(IOError, ImportError, OSError):
     long_description = open('README.md').read()
 
 setup(