Merge branch 'master' of https://github.com/KarrLab/wc_utils

.circleci/config.yml

@@ -3,7 +3,7 @@ jobs:
   build:
     working_directory: /root/project
     docker:
-      - image: karrlab/build:0.0.22
+      - image: karrlab/build:0.0.24
     steps:
       # Clone repository
       - checkout
@@ -71,8 +71,11 @@ jobs:
       - store_test_results:
           path: /root/project/tests/reports
       - store_artifacts:
-          path: /root/project/.coverage.3.6.4
-          destination: .coverage.3.6.4
+          path: /root/project/.coverage.3.6.5
+          destination: .coverage.3.6.5
+      - store_artifacts:
+          path: /root/project/logs
+          destination: logs
       - store_artifacts:
           path: /root/project/docs/_build/html
           destination: docs

.karr_lab_build_utils.yml

@@ -8,6 +8,7 @@ downstream_dependencies:
   - wc_kb
   - wc_lang
   - wc_sim
+  - wc
 email_notifications:
   - jonrkarr@gmail.com
   - arthur.p.goldberg@mssm.edu

README.md

@@ -1,5 +1,5 @@
 [//]: # ( [![PyPI package](https://img.shields.io/pypi/v/wc_utils.svg)](https://pypi.python.org/pypi/wc_utils) )
-[![Documentation](https://img.shields.io/badge/docs-latest-green.svg)](http://docs.karrlab.org/wc_utils)
+[![Documentation](https://readthedocs.org/projects/wc-utils/badge/?version=latest)](http://docs.karrlab.org/wc_utils)
 [![Test results](https://circleci.com/gh/KarrLab/wc_utils.svg?style=shield)](https://circleci.com/gh/KarrLab/wc_utils)
 [![Test coverage](https://coveralls.io/repos/github/KarrLab/wc_utils/badge.svg)](https://coveralls.io/github/KarrLab/wc_utils)
 [![Code analysis](https://api.codeclimate.com/v1/badges/8139298cdbc1e32dcde4/maintainability)](https://codeclimate.com/github/KarrLab/wc_utils)

setup.py

@@ -8,8 +8,8 @@
 try:
     import pkg_utils
 except ImportError:
-    import pip
-    pip.main(['install', '--process-dependency-links', 'git+https://github.com/KarrLab/pkg_utils.git#egg=pkg_utils'])
+    import pip._internal
+    pip._internal.main(['install', '--process-dependency-links', 'git+https://github.com/KarrLab/pkg_utils.git#egg=pkg_utils'])
     import pkg_utils
 import os
 

tests/test_cache.py

@@ -25,8 +25,6 @@ def tearDown(self):
     def test_memoize_positional_argument(self):
         cache = wc_utils.cache.Cache(directory=os.path.join(self.dir, 'cache'))
 
-        call_count = 0
-
         @cache.memoize()
         def func(input):
             print('func ran')
@@ -47,8 +45,6 @@ def func(input):
     def test_memoize_keyword_argument(self):
         cache = wc_utils.cache.Cache(directory=os.path.join(self.dir, 'cache'))
 
-        call_count = 0
-
         @cache.memoize()
         def func(input_1, input_2=1):
             print('func ran')
@@ -73,8 +69,6 @@ def func(input_1, input_2=1):
     def test_memoize_filename_argument(self):
         cache = wc_utils.cache.Cache(directory=os.path.join(self.dir, 'cache'))
 
-        call_count = 0
-
         @cache.memoize(filename_args=[0], filename_kwargs=['input_2'])
         def func(input_1, input_2=''):
             print('func ran')
@@ -138,8 +132,6 @@ def func(input_1, input_2=''):
     def test_memoize_filename_argument_glob(self):
         cache = wc_utils.cache.Cache(directory=os.path.join(self.dir, 'cache'))
 
-        call_count = 0
-
         @cache.memoize(filename_args=[0], filename_kwargs=['input_2'])
         def func(input_1, input_2=''):
             print('func ran')
@@ -164,8 +156,6 @@ def func(input_1, input_2=''):
     def test_memoize_typed(self):
         cache = wc_utils.cache.Cache(directory=os.path.join(self.dir, 'cache'))
 
-        call_count = 0
-
         @cache.memoize(typed=True)
         def func(input_1, input_2=None):
             print('func ran')
@@ -198,8 +188,6 @@ def func(input_1, input_2=None):
     def test_memoize_with_explicit_name(self):
         cache = wc_utils.cache.Cache(directory=os.path.join(self.dir, 'cache'))
 
-        call_count = 0
-
         @cache.memoize(name='func_1_2')
         def func_1(input):
             print('func ran')
@@ -233,8 +221,6 @@ def func_3(input):
     def test_memoize_tuple(self):
         cache = wc_utils.cache.Cache(directory=os.path.join(self.dir, 'cache'))
 
-        call_count = 0
-
         @cache.memoize()
         def func(input):
             print('func ran')
@@ -251,8 +237,6 @@ def func(input):
     def test_memoize_list(self):
         cache = wc_utils.cache.Cache(directory=os.path.join(self.dir, 'cache'))
 
-        call_count = 0
-
         @cache.memoize()
         def func(input):
             print('func ran')
@@ -273,8 +257,6 @@ def func(input):
     def test_memoize_dict(self):
         cache = wc_utils.cache.Cache(directory=os.path.join(self.dir, 'cache'))
 
-        call_count = 0
-
         @cache.memoize()
         def func(input):
             print('func ran')
@@ -299,8 +281,6 @@ def func(input):
     def test_memoize_ordereddict(self):
         cache = wc_utils.cache.Cache(directory=os.path.join(self.dir, 'cache'))
 
-        call_count = 0
-
         @cache.memoize()
         def func(input):
             print('func ran')
@@ -321,14 +301,14 @@ def func(input):
             self.assertEqual(captured.stdout.get_text(), '')
 
         with capturer.CaptureOutput(merged=False, relay=False) as captured:
-            odict = collections.OrderedDict()            
+            odict = collections.OrderedDict()
             odict['1'] = 2
             odict['0'] = 1
             self.assertEqual(func(odict), 3)
             self.assertEqual(captured.stdout.get_text(), 'func ran')
 
         with capturer.CaptureOutput(merged=False, relay=False) as captured:
-            odict2 = collections.OrderedDict()            
+            odict2 = collections.OrderedDict()
             odict2['1'] = 2
             odict2['0'] = 1
             self.assertEqual(func(odict2), 3)
@@ -351,8 +331,6 @@ def func(input):
     def test_memoize_nested_data_struct(self):
         cache = wc_utils.cache.Cache(directory=os.path.join(self.dir, 'cache'))
 
-        call_count = 0
-
         @cache.memoize()
         def func(input):
             print('func ran')
@@ -389,3 +367,53 @@ def func(input):
             ]
             self.assertEqual(func(data_struct4), 7)
             self.assertEqual(captured.stdout.get_text(), '')
+
+    def test_positional_arg_as_keyword_and_vice_versa(self):
+        cache = wc_utils.cache.Cache(directory=os.path.join(self.dir, 'cache'))
+
+        @cache.memoize()
+        def func(a, b, c=3, d=None, e=5):
+            print('func ran')
+            if d:
+                return a + b + c + d + e
+            else:
+                return a + b + c + e
+
+        with capturer.CaptureOutput(merged=False, relay=False) as captured:
+            self.assertEqual(func(1, 2, 3), 11)
+            self.assertEqual(captured.stdout.get_text(), 'func ran')
+
+        with capturer.CaptureOutput(merged=False, relay=False) as captured:
+            self.assertEqual(func(1, 2, 3), 11)
+            self.assertEqual(captured.stdout.get_text(), '')
+
+        with capturer.CaptureOutput(merged=False, relay=False) as captured:
+            self.assertEqual(func(1, 2, c=3), 11)
+            self.assertEqual(captured.stdout.get_text(), '')
+
+        with capturer.CaptureOutput(merged=False, relay=False) as captured:
+            self.assertEqual(func(1, 2, d=None), 11)
+            self.assertEqual(captured.stdout.get_text(), '')
+
+        with capturer.CaptureOutput(merged=False, relay=False) as captured:
+            self.assertEqual(func(1, b=2, e=5), 11)
+            self.assertEqual(captured.stdout.get_text(), '')
+
+        with capturer.CaptureOutput(merged=False, relay=False) as captured:
+            self.assertEqual(func(a=1, b=2, e=5), 11)
+            self.assertEqual(captured.stdout.get_text(), '')
+
+        with self.assertRaisesRegexp(TypeError, 'missing required positional argument'):
+            func(1, e=5)
+
+        @cache.memoize()
+        def func(a, **kwargs):
+            return a
+        with self.assertRaisesRegexp(NotImplementedError, 'only supports positional-or-keyword arguments'):
+            func(1, e=5)
+
+        @cache.memoize()
+        def func(*args, c=3, d=None):
+            return a
+        with self.assertRaisesRegexp(NotImplementedError, 'Submit issue to request support for optional arguments'):
+            func(1, 2, e=5)

tests/util/test_files.py

@@ -0,0 +1,62 @@
+""" Tests of the file utilities
+
+:Author: Jonathan Karr <jonrkarr@gmail.com>
+:Date: 2018-05-11
+:Copyright: 2018, Karr Lab
+:License: MIT
+"""
+
+from wc_utils.util import files
+import os
+import shutil
+import tempfile
+import unittest
+
+
+class FileUtilsTestCase(unittest.TestCase):
+    def setUp(self):
+        self.dirname = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.dirname)
+
+    def test_copytree_to_existing_destination(self):
+        base = self.dirname
+
+        os.mkdir(os.path.join(base, 'a'))
+        os.mkdir(os.path.join(base, 'a', 'b'))
+        with open(os.path.join(base, 'a', 'b', 'c'), 'w') as file:
+            file.write('3')
+        with open(os.path.join(base, 'a', 'b', 'd'), 'w') as file:
+            file.write('4')
+
+        # destination doesn't exist
+        files.copytree_to_existing_destination(os.path.join(base, 'a'), os.path.join(base, 'A'))
+        with open(os.path.join(base, 'A', 'b', 'c'), 'r') as file:
+            self.assertEqual(file.read(), '3')
+
+        # destination exists, but empty
+        os.mkdir(os.path.join(base, 'B'))
+        files.copytree_to_existing_destination(os.path.join(base, 'a'), os.path.join(base, 'B'))
+        with open(os.path.join(base, 'B', 'b', 'c'), 'r') as file:
+            self.assertEqual(file.read(), '3')
+
+        # destination exists, but not empty
+        os.mkdir(os.path.join(base, 'C'))
+        os.mkdir(os.path.join(base, 'C', 'b'))
+        with open(os.path.join(base, 'C', 'b', 'c'), 'w') as file:
+            file.write('4')
+        with open(os.path.join(base, 'C', 'b', 'e'), 'w') as file:
+            file.write('5')
+        files.copytree_to_existing_destination(os.path.join(base, 'a'), os.path.join(base, 'C'))
+        with open(os.path.join(base, 'C', 'b', 'c'), 'r') as file:
+            self.assertEqual(file.read(), '3')
+        with open(os.path.join(base, 'C', 'b', 'd'), 'r') as file:
+            self.assertEqual(file.read(), '4')
+        with open(os.path.join(base, 'C', 'b', 'e'), 'r') as file:
+            self.assertEqual(file.read(), '5')
+
+        #src and dest are files
+        files.copytree_to_existing_destination(os.path.join(base, 'a', 'b', 'c'), os.path.join(base, 'D'))
+        with open(os.path.join(base, 'D'), 'r') as file:
+            self.assertEqual(file.read(), '3')

wc_utils/cache.py

@@ -10,6 +10,7 @@
 import functools
 import glob
 import hashlib
+import inspect
 import os
 import types
 
@@ -56,9 +57,11 @@ def decorator(function):
             """ Decorator created by memoize call for callable. """
             if name is None:
                 try:
+                    # Python 3
                     reference = function.__qualname__
-                except AttributeError:
-                    reference = function.__name__
+                except AttributeError:  # pragma: no cover
+                    # Python 2
+                    reference = function.__name__  # pragma: no cover
 
                 reference = function.__module__ + reference
             else:
@@ -70,31 +73,58 @@ def decorator(function):
             def wrapper(*args, **kwargs):
                 "Wrapper for callable to cache arguments and return values."
 
-                key = reference + args
-
-                if kwargs:
+                # match arguments to function signature
+                func_signature = inspect.signature(function)
+                proc_args = []
+                proc_kwargs = {}
+                for i_param, (name, param) in enumerate(func_signature.parameters.items()):
+                    if param.kind is not inspect.Parameter.POSITIONAL_OR_KEYWORD:
+                        raise NotImplementedError('Memoize decorator only supports positional-or-keyword arguments. '
+                                                  'Submit issue to request support for optional arguments')
+
+                    if param.default is inspect._empty:
+                        if i_param < len(args):
+                            val = args[i_param]
+                        elif param.name in kwargs:
+                            val = kwargs[param.name]
+                        else:
+                            raise TypeError("{} missing required positional argument '{}'".format(function.__name__, param.name))
+                        proc_args.append(val)
+                    else:
+                        if i_param < len(args):
+                            val = args[i_param]
+                        elif param.name in kwargs:
+                            val = kwargs[param.name]
+                        else:
+                            val = param.default
+                        proc_kwargs[param.name] = val
+
+                # generate key from arguments
+                key = reference + tuple(proc_args)
+
+                if proc_kwargs:
                     key += (diskcache.core.ENOVAL,)
-                    sorted_items = sorted(kwargs.items())
+                    sorted_items = sorted(proc_kwargs.items())
 
                     for item in sorted_items:
                         key += item
 
                 if typed:
-                    key += tuple(type(arg) for arg in args)
+                    key += tuple(type(arg) for arg in proc_args)
 
-                    if kwargs:
+                    if proc_kwargs:
                         key += tuple(type(value) for _, value in sorted_items)
 
                 for filename_arg in filename_args:
                     stats = []
-                    for filename in glob.glob(args[filename_arg]):
+                    for filename in glob.glob(proc_args[filename_arg]):
                         stats.append((os.path.getmtime(filename), self._hash_file_content(filename)))
                     key += tuple(stats)
 
                 for filename_kwarg in filename_kwargs:
-                    if filename_kwarg in kwargs:
+                    if filename_kwarg in proc_kwargs:
                         stats = []
-                        for filename in glob.glob(kwargs[filename_kwarg]):
+                        for filename in glob.glob(proc_kwargs[filename_kwarg]):
                             stats.append((os.path.getmtime(filename), self._hash_file_content(filename)))
                         key += tuple(stats)
 

wc_utils/util/__init__.py

@@ -3,6 +3,7 @@
 from . import dict
 from . import enumerate
 from . import environ
+from . import files
 from . import git
 from . import list
 from . import misc

wc_utils/util/chem.py

@@ -140,7 +140,7 @@ def __div__(self, quantity):
         Returns:
             :obj:`EmpiricalFormula`: division of the empirical formula by :obj:`quantity`
         """
-        return self.__truediv__(quantity)
+        return self.__truediv__(quantity)  # pragma: no cover # only used in Python 2
 
     def __truediv__(self, quantity):
         """ Subtract two empirical formulae

wc_utils/util/enumerate.py

@@ -30,7 +30,7 @@ def __new__(metacls, cls, bases, classdict):
             lower_classdict = classdict
         else:
             # For Python 2
-            lower_classdict = {key.lower(): val for key, val in dict(classdict).items()}
+            lower_classdict = {key.lower(): val for key, val in dict(classdict).items()}  # pragma: no cover # Python 2 only
         return super(CaseInsensitiveEnumMeta, metacls).__new__(metacls, cls, bases, lower_classdict)
 
     def __getattr__(cls, name):