Merge pull request #96 from datmo/speed_up

Created Config.cache_setting decorator

appveyor.yml

@@ -48,4 +48,4 @@ install:
   - "%PYTHON%/python.exe setup.py install"
 test_script:
   - "%PYTHON%/Scripts/pip.exe --version"
-  - "%PYTHON%/Scripts/pytest "
+  - "%PYTHON%/Scripts/pytest "

datmo/config.py

@@ -1,7 +1,11 @@
 #!/usr/bin/python
 
 import logging
+import os
+import datetime
 from datmo.core.util.logger import DatmoLogger
+from datmo.core.util.json_store import JSONStore
+from datmo.core.util.misc_functions import parameterized
 
 
 class Config(object):
@@ -27,6 +31,8 @@ def __init__(self):
             self._home = None
             self.logging_level = logging.DEBUG
             DatmoLogger.get_logger(__name__).info("initalizing")
+            self.data_cache = JSONStore(
+                os.path.join(os.path.expanduser("~"), ".datmo", "cache.json"))
 
         @property
         def home(self):
@@ -36,6 +42,31 @@ def home(self):
         def home(self, home_path):
             self._home = home_path
 
+        def get_cache_item(self, key):
+            cache_expire_key = 'cache_key_expires.' + key
+            cache_key = 'cache_key.' + key
+            cache_expire_val = self.data_cache.get(cache_expire_key)
+            # no cache expire val, it's not stored
+            if cache_expire_val == None:
+                return None
+            # return value if item has not expired
+            elif int(cache_expire_val) > int(
+                    datetime.datetime.now().strftime('%s')):
+                return self.data_cache.get(cache_key)
+            # expire item and return None
+            else:
+                self.data_cache.remove(cache_expire_key)
+                self.data_cache.remove(cache_key)
+                return None
+
+        def set_cache_item(self, key, value, duration=60):
+            cache_expire_key = 'cache_key_expires.' + key
+            cache_key = 'cache_key.' + key
+            expire_val = (duration * 60) + int(
+                datetime.datetime.now().strftime('%s'))
+            self.data_cache.save(cache_expire_key, expire_val)
+            self.data_cache.save(cache_key, value)
+
     def __new__(cls):  # __new__ always a classmethod
         if not Config.instance:
             Config.instance = Config.__InternalConfig()
@@ -46,3 +77,20 @@ def __getattr__(self, name):
 
     def __setattr__(self, name, value):
         return setattr(self.instance, name, value)
+
+    @staticmethod
+    @parameterized
+    def cache_setting(method, key=None, expires_min=60, ignore_values=[]):
+        name = key if key is not None else method.__module__ + '.' + method.__name__
+        config = Config()
+
+        def fn(*args, **kw):
+            cached_val = config.get_cache_item(name)
+            if cached_val is not None:
+                return cached_val
+            result = method(*args, **kw)
+            if not result in ignore_values:
+                config.set_cache_item(name, result, expires_min)
+            return result
+
+        return fn

datmo/core/controller/code/driver/git.py

@@ -14,6 +14,7 @@
     PathDoesNotExist, GitUrlArgumentException, GitExecutionException,
     FileIOException, GitCommitDoesNotExist, DatmoFolderInWorkTree)
 from datmo.core.controller.code.driver import CodeDriver
+from datmo.config import Config
 
 
 class GitCodeDriver(CodeDriver):
@@ -686,6 +687,8 @@ def _netrc_exists(self):
         else:
             return False
 
+    @Config.cache_setting(
+        key="git.ssh_enabled", expires_min=1440, ignore_values=[False])
     def _check_for_ssh(self):
         cmd = "ssh-keyscan %s >> ~/.ssh/known_hosts" % (self.host)
         p = subprocess.Popen(

datmo/core/controller/code/driver/tests/test_git.py

@@ -503,7 +503,7 @@ def test_netrc(self):
 
     def test_ssh_git(self):
         hostm = GitHostDriver(self.ssh_temp_dir)
-        assert hostm.ssh_enabled == False
+        assert hostm.ssh_enabled == hostm._check_for_ssh()
         # If id_rsa already synced with remote account
         # if os.path.join(os.path.expanduser("~"), ".ssh", "id_rsa"):
         #     shutil.copytree(

datmo/core/controller/task.py

@@ -1,11 +1,11 @@
 import os
 from datetime import datetime
 
-from datmo.core.util.i18n import get as __
 from datmo.core.controller.base import BaseController
-from datmo.core.controller.environment.environment import EnvironmentController
 from datmo.core.controller.snapshot import SnapshotController
+from datmo.core.controller.environment.environment import EnvironmentController
 from datmo.core.entity.task import Task
+from datmo.core.util.i18n import get as __
 from datmo.core.util.exceptions import (
     TaskRunException, RequiredArgumentMissing, ProjectNotInitializedException,
     PathDoesNotExist, TaskInteractiveDetachException)

datmo/core/storage/driver/tests/test_blitzdb_dal_driver.py

@@ -219,7 +219,7 @@ def test_query_gte_datetime_should_fail(self):
 
         failed = False
         try:
-            items = self.database.query(
+            _ = self.database.query(
                 collection,
                 {"range_query2": {
                     "$gte": datetime.datetime(2017, 2, 1)

datmo/core/storage/local/tests/test_dal_snapshot.py

@@ -222,4 +222,4 @@ def test_sort_snapshots(self):
             sort_order='ascending')
         expected_ids = [item.id for item in expected_items]
         ids = [item.id for item in items]
-        assert set(expected_ids) == set(ids)
+        assert set(expected_ids) == set(ids)

datmo/core/storage/local/tests/test_dal_task.py

@@ -214,4 +214,4 @@ def test_sort_tasks(self):
             sort_order='ascending')
         expected_ids = [item.id for item in expected_items]
         ids = [item.id for item in items]
-        assert set(expected_ids) == set(ids)
+        assert set(expected_ids) == set(ids)

datmo/core/util/json_store.py

@@ -42,7 +42,7 @@ def save(self, key, value):
         self.in_memory_settings = False
         settings_dict = {}
         if not os.path.exists(self.filepath):
-            open(self.filepath, 'w').close()
+            open(self.filepath, 'w+').close()
         else:
             settings_dict = json.load(open(self.filepath, 'r'))
         settings_dict[key] = value
@@ -75,6 +75,22 @@ def get(self, name):
             else:
                 return None
 
+    def remove(self, name):
+        if not os.path.exists(self.filepath):
+            return None
+        else:
+            settings_dict = json.load(open(self.filepath, 'r'))
+        settings_dict.pop(name, None)
+        with open(self.filepath, 'w', encoding='utf8') as outfile:
+            str_ = json.dumps(
+                settings_dict,
+                indent=4,
+                sort_keys=True,
+                separators=(',', ': '),
+                ensure_ascii=False)
+            outfile.write(to_unicode(str_))
+        return
+
     def to_dict(self):
         output_dict = dict()
         # reading json file to stats

datmo/core/util/misc_functions.py

@@ -1,8 +1,8 @@
 #!/usr/bin/python
 
 import os
+import re
 import hashlib
-
 import textwrap
 import datetime
 from io import open
@@ -16,8 +16,6 @@
 from datmo.core.util.exceptions import (PathDoesNotExist,
                                         MutuallyExclusiveArguments)
 
-import re
-
 
 def grep(pattern, fileObj):
     r = []
@@ -127,6 +125,27 @@ def find_project_dir(starting_path=os.getcwd()):
         return find_project_dir(os.path.split(starting_path)[0])
 
 
+def parameterized(dec):
+    """Lifted from https://stackoverflow.com/questions/5929107/decorators-with-parameters
+
+    Parameters
+    ----------
+    dec : function
+
+    Returns
+    -------
+    function
+    """
+
+    def layer(*args, **kwargs):
+        def repl(f):
+            return dec(f, *args, **kwargs)
+
+        return repl
+
+    return layer
+
+
 def is_project_dir(path):
     return ".datmo" in os.listdir(path) and os.path.isdir(
         os.path.join(path, ".datmo"))

datmo/core/util/tests/test_logger.py

@@ -4,7 +4,6 @@
 """
 import shutil
 import os
-import logging
 from time import sleep
 from datmo.core.util.logger import DatmoLogger
 
@@ -17,71 +16,64 @@ def test_logger(self):
         assert logger.level == DatmoLogger().logging_level
 
     def test_datmo_logger_get_logfiles(self):
-        if hasattr(logging, 'handlers'):
-            files = DatmoLogger.get_logfiles()
-            assert len(list(files)) > 0
-            for f in files:
-                assert DatmoLogger().logging_path in f
+        files = DatmoLogger.get_logfiles()
+        assert len(list(files)) > 0
+        for f in files:
+            assert DatmoLogger().logging_path in f
 
     def test_find_text_in_logs(self):
-        if hasattr(logging, 'handlers'):
-            logger = DatmoLogger.get_logger()
-            logger.warning("can you find this")
-            results = DatmoLogger.find_text_in_logs("can you find this")
-            assert len(results) == 1
-            for r in results:
-                assert r["file"]
-                assert r["line_number"]
-                assert r["line"]
+        logger = DatmoLogger.get_logger()
+        logger.info("can you find this")
+        results = DatmoLogger.find_text_in_logs("can you find this")
+        assert len(results) == 1
+        for r in results:
+            assert r["file"]
+            assert r["line_number"]
+            assert r["line"]
 
     def test_get_logger_should_return_same_logger(self):
         f = DatmoLogger.get_logger("foobar")
         f2 = DatmoLogger.get_logger("foobar")
         assert f == f2
 
     def test_multiple_loggers(self):
-        if hasattr(logging, 'handlers'):
-            l1 = DatmoLogger.get_logger("logger1")
-            l2 = DatmoLogger.get_logger("logger2")
-            l1.warning("pizza")
-            l2.warning("party")
-            assert len(DatmoLogger.find_text_in_logs("pizza")) == 1
-            assert len(DatmoLogger.find_text_in_logs("logger2")) == 1
+        l1 = DatmoLogger.get_logger("logger1")
+        l2 = DatmoLogger.get_logger("logger2")
+        l1.info("pizza")
+        l2.info("party")
+        assert len(DatmoLogger.find_text_in_logs("pizza")) == 1
+        assert len(DatmoLogger.find_text_in_logs("logger2")) == 1
 
     def test_multiple_log_files(self):
-        if hasattr(logging, 'handlers'):
-            l1 = DatmoLogger.get_logger("logger3", "debug.txt")
-            l2 = DatmoLogger.get_logger("logger3", "info.txt")
-            l1.warning("green")
-            l2.warning("purple")
-            r = DatmoLogger.find_text_in_logs("green")
-            assert len(r) == 1
-            assert r[0]["file"].find("debug.txt")
-            r = DatmoLogger.find_text_in_logs("purple")
-            assert len(r) == 1
-            assert r[0]["file"].find("info.txt")
+        l1 = DatmoLogger.get_logger("logger3", "debug.txt")
+        l2 = DatmoLogger.get_logger("logger3", "info.txt")
+        l1.info("green")
+        l2.info("purple")
+        r = DatmoLogger.find_text_in_logs("green")
+        assert len(r) == 1
+        assert r[0]["file"].find("debug.txt")
+        r = DatmoLogger.find_text_in_logs("purple")
+        assert len(r) == 1
+        assert r[0]["file"].find("info.txt")
 
     def test_autocreate_dir(self):
-        if hasattr(logging, 'handlers'):
-            log_path = os.path.join(os.path.expanduser("~"), '.datmo')
-            shutil.rmtree(log_path)
-            assert not os.path.exists(log_path)
-            logger = DatmoLogger.get_logger("logger3", "new.txt")
-            logger.warning("testing")
-            assert len(DatmoLogger.find_text_in_logs("testing")) == 1
-            default_logger = DatmoLogger.get_logger()
-            default_logger.warning("default-logger")
-            assert len(DatmoLogger.find_text_in_logs("default-logger")) == 1
+        log_path = os.path.join(os.path.expanduser("~"), '.datmo')
+        shutil.rmtree(log_path)
+        assert not os.path.exists(log_path)
+        logger = DatmoLogger.get_logger("logger3", "new.txt")
+        logger.info("testing")
+        assert len(DatmoLogger.find_text_in_logs("testing")) == 1
+        default_logger = DatmoLogger.get_logger()
+        default_logger.info("default-logger")
+        assert len(DatmoLogger.find_text_in_logs("default-logger")) == 1
 
     def test_timeit_decorator(self):
         # NOTE:  If this test is failing be sure to add
         # LOGGING_LEVEL=DEBUG before pytest
         # or add as an environment variable
-        if hasattr(logging, 'handlers'):
-
-            @DatmoLogger.timeit
-            def slow_fn():
-                sleep(1)
+        @DatmoLogger.timeit
+        def slow_fn():
+            sleep(1)
 
-            slow_fn()
-            assert len(DatmoLogger.find_text_in_logs("slow_fn")) == 1
+        slow_fn()
+        assert len(DatmoLogger.find_text_in_logs("slow_fn")) == 1

devtools/README.md

@@ -31,6 +31,12 @@ $ export LOGGING_LEVEL=DEBUG # sets logging level to debug for tests
 $ python -m pytest --cov-config .coveragerc --cov=datmo
 ```
 
+If you run into issues with testing on docker, you might want to clean up your open docker containers 
+with the following command
+```
+$ docker rm -f $(docker ps -a -q)
+```
+
 ## Cleaning Up Code
 We use [yapf](https://github.com/google/yapf) to clean code and have added a pre-commit hook to
 ensure any changed files adhere to the styles specified in `.style.yapf` in the root of the project.