Merge pull request #128 from datmo/fix-alt-os-builds

Fix alternative os builds
datmo · May 15, 2018 · 14ff961 · 14ff961
2 parents 5fdebd7 + d8df8c2
commit 14ff961
Show file tree

Hide file tree

Showing 40 changed files with 452 additions and 269 deletions.
diff --git a/README.md b/README.md
@@ -7,6 +7,7 @@
 | OS | CI testing on `master` |
 |----|--------------------|
 | <img height="20" src="http://icons.iconarchive.com/icons/dakirby309/simply-styled/256/OS-Linux-icon.png"> | [![Linux](https://travis-ci.org/datmo/datmo.svg?branch=master)](https://travis-ci.org/datmo/datmo) |
+| <img height="20" src="http://icons.iconarchive.com/icons/icons8/windows-8/128/Systems-Mac-Os-icon.png"> | [![CircleCI branch](https://circleci.com/gh/datmo/datmo.svg?style=shield)](https://circleci.com/gh/datmo/datmo) |
 | <img height="20" src="http://icons.iconarchive.com/icons/dakirby309/windows-8-metro/128/Folders-OS-Windows-8-Metro-icon.png"> | [![Windows](https://ci.appveyor.com/api/projects/status/5302d8a23qr4ui4y/branch/master?svg=true)](https://ci.appveyor.com/project/asampat3090/datmo/branch/master) |
 
 **Datmo** is an open source model tracking and reproducibility tool for developers. Use `datmo init` to turn any repository into a trackable task record with reusable environments and metrics logging.

diff --git a/appveyor.yml b/appveyor.yml
@@ -12,24 +12,24 @@ environment:
     - PYTHON: "C:\\Python27"
       PYTHON_VERSION: "2.7.x"
 
-    - PYTHON: "C:\\Python34"
-      PYTHON_VERSION: "3.4.x"
-
-    - PYTHON: "C:\\Python35"
-      PYTHON_VERSION: "3.5.x"
-
-    - PYTHON: "C:\\Python27-x64"
-      PYTHON_VERSION: "2.7.x"
-
-    - PYTHON: "C:\\Python34-x64"
-      DISTUTILS_USE_SDK: "1"
-      PYTHON_VERSION: "3.4.x"
-
-    - PYTHON: "C:\\Python35-x64"
-      PYTHON_VERSION: "3.5.x"
-
-    - PYTHON: "C:\\Python36-x64"
-      PYTHON_VERSION: "3.6.x"
+#    - PYTHON: "C:\\Python34"
+#      PYTHON_VERSION: "3.4.x"
+#
+#    - PYTHON: "C:\\Python35"
+#      PYTHON_VERSION: "3.5.x"
+#
+#    - PYTHON: "C:\\Python27-x64"
+#      PYTHON_VERSION: "2.7.x"
+#
+#    - PYTHON: "C:\\Python34-x64"
+#      DISTUTILS_USE_SDK: "1"
+#      PYTHON_VERSION: "3.4.x"
+#
+#    - PYTHON: "C:\\Python35-x64"
+#      PYTHON_VERSION: "3.5.x"
+#
+#    - PYTHON: "C:\\Python36-x64"
+#      PYTHON_VERSION: "3.6.x"
 
 build: false
 init:

diff --git a/circle.yml b/circle.yml
@@ -32,8 +32,8 @@ test:
     # Avoiding loop here, so that each test shown in separate tab in circle-ci
     - echo "Running Test with 2.7.10"
     - export PATH="$HOME/miniconda/bin:$PATH" && source activate datmo_env2.7.10 && $TEST_PACKAGE  && source deactivate datmo_env2.7.10;
-    - echo "Running Test with 3.4.1"
-    - export PATH="$HOME/miniconda/bin:$PATH" && source activate datmo_env3.4.1 && $TEST_PACKAGE  && source deactivate datmo_env3.4.1;
+#    - echo "Running Test with 3.4.1"
+#    - export PATH="$HOME/miniconda/bin:$PATH" && source activate datmo_env3.4.1 && $TEST_PACKAGE  && source deactivate datmo_env3.4.1;
     - echo "Running Test with 3.5.1"
     - export PATH="$HOME/miniconda/bin:$PATH" && source activate datmo_env3.5.1 && $TEST_PACKAGE && source deactivate datmo_env3.5.1;
     - echo "Running Test with 3.6.1"

diff --git a/datmo/cli/command/base.py b/datmo/cli/command/base.py
@@ -3,13 +3,15 @@
 from datmo.core.util.i18n import get as __
 from datmo.core.util.exceptions import ClassMethodNotFound
 from datmo.cli.parser import get_datmo_parser
+from datmo.core.util.logger import DatmoLogger
 from datmo.core.util.misc_functions import parameterized
 
 
 class BaseCommand(object):
     def __init__(self, home, cli_helper):
         self.home = home
         self.cli_helper = cli_helper
+        self.logger = DatmoLogger.get_logger(__name__)
         self.parser = get_datmo_parser()
 
     def parse(self, args):

diff --git a/datmo/cli/command/project.py b/datmo/cli/command/project.py
@@ -50,7 +50,7 @@ def init(self, name, description):
                             "name": name,
                             "path": self.home
                         }))
-            except:
+            except Exception:
                 self.cli_helper.echo(
                     __("info", "cli.project.init.create.failure", {
                         "name": name,
@@ -83,7 +83,7 @@ def init(self, name, description):
                             "name": name,
                             "path": self.home
                         }))
-            except:
+            except Exception:
                 self.cli_helper.echo(
                     __("info", "cli.project.init.update.failure", {
                         "name": name,

diff --git a/datmo/cli/command/task.py b/datmo/cli/command/task.py
@@ -13,14 +13,12 @@
 from datmo.core.util.misc_functions import mutually_exclusive
 from datmo.cli.command.project import ProjectCommand
 from datmo.core.controller.task import TaskController
-from datmo.core.util.logger import DatmoLogger
 from datmo.core.util.exceptions import RequiredArgumentMissing
 
 
 class TaskCommand(ProjectCommand):
     def __init__(self, home, cli_helper):
         super(TaskCommand, self).__init__(home, cli_helper)
-        self.logger = DatmoLogger.get_logger(__name__)
         self.task_controller = TaskController(home=home)
 
     def task(self):
@@ -105,7 +103,7 @@ def stop(self, **kwargs):
                     self.cli_helper.echo(
                         __("info", "cli.task.stop.all.success"))
             return result
-        except:
+        except Exception:
             if "id" in input_dict:
                 self.cli_helper.echo(
                     __("error", "cli.task.stop", input_dict['id']))

diff --git a/datmo/cli/command/tests/test_session.py b/datmo/cli/command/tests/test_session.py
@@ -5,9 +5,8 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
-import shutil
 import tempfile
-from io import open
+import platform
 try:
     to_unicode = unicode
 except NameError:
@@ -23,7 +22,8 @@
 class TestSession():
     def setup_class(self):
         # provide mountable tmp directory for docker
-        tempfile.tempdir = '/tmp'
+        tempfile.tempdir = "/tmp" if not platform.system(
+        ) == "Windows" else None
         test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
                                         tempfile.gettempdir())
         self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)

diff --git a/datmo/cli/command/tests/test_snapshot.py b/datmo/cli/command/tests/test_snapshot.py
@@ -28,15 +28,15 @@
 from datmo.core.util.exceptions import (ProjectNotInitializedException,
                                         MutuallyExclusiveArguments,
                                         SnapshotCreateFromTaskArgs)
+from datmo.core.util.misc_functions import pytest_docker_environment_failed_instantiation
+
+# provide mountable tmp directory for docker
+tempfile.tempdir = "/tmp" if not platform.system() == "Windows" else None
+test_datmo_dir = os.environ.get('TEST_DATMO_DIR', tempfile.gettempdir())
 
 
 class TestSnapshot():
     def setup_class(self):
-        # provide mountable tmp directory for docker
-        tempfile.tempdir = "/tmp" if not platform.system(
-        ) == "Windows" else None
-        test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
-                                        tempfile.gettempdir())
         self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
         self.cli_helper = Helper()
 
@@ -106,7 +106,7 @@ def test_snapshot_help(self):
         self.snapshot.parse(["snapshot", "create", "--help"])
         assert self.snapshot.execute()
 
-    def test_snapshot_command(self):
+    def test_snapshot_create(self):
         self.__set_variables()
         test_message = "this is a test message"
         test_label = "test label"
@@ -183,10 +183,10 @@ def test_snapshot_command(self):
         snapshot_id_1 = self.snapshot.execute()
         assert snapshot_id_1
 
-    def test_datmo_snapshot_create_from_task(self):
+    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
+    def test_snapshot_create_from_task(self):
         self.__set_variables()
         test_message = "this is a test message"
-        test_code_id = "test_code_id"
 
         # create task
         test_command = "sh -c 'echo accuracy:0.45'"
@@ -213,6 +213,7 @@ def test_datmo_snapshot_create_from_task(self):
         snapshot_id = self.snapshot.execute()
         assert snapshot_id
 
+    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
     def test_snapshot_create_from_task_fail_user_inputs(self):
         self.__set_variables()
         test_message = "this is a test message"
@@ -546,7 +547,8 @@ def test_datmo_snapshot_checkout(self):
 
         # remove datmo_task folder to have no changes before checkout
         datmo_tasks_dirpath = os.path.join(self.snapshot.home, "datmo_tasks")
-        shutil.rmtree(datmo_tasks_dirpath)
+        if os.path.exists(datmo_tasks_dirpath):
+            shutil.rmtree(datmo_tasks_dirpath)
 
         # Test when optional parameters are not given
         self.snapshot.parse(["snapshot", "checkout", "--id", snapshot_id])

diff --git a/datmo/cli/command/tests/test_task.py b/datmo/cli/command/tests/test_task.py
@@ -29,15 +29,15 @@
 from datmo.cli.command.task import TaskCommand
 from datmo.core.entity.task import Task as CoreTask
 from datmo.core.util.exceptions import ProjectNotInitializedException, MutuallyExclusiveArguments, RequiredArgumentMissing
+from datmo.core.util.misc_functions import pytest_docker_environment_failed_instantiation
+
+# provide mountable tmp directory for docker
+tempfile.tempdir = "/tmp" if not platform.system() == "Windows" else None
+test_datmo_dir = os.environ.get('TEST_DATMO_DIR', tempfile.gettempdir())
 
 
 class TestTaskCommand():
     def setup_class(self):
-        # provide mountable tmp directory for docker
-        tempfile.tempdir = "/tmp" if not platform.system(
-        ) == "Windows" else None
-        test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
-                                        tempfile.gettempdir())
         self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
         self.cli_helper = Helper()
 
@@ -65,7 +65,7 @@ def test_task_project_not_init(self):
             failed = True
         assert failed
 
-    def test_snapshot_command(self):
+    def test_task_command(self):
         self.__set_variables()
         self.task.parse(["task"])
         assert self.task.execute()
@@ -77,6 +77,7 @@ def test_task_run_should_fail1(self):
         result = self.task.execute()
         assert not result
 
+    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
     def test_task_run(self):
         # TODO: Adding test with `--interactive` argument and terminate inside container
         self.__set_variables()
@@ -123,6 +124,7 @@ def test_task_run(self):
         assert result.results == {"accuracy": "0.45"}
         assert result.status == "SUCCESS"
 
+    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
     def test_task_run_string_command(self):
         # TODO: Adding test with `--interactive` argument and terminate inside container
         self.__set_variables()
@@ -191,6 +193,7 @@ def test_task_run_string_command(self):
     #         assert result.results == {"accuracy": "0.45"}
     #         assert result.status == "SUCCESS"
 
+    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
     def test_task_run_notebook(self):
         self.__set_variables()
         # Test success case
@@ -258,6 +261,7 @@ def test_task_ls_invalid_arg(self):
             exception_thrown = True
         assert exception_thrown
 
+    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
     def test_task_stop_success(self):
         # 1) Test stop with task_id
         # 2) Test stop with all
@@ -329,4 +333,4 @@ def test_task_stop_invalid_task_id(self):
 
         # test when wrong task id is passed to stop it
         result = self.task.execute()
-        assert not result
+        assert not result
diff --git a/datmo/core/controller/base.py b/datmo/core/controller/base.py
@@ -1,6 +1,7 @@
 import os
 
 from datmo.core.util.i18n import get as __
+from datmo.core.util.logger import DatmoLogger
 from datmo.core.util import get_class_contructor
 from datmo.core.util.json_store import JSONStore
 from datmo.core.util.exceptions import (InvalidProjectPathException,
@@ -47,6 +48,7 @@ def __init__(self, home):
             raise InvalidProjectPathException(
                 __("error", "controller.base.__init__", home))
         self.config = JSONStore(os.path.join(self.home, ".datmo", ".config"))
+        self.logger = DatmoLogger.get_logger(__name__)
         # property caches and initial values
         self._dal = None
         self._model = None
@@ -111,10 +113,9 @@ def environment_driver(self):
     def is_initialized(self):
         if not self._is_initialized:
             if self.code_driver.is_initialized and \
-                self.environment_driver.is_initialized and \
-                    self.file_driver.is_initialized:
-                if self.model:
-                    self._is_initialized = True
+                self.file_driver.is_initialized and \
+                 self.model:
+                self._is_initialized = True
         return self._is_initialized
 
     def dal_instantiate(self):

diff --git a/datmo/core/controller/code/code.py b/datmo/core/controller/code/code.py
@@ -1,7 +1,7 @@
 from datmo.core.util.i18n import get as __
 from datmo.core.controller.base import BaseController
 from datmo.core.entity.code import Code
-from datmo.core.util.exceptions import PathDoesNotExist
+from datmo.core.util.exceptions import PathDoesNotExist, EnvironmentInitFailed
 
 
 class CodeController(BaseController):
@@ -24,7 +24,11 @@ class CodeController(BaseController):
     """
 
     def __init__(self, home):
-        super(CodeController, self).__init__(home)
+        try:
+            super(CodeController, self).__init__(home)
+        except EnvironmentInitFailed:
+            self.logger.warning(
+                __("warn", "controller.general.environment.failed"))
 
     def create(self, commit_id=None):
         """Create a Code object

diff --git a/datmo/core/controller/code/driver/git.py b/datmo/core/controller/code/driver/git.py
@@ -133,7 +133,7 @@ def create_ref(self, commit_id=None):
                 self.add("-A")
                 new_commit_bool = self.commit(
                     options=["-m", "auto commit by datmo"])
-            except:
+            except Exception:
                 self.add("-A")
                 new_commit_bool = self.commit(
                     options=["-m", "auto initial commit by datmo"])

diff --git a/datmo/core/controller/code/driver/tests/test_git.py b/datmo/core/controller/code/driver/tests/test_git.py
@@ -264,7 +264,7 @@ def test_latest_commit(self, capsys):
         failed = False
         try:
             self.git_code_manager.latest_commit()
-        except:
+        except Exception:
             failed = True
         assert failed
 

diff --git a/datmo/core/controller/environment/driver/__init__.py b/datmo/core/controller/environment/driver/__init__.py
@@ -23,6 +23,20 @@ class EnvironmentDriver(with_metaclass(ABCMeta, object)):
     def __init__(self):
         pass
 
+    @abstractmethod
+    def init(self):
+        """Initialize the environment driver
+
+        Returns
+        -------
+        bool
+            returns True if success else False
+
+        Raises
+        ------
+        EnvironmentInitFailed
+        """
+
     @abstractmethod
     def create(self, path=None, output_path=None, language=None):
         """Create datmo environment definition