Merge pull request #55 from datmo/examples

Basic task run and snapshot create flows working

datmo/cli/command/snapshot.py

@@ -115,7 +115,7 @@ def ls(self, **kwargs):
             header_list = ["id", "created at", "config", "stats", "message", "label", "code id",
                            "environment id", "file collection id"]
             t = prettytable.PrettyTable(header_list)
-            snapshot_objs = self.snapshot_controller.list(session_id)
+            snapshot_objs = self.snapshot_controller.list(session_id=session_id, visible=True)
             for snapshot_obj in snapshot_objs:
                 t.add_row([snapshot_obj.id, snapshot_obj.created_at.strftime("%Y-%m-%d %H:%M:%S"),
                            snapshot_obj.config, snapshot_obj.stats,
@@ -125,7 +125,7 @@ def ls(self, **kwargs):
         else:
             header_list = ["id", "created at", "config", "stats", "message", "label"]
             t = prettytable.PrettyTable(header_list)
-            snapshot_objs = self.snapshot_controller.list(session_id)
+            snapshot_objs = self.snapshot_controller.list(session_id=session_id, visible=True)
             for snapshot_obj in snapshot_objs:
                 t.add_row([snapshot_obj.id, snapshot_obj.created_at.strftime("%Y-%m-%d %H:%M:%S"),
                            snapshot_obj.config, snapshot_obj.stats,

datmo/cli/command/task.py

@@ -66,13 +66,13 @@ def run(self, **kwargs):
 
         # Pass in the task
         try:
-            self.task_controller.run(task_obj.id, snapshot_dict=snapshot_dict)
+            updated_task_obj = self.task_controller.run(task_obj.id, snapshot_dict=snapshot_dict)
         except:
             self.cli_helper.echo(__("error",
                                     "cli.task.run",
                                     task_obj.id))
             return False
-        return task_obj.id
+        return updated_task_obj
 
     def ls(self, **kwargs):
         session_id = kwargs.get('session_id',

datmo/cli/command/test/test_snapshot.py

@@ -131,7 +131,8 @@ def test_datmo_snapshot_create(self):
         # Test when optional parameters are not given
         self.snapshot.parse([
             "snapshot",
-            "create"
+            "create",
+            "-m", "my test snapshot"
         ])
 
         snapshot_id_2 = self.snapshot.execute()
@@ -155,7 +156,8 @@ def test_datmo_snapshot_delete(self):
         # Test when optional parameters are not given
         self.snapshot.parse([
             "snapshot",
-            "create"
+            "create",
+            "-m", "my test snapshot"
         ])
 
         snapshot_id = self.snapshot.execute()
@@ -188,7 +190,8 @@ def test_datmo_snapshot_ls(self):
         # Test when optional parameters are not given
         self.snapshot.parse([
             "snapshot",
-            "create"
+            "create",
+            "-m", "my test snapshot"
         ])
 
         self.snapshot.execute()
@@ -231,7 +234,8 @@ def test_datmo_snapshot_checkout(self):
         # Test when optional parameters are not given
         self.snapshot.parse([
             "snapshot",
-            "create"
+            "create",
+            "-m", "my test snapshot"
         ])
 
         snapshot_id = self.snapshot.execute()

datmo/cli/command/test/test_task.py

@@ -24,6 +24,7 @@
 from datmo.cli.driver.helper import Helper
 from datmo.cli.command.project import ProjectCommand
 from datmo.cli.command.task import TaskCommand
+from datmo.core.entity.task import Task as CoreTask
 from datmo.core.util.exceptions import ProjectNotInitializedException
 
 
@@ -88,7 +89,7 @@ def test_datmo_task_run(self):
         assert not result
 
         # Test success case
-        test_command = ["sh", "-c", "echo yo"]
+        test_command = ["sh", "-c", "echo accuracy:0.45"]
         test_gpu = True # TODO: implement in controller
         test_ports = "8888:8888"
         test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
@@ -114,8 +115,14 @@ def test_datmo_task_run(self):
         # test proper execution of task run command
         result = self.task.execute()
         assert result
-
-    def test_datmo_task_run_invalid_arg(self):
+        assert isinstance(result, CoreTask)
+        assert result.logs
+        assert "accuracy" in result.logs
+        assert result.results
+        assert result.results == {"accuracy": "0.45"}
+        assert result.status == "SUCCESS"
+
+    def test_task_run_invalid_arg(self):
         self.__set_variables()
         exception_thrown = False
         try:
@@ -127,7 +134,7 @@ def test_datmo_task_run_invalid_arg(self):
             exception_thrown = True
         assert exception_thrown
 
-    def test_datmo_task_ls(self):
+    def test_task_ls(self):
         self.__set_variables()
         test_session_id = 'test_session_id'
 
@@ -143,7 +150,7 @@ def test_datmo_task_ls(self):
         task_ls_command = self.task.execute()
         assert task_ls_command == True
 
-    def test_datmo_task_ls_invalid_arg(self):
+    def test_task_ls_invalid_arg(self):
         self.__set_variables()
         exception_thrown = False
         try:
@@ -155,7 +162,7 @@ def test_datmo_task_ls_invalid_arg(self):
             exception_thrown = True
         assert exception_thrown
 
-    def test_datmo_task_stop(self):
+    def test_task_stop(self):
         self.__set_variables()
 
         test_command = ["sh", "-c", "echo yo"]
@@ -172,16 +179,16 @@ def test_datmo_task_stop(self):
             test_command
         ])
 
-        test_task_id = self.task.execute()
+        test_task_obj = self.task.execute()
 
         self.task.parse([
             "task",
             "stop",
-            "--id", test_task_id
+            "--id", test_task_obj.id
         ])
 
         # test for desired side effects
-        assert self.task.args.id == test_task_id
+        assert self.task.args.id == test_task_obj.id
 
         # test when task id is passed to stop it
         task_stop_command = self.task.execute()
@@ -199,7 +206,7 @@ def test_datmo_task_stop(self):
         result = self.task.execute()
         assert not result
 
-    def test_datmo_task_stop_invalid_arg(self):
+    def test_task_stop_invalid_arg(self):
         self.__set_variables()
         exception_thrown = False
         try:

datmo/core/controller/code/code.py

@@ -1,6 +1,7 @@
 from datmo.core.util.i18n import get as __
 from datmo.core.controller.base import BaseController
-from datmo.core.util.exceptions import DoesNotExistException
+from datmo.core.entity.code import Code
+from datmo.core.util.exceptions import PathDoesNotExist
 
 
 class CodeController(BaseController):
@@ -69,7 +70,7 @@ def create(self, commit_id=None):
                 raise NotImplementedError()
 
         # Create code and return
-        return self.dal.code.create(create_dict)
+        return self.dal.code.create(Code(create_dict))
 
     def list(self):
         # TODO: Add time filters
@@ -90,14 +91,14 @@ def delete(self, code_id):
 
         Raises
         ------
-        DoesNotExistException
+        PathDoesNotExist
             if the specified Code does not exist.
         """
         code_obj = self.dal.code.get_by_id(code_id)
         if not code_obj:
-            raise DoesNotExistException(__("error",
+            raise PathDoesNotExist(__("error",
                                            "controller.code.delete",
-                                           code_id))
+                                      code_id))
         # Remove code reference
         delete_code_success = self.code_driver.delete_ref(code_obj.commit_id)
         # Delete code object

datmo/core/controller/code/driver/git.py

@@ -10,7 +10,7 @@
 from giturlparse import parse
 
 from datmo.core.util.i18n import get as __
-from datmo.core.util.exceptions import DoesNotExistException,\
+from datmo.core.util.exceptions import PathDoesNotExist,\
     GitUrlArgumentException, GitExecutionException, \
     FileIOException, GitCommitDoesNotExist, \
     DatmoFolderInWorkTree
@@ -29,9 +29,9 @@ def __init__(self, filepath, execpath, remote_url=None):
         self.filepath = filepath
         # Check if filepath exists
         if not os.path.exists(self.filepath):
-            raise DoesNotExistException(__("error",
+            raise PathDoesNotExist(__("error",
                                            "controller.code.driver.git.__init__.dne",
-                                           filepath))
+                                      filepath))
         self.execpath = execpath
         # Check the execpath and the version
         try:

datmo/core/controller/code/driver/test/test_git.py

@@ -16,7 +16,7 @@
 from datmo.core.controller.code.driver.git import GitCodeDriver, \
     GitHostDriver
 from datmo.core.util.exceptions import GitCommitDoesNotExist, \
-    DoesNotExistException, GitExecutionException, \
+    PathDoesNotExist, GitExecutionException, \
     DatmoFolderInWorkTree
 
 
@@ -42,7 +42,7 @@ def test_instantiation_fail_dne(self):
         failed = False
         try:
             _ = GitCodeDriver(filepath="nonexistant_path", execpath="git")
-        except DoesNotExistException:
+        except PathDoesNotExist:
             failed = True
         assert failed
 

datmo/core/controller/environment/driver/dockerenv.py

@@ -10,9 +10,10 @@
 from docker import DockerClient
 
 from datmo.core.util.i18n import get as __
-from datmo.core.util.exceptions import DoesNotExistException, \
-    EnvironmentInitFailed, EnvironmentExecutionException, \
-    FileAlreadyExistsException, EnvironmentRequirementsCreateException
+from datmo.core.util.exceptions import PathDoesNotExist, \
+    EnvironmentDoesNotExist, EnvironmentInitFailed, \
+    EnvironmentExecutionException, FileAlreadyExistsException, \
+    EnvironmentRequirementsCreateException
 from datmo.core.controller.environment.driver import EnvironmentDriver
 
 
@@ -26,9 +27,9 @@ def __init__(self, filepath="", docker_execpath="docker", docker_socket="unix://
         self.filepath = filepath
         # Check if filepath exists
         if not os.path.exists(self.filepath):
-            raise DoesNotExistException(__("error",
-                                          "controller.environment.driver.docker.__init__.dne",
-                                          filepath))
+            raise PathDoesNotExist(__("error",
+                                      "controller.environment.driver.docker.__init__.dne",
+                                      filepath))
 
         # TODO: separate methods for instantiation into init function below
 
@@ -85,7 +86,7 @@ def create(self, path=None, output_path=None, language=None):
                 path = self.create_default_dockerfile(requirements_filepath=requirements_filepath,
                                                       language=language)
             else:
-                raise DoesNotExistException(__("error",
+                raise EnvironmentDoesNotExist(__("error",
                                               "controller.environment.driver.docker.create.dne",
                                               path))
         if os.path.isfile(output_path):
@@ -497,22 +498,37 @@ def stop_remove_containers_by_term(self, term, force=False):
         return True
 
     def create_requirements_file(self, execpath="pipreqs"):
-        """Create requirements txt file for the project
+        """Create python requirements txt file for the project
+
+        Parameters
+        ----------
+        execpath : str, optional
+            execpath for the pipreqs command to form requirements.txt file
+            (default is "pipreqs")
 
         Returns
         -------
         str
             absolute filepath for requirements file
+
+        Raises
+        ------
+        EnvironmentDoesNotExist
+            no python requirements found for environment
+        EnvironmentRequirementsCreateException
+            error in running pipreqs command to extract python requirements
         """
         try:
             subprocess.check_output([execpath, self.filepath, "--force"],
                                     cwd=self.filepath).strip()
             requirements_filepath = os.path.join(self.filepath, "requirements.txt")
-            return requirements_filepath
         except Exception as e:
             raise EnvironmentRequirementsCreateException(__("error",
-                                                           "controller.environment.requirements.create",
-                                                           str(e)))
+                                                            "controller.environment.requirements.create",
+                                                            str(e)))
+        if open(requirements_filepath, "r").read() == "\n":
+            raise EnvironmentDoesNotExist()
+        return requirements_filepath
 
     def create_default_dockerfile(self, requirements_filepath, language):
         """Create a default Dockerfile for a given language

datmo/core/controller/environment/driver/templates/python2Dockerfile

@@ -1,2 +1,3 @@
 FROM python:2
 RUN pip install --upgrade pip
+RUN pip install scipy

datmo/core/controller/environment/driver/templates/python3Dockerfile

@@ -1,3 +1,4 @@
 FROM python:3
 RUN pip install --upgrade pip
+RUN pip install scipy
 

datmo/core/controller/environment/driver/test/test_dockerenv.py

@@ -17,7 +17,8 @@
 
 from datmo.core.controller.environment.driver.dockerenv import DockerEnvironmentDriver
 from datmo.core.util.exceptions import EnvironmentInitFailed, \
-    FileAlreadyExistsException, EnvironmentRequirementsCreateException
+    FileAlreadyExistsException, EnvironmentRequirementsCreateException, \
+    EnvironmentDoesNotExist
 
 
 class TestDockerEnv():
@@ -504,6 +505,19 @@ def test_log_container(self):
         self.docker_environment_manager.remove_image(image_name, force=True)
 
     def test_create_requirements_file(self):
+        # 1) Test failure EnvironmentDoesNotExist
+        # 2) Test success
+        # 3) Test failure EnvironmentRequirementsCreateException
+
+        # 1) Test option 1
+        failed = False
+        try:
+            _ = self.docker_environment_manager.create_requirements_file()
+        except EnvironmentDoesNotExist:
+            failed = True
+        assert failed
+
+        # 2) Test option 2
         script_path = os.path.join(self.docker_environment_manager.filepath,
                                    "script.py")
         with open(script_path, "w") as f:
@@ -516,7 +530,7 @@ def test_create_requirements_file(self):
                "numpy" in open(result, "r").read() and \
                "scikit_learn" in open(result, "r").read()
 
-        # Test failure
+        # 3) Test option 3
         exception_thrown = False
         try:
             _ = self.docker_environment_manager.\