Merge pull request #227 from datmo/issue-216

Issue 216

datmo/cli/command/project.py

@@ -126,8 +126,8 @@ def version(self):
 
     @Helper.notify_no_project_found
     def status(self):
-        status_dict, latest_snapshot, ascending_unstaged_tasks = self.project_controller.status(
-        )
+        status_dict, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = \
+            self.project_controller.status()
 
         # Print out simple project meta data
         for k, v in status_dict.items():
@@ -142,24 +142,63 @@ def status(self):
 
         self.cli_helper.echo("")
 
-        # Print out latest snapshot info
-        self.cli_helper.echo("latest snapshot id: ")
-        if latest_snapshot:
-            self.cli_helper.echo(latest_snapshot.id)
+        # Print out info for the latest snapshot created by the user
+        self.cli_helper.echo(
+            "latest snapshot generated by the user (run `datmo snapshot ls` for more info): "
+        )
+        if latest_snapshot_user_generated:
+            self.cli_helper.echo(latest_snapshot_user_generated)
         else:
-            self.cli_helper.echo("no snapshots created yet")
+            self.cli_helper.echo("no user-generated snapshots created yet")
 
         self.cli_helper.echo("")
 
-        # Print out unstaged tasks
-        self.cli_helper.echo("unstaged task ids:")
-        if ascending_unstaged_tasks:
-            for task in ascending_unstaged_tasks:
-                self.cli_helper.echo(task.id)
+        # Print out info for the latest snapshot autogenerated by datmo
+        self.cli_helper.echo(
+            "latest snapshot autogenerated by datmo (run `datmo snapshot ls --all` for more info): "
+        )
+        if latest_snapshot_auto_generated:
+            self.cli_helper.echo(latest_snapshot_auto_generated)
+        else:
+            self.cli_helper.echo("no auto-generated snapshots created yet")
+
+        self.cli_helper.echo("")
+
+        # Print out the latest snapshot if no unstaged changes
+        if not unstaged_code and not unstaged_environment and not unstaged_files:
+            if not latest_snapshot_user_generated and not latest_snapshot_auto_generated:
+                self.cli_helper.echo(
+                    "all changes have been saved, no unstaged changes")
+                self.cli_helper.echo("no latest snapshot")
+            elif latest_snapshot_user_generated and not latest_snapshot_auto_generated:
+                self.cli_helper.echo(
+                    "all changes have been saved, no unstaged changes")
+                self.cli_helper.echo("latest snapshot:")
+                self.cli_helper.echo(latest_snapshot_user_generated)
+            elif not latest_snapshot_user_generated and latest_snapshot_auto_generated:
+                self.cli_helper.echo(
+                    "all changes have been saved, no unstaged changes")
+                self.cli_helper.echo("latest snapshot:")
+                self.cli_helper.echo(latest_snapshot_auto_generated)
+            else:
+                self.cli_helper.echo(
+                    "all changes have been saved, no unstaged changes")
+                self.cli_helper.echo("latest snapshot:")
+                if latest_snapshot_user_generated.created_at > latest_snapshot_auto_generated.created_at:
+                    self.cli_helper.echo(latest_snapshot_user_generated)
+                else:
+                    self.cli_helper.echo(latest_snapshot_auto_generated)
         else:
-            self.cli_helper.echo("no unstaged tasks")
+            # Print out the the unstaged components if unstaged
+            self.cli_helper.echo("unstaged changes since latest snapshot:")
+            if unstaged_code:
+                self.cli_helper.echo("code has been changed")
+            if unstaged_environment:
+                self.cli_helper.echo("environment has been changed")
+            if unstaged_files:
+                self.cli_helper.echo("files have been changed")
 
-        return status_dict, latest_snapshot, ascending_unstaged_tasks
+        return status_dict, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files
 
     def cleanup(self):
         # Prompt user to ensure they would like to remove datmo information along with environment and files folder
@@ -191,4 +230,4 @@ def cleanup(self):
                             "name": self.project_controller.model.name,
                             "path": self.project_controller.home
                         }))
-        return False
+        return False

datmo/cli/command/snapshot.py

@@ -174,14 +174,21 @@ def ls(self, **kwargs):
         session_id = kwargs.get('session_id',
                                 self.snapshot_controller.current_session.id)
         detailed_info = kwargs.get('details', None)
+        show_all = kwargs.get('show_all', None)
         print_format = kwargs.get('format', "table")
         download = kwargs.get('download', None)
         download_path = kwargs.get('download_path', None)
-        snapshot_objs = self.snapshot_controller.list(
-            session_id=session_id,
-            visible=True,
-            sort_key="created_at",
-            sort_order="descending")
+        if show_all:
+            snapshot_objs = self.snapshot_controller.list(
+                session_id=session_id,
+                sort_key="created_at",
+                sort_order="descending")
+        else:
+            snapshot_objs = self.snapshot_controller.list(
+                session_id=session_id,
+                visible=True,
+                sort_key="created_at",
+                sort_order="descending")
         item_dict_list = []
         if detailed_info:
             header_list = [

datmo/cli/command/tests/test_project.py

@@ -250,9 +250,13 @@ def dummy(self):
 
         self.project_command.parse(["status"])
         result = self.project_command.execute()
-        assert isinstance(result[0], dict)
-        assert not result[1]
-        assert not result[2]
+        status_dict, latest_snapshot_user_generated, latest_snapshot_auto_generated, unstaged_code, unstaged_environment, unstaged_files = result
+        assert isinstance(status_dict, dict)
+        assert not latest_snapshot_user_generated
+        assert not latest_snapshot_auto_generated
+        assert unstaged_code
+        assert not unstaged_environment
+        assert not unstaged_files
 
     def test_status_invalid_arg(self):
         exception_thrown = False

datmo/cli/command/tests/test_snapshot.py

@@ -40,8 +40,7 @@ def to_bytes(val):
 from datmo.cli.command.project import ProjectCommand
 from datmo.cli.command.snapshot import SnapshotCommand
 from datmo.cli.command.task import TaskCommand
-from datmo.core.util.exceptions import (ProjectNotInitialized,
-                                        MutuallyExclusiveArguments,
+from datmo.core.util.exceptions import (MutuallyExclusiveArguments,
                                         SnapshotCreateFromTaskArgs)
 from datmo.core.util.misc_functions import pytest_docker_environment_failed_instantiation
 
@@ -638,9 +637,41 @@ def test_snapshot_delete_invalid_arg(self):
             exception_thrown = True
         assert exception_thrown
 
+    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
+    def test_snapshot_ls_invisible(self):
+        self.__set_variables()
+
+        # Test invisible snapshot from task
+        # create task
+        test_command = "sh -c 'echo accuracy:0.45'"
+        test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
+        self.task = TaskCommand(self.cli_helper)
+        self.task.parse([
+            "task", "run", "--environment-paths", test_dockerfile, test_command
+        ])
+
+        # test proper execution of task run command
+        task_obj = self.task.execute()
+
+        # test no visible snapshots
+        self.snapshot_command.parse(["snapshot", "ls"])
+
+        result = self.snapshot_command.execute()
+
+        assert not result
+
+        # test invisible snapshot from task_id was created
+        self.snapshot_command.parse(["snapshot", "ls", "--all"])
+
+        result = self.snapshot_command.execute()
+
+        assert result
+        assert task_obj.after_snapshot_id in [obj.id for obj in result]
+
     def test_snapshot_ls(self):
         self.__set_variables()
-        # Test when optional parameters are not given
+
+        # create a visible snapshot
         self.snapshot_command.parse(
             ["snapshot", "create", "-m", "my test snapshot"])
 
@@ -655,7 +686,7 @@ def test_snapshot_ls(self):
         assert created_snapshot_obj in result
 
         # Test when optional parameters are not given
-        self.snapshot_command.parse(["snapshot", "ls", "-a"])
+        self.snapshot_command.parse(["snapshot", "ls", "--details"])
 
         result = self.snapshot_command.execute()
 

datmo/cli/parser.py

@@ -141,7 +141,10 @@ def get_datmo_parser():
         type=str,
         help="pass in the session id to list the tasks in that session")
     run_ls_parser.add_argument(
-        "--format", dest="format", default="table", help="output format")
+        "--format",
+        dest="format",
+        default="table",
+        help="output format ['table', 'csv']")
     run_ls_parser.add_argument(
         "--download",
         dest="download",
@@ -158,13 +161,16 @@ def get_datmo_parser():
     )
 
     # Rerun
-    rerun_parser = subparsers.add_parser("rerun", help="To rerun an experiment")
+    rerun_parser = subparsers.add_parser(
+        "rerun", help="To rerun an experiment")
     rerun_parser.add_argument("id", help="run id to be rerun")
     rerun_parser.add_argument(
         "--initial",
         dest="initial",
         action="store_true",
-        help="boolean if you want to rerun the experiment with the state at the beginning of the run")
+        help=
+        "boolean if you want to rerun the experiment with the state at the beginning of the run"
+    )
 
     # Session
     session_parser = subparsers.add_parser("session", help="session module")
@@ -198,7 +204,10 @@ def get_datmo_parser():
     session_ls = session_subcommand_parsers.add_parser(
         "ls", help="list sessions")
     session_ls.add_argument(
-        "--format", dest="format", default="table", help="output format")
+        "--format",
+        dest="format",
+        default="table",
+        help="output format ['table', 'csv']")
     session_ls.add_argument(
         "--download",
         dest="download",
@@ -283,7 +292,10 @@ def get_datmo_parser():
     environment_ls = environment_subcommand_parsers.add_parser(
         "ls", help="list environments")
     environment_ls.add_argument(
-        "--format", dest="format", default="table", help="output format")
+        "--format",
+        dest="format",
+        default="table",
+        help="output format ['table', 'csv']")
     environment_ls.add_argument(
         "--download",
         dest="download",
@@ -452,13 +464,20 @@ def get_datmo_parser():
         default=None,
         help="session id to filter")
     snapshot_ls.add_argument(
-        "--all",
-        "-a",
+        "--details",
         dest="details",
         action="store_true",
         help="show detailed snapshot information")
     snapshot_ls.add_argument(
-        "--format", dest="format", default="table", help="output format")
+        "--all",
+        dest="show_all",
+        action="store_true",
+        help="show all visible and hidden snapshots")
+    snapshot_ls.add_argument(
+        "--format",
+        dest="format",
+        default="table",
+        help="output format ['table', 'csv']")
     snapshot_ls.add_argument(
         "--download",
         dest="download",
@@ -564,7 +583,10 @@ def get_datmo_parser():
         type=str,
         help="pass in the session id to list the tasks in that session")
     task_ls.add_argument(
-        "--format", dest="format", default="table", help="output format")
+        "--format",
+        dest="format",
+        default="table",
+        help="output format ['table', 'csv']")
     task_ls.add_argument(
         "--download",
         dest="download",

datmo/core/controller/code/code.py

@@ -1,7 +1,7 @@
 from datmo.core.util.i18n import get as __
 from datmo.core.controller.base import BaseController
 from datmo.core.entity.code import Code
-from datmo.core.util.exceptions import PathDoesNotExist, EnvironmentInitFailed
+from datmo.core.util.exceptions import PathDoesNotExist, EnvironmentInitFailed, ArgumentError, CodeDoesNotExist
 
 
 class CodeController(BaseController):
@@ -106,3 +106,74 @@ def delete(self, code_id):
         delete_code_obj_success = self.dal.code.delete(code_obj.id)
 
         return delete_code_success and delete_code_obj_success
+
+    def exists(self, code_id=None, code_commit_id=None):
+        """Returns a boolean if the code exists
+
+        Parameters
+        ----------
+        code_id : str
+            code id to check for
+        code_commit_id : str
+            unique commit id for the code
+
+        Returns
+        -------
+        bool
+            True if exists else False
+        """
+        if code_id:
+            code_objs = self.dal.code.query({"id": code_id})
+        elif code_commit_id:
+            code_objs = self.dal.code.query({"commit_id": code_commit_id})
+        else:
+            raise ArgumentError()
+        if code_objs:
+            return True
+        return False
+
+    def check_unstaged_changes(self):
+        """Checks if there exists any unstaged changes for the code in the project.
+
+        Returns
+        -------
+        bool
+            False if it's already staged else error
+
+        Raises
+        ------
+        CodeNotInitialized
+            error if not initialized (must initialize first)
+        UnstagedChanges
+            error if not there exists unstaged changes in environment
+        """
+        return self.code_driver.check_unstaged_changes()
+
+    def checkout(self, code_id):
+        """Checkout to specific code id
+
+        Parameters
+        ----------
+        code_id : str
+            code id to checkout to
+
+        Returns
+        -------
+        bool
+            True if success
+
+        Raises
+        ------
+        CodeDoesNotExist
+            if code id does not exist
+        CodeNotInitialized
+            error if not initialized (must initialize first)
+        UnstagedChanges
+            error if not there exists unstaged changes in environment
+
+        """
+        if not self.exists(code_id):
+            raise CodeDoesNotExist(
+                __("error", "controller.code.checkout", code_id))
+        code_obj = self.dal.code.get_by_id(code_id)
+        return self.code_driver.checkout_ref(code_obj.commit_id)

datmo/core/controller/code/driver/__init__.py

@@ -19,6 +19,8 @@ class CodeDriver(with_metaclass(ABCMeta, object)):
         push commit reference given
     fetch_ref()
         fetch commit reference given
+    check_unstaged_changes()
+        check if there exists any unstaged changes for code
     checkout_ref()
         checkout to commit reference given
     """
@@ -158,6 +160,7 @@ def list_refs(self):
     #     """
     #     pass
 
+    @abstractmethod
     def check_unstaged_changes(self):
         """Checks if there exists any unstaged changes for code