Merge pull request #8 from datmo/command-controller

Task commands with merged changes from master

datmo/cli/command/base.py

@@ -59,4 +59,4 @@ def execute(self):
                                         "cli.general.method.not_found",
                                         (self.args.command, method)))
 
-        method(**command_args)
+        return method(**command_args)

datmo/cli/command/snapshot.py

@@ -39,7 +39,7 @@ def __init__(self, home, cli_helper):
         create.add_argument("--stats-filepath", dest="stats_filepath", default=None,
                             help="Absolute filepath to use to search for metrics JSON")
 
-        create.add_argument("--filepaths", dest="filepaths", default=None, nargs="*",
+        create.add_argument("--filepaths", dest="filepaths", default=[], nargs="*",
                             help="Absolute paths to files or folders to include within the files of the snapshot")
 
         delete = subcommand_parsers.add_parser("delete", help="Delete a snapshot by id")

datmo/cli/command/task.py

@@ -1,4 +1,7 @@
 from __future__ import print_function
+
+import prettytable
+
 from datmo.util.i18n import get as _
 from datmo.cli.command.project import ProjectCommand
 from datmo.controller.task import TaskController
@@ -14,31 +17,30 @@ def __init__(self, home, cli_helper):
 
         # Task run arguments
         run = subcommand_parsers.add_parser("run", help="Run task")
-
-        run.add_argument('--gpu', dest='gpu', action='store_true',
-                         help='Boolean if you want to train the Model leveraging GPUs')
-        run.add_argument('--ports', nargs='*', dest='ports', type=str, help='Network port(s) to open during the Task')
-        run.add_argument('--data', nargs='*', dest='data', type=str, help='Path for data to be used during the Task')
-        run.add_argument('--dockerfile', dest='dockerfile', default='Dockerfile', nargs='?', type=str,
-                         help='Pass in the Dockerfile with which you want to build the environment')
-        run.add_argument('--interactive', dest='interactive', action='store_true',
-                         help='Run the environment in interactive mode (keeps STDIN open)')
-        run.add_argument("command", nargs='?', default=None)
+        run.add_argument("--gpu", dest="gpu", action="store_true",
+                         help="Boolean if you want to run using GPUs")
+        run.add_argument("--ports", nargs="*", dest="ports", type=str, help="""
+            Network port mapping during task (e.g. 8888:8888). Left is the host machine port and right
+            is the environment port available during a run.
+        """)
+        # run.add_argument("--data", nargs="*", dest="data", type=str, help="Path for data to be used during the Task")
+        run.add_argument("--env-def", dest="environment_definition_filepath", default="",
+                         nargs="?", type=str,
+                         help="Pass in the Dockerfile with which you want to build the environment")
+        run.add_argument("--interactive", dest="interactive", action="store_true",
+                         help="Run the environment in interactive mode (keeps STDIN open)")
+        run.add_argument("cmd", nargs="?", default=None)
 
         # Task list arguments
         ls = subcommand_parsers.add_parser("ls", help="List tasks")
-        ls.add_argument('--running', dest='running', action='store_true',
-                         help='Boolean to filter for running Tasks')
-        ls.add_argument('--all', dest='all', action='store_true',
-                         help='Boolean to filter all running/stopped Tasks')
+        ls.add_argument("--session-id", dest="session_id", default=None, nargs="?", type=str,
+                         help="Pass in the session id to list the tasks in that session")
 
         # Task stop arguments
         stop = subcommand_parsers.add_parser("stop", help="Stop tasks")
-        stop.add_argument('--running', dest='running', action='store_true',
-                         help='Boolean to filter and stop running Tasks')
-        stop.add_argument('--id', dest='id', default=None, type=str, help='Task ID to stop')
+        stop.add_argument("--id", dest="id", default=None, type=str, help="Task ID to stop")
 
-        self.snapshot_controller = TaskController(home=home,
+        self.task_controller = TaskController(home=home,
                                                   dal_driver=self.project_controller.dal_driver)
         if not self.project_controller.is_initialized:
             raise ProjectNotInitializedException(_("error",
@@ -47,13 +49,51 @@ def __init__(self, home, cli_helper):
 
     def run(self, **kwargs):
         self.cli_helper.echo(_("info", "cli.task.run"))
-        print("run", kwargs)
+
+        # Create input dictionaries
+        snapshot_dict = {
+            "environment_definition_filepath":
+                kwargs['environment_definition_filepath']
+        }
+
+        task_dict = {
+            "gpu": kwargs['gpu'],
+            "ports": kwargs['ports'],
+            "interactive": kwargs['interactive'],
+            "command": kwargs['cmd']
+        }
+
+        # Create the task object
+        task_obj = self.task_controller.create(task_dict)
+
+        # Pass in the task
+        self.task_controller.run(task_obj.id, snapshot_dict=snapshot_dict)
+
+        return task_obj.id
 
     def ls(self, **kwargs):
-        print("ls", kwargs)
+        session_id = kwargs.get('session_id',
+                                self.task_controller.current_session.id)
+        # Get all snapshot meta information
+        header_list = ["id", "command", "status", "gpu", "created_at"]
+        t = prettytable.PrettyTable(header_list)
+        task_objs = self.task_controller.list(session_id)
+        for task_obj in task_objs:
+            t.add_row([task_obj.id, task_obj.command, task_obj.status, task_obj.gpu,
+                       task_obj.created_at.strftime("%Y-%m-%d %H:%M:%S")])
+        self.cli_helper.echo(t)
+        return True
 
     def stop(self, **kwargs):
-        print("stop", kwargs)
+        id = kwargs.get('id', None)
+        try:
+            task_delete_dict = {"id": id}
+            self.task_controller.delete(**task_delete_dict)
+        except Exception:
+            self.cli_helper.echo(_("error",
+                                   "cli.task.delete"))
+            return False
+        return True
 
 
 

datmo/cli/command/test/test_project.py

@@ -21,6 +21,8 @@
 
 class TestInit():
     def setup_class(self):
+        # provide mountable tmp directory for docker
+        tempfile.tempdir = '/tmp'
         test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
                                         tempfile.gettempdir())
         self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)

datmo/cli/command/test/test_snapshot.py

@@ -23,6 +23,8 @@
 
 class TestSnapshot():
     def setup_class(self):
+        # provide mountable tmp directory for docker
+        tempfile.tempdir = '/tmp'
         test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
                                         tempfile.gettempdir())
         self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)

datmo/cli/command/test/test_task.py

@@ -19,11 +19,14 @@
 from datmo.cli.driver.helper import Helper
 from datmo.cli.command.project import ProjectCommand
 from datmo.cli.command.task import TaskCommand
-from datmo.util.exceptions import ProjectNotInitializedException
+from datmo.util.exceptions import ProjectNotInitializedException, \
+    EntityNotFound
 
 
 class TestTaskCommand():
     def setup_class(self):
+        # provide mountable tmp directory for docker
+        tempfile.tempdir = '/tmp'
         test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
                                         tempfile.gettempdir())
         self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
@@ -49,32 +52,39 @@ def test_task_project_not_init(self):
 
     def test_datmo_task_run(self):
         self.__set_variables()
-        test_command = "python test.py"
+        # Create environment_driver definition
+        env_def_path = os.path.join(self.temp_dir,
+                                    "Dockerfile")
+        with open(env_def_path, "w") as f:
+            f.write(str("FROM datmo/xgboost:cpu"))
+
+        test_command = ["sh", "-c", "echo yo"]
         test_gpu = True
         test_ports = "8888:8888"
-        test_data = "data"
-        test_dockerfile = "Dockerfile"
+        test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
         test_interactive = True
 
         self.task.parse([
             "task",
             "run",
             "--gpu",
             "--ports", test_ports,
-            "--data", test_data,
-            "--dockerfile", test_dockerfile,
+            "--env-def", test_dockerfile,
             "--interactive",
             test_command
         ])
 
         # test for desired side effects
-        assert self.task.args.command == test_command
+        assert self.task.args.cmd == test_command
         assert self.task.args.gpu == test_gpu
         assert self.task.args.ports == [test_ports]
-        assert self.task.args.data == [test_data]
-        assert self.task.args.dockerfile == test_dockerfile
+        assert self.task.args.environment_definition_filepath == test_dockerfile
         assert self.task.args.interactive == test_interactive
 
+        # test proper execution of task run command
+        result = self.task.execute()
+        assert result
+
     def test_datmo_task_run_invalid_arg(self):
         self.__set_variables()
         exception_thrown = False
@@ -89,19 +99,19 @@ def test_datmo_task_run_invalid_arg(self):
 
     def test_datmo_task_ls(self):
         self.__set_variables()
-        test_running = True
-        test_all = True
+        test_session_id = 'test_session_id'
 
         self.task.parse([
             "task",
             "ls",
-            "--running",
-            "--all"
+            "--session-id", test_session_id
         ])
 
         # test for desired side effects
-        assert self.task.args.running == test_running
-        assert self.task.args.all == test_all
+        assert self.task.args.session_id == test_session_id
+
+        task_ls_command = self.task.execute()
+        assert task_ls_command == True
 
     def test_datmo_task_ls_invalid_arg(self):
         self.__set_variables()
@@ -117,19 +127,49 @@ def test_datmo_task_ls_invalid_arg(self):
 
     def test_datmo_task_stop(self):
         self.__set_variables()
-        test_running = True
-        test_id = 'test_id'
+
+        test_command = ["sh", "-c", "echo yo"]
+        test_ports = "8888:8888"
+        test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
+
+        self.task.parse([
+            "task",
+            "run",
+            "--gpu",
+            "--ports", test_ports,
+            "--env-def", test_dockerfile,
+            "--interactive",
+            test_command
+        ])
+
+        test_task_id = self.task.execute()
 
         self.task.parse([
             "task",
             "stop",
-            "--running",
-            "--id", test_id
+            "--id", test_task_id
         ])
 
         # test for desired side effects
-        assert self.task.args.running == test_running
-        assert self.task.args.id == test_id
+        assert self.task.args.id == test_task_id
+
+        # test when task id is passed to stop it
+        task_stop_command = self.task.execute()
+        assert task_stop_command == True
+
+        # Passing wrong task id
+        test_task_id = "task_id"
+        self.task.parse([
+            "task",
+            "stop",
+            "--id", test_task_id
+        ])
+
+        # test when wrong task id is passed to stop it
+        try:
+            self.task.execute()
+        except EntityNotFound:
+            assert True
 
     def test_datmo_task_stop_invalid_arg(self):
         self.__set_variables()

datmo/cli/driver/test/test_helper.py

@@ -23,6 +23,8 @@ class TestHelper():
     # https://stackoverflow.com/questions/35851323/pytest-how-to-test-a-function-with-input-call/36377194
 
     def setup_method(self):
+        # provide mountable tmp directory for docker
+        tempfile.tempdir = '/tmp'
         test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
                                         tempfile.gettempdir())
         self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)

datmo/controller/code/driver/test/test_git.py

@@ -18,6 +18,8 @@ class TestGitCodeDriver():
     Checks all functions of the GitCodeDriver
     """
     def setup_method(self):
+        # provide mountable tmp directory for docker
+        tempfile.tempdir = '/tmp'
         test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
                                         tempfile.gettempdir())
         self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)