Merge pull request #181 from datmo/notebook

Resolved #176, #180 and standardized predefined environments

MANIFEST.in

@@ -2,6 +2,7 @@ include datmo/core/controller/environment/driver/templates/baseDockerfile
 include datmo/core/controller/environment/driver/templates/python2Dockerfile
 include datmo/core/controller/environment/driver/templates/python3Dockerfile
 include datmo/core/util/validation/schemas.yml
+include datmo/core/controller/environment/driver/config/docker.json
 include datmo/VERSION
 prune examples
 prune docs

datmo/cli/command/environment.py

@@ -25,21 +25,8 @@ def setup(self, **kwargs):
         available_environments = self.environment_controller.get_supported_environments(
         )
         if not name:
-            for idx, n in enumerate(available_environments):
-                self.cli_helper.echo("(%s) %s" % (idx + 1, n))
-            input_name = self.cli_helper.prompt(
-                __("prompt", "cli.environment.setup.name"))
-            try:
-                name_index = int(input_name)
-            except ValueError:
-                name_index = 0
-            if name_index > 0 and name_index < len(available_environments):
-                name = available_environments[name_index - 1]
-            elif name_index == 0:
-                name = input_name
-            else:
-                self.cli_helper.echo(
-                    __("error", "cli.environment.setup.argument", input_name))
+            name = self.cli_helper.prompt_available_environments(
+                available_environments)
         try:
             options = {"name": name}
             environment_obj = self.environment_controller.setup(

datmo/cli/command/project.py

@@ -1,7 +1,10 @@
 from datmo import __version__
 from datmo.core.util.i18n import get as __
 from datmo.cli.command.base import BaseCommand
+from datmo.core.util.misc_functions import mutually_exclusive
 from datmo.core.controller.project import ProjectController
+from datmo.core.controller.environment.environment import EnvironmentController, EnvironmentDoesNotExist
+from datmo.core.controller.task import TaskController
 
 
 class ProjectCommand(BaseCommand):
@@ -97,7 +100,27 @@ def init(self, name, description):
         for k, v in self.project_controller.model.to_dictionary().items():
             if k != "config":
                 self.cli_helper.echo(str(k) + ": " + str(v))
-
+        # Ask question if the user would like to setup environment
+        environment_setup = self.cli_helper.prompt_bool(
+            __("prompt", "cli.project.environment.setup"))
+        if environment_setup:
+            # Setting up the environment definition file
+            self.environment_controller = EnvironmentController(home=self.home)
+            available_environments = self.environment_controller.get_supported_environments(
+            )
+            input_environment_name = self.cli_helper.prompt_available_environments(
+                available_environments)
+            try:
+                options = {"name": input_environment_name}
+                environment_obj = self.environment_controller.setup(
+                    options=options)
+                self.cli_helper.echo(
+                    __("info", "cli.environment.setup.success",
+                       (environment_obj.name, environment_obj.id)))
+            except EnvironmentDoesNotExist:
+                self.cli_helper.echo(
+                    __("error", "cli.environment.setup.argument",
+                       input_environment_name))
         return self.project_controller.model
 
     def version(self):
@@ -170,3 +193,41 @@ def cleanup(self):
                             "path": self.project_controller.home
                         }))
         return False
+
+    def notebook(self, **kwargs):
+        self.cli_helper.echo(__("info", "cli.project.notebook"))
+        self.task_controller = TaskController(
+            home=self.project_controller.home)
+
+        # Creating input dictionaries
+        snapshot_dict = {}
+
+        # Environment
+        if kwargs.get("environment_id", None) or kwargs.get(
+                "environment_paths", None):
+            mutually_exclusive_args = ["environment_id", "environment_paths"]
+            mutually_exclusive(mutually_exclusive_args, kwargs, snapshot_dict)
+
+        task_dict = {
+            "ports": ["8888:8888"],
+            "command_list": ["jupyter", "notebook"],
+            "mem_limit": kwargs["mem_limit"]
+        }
+
+        # Create the task object
+        task_obj = self.task_controller.create()
+
+        # Pass in the task
+        try:
+            updated_task_obj = self.task_controller.run(
+                task_obj.id, snapshot_dict=snapshot_dict, task_dict=task_dict)
+        except Exception as e:
+            self.logger.error("%s %s" % (e, task_dict))
+            self.cli_helper.echo(
+                __("error", "cli.project.notebook", task_obj.id))
+            return False
+
+        self.cli_helper.echo(
+            "Ran notebook with task id: %s" % updated_task_obj.id)
+
+        return updated_task_obj

datmo/cli/command/task.py

@@ -27,7 +27,6 @@ def task(self):
 
     def run(self, **kwargs):
         self.cli_helper.echo(__("info", "cli.task.run"))
-
         # Create input dictionaries
         snapshot_dict = {}
 
@@ -38,7 +37,8 @@ def run(self, **kwargs):
             mutually_exclusive(mutually_exclusive_args, kwargs, snapshot_dict)
         task_dict = {
             "ports": kwargs['ports'],
-            "interactive": kwargs['interactive']
+            "interactive": kwargs['interactive'],
+            "mem_limit": kwargs['mem_limit']
         }
         if not isinstance(kwargs['cmd'], list):
             if platform.system() == "Windows":

datmo/cli/command/tests/test_environment.py

@@ -49,7 +49,11 @@ def __set_variables(self):
         self.project = ProjectCommand(self.temp_dir, self.cli_helper)
         self.project.parse(
             ["init", "--name", "foobar", "--description", "test model"])
-        self.project.execute()
+        @self.project.cli_helper.input("\n")
+        def dummy(self):
+            return self.project.execute()
+
+        dummy(self)
         self.environment_command = EnvironmentCommand(self.temp_dir,
                                                       self.cli_helper)
 

datmo/cli/command/tests/test_project.py

@@ -11,6 +11,18 @@
 # except ImportError:
 #     # Python 3
 #     import builtins as __builtin__
+try:
+
+    def to_bytes(val):
+        return bytes(val)
+
+    to_bytes("test")
+except TypeError:
+
+    def to_bytes(val):
+        return bytes(val, "utf-8")
+
+    to_bytes("test")
 
 import os
 import tempfile
@@ -20,37 +32,73 @@
 from datmo.cli.driver.helper import Helper
 from datmo.cli.command.project import ProjectCommand
 from datmo.core.util.exceptions import UnrecognizedCLIArgument
+from datmo.core.util.misc_functions import pytest_docker_environment_failed_instantiation
 
+# provide mountable tmp directory for docker
+tempfile.tempdir = "/tmp" if not platform.system() == "Windows" else None
+test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
+                                tempfile.gettempdir())
 
 class TestProject():
     def setup_method(self):
-        # provide mountable tmp directory for docker
-        tempfile.tempdir = "/tmp" if not platform.system(
-        ) == "Windows" else None
-        test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
-                                        tempfile.gettempdir())
         self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
         self.cli_helper = Helper()
         self.project_command = ProjectCommand(self.temp_dir, self.cli_helper)
 
     def teardown_method(self):
         pass
 
-    def test_init_create_success(self):
+    def test_init_create_success_no_environment(self):
         test_name = "foobar"
         test_description = "test model"
         self.project_command.parse(
             ["init", "--name", test_name, "--description", test_description])
-        result = self.project_command.execute()
+
+        # Test when environment is not created
+        @self.project_command.cli_helper.input("\n")
+        def dummy_no_environment(self):
+            return self.project_command.execute()
+
+        result = dummy_no_environment(self)
+
+        definition_filepath = os.path.join(self.temp_dir, "datmo_environment", "Dockerfile")
+
+        assert result
+        assert not os.path.isfile(definition_filepath)
+        assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
+        assert result.name == test_name
+        assert result.description == test_description
+
+    def test_init_create_success_environment(self):
+        test_name = "foobar"
+        test_description = "test model"
+        self.project_command.parse(
+            ["init", "--name", test_name, "--description", test_description])
+
+        # Test when environment is created
+        @self.project_command.cli_helper.input("y\n1\n")
+        def dummy_environment(self):
+            return self.project_command.execute()
+
+        result = dummy_environment(self)
+
+        definition_filepath = os.path.join(self.temp_dir, "datmo_environment", "Dockerfile")
+
+        assert result
+        assert os.path.isfile(definition_filepath)
+        assert "FROM datmo/xgboost:cpu" in open(definition_filepath,
+                                                "r").read()
+
         # test for desired side effects
         assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
         assert result.name == test_name
         assert result.description == test_description
 
+
     def test_init_create_failure(self):
         self.project_command.parse(["init", "--name", "", "--description", ""])
         # test if prompt opens
-        @self.project_command.cli_helper.input("\n\n")
+        @self.project_command.cli_helper.input("\n\n\n")
         def dummy(self):
             return self.project_command.execute()
 
@@ -62,14 +110,20 @@ def test_init_update_success(self):
         test_description = "test model"
         self.project_command.parse(
             ["init", "--name", test_name, "--description", test_description])
-        result_1 = self.project_command.execute()
+
+        @self.project_command.cli_helper.input("\n")
+        def dummy(self):
+            return self.project_command.execute()
+
+        result_1 = dummy(self)
         updated_name = "foobar2"
         updated_description = "test model 2"
         self.project_command.parse([
             "init", "--name", updated_name, "--description",
             updated_description
         ])
-        result_2 = self.project_command.execute()
+
+        result_2 = dummy(self)
         # test for desired side effects
         assert os.path.exists(os.path.join(self.temp_dir, '.datmo'))
         assert result_2.id == result_1.id
@@ -81,12 +135,17 @@ def test_init_update_success_only_name(self):
         test_description = "test model"
         self.project_command.parse(
             ["init", "--name", test_name, "--description", test_description])
-        result_1 = self.project_command.execute()
+
+        @self.project_command.cli_helper.input("\n")
+        def dummy(self):
+            return self.project_command.execute()
+
+        result_1 = dummy(self)
         updated_name = "foobar2"
         self.project_command.parse(
             ["init", "--name", updated_name, "--description", ""])
 
-        @self.project_command.cli_helper.input("\n")
+        @self.project_command.cli_helper.input("\n\n")
         def dummy(self):
             return self.project_command.execute()
 
@@ -102,12 +161,17 @@ def test_init_update_success_only_description(self):
         test_description = "test model"
         self.project_command.parse(
             ["init", "--name", test_name, "--description", test_description])
-        result_1 = self.project_command.execute()
+
+        @self.project_command.cli_helper.input("\n")
+        def dummy(self):
+            return self.project_command.execute()
+
+        result_1 = dummy(self)
         updated_description = "test model 2"
         self.project_command.parse(
             ["init", "--name", "", "--description", updated_description])
 
-        @self.project_command.cli_helper.input("\n")
+        @self.project_command.cli_helper.input("\n\n")
         def dummy(self):
             return self.project_command.execute()
 
@@ -145,7 +209,12 @@ def test_status(self):
         test_description = "test model"
         self.project_command.parse(
             ["init", "--name", test_name, "--description", test_description])
-        _ = self.project_command.execute()
+
+        @self.project_command.cli_helper.input("\n")
+        def dummy(self):
+            return self.project_command.execute()
+
+        _ = dummy(self)
 
         self.project_command.parse(["status"])
         result = self.project_command.execute()
@@ -166,11 +235,16 @@ def test_cleanup(self):
         test_description = "test model"
         self.project_command.parse(
             ["init", "--name", test_name, "--description", test_description])
-        _ = self.project_command.execute()
+
+        @self.project_command.cli_helper.input("\n")
+        def dummy(self):
+            return self.project_command.execute()
+
+        _ = dummy(self)
 
         self.project_command.parse(["cleanup"])
 
-        @self.project_command.cli_helper.input("y\n")
+        @self.project_command.cli_helper.input("y\n\n")
         def dummy(self):
             return self.project_command.execute()
 
@@ -186,3 +260,34 @@ def test_cleanup_invalid_arg(self):
         except UnrecognizedCLIArgument:
             exception_thrown = True
         assert exception_thrown
+
+    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
+    def test_notebook(self):
+        self.project_command.parse(
+            ["init", "--name", "foobar", "--description", "test model"])
+
+        @self.project_command.cli_helper.input("\n")
+        def dummy(self):
+            self.project_command.execute()
+
+        dummy(self)
+
+        # Create environment_driver definition
+        env_def_path = os.path.join(self.temp_dir, "Dockerfile")
+        test_mem_limit = "4g"
+        with open(env_def_path, "wb") as f:
+            f.write(to_bytes(str("FROM datmo/xgboost:cpu\n")))
+
+        # test single ports option before command
+        self.project_command.parse(["notebook", "--gpu", "--environment-paths", env_def_path,
+                                    "--mem-limit", test_mem_limit,])
+
+        # test for desired side effects
+        assert self.project_command.args.gpu == True
+        assert self.project_command.args.environment_paths == [env_def_path]
+        assert self.project_command.args.mem_limit == test_mem_limit
+
+        # test multiple ports option before command
+        self.project_command.parse(["notebook"])
+
+        assert self.project_command.args.gpu == False

datmo/cli/command/tests/test_session.py

@@ -36,7 +36,12 @@ def __set_variables(self):
         self.project_command = ProjectCommand(self.temp_dir, self.cli_helper)
         self.project_command.parse(
             ["init", "--name", "foobar", "--description", "test model"])
-        self.project_command.execute()
+
+        @self.project_command.cli_helper.input("\n")
+        def dummy(self):
+            return self.project_command.execute()
+
+        dummy(self)
         self.session_command = SessionCommand(self.temp_dir, self.cli_helper)
 
     def test_session_project_not_init(self):