Merge fb4c738 into 771dfd9

.travis.yml

@@ -10,6 +10,9 @@ install:
 - pip install pytest pytest-cov
 - pip install yapf==0.20.0
 - pip install coveralls
+- pip install urllib3==1.23
+- pip install chardet==3.0.2
+- pip install requests==2.19.1
 - python setup.py install
 script:
 - pytest --cov-config .coveragerc --cov=datmo -s -v

MANIFEST.in

@@ -4,6 +4,7 @@ include datmo/core/controller/environment/driver/templates/python3Dockerfile
 include datmo/core/util/validation/schemas.yml
 include datmo/core/controller/environment/driver/config/docker.json
 include datmo/VERSION
+graft datmo/dashboard
 prune examples
 prune docs
 prune templates

appveyor.yml

@@ -44,6 +44,9 @@ install:
   - git config --global user.name "Datmo Devs"
   - "%PYTHON%/python.exe C:/get-pip.py"
   - "%PYTHON%/python.exe -m pip install pip==9.0.3"
+  - "%PYTHON%/python.exe -m pip install urllib3==1.23"
+  - "%PYTHON%/python.exe -m pip install chardet==3.0.2"
+  - "%PYTHON%/python.exe -m pip install requests==2.19.1"
   - "%PYTHON%/Scripts/pip.exe install pytest"
   - "%PYTHON%/Scripts/pip.exe install pypiwin32"
   - "%PYTHON%/python.exe setup.py install"

datmo/__init__.py

@@ -26,3 +26,4 @@
 import datmo.snapshot
 import datmo.logger
 import datmo.config
+import datmo.monitoring

datmo/cli/command/project.py

@@ -7,6 +7,7 @@
 from datmo.cli.command.base import BaseCommand
 from datmo.core.controller.project import ProjectController
 from datmo.core.controller.environment.environment import EnvironmentController
+from datmo.dashboard.app import app
 
 
 class ProjectCommand(BaseCommand):
@@ -77,15 +78,15 @@ def init(self, name, description, force):
                     }))
             # Prompt for the name and description and add default if not given
             if not name and not force:
-                    name = self.cli_helper.prompt(
-                        __("prompt", "cli.project.init.name"),
-                        default=self.project_controller.model.name)
+                name = self.cli_helper.prompt(
+                    __("prompt", "cli.project.init.name"),
+                    default=self.project_controller.model.name)
             elif force:
                 name = self.project_controller.model.name
             if not description and not force:
-                    description = self.cli_helper.prompt(
-                        __("prompt", "cli.project.init.description"),
-                        default=self.project_controller.model.description)
+                description = self.cli_helper.prompt(
+                    __("prompt", "cli.project.init.description"),
+                    default=self.project_controller.model.description)
             elif force:
                 description = self.project_controller.model.description
             # Update the project with the values given
@@ -113,7 +114,8 @@ def init(self, name, description, force):
                 self.cli_helper.echo(str(k) + ": " + str(v))
         # Ask question if the user would like to setup environment
         environment_setup = self.cli_helper.prompt_bool(
-            __("prompt", "cli.project.environment.setup")) if not force else False
+            __("prompt",
+               "cli.project.environment.setup")) if not force else False
         if environment_setup:
             # TODO: remove business logic from here and create common helper
             # Setting up the environment definition file
@@ -275,3 +277,9 @@ def cleanup(self):
                             "path": self.project_controller.home
                         }))
         return False
+
+    def dashboard(self):
+        dir_path = os.path.dirname(os.path.abspath(__file__))
+        os.chdir(os.path.join(dir_path, "../../dashboard"))
+        app.run(host='0.0.0.0')
+        return True

datmo/cli/command/run.py

@@ -309,9 +309,8 @@ def run(self, **kwargs):
 
         data_paths = kwargs['data']
         # Run task and return Task object result
-        task_obj = self.task_run_helper(task_dict, snapshot_dict,
-                                        "cli.run.run",
-                                        data_paths=data_paths)
+        task_obj = self.task_run_helper(
+            task_dict, snapshot_dict, "cli.run.run", data_paths=data_paths)
         if not task_obj:
             return False
         # Creating the run object
@@ -469,9 +468,9 @@ def delete(self, **kwargs):
             # Delete the task for the run
             result = self.task_controller.delete(task_id)
             if result:
-                self.cli_helper.echo(__("info", "cli.run.delete.success", task_id))
+                self.cli_helper.echo(
+                    __("info", "cli.run.delete.success", task_id))
             return result
         except Exception:
-            self.cli_helper.echo(
-                __("error", "cli.run.delete", task_id))
-            return False
+            self.cli_helper.echo(__("error", "cli.run.delete", task_id))
+            return False

datmo/cli/command/tests/test_project.py

@@ -27,6 +27,7 @@ def to_bytes(val):
 import os
 import tempfile
 import platform
+import timeout_decorator
 
 from datmo.config import Config
 from datmo import __version__
@@ -80,8 +81,7 @@ def dummy(self):
             definition_filepath, "r").read()
 
     def test_init_create_success_force(self):
-        self.project_command.parse(
-            ["init", "--force"])
+        self.project_command.parse(["init", "--force"])
 
         result = self.project_command.execute()
         assert result
@@ -160,8 +160,7 @@ def dummy(self):
 
         result_1 = dummy(self)
 
-        self.project_command.parse([
-            "init", "--force"])
+        self.project_command.parse(["init", "--force"])
 
         result_2 = self.project_command.execute()
         # test for desired side effects
@@ -550,3 +549,21 @@ def test_cleanup_invalid_arg(self):
         except UnrecognizedCLIArgument:
             exception_thrown = True
         assert exception_thrown
+
+    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
+    def test_dashboard(self):
+        # test dashboard command
+        self.project_command.parse(["dashboard"])
+
+        @timeout_decorator.timeout(10, use_signals=False)
+        def timed_run(timed_run_result):
+            if self.project_command.execute():
+                return timed_run_result
+
+        timed_run_result = False
+        try:
+            timed_run_result = timed_run(timed_run_result)
+        except timeout_decorator.timeout_decorator.TimeoutError:
+            timed_run_result = True
+
+        assert timed_run_result

datmo/cli/command/tests/test_run.py

@@ -182,9 +182,11 @@ def test_run_data_dir(self):
         test_dockerfile = os.path.join(self.temp_dir, "Dockerfile")
         test_mem_limit = "4g"
         # Test success for run with directory being passed
-        test_data_dir_1 = os.path.join(tempfile.mkdtemp(dir=test_datmo_dir), "data1")
+        test_data_dir_1 = os.path.join(
+            tempfile.mkdtemp(dir=test_datmo_dir), "data1")
         os.mkdir(test_data_dir_1)
-        test_data_dir_2 = os.path.join(tempfile.mkdtemp(dir=test_datmo_dir), "data2")
+        test_data_dir_2 = os.path.join(
+            tempfile.mkdtemp(dir=test_datmo_dir), "data2")
         os.mkdir(test_data_dir_2)
         with open(os.path.join(test_data_dir_1, "file.txt"), "wb") as f:
             f.write(to_bytes('my initial line in 1\n'))
@@ -210,9 +212,9 @@ def test_run_data_dir(self):
             f.write(to_bytes("    f.write('my test file in 2')\n"))
 
         self.run_command.parse([
-            "run", "--environment-paths", test_dockerfile,
-            "--data", test_data_dir_1, "--data", test_data_dir_2,
-            "--mem-limit", test_mem_limit, test_command
+            "run", "--environment-paths", test_dockerfile, "--data",
+            test_data_dir_1, "--data", test_data_dir_2, "--mem-limit",
+            test_mem_limit, test_command
         ])
 
         # test proper execution of run command
@@ -228,25 +230,26 @@ def test_run_data_dir(self):
         assert result.core_snapshot_id
         assert result.core_snapshot_id == result.after_snapshot_id
         assert result.environment_id
-        assert "my initial line in 1" in open(os.path.join(test_data_dir_1,
-                                                      "file.txt"), "r").read()
-        assert "my test file in 1" in open(os.path.join(test_data_dir_1,
-                                                      "file.txt"), "r").read()
-        assert "my initial line in 2" in open(os.path.join(test_data_dir_2,
-                                                           "file.txt"), "r").read()
-        assert "my test file in 2" in open(os.path.join(test_data_dir_2,
-                                                        "file.txt"), "r").read()
+        assert "my initial line in 1" in open(
+            os.path.join(test_data_dir_1, "file.txt"), "r").read()
+        assert "my test file in 1" in open(
+            os.path.join(test_data_dir_1, "file.txt"), "r").read()
+        assert "my initial line in 2" in open(
+            os.path.join(test_data_dir_2, "file.txt"), "r").read()
+        assert "my test file in 2" in open(
+            os.path.join(test_data_dir_2, "file.txt"), "r").read()
         # teardown
         self.run_command.parse(["stop", "--all"])
         # test when all is passed to stop all
         self.run_command.execute()
 
         # test failure
-        test_data_dir_dne = os.path.join(tempfile.mkdtemp(dir=test_datmo_dir), "data_dne")
+        test_data_dir_dne = os.path.join(
+            tempfile.mkdtemp(dir=test_datmo_dir), "data_dne")
         self.run_command.parse([
-                "run", "--environment-paths", test_dockerfile,
-                "--data", test_data_dir_dne
-            ])
+            "run", "--environment-paths", test_dockerfile, "--data",
+            test_data_dir_dne
+        ])
         result = self.run_command.execute()
         assert not result
 
@@ -540,8 +543,7 @@ def test_run_stop_failure_required_args(self):
     def test_run_stop_failure_mutually_exclusive_vars(self):
         self.__set_variables()
         # Passing wrong task id
-        self.run_command.parse(
-            ["stop", "--id", "invalid-task-id", "--all"])
+        self.run_command.parse(["stop", "--id", "invalid-task-id", "--all"])
         failed = False
         try:
             _ = self.run_command.execute()
@@ -614,17 +616,15 @@ def test_rerun(self):
 
         # test for single set of ports
         self.run_command.parse([
-            "run", "-p", test_ports[0],
-            "--environment-paths", test_dockerfile, "--mem-limit",
-            test_mem_limit, test_command
+            "run", "-p", test_ports[0], "--environment-paths", test_dockerfile,
+            "--mem-limit", test_mem_limit, test_command
         ])
 
         # test proper execution of run command
         run_obj = self.run_command.execute()
         run_id = run_obj.id
         # 1. Test success rerun
-        self.run_command.parse(
-            ["rerun", run_id])
+        self.run_command.parse(["rerun", run_id])
         result_run_obj = self.run_command.execute()
         assert result_run_obj
         assert isinstance(result_run_obj, RunObject)
@@ -647,4 +647,4 @@ def test_rerun_invalid_arg(self):
             self.run_command.execute()
         except DoesNotExist:
             exception_thrown = True
-        assert exception_thrown
+        assert exception_thrown

datmo/cli/driver/helper.py

@@ -151,9 +151,9 @@ def get_command_class(self, command_name):
     def get_command_choices(self):
         return [
             "init", "version", "--version", "-v", "status", "cleanup",
-            "configure", "snapshot", "session", "notebook", "jupyterlab",
-            "terminal", "rstudio", "environment", "run", "rerun", "stop",
-            "delete", "ls", "deploy"
+            "configure", "dashboard", "snapshot", "session", "notebook",
+            "jupyterlab", "terminal", "rstudio", "environment", "run", "rerun",
+            "stop", "delete", "ls", "deploy"
         ]
 
     def prompt_available_options(self, available_options, option_type):

datmo/cli/driver/tests/test_helper.py

@@ -310,7 +310,7 @@ def test_get_command_choices(self):
         # assert same as output
         assert self.cli.get_command_choices() == [
             "init", "version", "--version", "-v", "status", "cleanup",
-            "configure", "snapshot", "session", "notebook", "jupyterlab",
-            "terminal", "rstudio", "environment", "run", "rerun", "stop",
-            "delete", "ls", "deploy"
+            "configure", "dashboard", "snapshot", "session", "notebook",
+            "jupyterlab", "terminal", "rstudio", "environment", "run", "rerun",
+            "stop", "delete", "ls", "deploy"
         ]

datmo/cli/main.py

@@ -43,6 +43,9 @@ def main():
         elif command_name == "cleanup":
             command_name = "project"
             sys.argv[1] = "cleanup"
+        elif command_name == "dashboard":
+            command_name = "project"
+            sys.argv[1] = "dashboard"
         elif command_name == "configure":
             command_name = "project"
             sys.argv[1] = "configure"

datmo/cli/parser.py

@@ -13,15 +13,23 @@ def get_datmo_parser():
     init_parser = subparsers.add_parser("init", help="initialize project")
     init_parser.add_argument("--name", default=None)
     init_parser.add_argument("--description", default=None)
-    init_parser.add_argument("--force", "-f", "--no-prompt", dest="force", action="store_true",
-                             help="boolean if you want to run init without prompts")
+    init_parser.add_argument(
+        "--force",
+        "-f",
+        "--no-prompt",
+        dest="force",
+        action="store_true",
+        help="boolean if you want to run init without prompts")
 
     version_parser = subparsers.add_parser("version", help="datmo version")
 
     status_parser = subparsers.add_parser("status", help="project status")
 
     cleanup_parser = subparsers.add_parser("cleanup", help="remove project")
 
+    dashboard_parser = subparsers.add_parser(
+        "dashboard", help="start dashboard")
+
     configure_parser = subparsers.add_parser(
         "configure", help="configure datmo")
 

datmo/config.py

@@ -14,10 +14,9 @@ class Config(object):
     Parameters
     ----------
     home : string
-      project home directory
+        project home directory
+    remote_credentials : tuple
 
-    logging_level : int
-      logging level
     Returns
     -------
     Config

datmo/core/controller/base.py

@@ -14,15 +14,15 @@ class BaseController(object):
 
     Parameters
     ----------
-    home : str
-        home path of the project
+    home : str, optional
+        directory for the project, default is to pull from config
 
     Attributes
     ----------
     home : str
-        Filepath for the location of the project
+        absolute filepath for the location of the project
     config : JSONStore
-        This is the set of configurations used to create a project
+        this is the set of configurations used to create a project
     dal : datmo.core.storage.DAL
     model : datmo.core.entity.model.Model
     current_session : datmo.core.entity.session.Session
@@ -43,8 +43,8 @@ class BaseController(object):
         Return the configuration defaults
     """
 
-    def __init__(self):
-        self.home = Config().home
+    def __init__(self, home=None):
+        self.home = Config().home if not home else home
         if not os.path.isdir(self.home):
             raise InvalidProjectPath(
                 __("error", "controller.base.__init__", self.home))

datmo/core/controller/code/code.py

@@ -8,11 +8,6 @@ class CodeController(BaseController):
     """CodeController inherits from BaseController and manages business logic related to the
     code.
 
-    Parameters
-    ----------
-    home : str
-        home path of the project
-
     Methods
     -------
     create(code_id=None)

datmo/core/controller/deploy/deploy.py

@@ -4,7 +4,7 @@
 
 from datmo.core.controller.base import BaseController
 from datmo.core.util.misc_functions import Commands
-from datmo.core.controller.deploy.driver.microservice import MicroserviceDeployDriver
+from datmo.core.controller.deploy.driver.datmo_microservice import DatmoMicroserviceDeployDriver
 from datmo.config import Config
 
 
@@ -46,7 +46,7 @@ def __init__(self, service_container_management=False):
         self.config = Config()
         self.master_server_ip, self.datmo_api_key, self.datmo_end_point = self.config.remote_credentials
         self.service_container_management = service_container_management
-        self.driver = MicroserviceDeployDriver(
+        self.driver = DatmoMicroserviceDeployDriver(
             end_point=self.datmo_end_point, api_key=self.datmo_api_key)
 
     def cluster_deploy(self, cluster_name=None, server_type=None, size=None):