Merge pull request #242 from datmo/environment-workspace

environment workspace

README.rst

@@ -44,8 +44,6 @@ more granular control and workflow integration.
 Requirements
 ------------
 
--  `openssl <https://github.com/openssl/openssl/blob/master/INSTALL>`__
--  `git <https://git-scm.com/book/en/v2/Getting-Started-Installing-Git>`__
 -  `docker <https://docs.docker.com/engine/installation/>`__
 
 Installation

datmo/cli/command/environment.py

@@ -28,7 +28,7 @@ def setup(self, **kwargs):
         # TODO: remove business logic from here and create common helper
         # environment types
         environment_types = self.environment_controller.get_environment_types()
-        if not environment_type or environment_type not in environment_types:
+        if environment_types and environment_type not in environment_types:
             environment_type = self.cli_helper.prompt_available_options(
                 environment_types, option_type="type")
         # environment frameworks
@@ -37,13 +37,13 @@ def setup(self, **kwargs):
         available_frameworks = [
             item[0] for item in available_framework_details
         ]
-        if not environment_framework or environment_framework not in available_frameworks:
+        if available_frameworks and environment_framework not in available_frameworks:
             environment_framework = self.cli_helper.prompt_available_options(
                 available_framework_details, option_type="framework")
         # environment languages
         available_environment_languages = self.environment_controller.get_supported_languages(
             environment_type, environment_framework)
-        if available_environment_languages and not environment_language or environment_language not in available_environment_languages:
+        if available_environment_languages and environment_language not in available_environment_languages:
             environment_language = self.cli_helper.prompt_available_options(
                 available_environment_languages, option_type="language")
         try:

datmo/cli/command/run.py

@@ -404,7 +404,8 @@ def rerun(self, **kwargs):
             "ports": task_obj.ports,
             "interactive": task_obj.interactive,
             "mem_limit": task_obj.mem_limit,
-            "command_list": command
+            "command_list": command,
+            "workspace": task_obj.workspace
         }
         # Run task and return Task object result
         new_task_obj = self.task_run_helper(task_dict, snapshot_dict,

datmo/cli/command/tests/test_environment.py

@@ -8,6 +8,7 @@
 import os
 import glob
 import time
+import pytest
 import uuid
 import tempfile
 import shutil
@@ -138,6 +139,19 @@ def dummy(self):
         assert "FROM datmo/data-analytics:cpu-py27" in open(
             definition_filepath, "r").read()
 
+        # Test success with correct prompt input using numbers
+        self.environment_command.parse(["environment", "setup"])
+
+        @self.environment_command.cli_helper.input(
+            "cpu\ncaffe2\n")
+        def dummy(self):
+            return self.environment_command.execute()
+        result = dummy(self)
+        assert result
+        assert os.path.isfile(definition_filepath)
+        assert "FROM datmo/caffe2:cpu" in open(
+            definition_filepath, "r").read()
+
         # Test success with correct prompt input using string
         self.environment_command.parse(["environment", "setup"])
 
@@ -166,16 +180,38 @@ def dummy(self):
         # Test with prompt input string incorrect
         self.environment_command.parse(["environment", "setup"])
 
-        @self.environment_command.cli_helper.input("random\n\n\n")
+        # quit at environment type
+        @self.environment_command.cli_helper.input("quit\n")
         def dummy(self):
             return self.environment_command.execute()
 
-        result = dummy(self)
+        with pytest.raises(SystemExit) as pytest_wrapped_e:
+            dummy(self)
 
-        assert result
-        assert os.path.isfile(definition_filepath)
-        assert "FROM datmo/python-base:cpu-py27" in open(
-            definition_filepath, "r").read()
+        assert pytest_wrapped_e.type == SystemExit
+        assert pytest_wrapped_e.value.code == 0
+
+        # quit at environment framework
+        @self.environment_command.cli_helper.input("\nquit\n")
+        def dummy(self):
+            return self.environment_command.execute()
+
+        with pytest.raises(SystemExit) as pytest_wrapped_e:
+            dummy(self)
+
+        assert pytest_wrapped_e.type == SystemExit
+        assert pytest_wrapped_e.value.code == 0
+
+        # quit at environment language
+        @self.environment_command.cli_helper.input("\n\nquit\n")
+        def dummy(self):
+            return self.environment_command.execute()
+
+        with pytest.raises(SystemExit) as pytest_wrapped_e:
+            dummy(self)
+
+        assert pytest_wrapped_e.type == SystemExit
+        assert pytest_wrapped_e.value.code == 0
 
     def test_environment_create(self):
         # 1) Environment definition file in project environment directory (with name / description)

datmo/cli/command/workspace.py

@@ -28,7 +28,7 @@ def notebook(self, **kwargs):
             "mem_limit": kwargs["mem_limit"],
             "workspace": "notebook"
         }
-
+        self.cli_helper.echo(__("info", "cli.workspace.run.notebook"))
         # Run task and return Task object result
         return self.task_run_helper(task_dict, snapshot_dict,
                                     "cli.workspace.notebook")
@@ -52,7 +52,7 @@ def jupyterlab(self, **kwargs):
             "mem_limit": kwargs["mem_limit"],
             "workspace": "jupyterlab"
         }
-
+        self.cli_helper.echo(__("info", "cli.workspace.run.jupyterlab"))
         # Run task and return Task object result
         return self.task_run_helper(task_dict, snapshot_dict,
                                     "cli.workspace.jupyterlab")

datmo/cli/driver/helper.py

@@ -157,6 +157,10 @@ def get_command_choices(self):
 
     def prompt_available_options(self, available_options, option_type):
         """Prompt user to choose an available environment. Returns the environment name"""
+        # If there are no options
+        if not available_options:
+            return None
+        # If there exists list of options
         if option_type == "framework":
             available_options_info = available_options
             available_options = []
@@ -171,6 +175,9 @@ def prompt_available_options(self, available_options, option_type):
         option_environment_index = None
         valid_option = False
         while input_environment_option is not None and not valid_option:
+            # exit when user wants to quit
+            if input_environment_option in ["q", "quit"]:
+                sys.exit(0)
             try:
                 option_environment_index = int(input_environment_option)
                 if 0 < option_environment_index <= len(available_options):

datmo/cli/driver/tests/test_helper.py

@@ -10,6 +10,7 @@
 
 import os
 import sys
+import pytest
 import tempfile
 import platform
 try:
@@ -175,6 +176,79 @@ def dummy():
         i = dummy()
         assert i == False
 
+    def test_prompt_available_options(self):
+
+        # Setting up environment type
+        available_options = ["cpu", "gpu"]
+        option_type = "type"
+
+        @self.cli.input("cpu\n")
+        def dummy(self):
+            return self.cli.prompt_available_options(available_options, option_type)
+
+        result = dummy(self)
+        assert result
+        assert result == "cpu"
+
+        @self.cli.input("\n")
+        def dummy(self):
+            return self.cli.prompt_available_options(available_options, option_type)
+
+        result = dummy(self)
+        assert result
+        assert result == "cpu"
+
+        # Setting up environment framework
+        available_options = [["keras-tensorflow", "has libraries for keras(v2.1.6) and tensorflow(v1.9.0) along with sklearn, opencv etc."],
+                             ["mxnet", "has libraries for mxnet(v1.1.0) along with sklearn, opencv etc."]]
+        option_type = "framework"
+
+        @self.cli.input("mxnet\n")
+        def dummy(self):
+            return self.cli.prompt_available_options(available_options, option_type)
+
+        result = dummy(self)
+        assert result
+        assert result == "mxnet"
+
+        @self.cli.input("\n")
+        def dummy(self):
+            return self.cli.prompt_available_options(available_options, option_type)
+
+        result = dummy(self)
+        assert result
+        assert result == "python-base"
+
+        available_options = ["py27", "py35"]
+        option_type = "language"
+
+        @self.cli.input("py27\n")
+        def dummy(self):
+            return self.cli.prompt_available_options(available_options, option_type)
+
+        result = dummy(self)
+        assert result
+        assert result == "py27"
+
+        @self.cli.input("\n")
+        def dummy(self):
+            return self.cli.prompt_available_options(available_options, option_type)
+
+        result = dummy(self)
+        assert result
+        assert result == "py27"
+
+        # quit
+        @self.cli.input("quit\n")
+        def dummy(self):
+            return self.cli.prompt_available_options(available_options, option_type)
+
+        with pytest.raises(SystemExit) as pytest_wrapped_e:
+            dummy(self)
+
+        assert pytest_wrapped_e.type == SystemExit
+        assert pytest_wrapped_e.value.code == 0
+
     def test_prompt_validator(self):
         def validate_y(val):
             return True if val == "y" else False

datmo/cli/tests/test_main.py

@@ -101,20 +101,20 @@ def test_init(self):
             success = False
         assert success
 
-    @pytest_docker_environment_failed_instantiation(test_datmo_dir)
-    def test_run(self):
-        try:
-            success = True
-            p = self.command_run([self.execpath, "run", "python script.py"])
-            out, err = p.communicate()
-            out, err = out.decode(), err.decode()
-            if err:
-                success = False
-            elif 'hello' not in out:
-                success = False
-        except Exception:
-            success = False
-        assert success
+    # @pytest_docker_environment_failed_instantiation(test_datmo_dir)
+    # def test_run(self):
+    #     try:
+    #         success = True
+    #         p = self.command_run([self.execpath, "run", "python script.py"])
+    #         out, err = p.communicate()
+    #         out, err = out.decode(), err.decode()
+    #         if err:
+    #             success = False
+    #         elif 'hello' not in out:
+    #             success = False
+    #     except Exception:
+    #         success = False
+    #     assert success
 
     def test_run_ls(self):
         try:

datmo/core/controller/environment/driver/__init__.py

@@ -161,6 +161,24 @@ def build(self, name, path, workspace):
         """
         pass
 
+    @abstractmethod
+    def extract_workspace_url(self, container_name, workspace=None):
+        """Extract workspace url from the container
+
+        Parameters
+        ----------
+        container_name : str
+            name of the container being run
+        workspace : str
+            workspace being used for the run
+
+        Returns
+        -------
+        str
+            web url for the workspace being run, None if it doesn't exist
+        """
+        pass
+
     @abstractmethod
     def run(self, name, options, log_filepath):
         """Run and log an instance of the environment with the options given

datmo/core/controller/environment/driver/dockerenv.py

@@ -1,5 +1,7 @@
 import ast
 import os
+import re
+import time
 import json
 import subprocess
 import platform
@@ -437,6 +439,51 @@ def remove_images(self, name=None, all=False, filters=None, force=False):
                    str(e)))
         return True
 
+    def extract_workspace_url(self, container_name, workspace=None):
+        """Extract workspace url from the container
+
+        Parameters
+        ----------
+        container_name : str
+            name of the container being run
+        workspace : str
+            workspace being used for the run
+
+        Returns
+        -------
+        str
+            web url for the workspace being run, None if it doesn't exist
+
+        """
+        if workspace in ['notebook', 'jupyterlab']:
+            docker_shell_cmd_list = list(self.prefix)
+            docker_shell_cmd_list.append("exec")
+            docker_shell_cmd_list.append(container_name)
+            docker_shell_cmd_list.append("jupyter")
+            docker_shell_cmd_list.append("notebook")
+            docker_shell_cmd_list.append("list")
+            workspace_url = None
+            timeout_count = 0
+            while workspace_url is None and timeout_count < 10:
+                process = subprocess.Popen(
+                    docker_shell_cmd_list,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE)
+                stdout, stderr = process.communicate()
+                if process.returncode > 0:
+                    time.sleep(1)
+                    timeout_count += 1
+                result = re.search("(?P<url>https?://[^\s]+)", stdout.decode('utf-8')) if stdout else None
+                workspace_url = result.group("url") if result else None
+            return workspace_url
+        elif workspace == 'rstudio':
+            # Having a pause for two seconds before returning url
+            time.sleep(2)
+            workspace_url = 'http://localhost:8787'
+        else:
+            workspace_url = None
+        return workspace_url
+
     # running daemon needed
     def run_container(self,
                       image_name,