From 975af059d110d244a072ad9ed4bb7e52f8dd2ea5 Mon Sep 17 00:00:00 2001 From: Shabaz Patel Date: Sun, 22 Jul 2018 12:14:47 -0700 Subject: [PATCH 01/14] fixing environment setup while no language + adding quit --- datmo/cli/command/environment.py | 6 +++--- datmo/cli/driver/helper.py | 7 +++++++ datmo/core/controller/environment/environment.py | 14 ++++++++++---- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/datmo/cli/command/environment.py b/datmo/cli/command/environment.py index a5d1152a..3e4db946 100644 --- a/datmo/cli/command/environment.py +++ b/datmo/cli/command/environment.py @@ -28,7 +28,7 @@ def setup(self, **kwargs): # TODO: remove business logic from here and create common helper # environment types environment_types = self.environment_controller.get_environment_types() - if not environment_type or environment_type not in environment_types: + if environment_types and environment_type not in environment_types: environment_type = self.cli_helper.prompt_available_options( environment_types, option_type="type") # environment frameworks @@ -37,13 +37,13 @@ def setup(self, **kwargs): available_frameworks = [ item[0] for item in available_framework_details ] - if not environment_framework or environment_framework not in available_frameworks: + if available_frameworks and environment_framework not in available_frameworks: environment_framework = self.cli_helper.prompt_available_options( available_framework_details, option_type="framework") # environment languages available_environment_languages = self.environment_controller.get_supported_languages( environment_type, environment_framework) - if available_environment_languages and not environment_language or environment_language not in available_environment_languages: + if available_environment_languages and environment_language not in available_environment_languages: environment_language = self.cli_helper.prompt_available_options( available_environment_languages, option_type="language") try: diff --git a/datmo/cli/driver/helper.py b/datmo/cli/driver/helper.py index dd5b6e9f..a9099d65 100644 --- a/datmo/cli/driver/helper.py +++ b/datmo/cli/driver/helper.py @@ -157,6 +157,10 @@ def get_command_choices(self): def prompt_available_options(self, available_options, option_type): """Prompt user to choose an available environment. Returns the environment name""" + # If there are no options + if not available_options: + return None + # If there exists list of options if option_type == "framework": available_options_info = available_options available_options = [] @@ -171,6 +175,9 @@ def prompt_available_options(self, available_options, option_type): option_environment_index = None valid_option = False while input_environment_option is not None and not valid_option: + # exit when user wants to quit + if input_environment_option in ["q", "quit"]: + sys.exit(0) try: option_environment_index = int(input_environment_option) if 0 < option_environment_index <= len(available_options): diff --git a/datmo/core/controller/environment/environment.py b/datmo/core/controller/environment/environment.py index c3b44028..0c0f65be 100644 --- a/datmo/core/controller/environment/environment.py +++ b/datmo/core/controller/environment/environment.py @@ -125,11 +125,17 @@ def setup(self, options, save_hardware_file=True): definition_path=self.file_driver.environment_directory) except Exception: raise + name = options.get('name', None) + if name is None: + environment_framework = options['environment_framework'] + environment_type = options['environment_type'] + environment_language = options['environment_language'] + if environment_language: + name = "%s:%s-%s" % (environment_framework, environment_type, environment_language) + else: + name = "%s:%s" % (environment_framework, environment_type) create_dict = { - "name": - options['name'] if options.get('name', None) else "%s:%s-%s" % - (options['environment_framework'], options['environment_type'], - options['environment_language']), + "name": name, "description": "supported environment created by datmo" } From ed74a03808893376e91cd3d75db56296169565be Mon Sep 17 00:00:00 2001 From: Shabaz Patel Date: Sun, 22 Jul 2018 22:22:53 -0700 Subject: [PATCH 02/14] hotfix for rerun to be able to rerun for workspace --- datmo/cli/command/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datmo/cli/command/run.py b/datmo/cli/command/run.py index 945db2b1..5feed3a5 100644 --- a/datmo/cli/command/run.py +++ b/datmo/cli/command/run.py @@ -404,7 +404,8 @@ def rerun(self, **kwargs): "ports": task_obj.ports, "interactive": task_obj.interactive, "mem_limit": task_obj.mem_limit, - "command_list": command + "command_list": command, + "workspace": task_obj.workspace } # Run task and return Task object result new_task_obj = self.task_run_helper(task_dict, snapshot_dict, From ca75d8fc66701c84c196316ddf01e58b3558d45f Mon Sep 17 00:00:00 2001 From: Shabaz Patel Date: Mon, 23 Jul 2018 12:41:39 -0700 Subject: [PATCH 03/14] adding test for quit in environment setup --- datmo/cli/command/tests/test_environment.py | 35 +++++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/datmo/cli/command/tests/test_environment.py b/datmo/cli/command/tests/test_environment.py index 70abab1d..9149d045 100644 --- a/datmo/cli/command/tests/test_environment.py +++ b/datmo/cli/command/tests/test_environment.py @@ -8,6 +8,7 @@ import os import glob import time +import pytest import uuid import tempfile import shutil @@ -166,16 +167,38 @@ def dummy(self): # Test with prompt input string incorrect self.environment_command.parse(["environment", "setup"]) - @self.environment_command.cli_helper.input("random\n\n\n") + # quit at environment type + @self.environment_command.cli_helper.input("quit\n") def dummy(self): return self.environment_command.execute() - result = dummy(self) + with pytest.raises(SystemExit) as pytest_wrapped_e: + dummy(self) - assert result - assert os.path.isfile(definition_filepath) - assert "FROM datmo/python-base:cpu-py27" in open( - definition_filepath, "r").read() + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 0 + + # quit at environment framework + @self.environment_command.cli_helper.input("\nquit\n") + def dummy(self): + return self.environment_command.execute() + + with pytest.raises(SystemExit) as pytest_wrapped_e: + dummy(self) + + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 0 + + # quit at environment language + @self.environment_command.cli_helper.input("\n\nquit\n") + def dummy(self): + return self.environment_command.execute() + + with pytest.raises(SystemExit) as pytest_wrapped_e: + dummy(self) + + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 0 def test_environment_create(self): # 1) Environment definition file in project environment directory (with name / description) From d195b8ef2fd44fe172ace93f0a8a65ef077af65e Mon Sep 17 00:00:00 2001 From: Shabaz Patel Date: Mon, 23 Jul 2018 13:19:37 -0700 Subject: [PATCH 04/14] adding test for environment setup when there is no language --- datmo/cli/command/tests/test_environment.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/datmo/cli/command/tests/test_environment.py b/datmo/cli/command/tests/test_environment.py index 9149d045..56236a64 100644 --- a/datmo/cli/command/tests/test_environment.py +++ b/datmo/cli/command/tests/test_environment.py @@ -139,6 +139,19 @@ def dummy(self): assert "FROM datmo/data-analytics:cpu-py27" in open( definition_filepath, "r").read() + # Test success with correct prompt input using numbers + self.environment_command.parse(["environment", "setup"]) + + @self.environment_command.cli_helper.input( + "cpu\ncaffe2\n") + def dummy(self): + return self.environment_command.execute() + result = dummy(self) + assert result + assert os.path.isfile(definition_filepath) + assert "FROM datmo/caffe2:cpu" in open( + definition_filepath, "r").read() + # Test success with correct prompt input using string self.environment_command.parse(["environment", "setup"]) From fb17d3a37b7ba2e7a5f60739a76d5d5ecbca7076 Mon Sep 17 00:00:00 2001 From: Shabaz Patel Date: Mon, 23 Jul 2018 21:03:19 -0700 Subject: [PATCH 05/14] adding test for prompt_available_options, helper function --- datmo/cli/driver/tests/test_helper.py | 74 +++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/datmo/cli/driver/tests/test_helper.py b/datmo/cli/driver/tests/test_helper.py index 20658c12..89c15aaf 100644 --- a/datmo/cli/driver/tests/test_helper.py +++ b/datmo/cli/driver/tests/test_helper.py @@ -10,6 +10,7 @@ import os import sys +import pytest import tempfile import platform try: @@ -175,6 +176,79 @@ def dummy(): i = dummy() assert i == False + def test_prompt_available_options(self): + + # Setting up environment type + available_options = ["cpu", "gpu"] + option_type = "type" + + @self.cli.input("cpu\n") + def dummy(self): + return self.cli.prompt_available_options(available_options, option_type) + + result = dummy(self) + assert result + assert result == "cpu" + + @self.cli.input("\n") + def dummy(self): + return self.cli.prompt_available_options(available_options, option_type) + + result = dummy(self) + assert result + assert result == "cpu" + + # Setting up environment framework + available_options = [["keras-tensorflow", "has libraries for keras(v2.1.6) and tensorflow(v1.9.0) along with sklearn, opencv etc."], + ["mxnet", "has libraries for mxnet(v1.1.0) along with sklearn, opencv etc."]] + option_type = "framework" + + @self.cli.input("mxnet\n") + def dummy(self): + return self.cli.prompt_available_options(available_options, option_type) + + result = dummy(self) + assert result + assert result == "mxnet" + + @self.cli.input("\n") + def dummy(self): + return self.cli.prompt_available_options(available_options, option_type) + + result = dummy(self) + assert result + assert result == "python-base" + + available_options = ["py27", "py35"] + option_type = "language" + + @self.cli.input("py27\n") + def dummy(self): + return self.cli.prompt_available_options(available_options, option_type) + + result = dummy(self) + assert result + assert result == "py27" + + @self.cli.input("\n") + def dummy(self): + return self.cli.prompt_available_options(available_options, option_type) + + result = dummy(self) + assert result + assert result == "py27" + + # quit + @self.cli.input("quit\n") + def dummy(self): + return self.cli.prompt_available_options(available_options, option_type) + + with pytest.raises(SystemExit) as pytest_wrapped_e: + dummy(self) + + assert pytest_wrapped_e.type == SystemExit + assert pytest_wrapped_e.value.code == 0 + def test_prompt_validator(self): def validate_y(val): return True if val == "y" else False From 8d99b46a1c383be56699bd5ab11506c77af4e590 Mon Sep 17 00:00:00 2001 From: Shabaz Patel Date: Mon, 23 Jul 2018 21:04:55 -0700 Subject: [PATCH 06/14] adding method to extract workspace url in environment driver + test --- .../controller/environment/driver/__init__.py | 18 ++++++++ .../environment/driver/dockerenv.py | 45 +++++++++++++++++++ .../driver/tests/test_dockerenv.py | 29 ++++++++++++ 3 files changed, 92 insertions(+) diff --git a/datmo/core/controller/environment/driver/__init__.py b/datmo/core/controller/environment/driver/__init__.py index b4c5bad6..e643eebe 100644 --- a/datmo/core/controller/environment/driver/__init__.py +++ b/datmo/core/controller/environment/driver/__init__.py @@ -161,6 +161,24 @@ def build(self, name, path, workspace): """ pass + @abstractmethod + def extract_workspace_url(self, container_name, workspace=None): + """Extract workspace url from the container + + Parameters + ---------- + container_name : str + name of the container being run + workspace : str + workspace being used for the run + + Returns + ------- + str + web url for the workspace being run, None if it doesn't exist + """ + pass + @abstractmethod def run(self, name, options, log_filepath): """Run and log an instance of the environment with the options given diff --git a/datmo/core/controller/environment/driver/dockerenv.py b/datmo/core/controller/environment/driver/dockerenv.py index 16721217..281cd043 100644 --- a/datmo/core/controller/environment/driver/dockerenv.py +++ b/datmo/core/controller/environment/driver/dockerenv.py @@ -1,5 +1,7 @@ import ast import os +import re +import time import json import subprocess import platform @@ -437,6 +439,49 @@ def remove_images(self, name=None, all=False, filters=None, force=False): str(e))) return True + def extract_workspace_url(self, container_name, workspace=None): + """Extract workspace url from the container + + Parameters + ---------- + container_name : str + name of the container being run + workspace : str + workspace being used for the run + + Returns + ------- + str + web url for the workspace being run, None if it doesn't exist + + """ + if workspace in ['notebook', 'jupyterlab']: + docker_shell_cmd_list = list(self.prefix) + docker_shell_cmd_list.append("exec") + docker_shell_cmd_list.append(container_name) + docker_shell_cmd_list.append("jupyter") + docker_shell_cmd_list.append("notebook") + docker_shell_cmd_list.append("list") + workspace_url = None + timeout_count = 0 + while workspace_url is None and timeout_count < 10: + process = subprocess.Popen( + docker_shell_cmd_list, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + if process.returncode > 0: + time.sleep(1) + timeout_count += 1 + workspace_url = re.search("(?Phttps?://[^\s]+)", stdout).group("url") if stdout else None + return workspace_url + elif workspace == 'rstudio': + time.sleep(2) + workspace_url = 'http://localhost:8787' + else: + workspace_url = None + return workspace_url + # running daemon needed def run_container(self, image_name, diff --git a/datmo/core/controller/environment/driver/tests/test_dockerenv.py b/datmo/core/controller/environment/driver/tests/test_dockerenv.py index 3690d7ba..d6f6e3a2 100644 --- a/datmo/core/controller/environment/driver/tests/test_dockerenv.py +++ b/datmo/core/controller/environment/driver/tests/test_dockerenv.py @@ -8,6 +8,7 @@ import os import tempfile import platform +import threading import uuid import timeout_decorator from io import open @@ -663,6 +664,34 @@ def test_run_container(self): self.image_name, api=True, detach=True) assert container_obj + @pytest_docker_environment_failed_instantiation(test_datmo_dir) + def test_extract_workspace_url(self): + # TODO: test with all variables provided + with open(self.dockerfile_path, "wb") as f: + f.write(to_bytes("FROM datmo/python-base:cpu-py27-notebook" + os.linesep)) + f.write(to_bytes(str("RUN echo " + self.random_text))) + self.docker_environment_driver.build_image(self.image_name, + self.dockerfile_path) + + def dummy(self, name, workspace): + workspace_url = self.environment_driver.extract_workspace_url(name, workspace) + assert workspace_url + + thread = threading.Thread(target=dummy, args=(self.image_name, "notebook")) + thread.daemon = True # Daemonize thread + thread.start() # Start the execution + + @timeout_decorator.timeout(10, use_signals=False) + def timed_run(self): + return_code, container_id = \ + self.docker_environment_driver.run_container(self.image_name) + return return_code, container_id + + return_code, container_id = timed_run(self) + # teardown container + self.docker_environment_driver.stop(container_id, force=True) + + @pytest_docker_environment_failed_instantiation(test_datmo_dir) def test_get_container(self): failed = False From 128d485293b0a6b626da7626c65e38a3a0c2f2d0 Mon Sep 17 00:00:00 2001 From: Shabaz Patel Date: Mon, 23 Jul 2018 21:06:19 -0700 Subject: [PATCH 07/14] adding automatic opening of workspace in task controller + workspace messaging --- datmo/cli/command/workspace.py | 4 ++-- datmo/core/controller/task.py | 21 +++++++++++++++++++++ datmo/core/util/lang/en.py | 8 +++++++- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/datmo/cli/command/workspace.py b/datmo/cli/command/workspace.py index 4fc48c69..8bed8624 100644 --- a/datmo/cli/command/workspace.py +++ b/datmo/cli/command/workspace.py @@ -28,7 +28,7 @@ def notebook(self, **kwargs): "mem_limit": kwargs["mem_limit"], "workspace": "notebook" } - + self.cli_helper.echo(__("info", "cli.workspace.run.notebook")) # Run task and return Task object result return self.task_run_helper(task_dict, snapshot_dict, "cli.workspace.notebook") @@ -52,7 +52,7 @@ def jupyterlab(self, **kwargs): "mem_limit": kwargs["mem_limit"], "workspace": "jupyterlab" } - + self.cli_helper.echo(__("info", "cli.workspace.run.jupyterlab")) # Run task and return Task object result return self.task_run_helper(task_dict, snapshot_dict, "cli.workspace.jupyterlab") diff --git a/datmo/core/controller/task.py b/datmo/core/controller/task.py index 8db47cc0..b9385131 100644 --- a/datmo/core/controller/task.py +++ b/datmo/core/controller/task.py @@ -1,6 +1,8 @@ import os import time import shlex +import threading +import webbrowser from datetime import datetime from datmo.core.controller.base import BaseController @@ -131,12 +133,31 @@ def _run_helper(self, environment_id, options, log_filepath): } workspace = options.get('workspace', None) self.environment.build(environment_id, workspace) + # Start a daemon to run workspace on web browser + name = options.get('name', None) + if workspace is not None: + thread = threading.Thread(target=self._open_workspace, args=(name, workspace)) + thread.daemon = True # Daemonize thread + thread.start() # Start the execution + # Run container with environment return_code, run_id, logs = self.environment.run( environment_id, run_options, log_filepath) return return_code, run_id, logs + def _open_workspace(self, name, workspace): + """Run a daemon to open workspace + + :param name: name of the environment being run + :param workspace: name of the workspace + :return: + """ + workspace_url = self.environment_driver.extract_workspace_url(name, workspace) + result = webbrowser.open(workspace_url, new=2) + + return result + def _parse_logs_for_results(self, logs): """Parse log string to extract results and return dictionary. diff --git a/datmo/core/util/lang/en.py b/datmo/core/util/lang/en.py index f682fd2d..52dc90ff 100644 --- a/datmo/core/util/lang/en.py +++ b/datmo/core/util/lang/en.py @@ -16,14 +16,18 @@ "Failed to update project {name} @ ({path}) ", "cli.workspace.notebook": "Starting a notebook", + "cli.workspace.run.notebook": + "Automatically opens on the browser if it exists", "cli.workspace.jupyterlab": "Starting a jupyter lab", + "cli.workspace.run.jupyterlab": + "Automatically opens on the browser if it exists", "cli.workspace.terminal": "Starting a terminal", "cli.workspace.rstudio": "Starting a rstudio", "cli.workspace.run.rstudio": - "Open http://localhost:8787/auth-sign-in to login to rstudio, " + "Automatically opens http://localhost:8787 to login to rstudio," "enter username: rstudio and password: rstudio", "cli.project.pull": "Pulling information from the Datmo project url and adding it to local...", @@ -284,6 +288,8 @@ "Error in running multiple rmi commands: %s", "controller.environment.driver.docker.run_container": "Error running docker container. Failed command: %s", + "controller.environment.driver.docker.exec_container": + "Error executing inside docker container. Failed command: %s", "controller.environment.driver.docker.stop_container": "Error stopping docker container: %s", "controller.environment.driver.docker.remove_container": From a7b153e32b97b934d2ec443a1d59cf6558d8749e Mon Sep 17 00:00:00 2001 From: Shabaz Patel Date: Mon, 23 Jul 2018 21:07:03 -0700 Subject: [PATCH 08/14] adding extraction of workspace url in environment controller --- .../core/controller/environment/environment.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/datmo/core/controller/environment/environment.py b/datmo/core/controller/environment/environment.py index 0c0f65be..0fc7b50f 100644 --- a/datmo/core/controller/environment/environment.py +++ b/datmo/core/controller/environment/environment.py @@ -299,6 +299,23 @@ def build(self, environment_id, workspace=None): shutil.rmtree(_temp_env_dir) return result + def extract_workspace_url(self, name, workspace=None): + """Extract workspace url from the environment + + Parameters + ---------- + name : str + name of the environment being run + workspace : str + workspace being used for the run + + Returns + ------- + str + web url for the workspace being run, None if it doesn't exist + """ + return self.environment_driver.extract_workspace_url(name, workspace) + def run(self, environment_id, options, log_filepath): """Run and log an instance of the environment with the options given From 55a1c2f454d91cd1e5baa6db07ff29a5cd453e16 Mon Sep 17 00:00:00 2001 From: Shabaz Patel Date: Mon, 23 Jul 2018 23:17:32 -0700 Subject: [PATCH 09/14] updating readme --- README.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.rst b/README.rst index 5b6b4a49..3ad11cba 100644 --- a/README.rst +++ b/README.rst @@ -44,8 +44,6 @@ more granular control and workflow integration. Requirements ------------ -- `openssl `__ -- `git `__ - `docker `__ Installation From 75c3b3e966ac3c904077a8c954ae6674df180d93 Mon Sep 17 00:00:00 2001 From: Shabaz Patel Date: Tue, 24 Jul 2018 12:29:25 -0700 Subject: [PATCH 10/14] updating regex match for url in environment driver --- datmo/core/controller/environment/driver/dockerenv.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datmo/core/controller/environment/driver/dockerenv.py b/datmo/core/controller/environment/driver/dockerenv.py index 281cd043..edb217fd 100644 --- a/datmo/core/controller/environment/driver/dockerenv.py +++ b/datmo/core/controller/environment/driver/dockerenv.py @@ -473,9 +473,11 @@ def extract_workspace_url(self, container_name, workspace=None): if process.returncode > 0: time.sleep(1) timeout_count += 1 - workspace_url = re.search("(?Phttps?://[^\s]+)", stdout).group("url") if stdout else None + result = re.search("(?Phttps?://[^\s]+)", stdout) if stdout else None + workspace_url = result.group("url") if result else None return workspace_url elif workspace == 'rstudio': + # Having a pause for two seconds before returning url time.sleep(2) workspace_url = 'http://localhost:8787' else: From e7b5bbdd6eb5ba4518a331cd1ebe1b11750c7ccb Mon Sep 17 00:00:00 2001 From: Shabaz Patel Date: Thu, 26 Jul 2018 13:07:45 -0700 Subject: [PATCH 11/14] hot fix to decode stdout before regex search --- datmo/core/controller/environment/driver/dockerenv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datmo/core/controller/environment/driver/dockerenv.py b/datmo/core/controller/environment/driver/dockerenv.py index edb217fd..1b5220e0 100644 --- a/datmo/core/controller/environment/driver/dockerenv.py +++ b/datmo/core/controller/environment/driver/dockerenv.py @@ -473,7 +473,7 @@ def extract_workspace_url(self, container_name, workspace=None): if process.returncode > 0: time.sleep(1) timeout_count += 1 - result = re.search("(?Phttps?://[^\s]+)", stdout) if stdout else None + result = re.search("(?Phttps?://[^\s]+)", stdout.decode('utf-8')) if stdout else None workspace_url = result.group("url") if result else None return workspace_url elif workspace == 'rstudio': From 4650cdb2dc7d82b9febb9c8954b959da1fd8c13c Mon Sep 17 00:00:00 2001 From: Shabaz Patel Date: Fri, 27 Jul 2018 10:58:06 -0700 Subject: [PATCH 12/14] commenting out test run in main for test reliability --- datmo/cli/tests/test_main.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/datmo/cli/tests/test_main.py b/datmo/cli/tests/test_main.py index 5f688d70..38cf2149 100644 --- a/datmo/cli/tests/test_main.py +++ b/datmo/cli/tests/test_main.py @@ -101,20 +101,20 @@ def test_init(self): success = False assert success - @pytest_docker_environment_failed_instantiation(test_datmo_dir) - def test_run(self): - try: - success = True - p = self.command_run([self.execpath, "run", "python script.py"]) - out, err = p.communicate() - out, err = out.decode(), err.decode() - if err: - success = False - elif 'hello' not in out: - success = False - except Exception: - success = False - assert success + # @pytest_docker_environment_failed_instantiation(test_datmo_dir) + # def test_run(self): + # try: + # success = True + # p = self.command_run([self.execpath, "run", "python script.py"]) + # out, err = p.communicate() + # out, err = out.decode(), err.decode() + # if err: + # success = False + # elif 'hello' not in out: + # success = False + # except Exception: + # success = False + # assert success def test_run_ls(self): try: From c46e034d629098ad304ad51347ee3a5ee0bfe345 Mon Sep 17 00:00:00 2001 From: Shabaz Patel Date: Fri, 27 Jul 2018 15:53:52 -0700 Subject: [PATCH 13/14] updating test to extract workspace url --- .../driver/tests/test_dockerenv.py | 41 ++++++++++++++----- .../environment/tests/test_environment.py | 28 ++++++++++++- 2 files changed, 58 insertions(+), 11 deletions(-) diff --git a/datmo/core/controller/environment/driver/tests/test_dockerenv.py b/datmo/core/controller/environment/driver/tests/test_dockerenv.py index d6f6e3a2..120016cd 100644 --- a/datmo/core/controller/environment/driver/tests/test_dockerenv.py +++ b/datmo/core/controller/environment/driver/tests/test_dockerenv.py @@ -9,6 +9,7 @@ import tempfile import platform import threading +import Queue import uuid import timeout_decorator from io import open @@ -673,23 +674,43 @@ def test_extract_workspace_url(self): self.docker_environment_driver.build_image(self.image_name, self.dockerfile_path) - def dummy(self, name, workspace): - workspace_url = self.environment_driver.extract_workspace_url(name, workspace) - assert workspace_url + random_container_id = str(uuid.uuid1()) + my_queue = Queue.Queue() - thread = threading.Thread(target=dummy, args=(self.image_name, "notebook")) - thread.daemon = True # Daemonize thread - thread.start() # Start the execution + def dummy(self, name, workspace, out_queue): + workspace_url = self.docker_environment_driver.extract_workspace_url(name, workspace) + out_queue.put(workspace_url) - @timeout_decorator.timeout(10, use_signals=False) + @timeout_decorator.timeout(20, use_signals=False) def timed_run(self): return_code, container_id = \ - self.docker_environment_driver.run_container(self.image_name) + self.docker_environment_driver.run_container(self.image_name, + name=random_container_id, + command=['jupyter', 'notebook', '--allow-root']) return return_code, container_id - return_code, container_id = timed_run(self) + thread = threading.Thread(target=dummy, args=(self, random_container_id, + "notebook", my_queue)) + thread.daemon = True # Daemonize thread + thread.start() # Start the execution + + timed_run_result = False + try: + _, _ = timed_run(self) + except timeout_decorator.timeout_decorator.TimeoutError: + timed_run_result = True + thread.join() + assert timed_run_result + # asserting the workspace url to be passed + workspace_url = my_queue.get() # get the workspace url + assert 'http' in workspace_url + + # Test when there is no container being run + workspace_url = self.docker_environment_driver.extract_workspace_url(self.image_name, "notebook") + assert workspace_url == None + # teardown container - self.docker_environment_driver.stop(container_id, force=True) + self.docker_environment_driver.stop(random_container_id, force=True) @pytest_docker_environment_failed_instantiation(test_datmo_dir) diff --git a/datmo/core/controller/environment/tests/test_environment.py b/datmo/core/controller/environment/tests/test_environment.py index ae8ba2d6..2548a6c4 100644 --- a/datmo/core/controller/environment/tests/test_environment.py +++ b/datmo/core/controller/environment/tests/test_environment.py @@ -3,7 +3,7 @@ """ import os import uuid -import shutil +import threading import tempfile import platform import timeout_decorator @@ -474,6 +474,32 @@ def test_build(self): environment_obj_4.id, workspace="notebook") assert result + @pytest_docker_environment_failed_instantiation(test_datmo_dir) + def test_extract_workspace_url(self): + # Create environment definition + self.project_controller.init("test5", "test description") + self.environment_controller = EnvironmentController() + definition_filepath = os.path.join(self.environment_controller.home, + "Dockerfile") + random_text = str(uuid.uuid1()) + with open(definition_filepath, "wb") as f: + f.write(to_bytes("FROM datmo/python-base:cpu-py27-notebook" + os.linesep)) + f.write(to_bytes(str("RUN echo " + random_text))) + + image_name = "test" + input_dict = { + "name": image_name, + "description": "test description" + } + # Create environment in the project + environment_obj = self.environment_controller.create(input_dict, + save_hardware_file=False) + self.environment_controller.build(environment_obj.id) + + # Test when there is no container being run + workspace_url = self.environment_controller.extract_workspace_url(image_name, "notebook") + assert workspace_url == None + @pytest_docker_environment_failed_instantiation(test_datmo_dir) def test_run(self): # Test run simple command with simple Dockerfile From a89083b50eea046ae2a23cbde80e744a568db840 Mon Sep 17 00:00:00 2001 From: Shabaz Patel Date: Fri, 27 Jul 2018 16:14:14 -0700 Subject: [PATCH 14/14] fix for multiprocessing queue to work on py2 and py3 --- .../environment/driver/tests/test_dockerenv.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/datmo/core/controller/environment/driver/tests/test_dockerenv.py b/datmo/core/controller/environment/driver/tests/test_dockerenv.py index 120016cd..f6d46f85 100644 --- a/datmo/core/controller/environment/driver/tests/test_dockerenv.py +++ b/datmo/core/controller/environment/driver/tests/test_dockerenv.py @@ -9,7 +9,12 @@ import tempfile import platform import threading -import Queue +import sys +is_py2 = sys.version[0] == '2' +if is_py2: + import Queue as queue +else: + import queue as queue import uuid import timeout_decorator from io import open @@ -675,7 +680,7 @@ def test_extract_workspace_url(self): self.dockerfile_path) random_container_id = str(uuid.uuid1()) - my_queue = Queue.Queue() + my_queue = queue.Queue() def dummy(self, name, workspace, out_queue): workspace_url = self.docker_environment_driver.extract_workspace_url(name, workspace)