Merge pull request #11 from datmo/environment

updating environment to create requirements and default Dockerfile when not present

MANIFEST.in

@@ -1,2 +1,4 @@
 include datmo/controller/environment/driver/templates/baseDockerfile
+include datmo/controller/environment/driver/templates/python2Dockerfile
+include datmo/controller/environment/driver/templates/python3Dockerfile
 recursive-include templates *

datmo/cli/command/snapshot.py

@@ -54,10 +54,10 @@ def __init__(self, home, cli_helper):
         ls.add_argument("--session-id", dest="session_id", default=None, help="Session ID to filter")
         ls.add_argument("--session-name", dest="session_name", default=None, help="Session name to filter")
         ls.add_argument("--all", "-a", dest="details", action="store_true",
-                            help="Show detailed SnapshotCommand information")
+                            help="Show detailed snapshot information")
 
         checkout = subcommand_parsers.add_parser("checkout", help="Checkout a snapshot by id")
-        checkout.add_argument("--id", dest="id", default=None, help="SnapshotCommand ID")
+        checkout.add_argument("--id", dest="id", default=None, help="Snapshot ID")
 
         self.snapshot_controller = SnapshotController(home=home,
                                                       dal_driver=self.project_controller.dal_driver)

datmo/controller/environment/driver/dockerenv.py

@@ -7,7 +7,7 @@
 from datmo.util.i18n import get as _
 from datmo.util.exceptions import DoesNotExistException, \
     EnvironmentInitFailed, EnvironmentExecutionException, \
-    FileAlreadyExistsException
+    FileAlreadyExistsException, EnvironmentRequirementsCreateException
 from datmo.controller.environment.driver import EnvironmentDriver
 
 
@@ -60,25 +60,37 @@ def is_initialized(self):
         self._is_initialized = False
         return self._is_initialized
 
-    def create(self, path=None, output_path=None):
+    def create(self, path=None, output_path=None, language=None):
         if not path:
             path = os.path.join(self.filepath,
                                 "Dockerfile")
         if not output_path:
             directory, filename = os.path.split(path)
             output_path = os.path.join(directory,
                                        "datmo" + filename)
+        if not language:
+            language = "python3"
+
+        requirements_filepath = None
         if not os.path.isfile(path):
-            raise DoesNotExistException(_("error",
-                                          "controller.environment.driver.docker.create.dne",
-                                          path))
+            if language == "python3":
+                # Create requirements txt file for python
+                requirements_filepath = self.create_requirements_file()
+                # Create Dockerfile for ubuntu
+                path = self.create_default_dockerfile(requirements_filepath=requirements_filepath,
+                                                      language=language)
+            else:
+                raise DoesNotExistException(_("error",
+                                              "controller.environment.driver.docker.create.dne",
+                                              path))
         if os.path.isfile(output_path):
             raise FileAlreadyExistsException(_("error",
                                                "controller.environment.driver.docker.create.exists",
                                                output_path))
         success = self.form_datmo_definition_file(input_definition_path=path,
                                         output_definition_path=output_path)
-        return success, path, output_path
+
+        return success, path, output_path, requirements_filepath
 
     def build(self, name, path):
 
@@ -115,7 +127,6 @@ def init(self):
                                                   str(e)))
         return True
 
-
     def get_tags_for_docker_repository(self, repo_name):
         """Method to get tags for docker repositories
 
@@ -232,7 +243,7 @@ def run_container(self, image_name, command=None, ports=None, name=None, volumes
         image_name : str
             Docker image name
         command : list, optional
-            List with complete user-given command (e.g. ["python", "cool.py"])
+            List with complete user-given command (e.g. ["python3", "cool.py"])
         ports : list, optional
             Here are some example ports used for common applications.
                *  "jupyter notebook" - 8888
@@ -480,10 +491,57 @@ def stop_remove_containers_by_term(self, term, force=False):
                                                   str(e)))
         return True
 
+    def create_requirements_file(self, execpath="pipreqs"):
+        """Create requirements txt file for the project
+
+        Returns
+        -------
+        str
+            absolute filepath for requirements file
+        """
+        try:
+            subprocess.check_output([execpath, self.filepath, "--force"],
+                                    cwd=self.filepath).strip()
+            requirements_filepath = os.path.join(self.filepath, "requirements.txt")
+            return requirements_filepath
+        except Exception as e:
+            raise EnvironmentRequirementsCreateException(_("error",
+                                                           "controller.environment.requirements.create",
+                                                           str(e)))
+
+    def create_default_dockerfile(self, requirements_filepath, language):
+        """Create a default Dockerfile for a given language
+
+        Parameters
+        ----------
+        requirements_filepath : str
+            path for the requirements txt file
+        language : str
+            programming language used ("python2" and "python3" currently supported)
+
+        Returns
+        -------
+        str
+            absolute path for the new Dockerfile using requirements txt file
+        """
+        language_dockerfile = "%sDockerfile" % language
+        base_dockerfile_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                                                "templates", language_dockerfile)
+
+        # Combine dockerfile
+        destination_dockerfile = os.path.join(self.filepath, "Dockerfile")
+        destination = open(destination_dockerfile, "w")
+        shutil.copyfileobj(open(base_dockerfile_filepath, "r"), destination)
+        destination.write(str("ADD %s /tmp/requirements.txt\n" % requirements_filepath))
+        destination.write(str("RUN pip install --no-cache-dir -r /tmp/requirements.txt\n"))
+        destination.close()
+
+        return destination_dockerfile
+
     def form_datmo_definition_file(self, input_definition_path="Dockerfile",
                                    output_definition_path="datmoDockerfile"):
         """
-        n order to create intermediate dockerfile to run
+        In order to create intermediate dockerfile to run
         """
         base_dockerfile_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                                     "templates", "baseDockerfile")

datmo/controller/environment/driver/templates/python2Dockerfile

@@ -0,0 +1,2 @@
+FROM python:2
+RUN pip install --upgrade pip

datmo/controller/environment/driver/templates/python3Dockerfile

@@ -0,0 +1,3 @@
+FROM python:3
+RUN pip install --upgrade pip
+

datmo/controller/environment/driver/test/test_dockerenv.py

@@ -12,7 +12,8 @@
 
 from datmo.controller.environment.driver.dockerenv import DockerEnvironmentDriver
 from datmo.util.exceptions import EnvironmentInitFailed, \
-    DoesNotExistException, FileAlreadyExistsException
+    DoesNotExistException, FileAlreadyExistsException, \
+    EnvironmentRequirementsCreateException
 
 
 class TestDockerEnv():
@@ -57,36 +58,60 @@ def test_create(self):
         output_dockerfile_path = os.path.join(self.docker_environment_manager.filepath,
                                               "datmoDockerfile")
         # Test both default values
-        success, path, output_path = \
+        success, path, output_path, requirements_filepath = \
             self.docker_environment_manager.create()
 
         assert success and \
                os.path.isfile(output_dockerfile_path) and \
                "datmo" in open(output_dockerfile_path, "r").read()
         assert path == input_dockerfile_path
         assert output_path == output_dockerfile_path
+        assert requirements_filepath == None
+
         os.remove(output_dockerfile_path)
 
         # Test default values for output
-        success, path, output_path = \
+        success, path, output_path, requirements_filepath = \
             self.docker_environment_manager.create(input_dockerfile_path)
 
         assert success and \
                os.path.isfile(output_dockerfile_path) and \
                "datmo" in open(output_dockerfile_path, "r").read()
         assert path == input_dockerfile_path
         assert output_path == output_dockerfile_path
+        assert requirements_filepath == None
+
         os.remove(output_dockerfile_path)
 
         # Test both values given
-        success, path, output_path = \
+        success, path, output_path, requirements_filepath = \
             self.docker_environment_manager.create(input_dockerfile_path,
                                                         output_dockerfile_path)
         assert success and \
                os.path.isfile(output_dockerfile_path) and \
                "datmo" in open(output_dockerfile_path, "r").read()
         assert path == input_dockerfile_path
         assert output_path == output_dockerfile_path
+        assert requirements_filepath == None
+
+        # Test for language being passed in
+        os.remove(input_dockerfile_path)
+        os.remove(output_dockerfile_path)
+
+        script_path = os.path.join(self.docker_environment_manager.filepath,
+                            "script.py")
+        with open(script_path, "w") as f:
+            f.write("import numpy\n")
+            f.write("import sklearn\n")
+        success, path, output_path, requirements_filepath = \
+            self.docker_environment_manager.create(language="python3")
+        assert success and \
+               os.path.isfile(output_dockerfile_path) and \
+               "datmo" in open(output_dockerfile_path, "r").read()
+        assert path == input_dockerfile_path
+        assert output_path == output_dockerfile_path
+        assert requirements_filepath and os.path.isfile(requirements_filepath) and \
+               "numpy" in open(requirements_filepath, "r").read()
 
         # Test exception for path does not exist
         try:
@@ -161,6 +186,8 @@ def test_run(self):
         assert return_code == 0
         assert run_id
         assert logs
+        # teardown container
+        self.docker_environment_manager.stop(run_id, force=True)
         # teardown image
         self.docker_environment_manager.remove(image_name, force=True)
 
@@ -243,7 +270,7 @@ def test_get_image(self):
         self.docker_environment_manager.build_image(image_name, dockerfile_path)
 
         result = self.docker_environment_manager.get_image(image_name)
-        tags  = result.__dict__['attrs']['RepoTags']
+        tags = result.__dict__['attrs']['RepoTags']
         assert image_name + ":latest" in tags
         self.docker_environment_manager.remove_image(image_name, force=True)
 
@@ -334,13 +361,16 @@ def test_run_container(self):
             self.docker_environment_manager.run_container(image_name)
         assert return_code == 0 and \
             container_id
+        # teardown container
+        self.docker_environment_manager.stop(container_id, force=True)
         # With api=True, detach=False
         logs = self.docker_environment_manager.run_container(image_name, api=True)
         assert logs == ""
         # With api=True, detach=True
         container_obj = self.docker_environment_manager.run_container(image_name, api=True,
                                                                       detach=True)
         assert container_obj
+        self.docker_environment_manager.stop(container_obj.id, force=True)
         self.docker_environment_manager.remove_image(image_name, force=True)
 
     def test_get_container(self):
@@ -371,6 +401,7 @@ def test_list_containers(self):
                                                                      detach=True)
         result = self.docker_environment_manager.list_containers()
         assert container_id and len(result) > 0
+        self.docker_environment_manager.stop(container_id, force=True)
         self.docker_environment_manager.remove_image(image_name, force=True)
 
     def test_stop_container(self):
@@ -424,7 +455,6 @@ def test_log_container(self):
         _, container_id = \
             self.docker_environment_manager.run_container(image_name,
                                                           command=["sh", "-c", "echo yo"])
-        self.docker_environment_manager.stop_container(container_id)
         return_code, logs = self.docker_environment_manager.log_container(container_id,
                                                                log_filepath)
         assert return_code == 0
@@ -438,6 +468,46 @@ def test_log_container(self):
         self.docker_environment_manager.remove_container(container_id, force=True)
         self.docker_environment_manager.remove_image(image_name, force=True)
 
+    def test_create_requirements_file(self):
+        script_path = os.path.join(self.docker_environment_manager.filepath,
+                                   "script.py")
+        with open(script_path, "w") as f:
+            f.write("import numpy\n")
+            f.write("import sklearn\n")
+        # Test default
+        result = self.docker_environment_manager.create_requirements_file()
+        assert result
+        assert os.path.isfile(result) and \
+               "numpy" in open(result, "r").read() and \
+               "scikit_learn" in open(result, "r").read()
+
+        # Test failure
+        exception_thrown = False
+        try:
+            _ = self.docker_environment_manager.\
+                create_requirements_file(execpath="does_not_work")
+        except EnvironmentRequirementsCreateException:
+            exception_thrown = True
+
+        assert exception_thrown
+
+    def test_create_default_dockerfile(self):
+        script_path = os.path.join(self.docker_environment_manager.filepath,
+                                   "script.py")
+        with open(script_path, "w") as f:
+            f.write("import numpy\n")
+            f.write("import sklearn\n")
+        requirements_filepath = \
+            self.docker_environment_manager.create_requirements_file()
+        result = self.docker_environment_manager.\
+            create_default_dockerfile(requirements_filepath,
+                                      language="python3")
+
+        assert result
+        assert os.path.isfile(result)
+        assert "python" in open(result, "r").read()
+        assert "requirements.txt" in open(result, "r").read()
+
     def test_stop_remove_containers_by_term(self):
         # TODO: add more robust tests
         image_name = str(uuid.uuid1())