leverage file, code, and environment controllers in other controllers.

README.md

@@ -58,7 +58,7 @@ $ pip install recommonmark
 ```
 $ pip install pytest pytest-cov
 $ pip install coveralls
-$ pytest --cov-config .coveragerc --cov=datmo
+$ python -m pytest --cov-config .coveragerc --cov=datmo
 ```
 
 ## Project Structure

datmo/controller/environment/environment.py

@@ -3,6 +3,7 @@
 from datmo.util.i18n import get as _
 from datmo.util.misc_functions import get_filehash
 from datmo.controller.base import BaseController
+from datmo.controller.file.file_collection import FileCollectionController
 from datmo.util.exceptions import RequiredArgumentMissing, \
     DoesNotExistException
 
@@ -25,6 +26,7 @@ class EnvironmentController(BaseController):
     """
     def __init__(self, home, dal_driver=None):
         super(EnvironmentController, self).__init__(home, dal_driver)
+        self.file_collection = FileCollectionController(home, self.dal.driver)
 
     def create(self, dictionary):
         """Create an Environment
@@ -77,8 +79,7 @@ def create(self, dictionary):
                                                                        output_definition_path=datmo_definition_filepath)
                     # Add definition_filepath and datmo_definition_filepath to file_collection
                     filepaths = [dictionary['definition_filepath'], datmo_definition_filepath]
-                    create_dict['file_collection_id'] = self.file_driver. \
-                        create_collection(filepaths)
+                    create_dict['file_collection_id'] = self.file_collection.create(filepaths).id
                     create_dict['definition_filename'] = definition_filename
 
         ## Optional args
@@ -114,13 +115,55 @@ def build(self, id):
             raise DoesNotExistException(_("error",
                                           "controller.environment.build",
                                           id))
+        file_collection_obj = self.dal.file_collection.\
+            get_by_id(environment_obj.file_collection_id)
         # Build the Environment with the driver
-        datmo_definition_filepath = os.path.join(self.file_driver.
-                                                 get_collection_path(
-            environment_obj.file_collection_id), "datmo" + environment_obj.definition_filename)
+        datmo_definition_filepath = os.path.join(file_collection_obj.path,
+                                                 "datmo" + environment_obj.definition_filename)
         result = self.environment_driver.build_image(id, definition_path=datmo_definition_filepath)
         return result
 
+    def run(self, id, options, log_filepath):
+        """
+        Run and log an instance of the environment with the options given
+
+        Parameters
+        ----------
+        id : str
+        options : dict
+            can include the following values:
+
+            command : list
+            ports : list
+                Here are some example ports used for common applications.
+                   *  'jupyter notebook' - 8888
+                   *  flask API - 5000
+                   *  tensorboard - 6006
+            name : str
+            volumes : dict
+            detach : bool
+            stdin_open : bool
+            tty : bool
+            gpu : bool
+        log_filepath : str
+            filepath to the log file
+
+        Returns
+        -------
+        return_code : int
+            system return code for container and logs
+        container_id : str
+            identification for container run of the environment
+        logs : str
+            string version of output logs for the container
+        """
+        run_return_code, container_id = \
+            self.environment_driver.run_container(image_name=id, **options)
+        log_return_code, logs = self.environment_driver.log_container(container_id, filepath=log_filepath)
+        final_return_code = run_return_code and log_return_code
+
+        return final_return_code, container_id, logs
+
     def list(self):
         # TODO: Add time filters
         return self.dal.environment.query({})
@@ -149,7 +192,7 @@ def delete(self, id):
                                           "controller.environment.delete",
                                           id))
         # Remove file collection
-        file_collection_deleted = self.file_driver.delete_collection(environment_obj.file_collection_id)
+        file_collection_deleted = self.file_collection.delete(environment_obj.file_collection_id)
         # Remove images associated with the environment_driver
         image_removed = self.environment_driver.remove_image(id, force=True)
         # Remove running containers associated with environment_driver

datmo/controller/environment/test/test_environment.py

@@ -1,10 +1,6 @@
 """
 Tests for EnvironmentController
 """
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
 import os
 import shutil
 import tempfile
@@ -21,7 +17,7 @@ def setup_method(self):
         tempfile.tempdir = '/tmp'
         self.temp_dir = tempfile.mkdtemp('project')
         self.project = ProjectController(self.temp_dir)
-        self.environment = EnvironmentController(self.temp_dir)
+        self.environment = EnvironmentController(self.temp_dir, self.project.dal.driver)
 
     def teardown_method(self):
         shutil.rmtree(self.temp_dir)

datmo/controller/task.py

@@ -3,6 +3,7 @@
 
 from datmo.util.i18n import get as _
 from datmo.controller.base import BaseController
+from datmo.controller.environment.environment import EnvironmentController
 from datmo.controller.snapshot import SnapshotController
 from datmo.util.exceptions import TaskRunException
 
@@ -14,25 +15,26 @@ class TaskController(BaseController):
     Attributes
     ----------
     snapshot : SnapshotController
-        Snapshot controller is used to create snapshots before and after tasks
+        used to create snapshots before and after tasks
 
     Methods
     -------
     create(dictionary)
-        Create a Task object with the permanent parameters
+        creates a Task object with the permanent parameters
     _run_helper(environment_id, environment_file_collection_id,
-                    log_filepath, options_dict)
-        Helper for run to start environment and run with the appropriate parameters
+                    log_filepath, options)
+        helper for run to start environment and run with the appropriate parameters
     run(self, task_id, dictionary=None)
-        Run the task and track the run, logs, inputs and outputs
+        runs the task and tracks the run, logs, inputs and outputs
     list(session_id=None)
-        List all tasks within the project given filters
+        lists all tasks within the project given filters
     delete(id)
-        Delete the specified task from the project
+        deletes the specified task from the project
 
     """
     def __init__(self, home, dal_driver=None):
         super(TaskController, self).__init__(home, dal_driver)
+        self.environment = EnvironmentController(home, self.dal.driver)
         self.snapshot = SnapshotController(home, self.dal.driver)
 
     def create(self, dictionary):
@@ -41,12 +43,14 @@ def create(self, dictionary):
         Parameters
         ----------
         dictionary : dict
-            Options for task, command is required
+            options for Task object
+            command : str
+                full command used
 
         Returns
         -------
         Task
-            Object entity for Task
+            object entity for Task
 
         """
 
@@ -67,20 +71,16 @@ def create(self, dictionary):
         # Create Task
         return self.dal.task.create(create_dict)
 
-    def _run_helper(self, environment_id, environment_file_collection_id,
-                    log_filepath, options_dict):
+    def _run_helper(self, environment_id,
+                    options, log_filepath):
         """Run container with parameters
 
         Parameters
         ----------
         environment_id : str
-            the environment_driver id for definition
-        environment_file_collection_id : str
-            the file_collection_id for the environment_driver
-        log_filepath : str
-            filepath to the log file
-        options_dict : dict
-            Can include the following values:
+            the environment id for definition
+        options : dict
+            can include the following values:
 
             command : list
             ports : list
@@ -94,17 +94,19 @@ def _run_helper(self, environment_id, environment_file_collection_id,
             stdin_open : bool
             tty : bool
             gpu : bool
+        log_filepath : str
+            filepath to the log file
 
         Returns
         -------
+        hardware_info : str
+            hardware information of the device the environment was run on
         return_code : int
-            Return code of the environment that was run
+            system return code of the environment that was run
         container_id : str
-            ID of the container environment that was run
-        hardware_info : str
-            Information about the hardware of the device the environment was run on
+            id of the container environment that was run
         logs : str
-            Output logs from the run
+            output logs from the run
 
         """
         # TODO: Fix DAL to keep objects in sync and remove "environment_file_collection_id" as passed param
@@ -129,34 +131,26 @@ def _run_helper(self, environment_id, environment_file_collection_id,
         }
 
         # Run container with options provided
-        run_container_options = {
-            "command": options_dict.get('command', None),
-            "ports": options_dict.get('ports', None),
-            "name": options_dict.get('name', None),
-            "volumes": options_dict.get('volumes', None),
-            "detach": options_dict.get('detach', False),
-            "stdin_open": options_dict.get('stdin_open', False),
-            "tty": options_dict.get('tty', False),
-            "gpu": options_dict.get('gpu', False),
+        run_options = {
+            "command": options.get('command', None),
+            "ports": options.get('ports', None),
+            "name": options.get('name', None),
+            "volumes": options.get('volumes', None),
+            "detach": options.get('detach', False),
+            "stdin_open": options.get('stdin_open', False),
+            "tty": options.get('tty', False),
+            "gpu": options.get('gpu', False),
             "api": False
         }
 
-        # Build image for environment_driver
-        environment_file_collection_path = self.file_driver.\
-            get_collection_path(environment_file_collection_id)
-        self.environment_driver.build_image(
-            tag=environment_id,
-            definition_path=os.path.join(environment_file_collection_path, 'datmoDockerfile')
-        )
-
-        # Run image with environment_driver
-        run_return_code, container_id = \
-            self.environment_driver.run_container(image_name=environment_id, **run_container_options)
-        log_return_code, logs = self.environment_driver.log_container(container_id, filepath=log_filepath)
+        # Build image for environment
+        self.environment.build(environment_id)
 
-        final_return_code = run_return_code and log_return_code
+        # Run container with environment
+        return_code, container_id, logs = \
+            self.environment.run(environment_id, run_options, log_filepath)
 
-        return final_return_code, container_id, hardware_info, logs
+        return hardware_info, return_code, container_id, logs
 
     def run(self, task_id, dictionary=None):
         """Run a task with parameters. If dictionary specified, create a new task with new run parameters.
@@ -220,8 +214,9 @@ def run(self, task_id, dictionary=None):
         })
 
         # Copy over files from the before_snapshot file collection to task dir
-        self.file_driver.copytree(self.file_driver.get_collection_path(
-            before_snapshot_obj.file_collection_id),
+        file_collection_obj =  \
+            self.dal.file_collection.get_by_id(before_snapshot_obj.file_collection_id)
+        self.file_driver.copytree(file_collection_obj.path,
             task_obj.task_dirpath
         )
 
@@ -248,10 +243,9 @@ def run(self, task_id, dictionary=None):
         }
 
         # Run environment_driver
-        return_code, container_id, hardware_info, logs =  \
+        hardware_info, return_code, container_id, logs =  \
             self._run_helper(before_snapshot_obj.environment_id,
-                             dictionary['environment_file_collection_id'],
-                             task_obj.log_filepath, environment_run_options)
+                             environment_run_options, task_obj.log_filepath)
 
         # Create the after snapshot after execution is completed
         after_snapshot_create_dict = {

datmo/controller/test/test_task.py

@@ -1,10 +1,6 @@
 """
 Tests for TaskController
 """
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
 import os
 import random
 import string
@@ -23,8 +19,8 @@ def setup_method(self):
         tempfile.tempdir = '/tmp'
         self.temp_dir = tempfile.mkdtemp('project')
         self.project = ProjectController(self.temp_dir)
-        self.environment = EnvironmentController(self.temp_dir)
-        self.task = TaskController(self.temp_dir)
+        self.environment = EnvironmentController(self.temp_dir, self.project.dal.driver)
+        self.task = TaskController(self.temp_dir, self.project.dal.driver)
 
     def teardown_method(self):
         shutil.rmtree(self.temp_dir)
@@ -54,7 +50,6 @@ def test_run_helper(self):
             f.write(str("FROM datmo/xgboost:cpu"))
 
         environment_obj = self.environment.create({
-            "driver_type": "docker",
             "definition_filepath": env_def_path
         })
 
@@ -86,10 +81,9 @@ def test_run_helper(self):
             "api": False
         }
 
-        return_code, container_id, hardware_info, logs = \
+        hardware_info, return_code, container_id, logs = \
             self.task._run_helper(environment_obj.id,
-                                  environment_obj.file_collection_id,
-                                  log_filepath, options_dict)
+                                  options_dict, log_filepath)
         assert return_code == 0
         assert container_id and \
                self.task.environment_driver.get_container(container_id)
@@ -118,10 +112,9 @@ def test_run_helper(self):
             "api": True
         }
 
-        return_code, container_id, hardware_info, logs = \
+        hardware_info, return_code, container_id, logs = \
             self.task._run_helper(environment_obj.id,
-                                  environment_obj.file_collection_id,
-                                  log_filepath, options_dict)
+                                  options_dict, log_filepath)
         assert return_code == 0
         assert container_id and \
                self.task.environment_driver.get_container(container_id)