Merge pull request #6 from datmo/hardware-info

Moved hardware info to environment (and other changes)

datmo/controller/code/code.py

@@ -44,23 +44,25 @@ def create(self, commit_id=None):
             "model_id": self.model.id,
         }
 
-        ## Required args
+        ## Required args for Code entity
         required_args = ["driver_type", "commit_id"]
-        traversed_args = []
         for required_arg in required_args:
             # Handle Id if provided or not
-            if required_arg == "commit_id":
-                create_dict[required_arg] = commit_id if commit_id else \
-                    self.code_driver.create_code()
-                traversed_args.append(required_arg)
-            elif required_arg == "driver_type":
+            if required_arg == "driver_type":
                 create_dict[required_arg] = self.code_driver.type
-                traversed_args.append(required_arg)
-
-        # Error if required values not present
-        if not traversed_args == required_args:
-            raise RequiredArgumentMissing(_("error",
-                                            "controller.code.create"))
+            elif required_arg == "commit_id":
+                if commit_id:
+                    create_dict[required_arg] = commit_id
+                else:
+                    create_dict[required_arg] = \
+                        self.code_driver.create_code()
+                # If code object with commit id exists, return it
+                results = self.dal.code.query({
+                    "commit_id": create_dict[required_arg]
+                })
+                if results: return results[0];
+            else:
+                raise NotImplementedError()
 
         # Create code and return
         return self.dal.code.create(create_dict)

datmo/controller/code/test/test_code.py

@@ -47,6 +47,11 @@ def test_create(self):
         assert code_obj.id
         assert code_obj.driver_type
 
+        # Test should return same code_obj if same commit_id
+        code_obj_2 = self.code.create()
+
+        assert code_obj_2 == code_obj
+
         # Test failing with random id given
         random_commit_id = "random"
         try:

datmo/controller/environment/driver/__init__.py

@@ -7,6 +7,8 @@ class EnvironmentDriver(with_metaclass(ABCMeta, object)):
 
     Methods
     -------
+    create(path, output_path)
+        create datmo environment definition
     build(name, path)
         build the environment
     run(name, options, log_filepath)
@@ -21,6 +23,34 @@ class EnvironmentDriver(with_metaclass(ABCMeta, object)):
     def __init__(self):
         pass
 
+    @abstractmethod
+    def create(self, path, output_path):
+        """
+        Create datmo environment definition
+
+        Parameters
+        ----------
+        path : str, optional
+            absolute input definition file path
+            (default is to search for standard filename in project root,
+            e.g. `Dockerfile` in project root for docker driver)
+        output_path : str, optional
+            absolute datmo output definition file path
+            (default is to create name of above file with `datmo` prefixed
+            in the same directory as `path`. e.g. `datmoDockerfile` in
+            the project root for the default above for docker driver)
+
+        Returns
+        -------
+        tuple
+            success : bool
+                True if success
+            path : str
+                absolute path for original definition
+            output_path : str
+                absolute path for datmo definition
+        """
+
     @abstractmethod
     def build(self, name, path):
         """

datmo/controller/environment/driver/dockerenv.py

@@ -6,7 +6,8 @@
 
 from datmo.util.i18n import get as _
 from datmo.util.exceptions import DoesNotExistException, \
-    EnvironmentInitFailed, EnvironmentExecutionException
+    EnvironmentInitFailed, EnvironmentExecutionException, \
+    FileAlreadyExistsException
 from datmo.controller.environment.driver import EnvironmentDriver
 
 
@@ -59,7 +60,28 @@ def is_initialized(self):
         self._is_initialized = False
         return self._is_initialized
 
+    def create(self, path=None, output_path=None):
+        if not path:
+            path = os.path.join(self.filepath,
+                                "Dockerfile")
+        if not output_path:
+            directory, filename = os.path.split(path)
+            output_path = os.path.join(directory,
+                                       "datmo" + filename)
+        if not os.path.isfile(path):
+            raise DoesNotExistException(_("error",
+                                          "controller.environment.driver.docker.create.dne",
+                                          path))
+        if os.path.isfile(output_path):
+            raise FileAlreadyExistsException(_("error",
+                                               "controller.environment.driver.docker.create.exists",
+                                               output_path))
+        success = self.form_datmo_definition_file(input_definition_path=path,
+                                        output_definition_path=output_path)
+        return success, path, output_path
+
     def build(self, name, path):
+
         return self.build_image(name, path)
 
     def run(self, name, options, log_filepath):

datmo/controller/environment/driver/test/test_dockerenv.py

@@ -11,7 +11,8 @@
 import uuid
 
 from datmo.controller.environment.driver.dockerenv import DockerEnvironmentDriver
-from datmo.util.exceptions import EnvironmentInitFailed
+from datmo.util.exceptions import EnvironmentInitFailed, \
+    DoesNotExistException, FileAlreadyExistsException
 
 
 class TestDockerEnv():
@@ -48,6 +49,56 @@ def test_instantiation_not_connected(self):
             thrown = True
         assert thrown
 
+    def test_create(self):
+        input_dockerfile_path = os.path.join(self.docker_environment_manager.filepath,
+                                             "Dockerfile")
+        output_dockerfile_path = os.path.join(self.docker_environment_manager.filepath,
+                                              "datmoDockerfile")
+        # Test both default values
+        success, path, output_path = \
+            self.docker_environment_manager.create()
+
+        assert success and \
+               os.path.isfile(output_dockerfile_path) and \
+               "datmo" in open(output_dockerfile_path, "r").read()
+        assert path == input_dockerfile_path
+        assert output_path == output_dockerfile_path
+        os.remove(output_dockerfile_path)
+
+        # Test default values for output
+        success, path, output_path = \
+            self.docker_environment_manager.create(input_dockerfile_path)
+
+        assert success and \
+               os.path.isfile(output_dockerfile_path) and \
+               "datmo" in open(output_dockerfile_path, "r").read()
+        assert path == input_dockerfile_path
+        assert output_path == output_dockerfile_path
+        os.remove(output_dockerfile_path)
+
+        # Test both values given
+        success, path, output_path = \
+            self.docker_environment_manager.create(input_dockerfile_path,
+                                                        output_dockerfile_path)
+        assert success and \
+               os.path.isfile(output_dockerfile_path) and \
+               "datmo" in open(output_dockerfile_path, "r").read()
+        assert path == input_dockerfile_path
+        assert output_path == output_dockerfile_path
+
+        # Test exception for path does not exist
+        try:
+            self.docker_environment_manager.create("nonexistant_path")
+        except DoesNotExistException:
+            assert True
+
+        # Test exception for output path already exists
+        try:
+            self.docker_environment_manager.create(
+                output_path=output_dockerfile_path)
+        except FileAlreadyExistsException:
+            assert True
+
     def test_build(self):
         name = str(uuid.uuid1())
         path = os.path.join(self.docker_environment_manager.filepath,

datmo/controller/environment/environment.py

@@ -1,8 +1,10 @@
 import os
+import platform
 
-from datmo.util.i18n import get as _
+from datmo.util.i18n import get as __
 from datmo.controller.base import BaseController
 from datmo.controller.file.file_collection import FileCollectionController
+from datmo.util.json_store import JSONStore
 from datmo.util.exceptions import RequiredArgumentMissing, \
     DoesNotExistException
 
@@ -33,8 +35,18 @@ def create(self, dictionary):
         Parameters
         ----------
         dictionary : dict
-            definition_filepath : str
-                absolute filepath to the environment definition file (e.g. ./Dockerfile)
+            optional values to populate required environment entity args
+                definition_filepath : str, optional
+                    absolute filepath to the environment definition file
+                    (default is to use driver default filepath)
+                hardware_info : dict, optional
+                    information about the environment hardware
+                    (default is to extract hardware from platform currently running)
+            optional values to populate optional  environment entity args
+                description : str, optional
+                    description of the environment
+                    (default is blank)
+
 
         Returns
         -------
@@ -52,32 +64,80 @@ def create(self, dictionary):
             "model_id": self.model.id,
         }
 
-        ## Required args
-        required_args = ["driver_type", "file_collection_id", "definition_filename"]
+        ## Required args for Environment entity
+        required_args = ["driver_type", "definition_filename",
+                         "hardware_info", "file_collection_id", "unique_hash"]
         for required_arg in required_args:
-            # Add in any values that are
-            if required_arg in dictionary:
-                create_dict[required_arg] = dictionary[required_arg]
-            else:
-                # Pull in driver type from base
-                if required_arg == "driver_type":
-                    create_dict[required_arg] = self.environment_driver.type
-                # File setup
-                elif required_arg == "file_collection_id":
-                    if "definition_filepath" not in dictionary:
-                        raise RequiredArgumentMissing(_("error",
-                                                        "controller.environment.create"))
-                    #  Create datmo definition file
-                    definition_path, definition_filename = os.path.split(dictionary['definition_filepath'])
-                    datmo_definition_filepath = os.path.join(definition_path, "datmo" + definition_filename)
-                    self.environment_driver.form_datmo_definition_file(input_definition_path=dictionary['definition_filepath'],
-                                                                       output_definition_path=datmo_definition_filepath)
-                    # Add definition_filepath and datmo_definition_filepath to file_collection
-                    filepaths = [dictionary['definition_filepath'], datmo_definition_filepath]
-                    create_dict['file_collection_id'] = self.file_collection.create(filepaths).id
+            # Pull in driver type from base
+            if required_arg == "driver_type":
+                create_dict[required_arg] = self.environment_driver.type
+            elif required_arg == "definition_filename":
+                if "definition_filepath" in dictionary:
+                    original_definition_filepath = dictionary['definition_filepath']
+                    # Split up the given path and save definition filename
+                    definition_path, definition_filename = \
+                        os.path.split(original_definition_filepath)
+                    create_dict['definition_filename'] = definition_filename
+                    # Create datmo environment definition in the same dir as definition filepath
+                    datmo_definition_filepath = \
+                        os.path.join(definition_path, "datmo" + definition_filename)
+                    self.environment_driver.create(path=dictionary['definition_filepath'],
+                                                   output_path=datmo_definition_filepath)
+                else:
+                    # Use the default create to find environment definition
+                    _, original_definition_filepath, datmo_definition_filepath = \
+                        self.environment_driver.create()
+                    # Split up the default path obtained to save the definition name
+                    definition_path, definition_filename = \
+                        os.path.split(original_definition_filepath)
                     create_dict['definition_filename'] = definition_filename
 
-        ## Optional args
+            # Extract the hardware information
+            elif required_arg == "hardware_info":
+                if "hardware_info" in dictionary:
+                    create_dict['hardware_info'] = dictionary['hardware_info']
+                else:
+                    # Extract hardware info of the container (currently taking from system platform)
+                    # TODO: extract hardware information directly from the container
+                    (system, node, release, version, machine, processor) = platform.uname()
+                    create_dict['hardware_info'] = {
+                        'system': system,
+                        'node': node,
+                        'release': release,
+                        'version': version,
+                        'machine': machine,
+                        'processor': processor
+                    }
+                # Create hardware info file in definition path
+                hardware_info_filepath = os.path.join(definition_path, "hardware_info")
+                _ = JSONStore(hardware_info_filepath,
+                              initial_dict=create_dict['hardware_info'])
+            # File collection setup using files created above
+            elif required_arg == "file_collection_id":
+                # Add all environment files to collection:
+                # definition path, datmo_definition_path, hardware_info
+                filepaths = [original_definition_filepath, datmo_definition_filepath,
+                             hardware_info_filepath]
+                file_collection_obj = self.file_collection.create(filepaths)
+                create_dict['file_collection_id'] = file_collection_obj.id
+
+                # Delete temporary files created once transfered into file collection
+                os.remove(datmo_definition_filepath)
+                os.remove(hardware_info_filepath)
+            # Create new unique hash or find existing from the file collection above
+            elif required_arg == "unique_hash":
+                create_dict['unique_hash'] = file_collection_obj.filehash
+                # Check if unique hash is unique or not.
+                # If not, DO NOT CREATE Environment and return existing Environment object
+                results = self.dal.environment.query({
+                    "unique_hash": file_collection_obj.filehash
+                })
+                if results: return results[0];
+            else:
+                NotImplementedError()
+
+
+        ## Optional args for Environment entity
         optional_args = ["description"]
         for optional_arg in optional_args:
             if optional_arg in dictionary:
@@ -107,11 +167,12 @@ def build(self, id):
         """
         environment_obj = self.dal.environment.get_by_id(id)
         if not environment_obj:
-            raise DoesNotExistException(_("error",
+            raise DoesNotExistException(__("error",
                                           "controller.environment.build",
                                           id))
         file_collection_obj = self.dal.file_collection.\
             get_by_id(environment_obj.file_collection_id)
+        # TODO: Check hardware info here if different from creation time
         # Build the Environment with the driver
         datmo_definition_filepath = os.path.join(self.home, file_collection_obj.path,
                                                  "datmo" + environment_obj.definition_filename)
@@ -151,6 +212,7 @@ def run(self, id, options, log_filepath):
         logs : str
             string version of output logs for the container
         """
+        # TODO: Check hardware info here if different from creation time
         final_return_code, run_id, logs = \
             self.environment_driver.run(id, options, log_filepath)
         return final_return_code, run_id, logs
@@ -179,7 +241,7 @@ def delete(self, id):
         """
         environment_obj = self.dal.environment.get_by_id(id)
         if not environment_obj:
-            raise DoesNotExistException(_("error",
+            raise DoesNotExistException(__("error",
                                           "controller.environment.delete",
                                           id))
         # Remove file collection