Refactor to move DAG operations to the back end Conductor (#242)

* Removal of SimObject references.

* Addition of PickleInterface.

* Derive Study and ExecutionGraph from PickleInterface.

* Some style clean up.

* Clean up unused dill import.

* Checking in to develop on another machine.

* Start of pre-check.

* Removal of precheck.

* Tweaks to Maestro and Conductor.

* Initial interface and refactor to Conductor class.

* Refactor of monitor_study

* Tweaks to backend conductor to use Conductor class.

* Tweaks to Maestro frontend to use Conductor class.

* Minor bug fixes in Conductor class.

* Minor tweaks to user cancel in Conductor class.

* Port status to the Conductor class.

* Continued additions to port to Conductor class.

* Slight fix to fix flake8 violation.

* Removal of named argument *, python2.7 does not support it.

* Refactor to remove parser and logging from Conductor class.

* Style clean up to fix flake8 errors.

* Updates to the docstrings for PickleInterface.

* Updates to the Conductor docstrings.

* Small flake8 error fix.

maestrowf/abstracts/__init__.py

@@ -33,23 +33,56 @@
 This module contains all of the abstract classes and APIs for defining objects.
 Abstracts include abstract data stuctures (like a graph), APIs for concepts
 such as queueing adapters and environment APIs, as well as fundamental data
-structures like a SimObject.
+structures.
 """
 # NOTE: Some of these abstracts will be moved in the future. The Graph abstract
 # class does not belong here, and should be moved to something more general.
-# NOTE: The SimObject base class may not be required, since it basically
-# just requires objects to be dictionaries.
+import dill
+import logging
 
 from maestrowf.abstracts.abstractclassmethod import abstractclassmethod
 from maestrowf.abstracts.envobject import Dependency, Source, Substitution
 from maestrowf.abstracts.graph import Graph
-from maestrowf.abstracts.simobject import SimObject
 from maestrowf.abstracts.specification import Specification
 
 
-__all__ = ("abstractclassmethod", "Dependency", "Graph", "SimObject",
+__all__ = ("abstractclassmethod", "Dependency", "Graph", "PickleInterface",
            "Singleton", "Source", "Specification", "Substitution")
 
+LOGGER = logging.getLogger(__name__)
+
+
+class PickleInterface:
+    """A mixin class that implements a general pickle interface using dill."""
+
+    @classmethod
+    def unpickle(cls, path):
+        """
+        Load a pickled instance from a pickle file.
+
+        :param path: Path to a pickle file containing a class instance.
+        """
+        with open(path, 'rb') as pkl:
+            obj = dill.load(pkl)
+
+        if not isinstance(obj, cls):
+            msg = "Object loaded from {path} is of type {type}. Expected an" \
+                  " object of type '{cls}.'".format(path=path, type=type(obj),
+                                                    cls=type(cls))
+            LOGGER.error(msg)
+            raise TypeError(msg)
+
+        return obj
+
+    def pickle(self, path):
+        """
+        Generate a pickle file of of a class instance.
+
+        :param path: The path to write the pickle to.
+        """
+        with open(path, 'wb') as pkl:
+            dill.dump(self, pkl)
+
 
 class _Singleton(type):
     _instances = {}

maestrowf/abstracts/envobject.py

@@ -33,13 +33,11 @@
 import logging
 import six
 
-from maestrowf.abstracts.simobject import SimObject
-
-logger = logging.getLogger(__name__)
+LOGGER = logging.getLogger(__name__)
 
 
 @six.add_metaclass(ABCMeta)
-class EnvObject(SimObject):
+class EnvObject:
     """
     An abstract class representing objects that exist in a study's environment.
 
@@ -64,7 +62,6 @@ def _verify(self):
 
         :returns: True if the EnvObject is verified, False otherwise.
         """
-        pass
 
     def _verification(self, error):
         """
@@ -73,7 +70,7 @@ def _verification(self, error):
         :param error: String containing a custom error message.
         """
         if not self._verify():
-            logger.exception(error)
+            LOGGER.exception(error)
             raise ValueError(error)
 
 
@@ -94,10 +91,8 @@ def substitute(self, data):
         :returns: A string equal to the original string data with substitutions
             made (if any were performed).
         """
-        pass
 
 
-@six.add_metaclass(ABCMeta)
 class Source(EnvObject):
     """
     Abstract class representing classes that alter environment sourcing.
@@ -126,7 +121,6 @@ def apply(self, data):
         # NOTE: This functionality has not been settled yet. The use of this
         # class or this design may not be the best for applying script sources
         # to an environment.
-        pass
 
 
 @six.add_metaclass(ABCMeta)
@@ -158,4 +152,3 @@ def acquire(self, substitutions=None):
 
         :param substitutions: List of Substitution objects that can be applied.
         """
-        pass

maestrowf/abstracts/graph.py

@@ -33,7 +33,7 @@
 
 
 @six.add_metaclass(ABCMeta)
-class Graph(object):
+class Graph:
     """An abstract graph data structure."""
 
     # NOTE: fdinatal -- 04/07/2017
@@ -50,7 +50,6 @@ def add_node(self, name, obj):
         :param name: String identifier of the node.
         :param obj: An object representing the value of the node.
         """
-        pass
 
     @abstractmethod
     def add_edge(self, src, dest):
@@ -60,7 +59,6 @@ def add_edge(self, src, dest):
         :param src: Source vertex name.
         :param dest: Destination vertex name.
         """
-        pass
 
     @abstractmethod
     def remove_edge(self, src, dest):
@@ -70,4 +68,3 @@ def remove_edge(self, src, dest):
         :param src: Source vertex name.
         :param dest: Destination vertex name.
         """
-        pass

maestrowf/abstracts/specification.py

@@ -30,11 +30,11 @@
 from abc import ABCMeta, abstractmethod, abstractproperty
 import six
 
-from maestrowf.abstracts import abstractclassmethod
+from . import abstractclassmethod
 
 
 @six.add_metaclass(ABCMeta)
-class Specification(object):
+class Specification:
     """
     Abstract class for loading and verifying a Study Specification
     """
@@ -48,7 +48,6 @@ def load_specification(cls, path):
         :returns: A specification object containing the information loaded
         from path.
         """
-        pass
 
     @abstractclassmethod
     def load_specification_from_stream(cls, stream):
@@ -58,14 +57,12 @@ def load_specification_from_stream(cls, stream):
         :param stream: Raw text stream containing specification data.
         :returns: A specification object containing the information in string.
         """
-        pass
 
     @abstractmethod
     def verify(self):
         """
         Verify the whole specification.
         """
-        pass
 
     @abstractmethod
     def get_study_environment(self):
@@ -74,7 +71,6 @@ def get_study_environment(self):
 
         :returns: A StudyEnvironment object with the data in the specification.
         """
-        pass
 
     @abstractmethod
     def get_parameters(self):
@@ -83,7 +79,6 @@ def get_parameters(self):
 
         :returns: A ParameterGenerator with data from the specification.
         """
-        pass
 
     @abstractmethod
     def get_study_steps(self):
@@ -92,7 +87,6 @@ def get_study_steps(self):
 
         :returns: A list of StudyStep objects.
         """
-        pass
 
     @abstractproperty
     def output_path(self):
@@ -101,7 +95,6 @@ def output_path(self):
 
         :returns: Returns OUTPUT_PATH if it exists, empty string otherwise.
         """
-        pass
 
     @abstractproperty
     def name(self):
@@ -110,7 +103,6 @@ def name(self):
 
         :returns: The name of the study described by the specification.
         """
-        pass
 
     @name.setter
     def name(self, value):
@@ -119,7 +111,6 @@ def name(self, value):
 
         :param value: String value representing the new name.
         """
-        pass
 
     @abstractproperty
     def desc(self):
@@ -129,7 +120,6 @@ def desc(self):
         :returns: A string containing the description of the study
             specification.
         """
-        pass
 
     @desc.setter
     def desc(self, value):
@@ -138,4 +128,3 @@ def desc(self, value):
 
         :param value: String value representing the new description.
         """
-        pass