Improve the performance of expansion (#232)

* Addition of override for ExectionGraph to not check cycles.

* Addition of documentation for justification of override.

* Addition of a newline due to style.

* Addition of dill as a dependency.

* Fix pickle and unpickle to use dill.

maestrowf/datastructures/core/executiongraph.py

@@ -1,11 +1,11 @@
 """Module for the execution of DAG workflows."""
 from collections import deque, OrderedDict
 from datetime import datetime
+import dill
 from filelock import FileLock, Timeout
 import getpass
 import logging
 import os
-import pickle
 import shutil
 import tempfile
 
@@ -447,7 +447,7 @@ def unpickle(cls, path):
         :param path: Path to a ExecutionGraph pickle file.
         """
         with open(path, 'rb') as pkl:
-            dag = pickle.load(pkl)
+            dag = dill.load(pkl)
 
         if not isinstance(dag, cls):
             msg = "Object loaded from {path} is of type {type}. Expected an" \
@@ -472,7 +472,7 @@ def pickle(self, path):
             raise Exception(msg)
 
         with open(path, 'wb') as pkl:
-            pickle.dump(self, pkl)
+            dill.dump(self, pkl)
 
     @property
     def name(self):

maestrowf/datastructures/core/study.py

@@ -34,6 +34,7 @@
 import os
 import pickle
 import re
+from types import MethodType
 import yaml
 
 from maestrowf.abstracts import SimObject
@@ -831,4 +832,13 @@ def stage(self):
         dag.add_description(**self.description)
         dag.log_description()
 
+        # Because we're working within a Study class whose steps have already
+        # been verified to not contain a cycle, we can override the check for
+        # the execution graph. Because the execution graph is constructed from
+        # the study steps, it won't contain a cycle.
+        def _pass_detect_cycle(self):
+            pass
+
+        dag.detect_cycle = MethodType(_pass_detect_cycle, dag)
+
         return self._out_path, self._stage(dag)

setup.py

@@ -23,7 +23,8 @@
     'six',
     "filelock",
     "tabulate",
-    "enum34 ; python_version<'3.4'"
+    "enum34 ; python_version<'3.4'",
+    "dill",
   ],
   extras_require={},
   long_description_content_type='text/markdown',