Merge pull request #177 from ICRAR/speedup-remove-unmet-dependencies

Speedup instantiation of bigger graphs

daliuge-engine/dlg/drop.py

@@ -383,17 +383,25 @@ def __init__(self, oid, uid, **kwargs):
             DROPStates.INITIALIZED
         )  # no need to use synchronised self.status here
 
+    _members_cache = {}
+
+    def _get_members(self):
+        cls = self.__class__
+        if cls not in AbstractDROP._members_cache:
+            members = [
+                (name, val)
+                for c in cls.__mro__[:-1]
+                for name, val in vars(c).items()
+                if not (inspect.isfunction(val) or isinstance(val, property))
+            ]
+            AbstractDROP._members_cache[cls] = members
+        return AbstractDROP._members_cache[cls]
+
     def _extract_attributes(self, **kwargs):
         """
         Extracts component and app params then assigns them to class instance attributes.
         Component params take pro
         """
-        def getmembers(object, predicate=None):
-            for cls in object.__class__.__mro__[:-1]:
-                for k, v in vars(cls).items():
-                    if not predicate or predicate(v):
-                        yield k, v
-
         def get_param_value(attr_name, default_value):
             has_component_param = attr_name in kwargs
             has_app_param = 'applicationArgs' in kwargs \
@@ -410,9 +418,7 @@ def get_param_value(attr_name, default_value):
             return param
 
         # Take a class dlg defined parameter class attribute and create an instanced attribute on object
-        for attr_name, member in getmembers(
-            self, lambda a: not (inspect.isfunction(a) or isinstance(a, property))
-        ):
+        for attr_name, member in self._get_members():
             if isinstance(member, dlg_float_param):
                 value = get_param_value(attr_name, member.default_value)
                 if value is not None and value != "":

daliuge-engine/dlg/graph_loader.py

@@ -30,8 +30,6 @@
 
 from dlg.common.reproducibility.constants import ReproducibilityFlags
 
-from numpy import isin
-
 from . import droputils
 from .apps.socket_listener import SocketListenerApp
 from .common import Categories
@@ -53,7 +51,7 @@
 from dlg.parset_drop import ParameterSetDROP
 from .exceptions import InvalidGraphException
 from .json_drop import JsonDROP
-from .common import Categories, DropType
+from .common import DropType
 
 
 STORAGE_TYPES = {
@@ -136,19 +134,19 @@ def addLink(linkType, lhDropSpec, rhOID, force=False):
 def removeUnmetRelationships(dropSpecList):
     unmetRelationships = []
 
+    normalise_oid = lambda oid: next(iter(oid)) if isinstance(oid, dict) else oid
+
     # Step #1: Get all OIDs
-    oids = []
+    oids = set()
     for dropSpec in dropSpecList:
-        oid = dropSpec["oid"]
-        oid = list(oid.keys())[0] if isinstance(oid, dict) else oid
-        oids.append(oid)
+        oid = normalise_oid(dropSpec["oid"])
+        oids.add(oid)
 
     # Step #2: find unmet relationships and remove them from the original
     # DROP spec, keeping track of them
     for dropSpec in dropSpecList:
 
-        this_oid = dropSpec["oid"]
-        this_oid = list(this_oid.keys())[0] if isinstance(this_oid, dict) else this_oid
+        this_oid = normalise_oid(dropSpec["oid"])
         to_delete = []
 
         for rel in dropSpec:
@@ -162,7 +160,7 @@ def removeUnmetRelationships(dropSpecList):
                 # removing them from the current DROP spec
                 ds = dropSpec[rel]
                 if isinstance(ds[0], dict):
-                    ds = [list(d.keys())[0] for d in ds]
+                    ds = [next(iter(d)) for d in ds]
                 missingOids = [oid for oid in ds if oid not in oids]
                 for oid in missingOids:
                     unmetRelationships.append(DROPRel(oid, link, this_oid))
@@ -178,8 +176,7 @@ def removeUnmetRelationships(dropSpecList):
                 link = __TOONE[rel]
 
                 # Check if OID is missing
-                oid = dropSpec[rel]
-                oid = list(oid.keys())[0] if isinstance(oid, dict) else oid
+                oid = normalise_oid(dropSpec[rel])
                 if oid in oids:
                     continue
 
@@ -190,9 +187,7 @@ def removeUnmetRelationships(dropSpecList):
                 to_delete.append(rel)
 
         for rel in to_delete:
-            ds = dropSpec[rel]
-            ds = list(ds.keys())[0] if isinstance(ds, dict) else ds
-            del ds
+            del dropSpec[rel]
 
     return unmetRelationships