Changes to DLM for control of EHT data drop

daliuge-engine/dlg/data/drops/file.py

@@ -88,6 +88,17 @@ class FileDROP(DataDROP, PathBasedDrop):
     delete_parent_directory = dlg_bool_param("delete_parent_directory", False)
     check_filepath_exists = dlg_bool_param("check_filepath_exists", False)
 
+    # Make sure files are not deleted by default and certainly not in they are
+    # marked as precious
+    def __init__(self, *args, **kwargs):
+        if "precious" not in kwargs:
+            kwargs["precious"] = True
+        if "expireAfterUse" not in kwargs:
+            kwargs["expireAfterUse"] = False
+        if kwargs["precious"]: kwargs["expireAfterUse"] = False
+        super().__init__(*args, **kwargs)
+
+
     def sanitize_paths(self, filepath, dirname):
 
         # first replace any ENV_VARS on the names

daliuge-engine/dlg/data/drops/memory.py

@@ -53,6 +53,8 @@ def __init__(self, *args, **kwargs):
             kwargs["precious"] = False
         if "expireAfterUse" not in kwargs:
             kwargs["expireAfterUse"] = True
+        if kwargs["precious"]:
+            kwargs["expireAfterUse"] = False
         super().__init__(*args, **kwargs)
 
     def initialize(self, **kwargs):

daliuge-engine/dlg/drop.py

@@ -335,7 +335,7 @@ def __init__(self, oid, uid, **kwargs):
                 "but they are mutually exclusive" % (self,),
             )
 
-        self._expireAfterUse = self._popArg(kwargs, "expireAfterUse", True)
+        self._expireAfterUse = self._popArg(kwargs, "expireAfterUse", False)
         self._expirationDate = -1
         if not self._expireAfterUse:
             lifespan = float(self._popArg(kwargs, "lifespan", -1))
@@ -348,8 +348,11 @@ def __init__(self, oid, uid, **kwargs):
         if "expectedSize" in kwargs and kwargs["expectedSize"]:
             self._expectedSize = int(kwargs.pop("expectedSize"))
 
-        # All DROPs are precious unless stated otherwise; used for replication
-        self._precious = self._popArg(kwargs, "precious", True)
+        # No DROP is precious unless stated otherwise; used for replication
+        self._precious = self._popArg(kwargs, "precious", False)
+        # If DROP is precious, don't expire (delete) it.
+        if self._precious:
+            self._expireAfterUse = False
 
         # Useful to have access to all EAGLE parameters without a prior knowledge
         self._parameters = dict(kwargs)

daliuge-engine/dlg/lifecycle/dlm.py

@@ -290,7 +290,7 @@ def expireCompletedDrops(self):
 
             # Expire-after-use: mark as expired if all consumers
             # are finished using this DROP
-            if drop.expireAfterUse:
+            if not drop.precious and drop.expireAfterUse:
                 allDone = all(
                     c.execStatus in [AppDROPStates.FINISHED, AppDROPStates.ERROR]
                     for c in drop.consumers