diff --git a/daliuge-engine/dlg/data/drops/file.py b/daliuge-engine/dlg/data/drops/file.py index 3d38a4eee..ed4e640ef 100644 --- a/daliuge-engine/dlg/data/drops/file.py +++ b/daliuge-engine/dlg/data/drops/file.py @@ -88,6 +88,17 @@ class FileDROP(DataDROP, PathBasedDrop): delete_parent_directory = dlg_bool_param("delete_parent_directory", False) check_filepath_exists = dlg_bool_param("check_filepath_exists", False) + # Make sure files are not deleted by default and certainly not in they are + # marked as precious + def __init__(self, *args, **kwargs): + if "precious" not in kwargs: + kwargs["precious"] = True + if "expireAfterUse" not in kwargs: + kwargs["expireAfterUse"] = False + if kwargs["precious"]: kwargs["expireAfterUse"] = False + super().__init__(*args, **kwargs) + + def sanitize_paths(self, filepath, dirname): # first replace any ENV_VARS on the names diff --git a/daliuge-engine/dlg/data/drops/memory.py b/daliuge-engine/dlg/data/drops/memory.py index fecfd4735..48399e3a9 100644 --- a/daliuge-engine/dlg/data/drops/memory.py +++ b/daliuge-engine/dlg/data/drops/memory.py @@ -53,6 +53,8 @@ def __init__(self, *args, **kwargs): kwargs["precious"] = False if "expireAfterUse" not in kwargs: kwargs["expireAfterUse"] = True + if kwargs["precious"]: + kwargs["expireAfterUse"] = False super().__init__(*args, **kwargs) def initialize(self, **kwargs): diff --git a/daliuge-engine/dlg/drop.py b/daliuge-engine/dlg/drop.py index 284c63f64..8825794dd 100644 --- a/daliuge-engine/dlg/drop.py +++ b/daliuge-engine/dlg/drop.py @@ -335,7 +335,7 @@ def __init__(self, oid, uid, **kwargs): "but they are mutually exclusive" % (self,), ) - self._expireAfterUse = self._popArg(kwargs, "expireAfterUse", True) + self._expireAfterUse = self._popArg(kwargs, "expireAfterUse", False) self._expirationDate = -1 if not self._expireAfterUse: lifespan = float(self._popArg(kwargs, "lifespan", -1)) @@ -348,8 +348,11 @@ def __init__(self, oid, uid, **kwargs): if "expectedSize" in kwargs and kwargs["expectedSize"]: self._expectedSize = int(kwargs.pop("expectedSize")) - # All DROPs are precious unless stated otherwise; used for replication - self._precious = self._popArg(kwargs, "precious", True) + # No DROP is precious unless stated otherwise; used for replication + self._precious = self._popArg(kwargs, "precious", False) + # If DROP is precious, don't expire (delete) it. + if self._precious: + self._expireAfterUse = False # Useful to have access to all EAGLE parameters without a prior knowledge self._parameters = dict(kwargs) diff --git a/daliuge-engine/dlg/lifecycle/dlm.py b/daliuge-engine/dlg/lifecycle/dlm.py index bf751e2ea..e28049d48 100644 --- a/daliuge-engine/dlg/lifecycle/dlm.py +++ b/daliuge-engine/dlg/lifecycle/dlm.py @@ -290,7 +290,7 @@ def expireCompletedDrops(self): # Expire-after-use: mark as expired if all consumers # are finished using this DROP - if drop.expireAfterUse: + if not drop.precious and drop.expireAfterUse: allDone = all( c.execStatus in [AppDROPStates.FINISHED, AppDROPStates.ERROR] for c in drop.consumers