LIU-308: Ensure drop precious flag is still handled

To ensure backwards compatibility rename it to persist when constructing
the drop instances. We also could add duplicated logic to do this during
translation from LG->PG, however we still need to handle this here as
old PGs might still have the flag. Additionally there don't seem to be
any obvious locations in the translator to add this logic as it is
exclusively dealing with graph traversal and just passes through any
properties is doesn't deal with straight through to the PG.

daliuge-engine/dlg/droputils.py

@@ -167,7 +167,7 @@ def copyDropContents(source: DataDROP, target: DataDROP, bufsize=65536):
     return
 
 
-def getUpstreamObjects(drop):
+def getUpstreamObjects(drop: AbstractDROP):
     """
     Returns a list of all direct "upstream" DROPs for the given+
     DROP. An DROP A is "upstream" with respect to DROP B if
@@ -179,7 +179,7 @@ def getUpstreamObjects(drop):
     In practice if A is an upstream DROP of B means that it must be moved
     to the COMPLETED state before B can do so.
     """
-    upObjs = []
+    upObjs: list[AbstractDROP] = []
     if isinstance(drop, AppDROP):
         upObjs += drop.inputs
         upObjs += drop.streamingInputs

daliuge-engine/dlg/graph_loader.py

@@ -36,6 +36,7 @@
 from .common import Categories
 from .ddap_protocol import DROPRel, DROPLinkType
 from .drop import (
+    AbstractDROP,
     ContainerDROP,
     LINKTYPE_NTO1_PROPERTY,
     LINKTYPE_1TON_APPEND_METHOD,
@@ -326,7 +327,7 @@ def createGraphFromDropSpecList(dropSpecList, session=None):
 
     # We're done! Return the roots of the graph to the caller
     logger.info("Calculating graph roots")
-    roots = []
+    roots: list[AbstractDROP] = []
     for drop in drops.values():
         if not droputils.getUpstreamObjects(drop):
             roots.append(drop)
@@ -345,7 +346,7 @@ def _createData(dropSpec, dryRun=False, session=None):
         module = importlib.import_module(".".join(parts[:-1]))
         storageType = getattr(module, parts[-1])
     else:
-        # Fall back to old behaviour or to FileDROP 
+        # Fall back to old behaviour or to FileDROP
         # if nothing else is specified
         if "storage" in dropSpec:
             storageType = STORAGE_TYPES[dropSpec["storage"]]
@@ -424,18 +425,29 @@ def _getIds(dropSpec):
 
 
 def _getKwargs(dropSpec):
+    kwargs = dict(dropSpec)
+
     REMOVE = [
         "oid",
         "uid",
         "app",
         "appclass",
     ]
-    kwargs = dict(dropSpec)
     for kw in REMOVE:
         if kw in kwargs:
             del kwargs[kw]
+
+    RENAME = {
+        "precious": "persist",
+    }
+    for find, replace in RENAME.items():
+        if find in kwargs:
+            kwargs[replace] = kwargs[find]
+            del kwargs[find]
+
     for name, spec in dropSpec.get("applicationArgs", dict()).items():
         kwargs[name] = spec["value"]
+
     return kwargs
 
 

daliuge-engine/test/test_graph_loader.py

@@ -186,3 +186,26 @@ def test_applicationArgs(self):
         self.assertEqual(app.integer, True)
         self.assertEqual(app.low, 34)
         self.assertEqual(app.high, 3456)
+
+    def test_backwardsCompatibilityWithPreciousFlag(self):
+        """
+        Ensure that precious is detected and exposed as the persist flag
+        """
+        testCases = [
+            ("precious", True),
+            ("precious", False),
+            ("persist", True),
+            ("persist", False),
+            (None, False), # Default of False is specific to MemoryDrops
+        ]
+        for key, value in testCases:
+            with self.subTest(key=key, value=value):
+                dropSpec = {"oid": "A", "type": "data", "storage": Categories.MEMORY}
+                if key is not None:
+                    dropSpec[key] = value
+
+                graph = graph_loader.createGraphFromDropSpecList([dropSpec])
+                data = graph[0]
+                self.assertIsInstance(data, InMemoryDROP)
+                self.assertEqual(value, data.persist)
+                self.assertFalse(hasattr(data, 'precious'))

daliuge-translator/dlg/dropmake/lg.graph.schema

@@ -118,6 +118,13 @@
                     "expanded": {
                         "type": "boolean"
                     },
+                    "precious": {
+                        "type": "boolean",
+                        "deprecated": true
+                    },
+                    "persist": {
+                        "type": "boolean"
+                    },
                     "inputApplicationName": {
                         "type": "string"
                     },