Merge pull request #205 from ICRAR/LIU-306

Refactors reprodata to make storage consistent
ICRAR · Oct 14, 2022 · 8871be1 · 8871be1
2 parents 2405183 + 1f93cef
commit 8871be1
Show file tree

Hide file tree

Showing 15 changed files with 637 additions and 1,708 deletions.
diff --git a/daliuge-common/dlg/common/reproducibility/reproducibility.py b/daliuge-common/dlg/common/reproducibility/reproducibility.py
diff --git a/daliuge-engine/test/deploy/test_common.py b/daliuge-engine/test/deploy/test_common.py
@@ -33,15 +33,19 @@
 
 default_repro = {
     "rmode": "1",
-    "lg_blockhash": "x",
-    "pgt_blockhash": "y",
-    "pg_blockhash": "z",
+    "RERUN":{
+        "lg_blockhash": "x",
+        "pgt_blockhash": "y",
+        "pg_blockhash": "z",
+    }
 }
 default_graph_repro = {
     "rmode": "1",
     "meta_data": {"repro_protocol": 0.1, "hashing_alg": "_sha3.sha3_256"},
     "merkleroot": "a",
-    "signature": "b",
+    "RERUN": {
+        "signature": "b",
+    }
 }
 
 

diff --git a/daliuge-engine/test/manager/test_dim.py b/daliuge-engine/test/manager/test_dim.py
@@ -19,16 +19,15 @@
 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 #    MA 02111-1307  USA
 #
-from asyncio.log import logger
 import codecs
 import json
 import os
 import time
 import unittest
+from asyncio.log import logger
 
 import pkg_resources
 
-from dlg import runtime
 from dlg import droputils
 from dlg import utils
 from dlg.common import tool, Categories
@@ -38,21 +37,23 @@
 from dlg.testutils import ManagerStarter
 from test.manager import testutils
 
-
 hostname = "localhost"
 
-
 default_repro = {
     "rmode": "1",
-    "lg_blockhash": "x",
-    "pgt_blockhash": "y",
-    "pg_blockhash": "z",
+    "RERUN": {
+        "lg_blockhash": "x",
+        "pgt_blockhash": "y",
+        "pg_blockhash": "z",
+    }
 }
 default_graph_repro = {
     "rmode": "1",
     "meta_data": {"repro_protocol": 0.1, "hashing_alg": "_sha3.sha3_256"},
     "merkleroot": "a",
-    "signature": "b",
+    "RERUN": {
+        "signature": "b",
+    }
 }
 
 
@@ -384,7 +385,7 @@ def test_fullRound(self):
             # we need to add it manually before submitting -- otherwise it will
             # get rejected by the DIM.
             with pkg_resources.resource_stream(
-                "test", "graphs/complex.js"
+                    "test", "graphs/complex.js"
             ) as f:  # @UndefinedVariable
                 complexGraphSpec = json.load(codecs.getreader("utf-8")(f))
                 logger.debug(f"Loaded graph: {f}")
@@ -425,10 +426,10 @@ def test_fullRound(self):
             # Wait until the graph has finished its execution. We'll know
             # it finished by polling the status of the session
             while (
-                SessionStates.RUNNING
-                in testutils.get(
-                    self, "/sessions/%s/status" % (sessionId), restPort
-                ).values()
+                    SessionStates.RUNNING
+                    in testutils.get(
+                self, "/sessions/%s/status" % (sessionId), restPort
+            ).values()
             ):
                 time.sleep(0.2)
 

diff --git a/daliuge-engine/test/manager/test_dm.py b/daliuge-engine/test/manager/test_dm.py
@@ -44,15 +44,19 @@
 hostname = "localhost"
 default_repro = {
     "rmode": "1",
-    "lg_blockhash": "x",
-    "pgt_blockhash": "y",
-    "pg_blockhash": "z",
+    "RERUN":{
+        "lg_blockhash": "x",
+        "pgt_blockhash": "y",
+        "pg_blockhash": "z",
+    }
 }
 default_graph_repro = {
     "rmode": "1",
     "meta_data": {"repro_protocol": 0.1, "hashing_alg": "_sha3.sha3_256"},
     "merkleroot": "a",
-    "signature": "b",
+    "RERUN":{
+        "signature": "b",
+    }
 }
 
 

diff --git a/daliuge-engine/test/manager/test_mm.py b/daliuge-engine/test/manager/test_mm.py
@@ -41,15 +41,19 @@
 
 default_repro = {
     "rmode": "1",
-    "lg_blockhash": "x",
-    "pgt_blockhash": "y",
-    "pg_blockhash": "z",
+    "RERUN": {
+        "lg_blockhash": "x",
+        "pgt_blockhash": "y",
+        "pg_blockhash": "z",
+    }
 }
 default_graph_repro = {
     "rmode": "1",
     "meta_data": {"repro_protocol": 0.1, "hashing_alg": "_sha3.sha3_256"},
     "merkleroot": "a",
-    "signature": "b",
+    "RERUN": {
+        "signature": "b",
+    }
 }
 
 

diff --git a/daliuge-engine/test/manager/test_rest.py b/daliuge-engine/test/manager/test_rest.py
@@ -36,15 +36,19 @@
 
 default_repro = {
     "rmode": "1",
-    "lg_blockhash": "x",
-    "pgt_blockhash": "y",
-    "pg_blockhash": "z",
+    "RERUN": {
+        "lg_blockhash": "x",
+        "pgt_blockhash": "y",
+        "pg_blockhash": "z",
+    }
 }
 default_graph_repro = {
     "rmode": "1",
     "meta_data": {"repro_protocol": 0.1, "hashing_alg": "_sha3.sha3_256"},
     "merkleroot": "a",
-    "signature": "b",
+    "RERUN": {
+        "signature": "b",
+    }
 }
 
 
@@ -221,7 +225,7 @@ def test_reprodata_get(self):
         c.addGraphSpec(sid, graph_spec)
         c.deploySession(sid, completed_uids=["a"])
         response = c.session_repro_data(sid)
-        self.assertIsNotNone(response["graph"]["a"]["reprodata"]["rg_blockhash"])
+        self.assertIsNotNone(response["graph"]["a"]["reprodata"]["RERUN"]["rg_blockhash"])
         self.assertIsNotNone(response["reprodata"])
         c.destroySession(sid)
         # Test without reprodata

diff --git a/daliuge-engine/test/manager/test_scalability.py b/daliuge-engine/test/manager/test_scalability.py
@@ -34,15 +34,19 @@
 
 default_repro = {
     "rmode": "1",
-    "lg_blockhash": "x",
-    "pgt_blockhash": "y",
-    "pg_blockhash": "z",
+    "RERUN": {
+        "lg_blockhash": "x",
+        "pgt_blockhash": "y",
+        "pg_blockhash": "z",
+    }
 }
 default_graph_repro = {
     "rmode": "1",
     "meta_data": {"repro_protocol": 0.1, "hashing_alg": "_sha3.sha3_256"},
     "merkleroot": "a",
-    "signature": "b",
+    "RERUN": {
+        "signature": "b",
+    }
 }