Merge e2aefd4 into 91d6c28

.travis.yml

@@ -69,7 +69,7 @@ matrix:
       python: "3.8"
       before_install:
       install:
-        - pip install sphinx sphinx-rtd-theme
+        - pip install sphinx sphinx-rtd-theme gputil merklelib
       script:
         - READTHEDOCS=True make -C docs html SPHINXOPTS="-W --keep-going"
 

Dockerfile

@@ -12,4 +12,4 @@ RUN cd /home/ray/daliuge/daliuge-common && pip install . \
     && sudo apt-get remove cmake gcc -y \
     && sudo apt-get clean 
 
-CMD ["dlg", "daemon", "-vv", "--no-nm"]
+CMD ["dlg", "daemon", "-vv", "--no-nm"]

OpenAPI/tests/managers_test_client.py

@@ -2,7 +2,6 @@
 import node_manager_client as nmc
 from composite_manager_client.api.default_api import DefaultApi
 
-
 nm_config = nmc.Configuration()
 nm_config.host = "127.0.0.1:8000"
 dim_config = cmc.Configuration()

OpenAPI/tests/translator_test_client.py

@@ -2,7 +2,6 @@
 
 import translator_client as tc
 
-
 translator_config = tc.Configuration()
 translator_config.host = "127.0.0.1:8084"
 

daliuge-common/dlg/__init__.py

@@ -25,7 +25,8 @@
 # set the version
 try:
     from dlg.common import version
+
     __version__ = version.full_version
 except:
     # This can happen when running from source
-    __version__ = 'unknown'
+    __version__ = "unknown"

daliuge-common/dlg/clients.py

@@ -165,6 +165,33 @@ def session_status(self, sessionId):
         )
         return status
 
+    def session_repro_status(self, sessionId):
+        """
+        Returns the reproducibility status of session `sessionId`.
+        """
+        status = self._get_json("/sessions/%s/repro/status" % (quote(sessionId),))
+        logger.debug(
+            "Successfully read session %s reproducibility status (%s) from %s:%s",
+            sessionId,
+            status,
+            self.host,
+            self.port,
+        )
+        return status
+
+    def session_repro_data(self, sessionId):
+        """
+        Returns the graph-wide reproducibility information of session `sessionId`.
+        """
+        data = self._get_json("/sessions/%s/repro/data" % (quote(sessionId),))
+        logger.debug(
+            "Successfully read session %s reproducibility data from %s:%s",
+            sessionId,
+            self.host,
+            self.port,
+        )
+        return data
+
     def graph_size(self, sessionId):
         """
         Returns the size of the graph of session `sessionId`

daliuge-common/dlg/common/__init__.py

@@ -20,7 +20,6 @@
 #    MA 02111-1307  USA
 #
 """Common utilities used by daliuge packages"""
-from numpy import isin
 from .osutils import terminate_or_kill, wait_or_kill
 from .network import check_port, connect_to, portIsClosed, portIsOpen, write_to
 from .streams import ZlibCompressedStream, JSONStream
@@ -75,7 +74,7 @@ class Categories:
     Categories.PLASMA,
     Categories.PLASMAFLIGHT,
     Categories.PARSET,
-    Categories.ENVIRONMENTVARS
+    Categories.ENVIRONMENTVARS,
 }
 APP_DROP_TYPES = [
     Categories.COMPONENT,
@@ -130,7 +129,7 @@ def _addSomething(self, other, key, IdText=None):
         if key not in self:
             self[key] = []
         if other["oid"] not in self[key]:
-            append = {other["oid"]:IdText} if IdText else other["oid"]
+            append = {other["oid"]: IdText} if IdText else other["oid"]
             self[key].append(append)
 
     def addConsumer(self, other, IdText=None):
@@ -151,21 +150,22 @@ def addOutput(self, other, IdText=None):
     def addProducer(self, other, IdText=None):
         self._addSomething(other, "producers", IdText=IdText)
 
+
 def _sanitize_links(links):
     """
     Links can now be dictionaries, but we only need
     the key.
     """
-    if isinstance(links,list):
+    if isinstance(links, list):
         nlinks = []
         for l in links:
-            if isinstance(l,dict): # could be a list of dicts
+            if isinstance(l, dict):  # could be a list of dicts
                 nlinks.extend(list(l.keys()))
             else:
-                nlinks.extend(l) if isinstance(l,list) else nlinks.append(l)
+                nlinks.extend(l) if isinstance(l, list) else nlinks.append(l)
         return nlinks
     elif isinstance(links, dict):
-        return list(links.keys()) if isinstance(links,dict) else links
+        return list(links.keys()) if isinstance(links, dict) else links
 
 
 def get_roots(pg_spec):

daliuge-common/dlg/common/network.py

@@ -19,12 +19,11 @@
 #    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 #    MA 02111-1307  USA
 #
+import contextlib
 import errno
 import logging
 import socket
 import time
-import contextlib
-
 
 logger = logging.getLogger(__name__)
 

daliuge-common/dlg/common/osutils.py

@@ -24,7 +24,6 @@
 import math
 import time
 
-
 logger = logging.getLogger(__name__)
 
 

daliuge-common/dlg/common/reproducibility/apps.py

@@ -0,0 +1,83 @@
+"""
+Contains several very basic apps to test python function reproducibility.
+"""
+import numpy as np
+
+from dlg.apps.pyfunc import PyFuncApp
+
+
+def write_in():
+    """
+    :return: "world" always
+    """
+    return "world"
+
+
+def write_out(phrase="everybody"):
+    """
+    Appends s to "Hello "
+    :param phrase: The string to be appended
+    :return: "Hello " + s
+    """
+    return "Hello " + phrase
+
+
+def numpy_av(nums):
+    """
+    Finds the mean of a list of numbers using numpy.
+    :param nums: The numbers to be averaged.
+    :return: The mean.
+    """
+    return np.asscalar(np.mean(nums))
+
+
+def my_av(nums):
+    """
+    Finds the mean of a list of numbers manually
+    :param nums: The numbers to be averaged
+    :return: The mean.
+    """
+    res = 0.0
+    for num in nums:
+        res += num
+    return res / len(nums)
+
+
+class HelloWorldPythonIn(PyFuncApp):
+    """
+    Wrapper app turning writeIn into a Python function app
+    """
+
+    def initialize(self, **kwargs):
+        fname = "dlg.common.reproducibility.apps.write_in"
+        super().initialize(func_name=fname)
+
+
+class HelloWorldPythonOut(PyFuncApp):
+    """
+    Wrapper app turning writeOut into a Python function app
+    """
+
+    def initialize(self, **kwargs):
+        fname = "dlg.common.reproducibility.apps.write_out"
+        super().initialize(func_name=fname)
+
+
+class NumpyAverage(PyFuncApp):
+    """
+    Wrapper app turning numpy_av into a Python function app
+    """
+
+    def initialize(self, **kwargs):
+        fname = "dlg.common.reproducibility.apps.numpy_av"
+        super().initialize(func_name=fname)
+
+
+class MyAverage(PyFuncApp):
+    """
+    Wrapper app turning my_av into a Python function app
+    """
+
+    def initialize(self, **kwargs):
+        fname = "dlg.common.reproducibility.apps.my_av"
+        super().initialize(func_name=fname)