daliuge 1.0.0

.gitignore

@@ -10,10 +10,12 @@ dist
 .settings
 .pytest_cache
 .vscode
-dlg/version.py
-test/apps/libdynlib_example.so
-test/apps/dynlib_example.o
-test/apps/libdynlib_example2.so
-test/apps/dynlib_example2.o
+daliuge-runtime/dlg/runtime/version.py
+daliuge-common/dlg/common/version.py
+daliuge-translator/dlg/translator/version.py
+daliuge-runtime/test/apps/libdynlib_example.so
+daliuge-runtime/test/apps/dynlib_example.o
+daliuge-runtime/test/apps/libdynlib_example2.so
+daliuge-runtime/test/apps/dynlib_example2.o
 .idea
 daliuge.iml

.travis.yml

@@ -27,13 +27,19 @@ sudo: required
 
 # let's go!
 language: python
-python:
- - "2.7"
- - "3.4"
- - "3.5"
- - "3.6"
- - "3.7"
- - "3.8"
+matrix:
+  include:
+    - python: "3.8"
+      env: NO_DLG_RUNTIME=1
+    - python: "3.8"
+      env: NO_DLG_TRANSLATOR=1
+    - python: "3.8"
+      env: TEST_OPENAPI=1
+    - python: "3.7"
+    - python: "3.6"
+    - python: "3.5"
+    - python: "3.4"
+    - python: "2.7"
 
 # We want to use docker during the tests
 services:
@@ -42,22 +48,22 @@ services:
 # Try to speed up builds by caching our dependencies
 cache: pip
 
-# Support for coveralls
 before_install:
  - pip install -U coveralls pytest pytest-cov
  - pip install -U setuptools pip wheel
 
-# install daliuge in editable mode
-# This will ensure both that our dependencies are installed
-# and that our version file is generated here.
- - pip install -e .
+install:
+ - pip install -e daliuge-common/
+ - test -n "$NO_DLG_TRANSLATOR" || pip install -e daliuge-translator/
+ - test -n "$NO_DLG_RUNTIME" || pip install -e daliuge-runtime/
 
 # run the tests, making sure subprocesses generate coverage information
 script:
- - mkdir tmp
- - echo -e "import coverage\ncoverage.process_startup()" > tmp/sitecustomize.py
- - PYTHONPATH=$PWD/tmp py.test --cov
- - test $TRAVIS_PYTHON_VERSION != 3.8 || (cd OpenAPI/tests && ./test_managers_openapi.sh)
+ - COVFILES=
+ - test -n "$NO_DLG_TRANSLATOR" || { (cd daliuge-translator && py.test --cov) && COVFILES+=" daliuge-translator/.coverage"; }
+ - test -n "$NO_DLG_RUNTIME" || { (cd daliuge-runtime && py.test --cov) && COVFILES+=" daliuge-runtime/.coverage"; }
+ - coverage combine $COVFILES
+ - test -z "$TEST_OPENAPI" || (cd OpenAPI/tests && ./test_managers_openapi.sh)
 
 # Publish to coveralls (only once per commit, so only using one environment)
 after_success:

README.rst

@@ -31,19 +31,49 @@ documentation <https://daliuge.readthedocs.io/>`_
 Installation
 ------------
 
-To get the latest stable version::
+To get the latest stable version of the full package::
 
  pip install daliuge
 
-You can also install directly from GitHub::
+If you only want the translator engine and don't need the runtime,
+or vice-versa, you can install them separately::
 
- pip install git+https://github.com/ICRAR/daliuge
+ pip install daliuge-translator
+ pip install daliuge-runtime
+
+.. You can also install each directly from GitHub::
+..
+..  pip install "git+https://github.com/ICRAR/daliuge#egg=daliuge-common&subdirectory=daliuge-common"
+..  pip install "git+https://github.com/ICRAR/daliuge#egg=daliuge-translator&subdirectory=daliuge-translator"
+..  pip install "git+https://github.com/ICRAR/daliuge#egg=daliuge-runtime&subdirectory=daliuge-runtime"
+..  pip install "git+https://github.com/ICRAR/daliuge"
 
 Or if you plan to develop |daliuge|::
 
  git clone https://github.com/ICRAR/daliuge
  cd daliuge
- pip install -e .
+ pip install -e daliuge-common
+ pip install -e daliuge-translator # optional
+ pip install -e daliuge-runtime    # optional
+
+
+Porting from |daliuge| 0.X
+--------------------------
+
+With the release of |daliuge| 1.0.0
+the code has been broken down into separate packages
+to accommodate leaner and easier installations
+when only a subset of the functionality is required.
+In doing so we tried to maintain
+as much backward compatibility as possible,
+but there are some minor exceptions:
+
+ * Code doing ``from dlg import delayed`` or similar must be changed
+   to ``from dlg.runtime import delayed``.
+ * Scripts finding the include directory path for daliuge headers
+   using code like ``python -c 'import dlg; print(dlg.get_include_dir())``
+   should switch to invoke ``dlg include_dir`` instead.
+
 
 .. |daliuge| replace:: DALiuGE
 .. _ICRAR: http://www.icrar.org

RELEASE.rst

@@ -0,0 +1,53 @@
+Release procedure
+=================
+
+* Make sure you are in the master branch, with the latest changes,
+  and a clean copy::
+
+  git checkout master
+  git pull
+  git status
+
+* Decide a new version number (with major, minor, patch components) and apply it
+  in all packages' ``setup.py``::
+
+  VERSION_MAJOR=X
+  VERSION_MINOR=Y
+  VERSION_PATCH=Z
+  VERSION=$VERSION_MAJOR.$VERSION_MINOR.$VERSION_PATCH
+  SETUP_FILES="daliuge-common/setup.py daliuge-translator/setup.py daliuge-runtime/setup.py setup.py"
+  sed -i "s/MAJOR = .*/MAJOR = $VERSION_MAJOR/; s/MINOR = .*/MINOR = $VERSION_MINOR/; s/PATCH = .*/PATCH = $VERSION_PATCH/" $SETUP_FILES
+
+* Double-check these are the only changes you will commit::
+
+  git diff
+
+* Commit the version changes::
+
+  git commit $SETUP_FILES -m "daliuge $VERSION"
+
+* Tag the repository with a new version name tag (using ``vX.Y.Z`` format)::
+
+  git tag v$VERSION -m "daliuge $VERSION"
+  # Alternatively, if you want to sign it
+  git tag v$VERSION -m "daliuge $VERSION" -s
+
+* Push the new commit and the tag to GitHub::
+
+  git push origin master v$VERSION
+
+* Produce and collect source distributions for all packages::
+
+  python setup.py sdist
+  for package in daliuge-common daliuge-translator daliuge-runtime; do
+    cd $package
+    python setup.py sdist
+    cp dist/*-$VERSION.tar.gz $OLDPWD/dist
+    cd $OLDPWD
+  done
+
+* Upload to PyPI::
+
+  pip install twine
+  # Adjust credentials in ~/.pypirc, then
+  twine upload dist/daliuge*-$VERSION.tar.gz

daliuge-common/dlg/__init__.py

@@ -0,0 +1,24 @@
+#
+#    ICRAR - International Centre for Radio Astronomy Research
+#    (c) UWA - The University of Western Australia, 2020
+#    Copyright by UWA (in the framework of the ICRAR)
+#    All rights reserved
+#
+#    This library is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU Lesser General Public
+#    License as published by the Free Software Foundation; either
+#    version 2.1 of the License, or (at your option) any later version.
+#
+#    This library is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#    Lesser General Public License for more details.
+#
+#    You should have received a copy of the GNU Lesser General Public
+#    License along with this library; if not, write to the Free Software
+#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+#    MA 02111-1307  USA
+#
+
+# Declaring this as a namespace package
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # @ReservedAssignment

dlg/manager/client.py → daliuge-common/dlg/clients.py

@@ -25,7 +25,7 @@
 from six.moves import urllib_parse as urllib  # @UnresolvedImport
 
 from . import constants
-from ..restutils import RestClient
+from .restutils import RestClient
 
 
 logger = logging.getLogger(__name__)

daliuge-common/dlg/common/__init__.py

@@ -0,0 +1,141 @@
+#
+#    ICRAR - International Centre for Radio Astronomy Research
+#    (c) UWA - The University of Western Australia, 2020
+#    Copyright by UWA (in the framework of the ICRAR)
+#    All rights reserved
+#
+#    This library is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU Lesser General Public
+#    License as published by the Free Software Foundation; either
+#    version 2.1 of the License, or (at your option) any later version.
+#
+#    This library is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#    Lesser General Public License for more details.
+#
+#    You should have received a copy of the GNU Lesser General Public
+#    License along with this library; if not, write to the Free Software
+#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+#    MA 02111-1307  USA
+#
+"""Common utilities used by daliuge packages"""
+import sys
+
+from .osutils import terminate_or_kill, wait_or_kill
+from .network import check_port, connect_to, portIsClosed, portIsOpen, write_to
+from .streams import ZlibCompressedStream, JSONStream
+
+STORAGE_TYPES = {'memory', 'file', 'ngas', 'null', 'json'}
+
+if sys.version_info[0] > 2:
+    def b2s(b, enc='utf8'):
+        return b.decode(enc)
+else:
+    def b2s(b, enc='utf8'):
+        return b
+b2s.__doc__ = "Converts bytes into a string"
+
+
+class dropdict(dict):
+    """
+    An intermediate representation of a DROP that can be easily serialized
+    into a transport format such as JSON or XML.
+
+    This dictionary holds all the important information needed to call any given
+    DROP constructor. The most essential pieces of information are the
+    DROP's OID, and its type (which determines the class to instantiate).
+    Depending on the type more fields will be required. This class doesn't
+    enforce these requirements though, as it only acts as an information
+    container.
+
+    This class also offers a few utility methods to make it look more like an
+    actual DROP class. This way, users can use the same set of methods
+    both to create DROPs representations (i.e., instances of this class)
+    and actual DROP instances.
+
+    Users of this class are, for example, the graph_loader module which deals
+    with JSON -> DROP representation transformations, and the different
+    repositories where graph templates are expected to be found by the
+    DROPManager.
+    """
+    def _addSomething(self, other, key):
+        if key not in self:
+            self[key] = []
+        if other['oid'] not in self[key]:
+            self[key].append(other['oid'])
+
+    def addConsumer(self, other):
+        self._addSomething(other, 'consumers')
+    def addStreamingConsumer(self, other):
+        self._addSomething(other, 'streamingConsumers')
+    def addInput(self, other):
+        self._addSomething(other, 'inputs')
+    def addStreamingInput(self, other):
+        self._addSomething(other, 'streamingInputs')
+    def addOutput(self, other):
+        self._addSomething(other, 'outputs')
+    def addProducer(self, other):
+        self._addSomething(other, 'producers')
+
+
+def get_roots(pg_spec):
+    """
+    Returns a set with the OIDs of the dropspecs that are the roots of the given physical
+    graph specification.
+    """
+
+    # We find all the nonroots first, which are easy to spot.
+    # The rest are the roots
+    all_oids = set()
+    nonroots = set()
+    for dropspec in pg_spec:
+
+        oid = dropspec['oid']
+        all_oids.add(oid)
+
+        if dropspec['type'] in ('app', 'socket'):
+            if dropspec.get('inputs', None) or dropspec.get('streamingInputs', None):
+                nonroots.add(oid)
+            if dropspec.get('outputs', None):
+                nonroots |= set(dropspec['outputs'])
+        elif dropspec['type'] == 'plain':
+            if dropspec.get('producers', None):
+                nonroots.add(oid)
+            if dropspec.get('consumers', None):
+                nonroots |= set(dropspec['consumers'])
+            if dropspec.get('streamingConsumers', None):
+                nonroots |= set(dropspec['streamingConsumers'])
+
+    return all_oids - nonroots
+
+
+def get_leaves(pg_spec):
+    """
+    Returns a set with the OIDs of the dropspecs that are the leaves of the given physical
+    graph specification.
+    """
+
+    # We find all the nonleaves first, which are easy to spot.
+    # The rest are the leaves
+    all_oids = set()
+    nonleaves = set()
+    for dropspec in pg_spec:
+
+        oid = dropspec['oid']
+        all_oids.add(oid)
+
+        if dropspec['type'] == 'app':
+            if dropspec.get('outputs', None):
+                nonleaves.add(oid)
+            if dropspec.get('streamingInputs', None):
+                nonleaves |= set(dropspec['streamingInputs'])
+            if dropspec.get('inputs', None):
+                nonleaves |= set(dropspec['inputs'])
+        elif dropspec['type'] == 'plain':
+            if dropspec.get('producers', None):
+                nonleaves |= set(dropspec['producers'])
+            if dropspec.get('consumers', None) or dropspec.get('streamingConsumers', None):
+                nonleaves.add(oid)
+
+    return all_oids - nonleaves