diff --git a/.github/workflows/build-documentation.yml b/.github/workflows/build-documentation.yml new file mode 100644 index 000000000..74dfb1ba6 --- /dev/null +++ b/.github/workflows/build-documentation.yml @@ -0,0 +1,25 @@ +name: Build documentation + +on: [push, pull_request] + +jobs: + + build_docs: + name: Build RTD documentation + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + name: Install Python + with: + python-version: '3.9' + + - name: Install documentation dependencies + run: | + pip install -r docs/requirements.txt + + - name: Build documentation + run: | + READTHEDOCS=True make -C docs html SPHINXOPTS="-W --keep-going" diff --git a/.github/workflows/create-palettes.yml b/.github/workflows/create-palettes.yml new file mode 100644 index 000000000..71522a090 --- /dev/null +++ b/.github/workflows/create-palettes.yml @@ -0,0 +1,51 @@ +name: Generate component palettes + +on: [push, pull_request] + +jobs: + + run_tests: + name: Generate component palettes + runs-on: ubuntu-20.04 + env: + PROJECT_NAME: daliuge + GIT_REPO: https://github.com/ICRAR/daliuge + GITHUB_USERNAME: eagle.updater + + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + name: Install Python + with: + python-version: '3.9' + + - name: Install system dependencies + run: | + sudo apt-get update && sudo apt-get install -y doxygen xsltproc + + - name: Configure git + run: | + git config --global user.name $GITHUB_USERNAME + git config --global user.email $GITHUB_USERNAME@gmail.com + OUTPUT_FILENAME=$PROJECT_NAME-${GITHUB_REF_NAME/\//_} + echo "PROJECT_VERSION=$(git rev-parse --short HEAD)" >> $GITHUB_ENV + echo "OUTPUT_FILENAME=$OUTPUT_FILENAME" >> $GITHUB_ENV + + - name: Create palettes + run: | + python3 tools/xml2palette/xml2palette.py -t daliuge -r ./ $OUTPUT_FILENAME.palette + python3 tools/xml2palette/xml2palette.py -t template -r ./ $OUTPUT_FILENAME-template.palette + + - name: Commit palettes to EAGLE + env: + EAGLE_UPDATER_TOKEN: ${{ secrets.EAGLE_UPDATER_TOKEN }} + run: | + git clone https://$EAGLE_UPDATER_TOKEN@github.com/ICRAR/EAGLE_test_repo + mkdir -p EAGLE_test_repo/$PROJECT_NAME + mv $OUTPUT_FILENAME.palette EAGLE_test_repo/$PROJECT_NAME/ + mv $OUTPUT_FILENAME-template.palette EAGLE_test_repo/$PROJECT_NAME/ + cd EAGLE_test_repo + git add * + git diff-index --quiet HEAD || git commit -m "Automatically generated DALiuGE palette (branch $GITHUB_REF_NAME, commit $PROJECT_VERSION)" + git push diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml new file mode 100644 index 000000000..6439f8cb5 --- /dev/null +++ b/.github/workflows/linting.yml @@ -0,0 +1,25 @@ +name: Run pylint + +on: [push, pull_request] + +jobs: + + pylint: + name: Build pylint + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + name: Install Python + with: + python-version: '3.9' + + - name: Install dependencies + run: | + pip install pylint + + - name: Run pylint + run: | + pylint daliuge-common daliuge-translator daliuge-engine diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml new file mode 100644 index 000000000..745677604 --- /dev/null +++ b/.github/workflows/run-unit-tests.yml @@ -0,0 +1,88 @@ +name: Run unit tests + +on: [push, pull_request] + +jobs: + + run_tests: + name: Run unit tests with python ${{matrix.python-version}} - ${{ matrix.desc }} + runs-on: ubuntu-20.04 + strategy: + matrix: + include: + - python-version: '3.8' + test_number: 0 + engine: no + translator: yes + desc: "no engine" + - python-version: '3.8' + test_number: 1 + desc: "no translator" + engine: yes + translator: no + - python-version: '3.9' + test_number: 2 + desc: "full package" + engine: yes + translator: yes + + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + name: Install Python + with: + python-version: ${{ matrix.python-version }} + + - name: Install test dependencies + run: | + pip install -U coveralls pytest pytest-cov + pip install -U setuptools pip wheel dask dlg-example-cmpts + + - name: Install daliuge-common + run: pip install -e daliuge-common/ + + - name: Install daliuge-translator + if: ${{ matrix.translator == 'yes' }} + run: pip install -e daliuge-translator/ + + - name: Install daliuge-engine + if: ${{ matrix.engine == 'yes' }} + run: pip install -e daliuge-engine/ + + - name: Run daliuge-translator tests + if: ${{ matrix.translator == 'yes' }} + run: | + COVFILES=" daliuge-translator/.coverage" + echo "COVFILES=$COVFILES" >> $GITHUB_ENV + cd daliuge-translator + pip install -r test-requirements.txt + py.test --cov + + - name: Run daliugen-engine tests + if: ${{ matrix.engine == 'yes' }} + run: | + COVFILES="$COVFILES daliuge-engine/.coverage" + echo "COVFILES=$COVFILES" >> $GITHUB_ENV + cd daliuge-engine + py.test --cov + + - name: Combine coverage + run: coverage combine $COVFILES + + - name: Update to coveralls + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COVERALLS_FLAG_NAME: ${{ matrix.test_number }} + COVERALLS_PARALLEL: true + run: coveralls --service=github + + finish: + needs: run_tests + runs-on: ubuntu-20.04 + steps: + - name: Coveralls Finished + uses: coverallsapp/github-action@master + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + parallel-finished: true diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 000000000..1820402b7 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,3 @@ +[Main] +disable=all +enable=logging-not-lazy,logging-format-interpolation diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 6b266ebb0..000000000 --- a/.travis.yml +++ /dev/null @@ -1,104 +0,0 @@ -# -# Travis CI configuration file -# -# ICRAR - International Centre for Radio Astronomy Research -# (c) UWA - The University of Western Australia, 2016 -# Copyright by UWA (in the framework of the ICRAR) -# All rights reserved -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, -# MA 02111-1307 USA -# - -# To use docker later... -sudo: required -dist: focal - -# let's go! -language: python -matrix: - include: - # - python: "3.10" - - python: "3.8" - env: NO_DLG_RUNTIME=1 - - python: "3.8" - env: NO_DLG_TRANSLATOR=1 - - python: "3.9" - # NOTE: The OpenAPI code still needs to be removed - # - python: "3.8" - # env: TEST_OPENAPI=1 - # NOTE: commenting 3.7 for now. - # travis early build stage fails non-reproducible - # - python: "3.7" - - name: doxygen - before_install: - install: - - sudo apt-get update && sudo apt-get install -y doxygen xsltproc - before_script: - - export PROJECT_NAME=daliuge - - export PROJECT_VERSION=$(git rev-parse --short HEAD) - - export GIT_REPO=$(git config --get remote.origin.url) - - git config --global user.name $GITHUB_USERNAME - - git config --global user.email "$GITHUB_USERNAME@gmail.com" - script: - - python3 tools/xml2palette/xml2palette.py -i ./ -t daliuge -o $PROJECT_NAME-$TRAVIS_BRANCH.palette - - python3 tools/xml2palette/xml2palette.py -i ./ -t template -o $PROJECT_NAME-$TRAVIS_BRANCH-template.palette - after_success: - - git clone https://$GITHUB_TOKEN@github.com/ICRAR/EAGLE_test_repo - - mkdir -p EAGLE_test_repo/$PROJECT_NAME - - mv $PROJECT_NAME-$TRAVIS_BRANCH.palette EAGLE_test_repo/$PROJECT_NAME/ - - mv $PROJECT_NAME-$TRAVIS_BRANCH-template.palette EAGLE_test_repo/$PROJECT_NAME/ - - cd EAGLE_test_repo - - git add * - - git diff-index --quiet HEAD || git commit -m "Automatically generated DALiuGE palette (branch $TRAVIS_BRANCH, commit $PROJECT_VERSION)" - - git push - - # Build documentation similarly to how RTD does - - name: documentation - python: "3.8" - before_install: - install: - - pip install sphinx sphinx-rtd-theme gputil merklelib - script: - - READTHEDOCS=True make -C docs html SPHINXOPTS="-W --keep-going" - - -# We want to use docker during the tests -services: - - docker - -# Try to speed up builds by caching our dependencies -cache: pip - -before_install: - - pip install -U coveralls pytest pytest-cov - - pip install -U setuptools pip wheel dask - -install: - - pip install -e daliuge-common/ - - test -n "$NO_DLG_TRANSLATOR" || pip install -e daliuge-translator/ - - test -n "$NO_DLG_RUNTIME" || pip install -e daliuge-engine/ - -# run the tests, making sure subprocesses generate coverage information -script: - - COVFILES= - - test -n "$NO_DLG_TRANSLATOR" || { (cd daliuge-translator && pip install -r test-requirements.txt && py.test --cov) && COVFILES+=" daliuge-translator/.coverage"; } - - test -n "$NO_DLG_RUNTIME" || { (cd daliuge-engine && py.test --cov) && COVFILES+=" daliuge-engine/.coverage"; } - - coverage combine $COVFILES - - test -z "$TEST_OPENAPI" || (cd OpenAPI/tests && ./test_managers_openapi.sh) - -# Publish to coveralls (only once per commit, so only using one environment) -after_success: - - coveralls diff --git a/daliuge-common/dlg/clients.py b/daliuge-common/dlg/clients.py index bfef4ef66..ed9f2bb82 100644 --- a/daliuge-common/dlg/clients.py +++ b/daliuge-common/dlg/clients.py @@ -26,7 +26,6 @@ from . import constants from .restutils import RestClient - logger = logging.getLogger(__name__) compress = os.environ.get("DALIUGE_COMPRESSED_JSON", True) @@ -49,7 +48,7 @@ def stop(self): self._POST("/stop") def cancelSession(self, sessionId): - self._POST("/sessions/%s/cancel" % quote(sessionId)) + self._POST(f"/sessions/{quote(sessionId)}/cancel") def create_session(self, sessionId): """ @@ -68,7 +67,7 @@ def deploy_session(self, sessionId, completed_uids=[]): content = None if completed_uids: content = {"completed": ",".join(completed_uids)} - self._post_form("/sessions/%s/deploy" % (quote(sessionId),), content) + self._post_form(f"/sessions/{quote(sessionId)}/deploy", content) logger.debug( "Successfully deployed session %s on %s:%s", sessionId, self.host, self.port ) @@ -79,7 +78,7 @@ def append_graph(self, sessionId, graphSpec): but checking that the graph looks correct """ self._post_json( - "/sessions/%s/graph/append" % (quote(sessionId),), + f"/sessions/{quote(sessionId)}/graph/append", graphSpec, compress=compress, ) @@ -94,7 +93,7 @@ def destroy_session(self, sessionId): """ Destroys session `sessionId` """ - self._DELETE("/sessions/%s" % (quote(sessionId),)) + self._DELETE(f"/sessions/{quote(sessionId)}") logger.debug( "Successfully deleted session %s on %s:%s", sessionId, self.host, self.port ) @@ -104,7 +103,7 @@ def graph_status(self, sessionId): Returns a dictionary where the keys are DROP UIDs and the values are their corresponding status. """ - ret = self._get_json("/sessions/%s/graph/status" % (quote(sessionId),)) + ret = self._get_json(f"/sessions/{quote(sessionId)}/graph/status") logger.debug( "Successfully read graph status from session %s on %s:%s", sessionId, @@ -118,7 +117,7 @@ def graph(self, sessionId): Returns a dictionary where the key are the DROP UIDs, and the values are the DROP specifications. """ - graph = self._get_json("/sessions/%s/graph" % (quote(sessionId),)) + graph = self._get_json(f"/sessions/{quote(sessionId)}/graph") logger.debug( "Successfully read graph (%d nodes) from session %s on %s:%s", len(graph), @@ -137,7 +136,7 @@ def sessions(self): "Successfully read %d sessions from %s:%s", len(sessions), self.host, - self.port, + self.port ) return sessions @@ -145,7 +144,7 @@ def session(self, sessionId): """ Returns the details of sessions `sessionId` """ - session = self._get_json("/sessions/%s" % (quote(sessionId),)) + session = self._get_json(f"/sessions/{quote(sessionId)}") logger.debug( "Successfully read session %s from %s:%s", sessionId, self.host, self.port ) @@ -155,7 +154,7 @@ def session_status(self, sessionId): """ Returns the status of session `sessionId` """ - status = self._get_json("/sessions/%s/status" % (quote(sessionId),)) + status = self._get_json(f"/sessions/{quote(sessionId)}/status") logger.debug( "Successfully read session %s status (%s) from %s:%s", sessionId, @@ -169,7 +168,7 @@ def session_repro_status(self, sessionId): """ Returns the reproducibility status of session `sessionId`. """ - status = self._get_json("/sessions/%s/repro/status" % (quote(sessionId),)) + status = self._get_json(f"/sessions/{quote(sessionId)}/repro/status") logger.debug( "Successfully read session %s reproducibility status (%s) from %s:%s", sessionId, @@ -183,7 +182,7 @@ def session_repro_data(self, sessionId): """ Returns the graph-wide reproducibility information of session `sessionId`. """ - data = self._get_json("/sessions/%s/repro/data" % (quote(sessionId),)) + data = self._get_json(f"/sessions/{quote(sessionId)}/repro/data") logger.debug( "Successfully read session %s reproducibility data from %s:%s", sessionId, @@ -196,7 +195,7 @@ def graph_size(self, sessionId): """ Returns the size of the graph of session `sessionId` """ - count = self._get_json("/sessions/%s/graph/size" % (quote(sessionId))) + count = self._get_json(f"/sessions/{quote(sessionId)}/graph/size") logger.debug( "Successfully read session %s graph size (%d) from %s:%s", sessionId, @@ -223,17 +222,17 @@ class NodeManagerClient(BaseDROPManagerClient): """ def __init__( - self, host="localhost", port=constants.NODE_DEFAULT_REST_PORT, timeout=10 + self, host="localhost", port=constants.NODE_DEFAULT_REST_PORT, timeout=10 ): super(NodeManagerClient, self).__init__(host=host, port=port, timeout=timeout) def add_node_subscriptions(self, sessionId, node_subscriptions): self._post_json( - "/sessions/%s/subscriptions" % (quote(sessionId),), node_subscriptions + f"/sessions/{quote(sessionId)}/subscriptions", node_subscriptions ) def trigger_drops(self, sessionId, drop_uids): - self._post_json("/sessions/%s/trigger" % (quote(sessionId),), drop_uids) + self._post_json(f"/sessions/{quote(sessionId)}/trigger", drop_uids) def shutdown_node_manager(self): self._GET("/shutdown") @@ -247,10 +246,10 @@ def nodes(self): return self._get_json("/nodes") def add_node(self, node): - self._POST("/nodes/%s" % (node,), content=None) + self._POST(f"/nodes/{node}", content=None) def remove_node(self, node): - self._DELETE("/nodes/%s" % (node,)) + self._DELETE(f"/nodes/{node}") class DataIslandManagerClient(CompositeManagerClient): @@ -259,7 +258,7 @@ class DataIslandManagerClient(CompositeManagerClient): """ def __init__( - self, host="localhost", port=constants.ISLAND_DEFAULT_REST_PORT, timeout=10 + self, host="localhost", port=constants.ISLAND_DEFAULT_REST_PORT, timeout=10 ): super(DataIslandManagerClient, self).__init__( host=host, port=port, timeout=timeout @@ -272,11 +271,33 @@ class MasterManagerClient(CompositeManagerClient): """ def __init__( - self, host="localhost", port=constants.MASTER_DEFAULT_REST_PORT, timeout=10 + self, host="localhost", port=constants.MASTER_DEFAULT_REST_PORT, timeout=10 ): super(MasterManagerClient, self).__init__(host=host, port=port, timeout=timeout) def create_island(self, island_host, nodes): self._post_json( - "/managers/%s/dataisland" % (quote(island_host)), {"nodes": nodes} + f"/managers/{quote(island_host)}/dataisland", {"nodes": nodes} ) + + def dims(self): + return self._get_json("/islands") + + def add_dim(self, dim): + self._POST(f"/islands/{dim}", content=None) + + def remove_dim(self, dim): + self._DELETE(f"/islands/{dim}") + + def add_node_to_dim(self, dim, nm): + """ + Adds a nm to a dim + """ + self._POST( + f"managers/{dim}/nodes/{nm}", content=None, ) + + def remove_node_from_dim(self, dim, nm): + """ + Removes a nm from a dim + """ + self._DELETE(f"managers/{dim}/nodes/{nm}") diff --git a/daliuge-common/dlg/common/__init__.py b/daliuge-common/dlg/common/__init__.py index 5c6bbfd11..e1c2767f3 100644 --- a/daliuge-common/dlg/common/__init__.py +++ b/daliuge-common/dlg/common/__init__.py @@ -180,6 +180,17 @@ def get_roots(pg_spec): nonroots = set() for dropspec in pg_spec: + # Assumed to be reprodata / other non-drop elements + # + # TODO (rtobar): Note that this should be a temporary measure. + # In principle the pg_spec given here should be a graph, which (until + # recently) consisted on drop specifications only. The fact that repro + # data is now appended at the end of some graphs highlights the need for + # a more formal specification of graphs and other pieces of data that we + # move through the system. + if "oid" not in dropspec: + continue + oid = dropspec["oid"] all_oids.add(oid) diff --git a/daliuge-common/dlg/common/reproducibility/apps.py b/daliuge-common/dlg/common/reproducibility/apps.py deleted file mode 100644 index 03bb9773a..000000000 --- a/daliuge-common/dlg/common/reproducibility/apps.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -Contains several very basic apps to test python function reproducibility. -""" -import numpy as np - -from dlg.apps.pyfunc import PyFuncApp - - -def write_in(): - """ - :return: "world" always - """ - return "world" - - -def write_out(phrase="everybody"): - """ - Appends s to "Hello " - :param phrase: The string to be appended - :return: "Hello " + s - """ - return "Hello " + phrase - - -def numpy_av(nums): - """ - Finds the mean of a list of numbers using numpy. - :param nums: The numbers to be averaged. - :return: The mean. - """ - return np.asscalar(np.mean(nums)) - - -def my_av(nums): - """ - Finds the mean of a list of numbers manually - :param nums: The numbers to be averaged - :return: The mean. - """ - res = 0.0 - for num in nums: - res += num - return res / len(nums) - - -class HelloWorldPythonIn(PyFuncApp): - """ - Wrapper app turning writeIn into a Python function app - """ - - def initialize(self, **kwargs): - fname = "dlg.common.reproducibility.apps.write_in" - super().initialize(func_name=fname) - - -class HelloWorldPythonOut(PyFuncApp): - """ - Wrapper app turning writeOut into a Python function app - """ - - def initialize(self, **kwargs): - fname = "dlg.common.reproducibility.apps.write_out" - super().initialize(func_name=fname) - - -class NumpyAverage(PyFuncApp): - """ - Wrapper app turning numpy_av into a Python function app - """ - - def initialize(self, **kwargs): - fname = "dlg.common.reproducibility.apps.numpy_av" - super().initialize(func_name=fname) - - -class MyAverage(PyFuncApp): - """ - Wrapper app turning my_av into a Python function app - """ - - def initialize(self, **kwargs): - fname = "dlg.common.reproducibility.apps.my_av" - super().initialize(func_name=fname) diff --git a/daliuge-common/dlg/common/reproducibility/apps_lowpass.py b/daliuge-common/dlg/common/reproducibility/apps_lowpass.py deleted file mode 100644 index 0ba06b459..000000000 --- a/daliuge-common/dlg/common/reproducibility/apps_lowpass.py +++ /dev/null @@ -1,457 +0,0 @@ -""" -Implements several DALiuGE drops to build low-pass filters with various methods. -""" - -import numpy as np -import pyfftw -from dlg import droputils -from dlg.apps.simple import BarrierAppDROP -from dlg.common.reproducibility.constants import system_summary -from dlg.meta import dlg_batch_output, dlg_streaming_input -from dlg.meta import dlg_component, dlg_batch_input -from dlg.meta import dlg_int_param, dlg_list_param, dlg_float_param, dlg_bool_param - - -def determine_size(length): - """ - :param length: - :return: Computes the next largest power of two needed to contain |length| elements - """ - return int(2 ** np.ceil(np.log2(length))) - 1 - - -class LP_SignalGenerator(BarrierAppDROP): - """ - Generates a noisy sine signal for filtering. Effectively an input generator. - """ - - component_meta = dlg_component( - "LPSignalGen", - "Low-pass filter example signal generator", - [None], - [dlg_batch_output("binary/*", [])], - [dlg_streaming_input("binary/*")], - ) - - # default values - length = dlg_int_param("length", 256) - srate = dlg_int_param("sample rate", 5000) - freqs = dlg_list_param("frequencies", [440, 800, 1000, 2000]) - noise = dlg_list_param("noise", []) - series = None - - def add_noise( - self, series: np.array, mean, std, freq, sample_rate, seed, alpha=0.1 - ): - """ - A noise to the provided signal by producing random values of a given frequency - :param series: The input (and output) numpy array signal series - :param mean: The average value - :param std: The standard deviation of the value - :param freq: The frequency of the noisy signal - :param sample_rate: The sample rate of the input series - :param seed: The random seed - :param alpha: The multiplier - :return: The input series with noisy values added - """ - np.random.seed(seed) - samples = alpha * np.random.normal(mean, std, size=len(series)) - for i in range(len(series)): - samples[i] += np.sin(2 * np.pi * i * freq / sample_rate) - np.add(series, samples, out=series) - return series - - def gen_sig(self): - """ - Generates an initial signal - :return: Numpy array of signal values. - """ - series = np.zeros(self.length, dtype=np.float64) - for freq in self.freqs: - for i in range(self.length): - series[i] += np.sin(2 * np.pi * i * freq / self.srate) - return series - - def run(self): - """ - Called by DALiuGE to start signal generation. Conditionally adds noise if parameters are set - :return: Writes signal to output ports. - """ - outs = self.outputs - if len(outs) < 1: - raise Exception("At least one output required for %r" % self) - self.series = self.gen_sig() - if len(self.noise) > 0: - self.noise[0] = 1 / self.noise[0] - self.series = self.add_noise( - self.series, - self.noise[2], - self.noise[4], - self.noise[1], - self.srate, - self.noise[3], - self.noise[0], - ) - - data = self.series.tostring() - for output in outs: - output.len = len(data) - output.write(data) - - def generate_recompute_data(self): - # This will do for now - return { - "length": self.length, - "sample_rate": self.srate, - "frequencies": self.freqs, - "status": self.status, - "system": system_summary(), - } - - -class LP_WindowGenerator(BarrierAppDROP): - """ - Generates a Hann window for low-pass filtering. - """ - - component_meta = dlg_component( - "LPWindowGen", - "Low-pass filter example window generator", - [None], - [dlg_batch_output("binary/*", [])], - [dlg_streaming_input("binary/*")], - ) - - # default values - length = dlg_int_param("length", 256) - cutoff = dlg_int_param("cutoff", 600) - srate = dlg_int_param("sample_rate", 5000) - series = None - - def sinc(self, x_val: np.float64): - """ - Computes the sin_c value for the input float - :param x_val: - """ - if np.isclose(x_val, 0.0): - return 1.0 - return np.sin(np.pi * x_val) / (np.pi * x_val) - - def gen_win(self): - """ - Generates the window values. - :return: Numpy array of window series. - """ - alpha = 2 * self.cutoff / self.srate - win = np.zeros(self.length, dtype=np.float64) - for i in range(int(self.length)): - ham = 0.54 - 0.46 * np.cos( - 2 * np.pi * i / int(self.length) - ) # Hamming coefficient - hsupp = i - int(self.length) / 2 - win[i] = ham * alpha * self.sinc(alpha * hsupp) - return win - - def run(self): - """ - Called by DALiuGE to start drop execution - :return: - """ - outs = self.outputs - if len(outs) < 1: - raise Exception("At least one output required for %r" % self) - self.series = self.gen_win() - data = self.series.tostring() - for output in outs: - output.len = len(data) - output.write(data) - - def generate_recompute_data(self): - output = dict() - output["length"] = self.length - output["cutoff"] = self.cutoff - output["sample_rate"] = self.srate - output["status"] = self.status - output["system"] = system_summary() - return output - - -class LP_AddNoise(BarrierAppDROP): - """ - Component to add additional noise to a signal array. - """ - - component_meta = dlg_component( - "LPAddNoise", - "Adds noise to a signal generated " "for the low-pass filter example", - [dlg_batch_input("binary/*", [])], - [dlg_batch_output("binary/*", [])], - [dlg_streaming_input("binary/*")], - ) - - # default values - mean = dlg_float_param("avg_noise", 0.0) - std = dlg_float_param("std_deviation", 1.0) - freq = dlg_int_param("frequency", 1200) - srate = dlg_int_param("sample_rate", 5000) - seed = dlg_int_param("random_seed", 42) - alpha = dlg_float_param("noise_multiplier", 0.1) - signal = np.empty([1]) - - def add_noise(self): - """ - Adds noise at a specified frequency. - :return: Modified signal - """ - np.random.seed(self.seed) - samples = self.alpha * np.random.normal( - self.mean, self.std, size=len(self.signal) - ) - for i in range(len(self.signal)): - samples[i] += np.sin(2 * np.pi * i * self.freq / self.srate) - np.add(self.signal, samples, out=self.signal) - return self.signal - - def get_inputs(self): - """ - Reads input data into a numpy array. - :return: - """ - ins = self.inputs - if len(ins) != 1: - raise Exception("Precisely one input required for %r" % self) - - array = np.fromstring(droputils.allDropContents(ins[0])) - self.signal = np.frombuffer(array) - - def run(self): - """ - Called by DALiuGE to start drop execution. - :return: - """ - outs = self.outputs - if len(outs) < 1: - raise Exception("At least one output required for %r" % self) - self.get_inputs() - sig = self.add_noise() - data = sig.tobytes() - for output in outs: - output.len = len(data) - output.write(data) - - def generate_recompute_data(self): - return { - "mean": self.mean, - "std": self.std, - "sample_rate": self.srate, - "seed": self.seed, - "alpha": self.alpha, - "system": system_summary(), - "status": self.status, - } - - -class LP_filter_fft_np(BarrierAppDROP): - """ - Uses numpy to filter a nosiy signal. - """ - - component_meta = dlg_component( - "LP_filter_np", - "Filters a signal with " "a provided window using numpy", - [dlg_batch_input("binary/*", [])], - [dlg_batch_output("binary/*", [])], - [dlg_streaming_input("binary/*")], - ) - - PRECISIONS = { - "double": {"float": np.float64, "complex": np.complex128}, - "single": {"float": np.float32, "complex": np.complex64}, - } - precision = {} - # default values - double_prec = dlg_bool_param("doublePrec", True) - series = [] - output = np.zeros([1]) - - def initialize(self, **kwargs): - super().initialize(**kwargs) - if self.double_prec: - self.precision = self.PRECISIONS["double"] - else: - self.precision = self.PRECISIONS["single"] - - def get_inputs(self): - """ - Reads input arrays into numpy array - :return: Sets class series variable. - """ - ins = self.inputs - if len(ins) != 2: - raise Exception("Precisely two input required for %r" % self) - - array = [np.fromstring(droputils.allDropContents(inp)) for inp in ins] - self.series = array - - def filter(self): - """ - Actually performs the filtering - :return: Numpy array of filtered signal. - """ - signal = self.series[0] - window = self.series[1] - nfft = determine_size(len(signal) + len(window) - 1) - print(nfft) - sig_zero_pad = np.zeros(nfft, dtype=self.precision["float"]) - win_zero_pad = np.zeros(nfft, dtype=self.precision["float"]) - sig_zero_pad[0 : len(signal)] = signal - win_zero_pad[0 : len(window)] = window - sig_fft = np.fft.fft(sig_zero_pad) - win_fft = np.fft.fft(win_zero_pad) - out_fft = np.multiply(sig_fft, win_fft) - out = np.fft.ifft(out_fft) - return out.astype(self.precision["complex"]) - - def run(self): - """ - Called by DALiuGE to start execution - :return: - """ - outs = self.outputs - if len(outs) < 1: - raise Exception("At least one output required for %r" % self) - self.get_inputs() - self.output = self.filter() - data = self.output.tostring() - for output in outs: - output.len = len(data) - output.write(data) - - def generate_recompute_data(self): - return { - "precision_float": str(self.precision["float"]), - "precision_complex": str(self.precision["complex"]), - "system": system_summary(), - "status": self.status, - } - - -class LP_filter_fft_fftw(LP_filter_fft_np): - """ - Uses fftw to implement a low-pass filter - """ - - component_meta = dlg_component( - "LP_filter_fftw", - "Filters a signal with " "a provided window using FFTW", - [dlg_batch_input("binary/*", [])], - [dlg_batch_output("binary/*", [])], - [dlg_streaming_input("binary/*")], - ) - - def filter(self): - """ - Actually performs the filtering - :return: Filtered signal as numpy array. - """ - pyfftw.interfaces.cache.disable() - signal = self.series[0] - window = self.series[1] - nfft = determine_size(len(signal) + len(window) - 1) - sig_zero_pad = pyfftw.empty_aligned(len(signal), dtype=self.precision["float"]) - win_zero_pad = pyfftw.empty_aligned(len(window), dtype=self.precision["float"]) - sig_zero_pad[0 : len(signal)] = signal - win_zero_pad[0 : len(window)] = window - sig_fft = pyfftw.interfaces.numpy_fft.fft(sig_zero_pad, n=nfft) - win_fft = pyfftw.interfaces.numpy_fft.fft(win_zero_pad, n=nfft) - out_fft = np.multiply(sig_fft, win_fft) - out = pyfftw.interfaces.numpy_fft.ifft(out_fft, n=nfft) - return out.astype(self.precision["complex"]) - - -class LP_filter_fft_cuda(LP_filter_fft_np): - """ - Uses pycuda to implement a low-pass filter - """ - - component_meta = dlg_component( - "LP_filter_fft_cuda", - "Filters a signal with " "a provided window using cuda", - [dlg_batch_input("binary/*", [])], - [dlg_batch_output("binary/*", [])], - [dlg_streaming_input("binary/*")], - ) - - def filter(self): - """ - Actually performs the filtering - :return: - """ - import pycuda.gpuarray as gpuarray - import skcuda.fft as cu_fft - import skcuda.linalg as linalg - import pycuda.driver as cuda - from pycuda.tools import make_default_context - - cuda.init() - context = make_default_context() - device = context.get_device() - signal = self.series[0] - window = self.series[1] - linalg.init() - nfft = determine_size(len(signal) + len(window) - 1) - # Move data to GPU - sig_zero_pad = np.zeros(nfft, dtype=self.precision["float"]) - win_zero_pad = np.zeros(nfft, dtype=self.precision["float"]) - sig_gpu = gpuarray.zeros(sig_zero_pad.shape, dtype=self.precision["float"]) - win_gpu = gpuarray.zeros(win_zero_pad.shape, dtype=self.precision["float"]) - sig_zero_pad[0 : len(signal)] = signal - win_zero_pad[0 : len(window)] = window - sig_gpu.set(sig_zero_pad) - win_gpu.set(win_zero_pad) - - # Plan forwards - sig_fft_gpu = gpuarray.zeros(nfft, dtype=self.precision["complex"]) - win_fft_gpu = gpuarray.zeros(nfft, dtype=self.precision["complex"]) - sig_plan_forward = cu_fft.Plan( - sig_fft_gpu.shape, self.precision["float"], self.precision["complex"] - ) - win_plan_forward = cu_fft.Plan( - win_fft_gpu.shape, self.precision["float"], self.precision["complex"] - ) - cu_fft.fft(sig_gpu, sig_fft_gpu, sig_plan_forward) - cu_fft.fft(win_gpu, win_fft_gpu, win_plan_forward) - - # Convolve - out_fft = linalg.multiply(sig_fft_gpu, win_fft_gpu, overwrite=True) - linalg.scale(2.0, out_fft) - - # Plan inverse - out_gpu = gpuarray.zeros_like(out_fft) - plan_inverse = cu_fft.Plan( - out_fft.shape, self.precision["complex"], self.precision["complex"] - ) - cu_fft.ifft(out_fft, out_gpu, plan_inverse, True) - out_np = np.zeros(len(out_gpu), self.precision["complex"]) - out_gpu.get(out_np) - context.pop() - return out_np - - -class LP_filter_pointwise_np(LP_filter_fft_np): - """ - Uses raw numpy to implement a low-pass filter - """ - - component_meta = dlg_component( - "LP_filter_pointwise_np", - "Filters a signal with " "a provided window using cuda", - [dlg_batch_input("binary/*", [])], - [dlg_batch_output("binary/*", [])], - [dlg_streaming_input("binary/*")], - ) - - def filter(self): - return np.convolve(self.series[0], self.series[1], mode="full").astype( - self.precision["complex"] - ) diff --git a/daliuge-common/dlg/common/reproducibility/reprodata_compare.py b/daliuge-common/dlg/common/reproducibility/reprodata_compare.py index 926b6dad5..f51db3ab2 100644 --- a/daliuge-common/dlg/common/reproducibility/reprodata_compare.py +++ b/daliuge-common/dlg/common/reproducibility/reprodata_compare.py @@ -153,6 +153,8 @@ def write_outfile(data, outfilepath, outfilesuffix="summary", verbose=False): writer.writerow(fieldnames) for filepath, signature_data in data.items(): + if signature_data == {}: + continue row = [filepath] + [signature_data[rmode.value] for rmode in ALL_RMODES] writer.writerow(row) if verbose: diff --git a/daliuge-common/dlg/common/reproducibility/reproducibility.py b/daliuge-common/dlg/common/reproducibility/reproducibility.py index effb5d143..39b159028 100644 --- a/daliuge-common/dlg/common/reproducibility/reproducibility.py +++ b/daliuge-common/dlg/common/reproducibility/reproducibility.py @@ -92,14 +92,13 @@ def accumulate_lg_drop_data(drop: dict, level: ReproducibilityFlags): raise NotImplementedError( f"Reproducibility level {level.name} not yet supported" ) - category_type = drop.get( - "categoryType", "" - ) # Made conditional to support older graphs category = drop.get("category", "") # Cheeky way to get field list into dicts. map(dict, drop...) makes a copy - fields = {e.pop("name"): e["value"] for e in map(dict, drop["fields"])} - lg_fields = lg_block_fields(category_type, category, level) + fields = {e.pop("name"): e["value"] for e in map(dict, drop.get("fields", {}))} + app_fields = {e.pop("name"): e["value"] for e in map(dict, drop.get("applicationArgs", {}))} + fields.update(app_fields) + lg_fields = lg_block_fields(category, level, app_fields.keys()) data = extract_fields(fields, lg_fields) return data diff --git a/daliuge-common/dlg/common/reproducibility/reproducibility_fields.py b/daliuge-common/dlg/common/reproducibility/reproducibility_fields.py index 3d17d5be1..3de9a44b7 100644 --- a/daliuge-common/dlg/common/reproducibility/reproducibility_fields.py +++ b/daliuge-common/dlg/common/reproducibility/reproducibility_fields.py @@ -85,34 +85,29 @@ def lgt_block_fields(rmode: ReproducibilityFlags): return data -def lg_block_fields( - category: Categories, category_type: str, rmode: ReproducibilityFlags -): +def lg_block_fields(category_type: str, rmode: ReproducibilityFlags, custom_fields=None): """ Collects dict of fields and operations for all drop types at the lg layer for the supplied reproducibility standard. :param category: The broad type of drop - :param category_type: The specific type of drop :param rmode: The reproducibility level in question + :param custom_fields: Additional application args (used in custom components) :return: Dictionary of pairs """ data = {} if rmode in ( - ReproducibilityFlags.NOTHING, - ReproducibilityFlags.RERUN, - ReproducibilityFlags.REPRODUCE, - ReproducibilityFlags.REPLICATE_SCI, + ReproducibilityFlags.NOTHING, + ReproducibilityFlags.RERUN, + ReproducibilityFlags.REPRODUCE, + ReproducibilityFlags.REPLICATE_SCI, ): return data - # Drop category considerations - if category == "Application": - data["execution_time"] = FieldOps.STORE - data["num_cpus"] = FieldOps.STORE - elif category == "Group": - data["inputApplicationName"] = FieldOps.STORE - data["inputApplicationType"] = FieldOps.STORE - elif category == Categories.DATA: # An anomaly, I know - data["data_volume"] = FieldOps.STORE + # Drop category considerations - Just try to get everything we can, will be filtered later + data["execution_time"] = FieldOps.STORE + data["num_cpus"] = FieldOps.STORE + data["inputApplicationName"] = FieldOps.STORE + data["inputApplicationType"] = FieldOps.STORE + data["data_volume"] = FieldOps.STORE # Drop type considerations if category_type == Categories.START: @@ -126,8 +121,8 @@ def lg_block_fields( elif category_type == Categories.FILE: data["check_filepath_exists"] = FieldOps.STORE if rmode in ( - ReproducibilityFlags.RECOMPUTE, - ReproducibilityFlags.REPLICATE_COMP, + ReproducibilityFlags.RECOMPUTE, + ReproducibilityFlags.REPLICATE_COMP, ): data["filepath"] = FieldOps.STORE data["dirname"] = FieldOps.STORE @@ -188,6 +183,10 @@ def lg_block_fields( data["libpath"] = FieldOps.STORE elif category_type == Categories.DYNLIB_PROC_APP: data["libpath"] = FieldOps.STORE + if custom_fields is not None and rmode in ( + ReproducibilityFlags.RECOMPUTE, ReproducibilityFlags.REPLICATE_COMP): + for name in custom_fields: + data[name] = FieldOps.STORE return data diff --git a/daliuge-common/docker/Dockerfile b/daliuge-common/docker/Dockerfile index 8750cab18..c30474e35 100644 --- a/daliuge-common/docker/Dockerfile +++ b/daliuge-common/docker/Dockerfile @@ -8,7 +8,7 @@ ARG BUILD_ID LABEL stage=builder LABEL build=$BUILD_ID RUN apt-get update && \ - apt-get install -y gcc python3 python3.8-venv python3-pip python3-distutils python3-appdirs libmetis-dev curl && \ + apt-get install -y gcc python3 python3.8-venv python3-pip python3-distutils python3-appdirs libmetis-dev curl git sudo && \ apt-get clean COPY / /daliuge diff --git a/daliuge-common/setup.py b/daliuge-common/setup.py index feb01e55a..4923a9829 100644 --- a/daliuge-common/setup.py +++ b/daliuge-common/setup.py @@ -32,7 +32,7 @@ # by setuptools/pkg_resources or "final" versions. MAJOR = 2 MINOR = 3 -PATCH = 0 +PATCH = 2 VERSION = (MAJOR, MINOR, PATCH) VERSION_FILE = "dlg/common/version.py" RELEASE = True diff --git a/daliuge-engine/build_engine.sh b/daliuge-engine/build_engine.sh index 00e674eff..efc4f44ba 100755 --- a/daliuge-engine/build_engine.sh +++ b/daliuge-engine/build_engine.sh @@ -39,16 +39,15 @@ case "$1" in exit 0;; "slim") C_TAG="master" - [[ ! -z $2 ]] && C_TAG=$2 - echo "Building daliuge-engine slim version ${VCS_TAG} using daliuge-common:${C_TAG}" - docker build --build-arg VCS_TAG=${VCS_TAG} --no-cache -t icrar/daliuge-engine:${DEV_TAG} -f docker/Dockerfile.dev . + echo "Building daliuge-engine slim version ${VCS_TAG} using daliuge-common:${VCS_TAG}" + docker build --build-arg VCS_TAG=${VCS_TAG} --no-cache -t icrar/daliuge-engine.big:${VCS_TAG} -f docker/Dockerfile . echo "Build finished! Slimming the image now" echo ">>>>> docker-slim output <<<<<<<<<" docker run -it --rm -v /var/run/docker.sock:/var/run/docker.sock dslim/docker-slim build --include-shell \ --include-path /etc --include-path /usr/local/lib --include-path /usr/local/bin --include-path /usr/lib/python3.8 \ --include-path /usr/lib/python3 --include-path /dlg --include-path /daliuge --publish-exposed-ports=true \ - --http-probe-exec start_local_managers.sh --http-probe=true --tag=icrar/daliuge-engine.slim:${DEV_TAG}\ - icrar/daliuge-engine:${DEV_TAG} \ + --http-probe-exec start_local_managers.sh --http-probe=true --tag=icrar/daliuge-engine:${VCS_TAG}\ + icrar/daliuge-engine.big:${VCS_TAG} \ ;; *) echo "Usage: build_engine.sh " diff --git a/daliuge-engine/dlg/apps/DALIUGE/xml/simple.xml b/daliuge-engine/dlg/apps/DALIUGE/xml/simple.xml deleted file mode 100644 index e69de29bb..000000000 diff --git a/daliuge-engine/dlg/apps/archiving.py b/daliuge-engine/dlg/apps/archiving.py index 1ee3ff3bd..4c31cf166 100644 --- a/daliuge-engine/dlg/apps/archiving.py +++ b/daliuge-engine/dlg/apps/archiving.py @@ -82,30 +82,18 @@ def store(self, inputDrop): # @par EAGLE_START # @param category PythonApp # @param tag daliuge -# @param[in] cparam/appclass Application class/dlg.apps.archiving.NgasArchivingApp/String/readonly/False//False/ -# \~English Application class -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? -# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/ -# \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed -# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/ -# \~English Specifies the number of times the 'run' method will be executed before finally giving up -# @param[in] aparam/ngasSrv NGAS Server URL/localhost/String/readwrite/False//False/ -# \~English URL of the NGAS Server -# @param[in] aparam/ngasPort NGAS Server Port/7777/Integer/readwrite/False//False/ -# \~English TCP/IP Port on the NGAS Server -# @param[in] aparam/ngasMime NGAS Mime Type/"application/octet-stream"/String/readwrite/False//False/ -# \~English Mime-type of the NGAS payload -# @param[in] aparam/ngasTimeout NGAS Server Timeout/2/Integer/readonly/False//False/ -# \~English Archiving request timeout -# @param[in] aparam/ngasConnectTimeout NGAS Server Connect Timeout/2/Integer/readonly/False//False/ -# \~English NGAS Server connection timeout -# @param[in] port/fileObject File Object/File/ -# \~English Input File Object +# @param appclass Application class/dlg.apps.archiving.NgasArchivingApp/String/ComponentParameter/readonly//False/False/Application class +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? +# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up +# @param ngasSrv NGAS Server URL/localhost/String/ApplicationArgument/readwrite//False/False/URL of the NGAS Server +# @param ngasPort NGAS Server Port/7777/Integer/ApplicationArgument/readwrite//False/False/TCP/IP Port on the NGAS Server +# @param ngasMime NGAS Mime Type/"application/octet-stream"/String/ApplicationArgument/readwrite//False/False/Mime-type of the NGAS payload +# @param ngasTimeout NGAS Server Timeout/2/Integer/ApplicationArgument/readonly//False/False/Archiving request timeout +# @param ngasConnectTimeout NGAS Server Connect Timeout/2/Integer/ApplicationArgument/readonly//False/False/NGAS Server connection timeout +# @param fileObject File Object//Object.File/InputPort/readwrite//False/False/Input File Object # @par EAGLE_END class NgasArchivingApp(ExternalStoreApp): """ diff --git a/daliuge-engine/dlg/apps/bash_shell_app.py b/daliuge-engine/dlg/apps/bash_shell_app.py index d479894c7..9ae934574 100644 --- a/daliuge-engine/dlg/apps/bash_shell_app.py +++ b/daliuge-engine/dlg/apps/bash_shell_app.py @@ -38,6 +38,7 @@ import threading import time import types +import collections from .. import droputils, utils from ..ddap_protocol import AppDROPStates, DROPStates @@ -181,6 +182,7 @@ def initialize(self, **kwargs): self, "No command specified, cannot create BashShellApp" ) + self.appArgs = self._applicationArgs self._recompute_data = {} def _run_bash(self, inputs, outputs, stdin=None, stdout=subprocess.PIPE): @@ -196,15 +198,32 @@ def _run_bash(self, inputs, outputs, stdin=None, stdout=subprocess.PIPE): output of the process is piped to. If not given it is consumed by this method and potentially logged. """ + logger.debug("Parameters found: %s", self.parameters) + # we only support passing a path for bash apps + fsInputs = {uid: i for uid, i in inputs.items() if droputils.has_path(i)} + fsOutputs = {uid: o for uid, o in outputs.items() if droputils.has_path(o)} + dataURLInputs = { + uid: i for uid, i in inputs.items() if not droputils.has_path(i) + } + dataURLOutputs = { + uid: o for uid, o in outputs.items() if not droputils.has_path(o) + } - session_id = ( - self._dlg_session.sessionId if self._dlg_session is not None else "" - ) - argumentString = droputils.serialize_applicationArgs( - self._applicationArgs, self._argumentPrefix, self._paramValueSeparator - ) + # deal with named ports + inport_names = self.parameters['inputs'] \ + if "inputs" in self.parameters else [] + outport_names = self.parameters['outputs'] \ + if "outputs" in self.parameters else [] + keyargs, pargs = droputils.replace_named_ports(inputs.items(), outputs.items(), + inport_names, outport_names, self.appArgs, argumentPrefix=self._argumentPrefix, + separator=self._paramValueSeparator) + argumentString = f"{' '.join(pargs + keyargs)}" # add kwargs to end of pargs # complete command including all additional parameters and optional redirects - cmd = f"{self.command} {argumentString} {self._cmdLineArgs} " + if len(argumentString.strip()) > 0: + # the _cmdLineArgs would very likely make the command line invalid + cmd = f"{self.command} {argumentString} " + else: + cmd = f"{self.command} {argumentString} {self._cmdLineArgs} " if self._outputRedirect: cmd = f"{cmd} > {self._outputRedirect}" if self._inputRedirect: @@ -212,25 +231,19 @@ def _run_bash(self, inputs, outputs, stdin=None, stdout=subprocess.PIPE): cmd = cmd.strip() app_uid = self.uid - # self.run_bash(self._command, self.uid, session_id, *args, **kwargs) # Replace inputs/outputs in command line with paths or data URLs - fsInputs = {uid: i for uid, i in inputs.items() if droputils.has_path(i)} - fsOutputs = {uid: o for uid, o in outputs.items() if droputils.has_path(o)} cmd = droputils.replace_path_placeholders(cmd, fsInputs, fsOutputs) - dataURLInputs = { - uid: i for uid, i in inputs.items() if not droputils.has_path(i) - } - dataURLOutputs = { - uid: o for uid, o in outputs.items() if not droputils.has_path(o) - } cmd = droputils.replace_dataurl_placeholders(cmd, dataURLInputs, dataURLOutputs) # Pass down daliuge-specific information to the subprocesses as environment variables env = os.environ.copy() - env["DLG_UID"] = app_uid - env["DLG_SESSION_ID"] = session_id + env.update({"DLG_UID": self._uid}) + if self._dlg_session: + env.update({"DLG_SESSION_ID": self._dlg_session.sessionId}) + + env.update({"DLG_ROOT": utils.getDlgDir()}) # Wrap everything inside bash cmd = ("/bin/bash", "-c", cmd) @@ -319,7 +332,7 @@ def execute(self, data): drop_state = DROPStates.COMPLETED execStatus = AppDROPStates.FINISHED except: - logger.exception("Error while executing %r" % (self,)) + logger.exception("Error while executing %r", self) drop_state = DROPStates.ERROR execStatus = AppDROPStates.ERROR finally: @@ -342,28 +355,17 @@ def execute(self, data): # @par EAGLE_START # @param category BashShellApp # @param tag template -# @param[in] cparam/command Command//String/readwrite/False//False/ -# \~English The command to be executed -# @param[in] cparam/input_redirection Input Redirection//String/readwrite/False//False/ -# \~English The command line argument that specifies the input into this application -# @param[in] cparam/output_redirection Output Redirection//String/readwrite/False//False/ -# \~English The command line argument that specifies the output from this application -# @param[in] cparam/command_line_arguments Command Line Arguments//String/readwrite/False//False/ -# \~English Additional command line arguments to be added to the command line to be executed -# @param[in] cparam/paramValueSeparator Param value separator/ /String/readwrite/False//False/ -# \~English Separator character(s) between parameters on the command line -# @param[in] cparam/argumentPrefix Argument prefix/"--"/String/readwrite/False//False/ -# \~English Prefix to each keyed argument on the command line -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? -# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/ -# \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed -# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/ -# \~English Specifies the number of times the 'run' method will be executed before finally giving up +# @param command Command//String/ComponentParameter/readwrite//False/False/The command to be executed +# @param input_redirection Input Redirection//String/ComponentParameter/readwrite//False/False/The command line argument that specifies the input into this application +# @param output_redirection Output Redirection//String/ComponentParameter/readwrite//False/False/The command line argument that specifies the output from this application +# @param command_line_arguments Command Line Arguments//String/ComponentParameter/readwrite//False/False/Additional command line arguments to be added to the command line to be executed +# @param paramValueSeparator Param value separator/ /String/ComponentParameter/readwrite//False/False/Separator character(s) between parameters on the command line +# @param argumentPrefix Argument prefix/"--"/String/ComponentParameter/readwrite//False/False/Prefix to each keyed argument on the command line +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? +# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up # @par EAGLE_END class BashShellApp(BashShellBase, BarrierAppDROP): """ diff --git a/daliuge-engine/dlg/apps/crc.py b/daliuge-engine/dlg/apps/crc.py index ac63b9a55..bda09a369 100644 --- a/daliuge-engine/dlg/apps/crc.py +++ b/daliuge-engine/dlg/apps/crc.py @@ -81,19 +81,13 @@ def run(self): # @par EAGLE_START # @param category PythonApp # @param tag daliuge -# @param[in] cparam/appclass Application Class/dlg.apps.crc.CRCStreamApp/String/readonly/False//False/ -# \~English Application class -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? -# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/ -# \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed -# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/ -# \~English Specifies the number of times the 'run' method will be executed before finally giving up -# @param[out] port/data Data/String/ +# @param appclass Application Class/dlg.apps.crc.CRCStreamApp/String/ComponentParameter/readonly//False/False/Application class +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? +# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up +# @param data Data//String/OutputPort/readwrite//False/False/Input data stream # @par EAGLE_END class CRCStreamApp(AppDROP): """ diff --git a/daliuge-engine/dlg/apps/dockerapp.py b/daliuge-engine/dlg/apps/dockerapp.py index 6177fc72d..078bde892 100644 --- a/daliuge-engine/dlg/apps/dockerapp.py +++ b/daliuge-engine/dlg/apps/dockerapp.py @@ -78,44 +78,25 @@ def waitForIp(self, timeout=None): # @par EAGLE_START # @param category Docker # @param tag template -# @param[in] cparam/image Image//String/readwrite/False//False/ -# \~English The name of the docker image to be used for this application -# @param[in] cparam/tag Tag/1.0/String/readwrite/False//False/ -# \~English The tag of the docker image to be used for this application -# @param[in] cparam/digest Digest//String/readwrite/False//False/ -# \~English The hexadecimal hash (long version) of the docker image to be used for this application -# @param[in] cparam/command Command//String/readwrite/False//False/ -# \~English The command line to run within the docker instance. The specified command will be executed in a bash shell. That means that images will need a bash shell. -# @param[in] cparam/input_redirection Input Redirection//String/readwrite/False//False/ -# \~English The command line argument that specifies the input into this application -# @param[in] cparam/output_redirection Output Redirection//String/readwrite/False//False/ -# \~English The command line argument that specifies the output from this application -# @param[in] cparam/command_line_arguments Command Line Arguments//String/readwrite/False//False/ -# \~English Additional command line arguments to be added to the command line to be executed -# @param[in] cparam/paramValueSeparator Param value separator/ /String/readwrite/False//False/ -# \~English Separator character(s) between parameters and their respective values on the command line -# @param[in] cparam/argumentPrefix Argument prefix/"--"/String/readwrite/False//False/ -# \~English Prefix to each keyed argument on the command line -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? -# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/ -# \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed -# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/ -# \~English Specifies the number of times the 'run' method will be executed before finally giving up -# @param[in] cparam/user User//String/readwrite/False//False/ -# \~English Username of the user who will run the application within the docker image -# @param[in] cparam/ensureUserAndSwitch Ensure User And Switch/False/Boolean/readwrite/False//False/ -# \~English Make sure the user specified in the User parameter exists and then run the docker container as that user -# @param[in] cparam/removeContainer Remove Container/True/Boolean/readwrite/False//False/ -# \~English Instruct Docker engine to delete the container after execution is complete -# @param[in] cparam/additionalBindings Additional Bindings//String/readwrite/False//False/ -# \~English Directories which will be visible inside the container during run-time. Format is srcdir_on_host:trgtdir_on_container. Multiple entries can be separated by commas. -# @param[in] cparam/portMappings Port Mappings//String/readwrite/False//False/ -# \~English Port mappings on the host machine +# @param image Image//String/ComponentParameter/readwrite//False/False/The name of the docker image to be used for this application +# @param tag Tag/1.0/String/ComponentParameter/readwrite//False/False/The tag of the docker image to be used for this application +# @param digest Digest//String/ComponentParameter/readwrite//False/False/The hexadecimal hash (long version) of the docker image to be used for this application +# @param command Command//String/ComponentParameter/readwrite//False/False/The command line to run within the docker instance. The specified command will be executed in a bash shell. That means that images will need a bash shell. +# @param input_redirection Input Redirection//String/ComponentParameter/readwrite//False/False/The command line argument that specifies the input into this application +# @param output_redirection Output Redirection//String/ComponentParameter/readwrite//False/False/The command line argument that specifies the output from this application +# @param command_line_arguments Command Line Arguments//String/ComponentParameter/readwrite//False/False/Additional command line arguments to be added to the command line to be executed +# @param paramValueSeparator Param value separator/ /String/ComponentParameter/readwrite//False/False/Separator character(s) between parameters and their respective values on the command line +# @param argumentPrefix Argument prefix/"--"/String/ComponentParameter/readwrite//False/False/Prefix to each keyed argument on the command line +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? +# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up +# @param user User//String/ComponentParameter/readwrite//False/False/Username of the user who will run the application within the docker image +# @param ensureUserAndSwitch Ensure User And Switch/False/Boolean/ComponentParameter/readwrite//False/False/Make sure the user specified in the User parameter exists and then run the docker container as that user +# @param removeContainer Remove Container/True/Boolean/ComponentParameter/readwrite//False/False/Instruct Docker engine to delete the container after execution is complete +# @param additionalBindings Additional Bindings//String/ComponentParameter/readwrite//False/False/Directories which will be visible inside the container during run-time. Format is srcdir_on_host:trgtdir_on_container. Multiple entries can be separated by commas. +# @param portMappings Port Mappings//String/ComponentParameter/readwrite//False/False/Port mappings on the host machine # @par EAGLE_END class DockerApp(BarrierAppDROP): """ @@ -276,22 +257,14 @@ def initialize(self, **kwargs): self._command = self._popArg(kwargs, "command", None) self._noBash = False - if not self._command or self._command[:2].strip() == "%%": + if not self._command or self._command.strip()[:2] == "%%": logger.warning("Assume a default command is executed in the container") - self._command = self._command.strip()[2:].strip() if self._command else "" + self._command = self._command.strip().strip()[:2] if self._command else "" self._noBash = True # This makes sure that we can retain any command defined in the image, but still be # able to add any arguments straight after. This requires to use the placeholder string # "%%" at the start of the command, else it is interpreted as a normal command. - # construct the actual command line from all application parameters - argumentString = droputils.serialize_applicationArgs( - self._applicationArgs, self._argumentPrefix, self._paramValueSeparator - ) - # complete command including all additional parameters and optional redirects - cmd = f"{self._command} {argumentString} {self._cmdLineArgs} " - cmd = cmd.strip() - self._command = cmd # The user used to run the process in the docker container is now always the user # who originally started the DALiuGE process as well. The information is passed through @@ -428,28 +401,18 @@ def run(self): fsInputs = {uid: i for uid, i in iitems if droputils.has_path(i)} fsOutputs = {uid: o for uid, o in oitems if droputils.has_path(o)} dockerInputs = { - # uid: DockerPath(utils.getDlgDir() + i.path) for uid, i in fsInputs.items() + # uid: DockerPath(utils.getDlgDir() + i.path) for uid, i in fsInputs.items() uid: DockerPath(i.path) for uid, i in fsInputs.items() } dockerOutputs = { - # uid: DockerPath(utils.getDlgDir() + o.path) for uid, o in fsOutputs.items() + # uid: DockerPath(utils.getDlgDir() + o.path) for uid, o in fsOutputs.items() uid: DockerPath(o.path) for uid, o in fsOutputs.items() } dataURLInputs = {uid: i for uid, i in iitems if not droputils.has_path(i)} dataURLOutputs = {uid: o for uid, o in oitems if not droputils.has_path(o)} - if self._command: - cmd = droputils.replace_path_placeholders( - self._command, dockerInputs, dockerOutputs - ) - cmd = droputils.replace_dataurl_placeholders( - cmd, dataURLInputs, dataURLOutputs - ) - else: - cmd = "" - # We bind the inputs and outputs inside the docker under the utils.getDlgDir() # directory, maintaining the rest of their original paths. # Outputs are bound only up to their dirname (see class doc for details) @@ -493,13 +456,6 @@ def run(self): ) logger.debug(f"port mappings: {portMappings}") - # Wait until the DockerApps this application runtime depends on have - # started, and replace their IP placeholders by the real IPs - for waiter in self._waiters: - uid, ip = waiter.waitForIp() - cmd = cmd.replace("%containerIp[{0}]%".format(uid), ip) - logger.debug("Command after IP replacement is: %s", cmd) - # deal with environment variables env = {} env.update({"DLG_UID": self._uid}) @@ -520,7 +476,7 @@ def run(self): logger.warning( "Ignoring provided environment variables: Format wrong? Check documentation" ) - addEnv = {} + addEnv = {} if isinstance(addEnv, dict): # if it is a dict populate directly # but replace placeholders first for key in addEnv: @@ -543,6 +499,38 @@ def run(self): ) logger.debug(f"Adding environment variables: {env}") + # deal with named ports + appArgs = self._applicationArgs + inport_names = self.parameters['inputs'] \ + if "inputs" in self.parameters else [] + outport_names = self.parameters['outputs'] \ + if "outputs" in self.parameters else [] + keyargs, pargs = droputils.replace_named_ports(iitems, oitems, + inport_names, outport_names, appArgs, + argumentPrefix=self._argumentPrefix, + separator=self._paramValueSeparator) + + argumentString = f"{' '.join(keyargs + pargs)}" + + # complete command including all additional parameters and optional redirects + cmd = f"{self._command} {argumentString} {self._cmdLineArgs} " + if cmd: + cmd = droputils.replace_path_placeholders( + cmd, dockerInputs, dockerOutputs + ) + cmd = droputils.replace_dataurl_placeholders( + cmd, dataURLInputs, dataURLOutputs + ) + else: + cmd = "" + ############### + # Wait until the DockerApps this application runtime depends on have + # started, and replace their IP placeholders by the real IPs + for waiter in self._waiters: + uid, ip = waiter.waitForIp() + cmd = cmd.replace("%containerIp[{0}]%".format(uid), ip) + logger.debug("Command after IP replacement is: %s", cmd) + # Wrap everything inside bash if len(cmd) > 0 and not self._noBash: cmd = '/bin/bash -c "%s"' % ( diff --git a/daliuge-engine/dlg/apps/dynlib.py b/daliuge-engine/dlg/apps/dynlib.py index 421974a8c..f49ce536a 100644 --- a/daliuge-engine/dlg/apps/dynlib.py +++ b/daliuge-engine/dlg/apps/dynlib.py @@ -213,7 +213,7 @@ def load_and_init(libname, oid, uid, params): libname = find_library(libname) or libname lib = ctypes.cdll.LoadLibrary(libname) - logger.info("Loaded {} as {!r}".format(libname, lib)) + logger.info("Loaded %s as %r", libname, lib) one_of_functions = [["run", "run2"], ["init", "init2"]] for functions in one_of_functions: @@ -251,7 +251,7 @@ def load_and_init(libname, oid, uid, params): if hasattr(lib, "init2"): # With init2 we pass the params as a PyObject* - logger.info("Extra parameters passed to application: {}".format(params)) + logger.info("Extra parameters passed to application: %r", params) init2 = lib.init2 init2.restype = ctypes.py_object result = init2(ctypes.pointer(c_app), ctypes.py_object(params)) @@ -367,18 +367,12 @@ def generate_recompute_data(self): # @par EAGLE_START # @param category DynlibApp # @param tag template -# @param[in] cparam/libpath Library Path//String/readwrite/False//False/ -# \~English The location of the shared object/DLL that implements this application -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? -# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/ -# \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed -# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/ -# \~English Specifies the number of times the 'run' method will be executed before finally giving up +# @param libpath Library Path//String/ComponentParameter/readwrite//False/False/The location of the shared object/DLL that implements this application +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? +# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up # @par EAGLE_END class DynlibApp(DynlibAppBase, BarrierAppDROP): """Loads a dynamic library into the current process and runs it""" @@ -465,18 +459,12 @@ def get_from_subprocess(proc, q): # @par EAGLE_START # @param category DynlibProcApp # @param tag template -# @param[in] cparam/libpath Library Path//String/readwrite/False//False/ -# \~English The location of the shared object/DLL that implements this application -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? -# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/ -# \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed -# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/ -# \~English Specifies the number of times the 'run' method will be executed before finally giving up +# @param libpath Library Path//String/ComponentParameter/readwrite//False/False/The location of the shared object/DLL that implements this application +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? +# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up # @par EAGLE_END class DynlibProcApp(BarrierAppDROP): """Loads a dynamic library in a different process and runs it""" @@ -527,7 +515,7 @@ def run(self): logger.info("Subprocess %s", step) error = get_from_subprocess(self.proc, queue) if error is not None: - logger.error("Error in sub-process when " + step) + logger.error("Error in sub-process when %s", step) raise error finally: self.proc.join(self.timeout) diff --git a/daliuge-engine/dlg/apps/mpi.py b/daliuge-engine/dlg/apps/mpi.py index ee53d84c1..07360c87f 100644 --- a/daliuge-engine/dlg/apps/mpi.py +++ b/daliuge-engine/dlg/apps/mpi.py @@ -37,30 +37,18 @@ # @par EAGLE_START # @param category Mpi # @param tag template -# @param[in] cparam/num_of_procs Num procs/1/Integer/readwrite/False//False/ -# \~English Number of processes used for this application -# @param[in] cparam/command Command//String/readwrite/False//False/ -# \~English The command to be executed -# @param[in] cparam/input_redirection Input Redirection//String/readwrite/False//False/ -# \~English The command line argument that specifies the input into this application -# @param[in] cparam/output_redirection Output Redirection//String/readwrite/False//False/ -# \~English The command line argument that specifies the output from this application -# @param[in] cparam/command_line_arguments Command Line Arguments//String/readwrite/False//False/ -# \~English Additional command line arguments to be added to the command line to be executed -# @param[in] cparam/paramValueSeparator Param value separator/ /String/readwrite/False//False/ -# \~English Separator character(s) between parameters on the command line -# @param[in] cparam/argumentPrefix Argument prefix/"--"/String/readwrite/False//False/ -# \~English Prefix to each keyed argument on the command line -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/group_end Group end/False/Boolean/readwrite/False//False/ -# \~English Is this node the end of a group? -# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/ -# \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed -# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/ -# \~English Specifies the number of times the 'run' method will be executed before finally giving up +# @param num_of_procs Num procs/1/Integer/ComponentParameter/readwrite//False/False/Number of processes used for this application +# @param command Command//String/ComponentParameter/readwrite//False/False/The command to be executed +# @param input_redirection Input Redirection//String/ComponentParameter/readwrite//False/False/The command line argument that specifies the input into this application +# @param output_redirection Output Redirection//String/ComponentParameter/readwrite//False/False/The command line argument that specifies the output from this application +# @param command_line_arguments Command Line Arguments//String/ComponentParameter/readwrite//False/False/Additional command line arguments to be added to the command line to be executed +# @param paramValueSeparator Param value separator/ /String/ComponentParameter/readwrite//False/False/Separator character(s) between parameters on the command line +# @param argumentPrefix Argument prefix/"--"/String/ComponentParameter/readwrite//False/False/Prefix to each keyed argument on the command line +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group? +# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up # @par EAGLE_END class MPIApp(BarrierAppDROP): """ diff --git a/daliuge-engine/dlg/apps/pyfunc.py b/daliuge-engine/dlg/apps/pyfunc.py index 2a3b483cc..87bb1a396 100644 --- a/daliuge-engine/dlg/apps/pyfunc.py +++ b/daliuge-engine/dlg/apps/pyfunc.py @@ -27,6 +27,7 @@ from enum import Enum import importlib import inspect +import json import logging import pickle @@ -39,10 +40,8 @@ from dlg.drop import BarrierAppDROP from dlg.exceptions import InvalidDropException from dlg.meta import ( - dlg_bool_param, dlg_string_param, dlg_enum_param, - dlg_float_param, dlg_dict_param, dlg_component, dlg_batch_input, @@ -116,10 +115,10 @@ def import_using_code(code): class DropParser(Enum): PICKLE = 'pickle' EVAL = 'eval' - PATH = 'path' - DATAURL = 'dataurl' NPY = 'npy' #JSON = "json" + PATH = 'path' # input only + DATAURL = 'dataurl' # input only ## # @brief PyFuncApp @@ -132,30 +131,20 @@ class DropParser(Enum): # @par EAGLE_START # @param category PythonApp # @param tag template -# @param[in] cparam/appclass Application Class/dlg.apps.pyfunc.PyFuncApp/String/readonly/False//False/ -# \~English Application class -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? -# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/ -# \~English The allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed -# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/ -# \~English Specifies the number of times the 'run' method will be executed before finally giving up -# @param[in] aparam/func_name Function Name//String/readwrite/False//False/ -# \~English Python function name -# @param[in] aparam/func_code Function Code//String/readwrite/False//False/ -# \~English Python function code, e.g. 'def function_name(args): return args' -# @param[in] aparam/input_parser Input Parser/pickle/Select/readwrite/False/pickle,eval,path,dataurl,npy/False/ -# \~English Input port parsing technique -# @param[in] aparam/output_parser Output Parser/pickle/Select/readwrite/False/pickle,eval,path,dataurl,npy/False/ -# \~English output port parsing technique -# @param[in] aparam/func_defaults Function Defaults//String/readwrite/False//False/ +# @param appclass Application Class/dlg.apps.pyfunc.PyFuncApp/String/ComponentParameter/readonly//False/False/Application class +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? +# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/The allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up +# @param func_name Function Name//String/ApplicationArgument/readwrite//False/False/Python function name +# @param func_code Function Code//String/ApplicationArgument/readwrite//False/False/Python function code, e.g. 'def function_name(args): return args' +# @param input_parser Input Parser/pickle/Select/ApplicationArgument/readwrite/pickle,eval,npy,path,dataurl/False/False/Input port parsing technique +# @param output_parser Output Parser/pickle/Select/ApplicationArgument/readwrite/pickle,eval,npy/False/False/Output port parsing technique +# @param func_defaults Function Defaults//String/ApplicationArgument/readwrite//False/False/ # \~English Mapping from argname to default value. Should match only the last part of the argnames list. # Values are interpreted as Python code literals and that means string values need to be quoted. -# @param[in] aparam/func_arg_mapping Function Arguments Mapping//String/readwrite/False//False/ +# @param func_arg_mapping Function Arguments Mapping//String/ApplicationArgument/readwrite//False/False/ # \~English Mapping between argument name and input drop uids # @par EAGLE_END class PyFuncApp(BarrierAppDROP): @@ -239,11 +228,11 @@ def _init_func_defaults(self): # we came all this way, now assume that any resulting dict is correct if not isinstance(self.func_defaults, dict): logger.error( - f"Wrong format or type for function defaults for " - + "{self.f.__name__}: {self.func_defaults}, {type(self.func_defaults)}" + "Wrong format or type for function defaults for %s: %r, %r", + self.f.__name__, self.func_defaults, type(self.func_defaults) ) raise ValueError - if DropParser(self.input_parser) is DropParser.PICKLE: + if self.input_parser is DropParser.PICKLE: # only values are pickled, get them unpickled for name, value in self.func_defaults.items(): self.func_defaults[name] = deserialize_data(value) @@ -294,26 +283,29 @@ def initialize(self, **kwargs): "func_arg_mapping", "input_parser", "output_parser", - "func_defaults" + "func_defaults", + "pickle", ] + + # backwards compatibility + if "pickle" in self._applicationArgs: + if self._applicationArgs["pickle"]["value"] == True: + self.input_parser = DropParser.PICKLE + self.output_parser = DropParser.PICKLE + else: + self.input_parser = DropParser.EVAL + self.output_parser = DropParser.EVAL + self._applicationArgs.pop("pickle") + for kw in self.func_def_keywords: - dum_arg = new_arg = "gIbbERiSH:askldhgol" if kw in self._applicationArgs: # these are the preferred ones now if isinstance( self._applicationArgs[kw]["value"], bool - ): # always transfer booleans - new_arg = self._applicationArgs.pop(kw) - elif ( - self._applicationArgs[kw]["value"] + or self._applicationArgs[kw]["value"] or self._applicationArgs[kw]["precious"] ): # only transfer if there is a value or precious is True - new_arg = self._applicationArgs.pop(kw) - - if new_arg != dum_arg: - logger.debug(f"Setting {kw} to {new_arg['value']}") - # we allow python expressions as values, means that strings need to be quoted - self.__setattr__(kw, new_arg["value"]) + self._applicationArgs.pop(kw) self.num_args = len( self._applicationArgs @@ -340,7 +332,7 @@ def initialize(self, **kwargs): self.arguments = inspect.getfullargspec(self.f) logger.debug(f"Function inspection revealed {self.arguments}") # we don't want to mess with the 'self' argument - if self.arguments.args.count('self'): + if self.arguments.args.count('self'): self.arguments.args.remove('self') self.fn_nargs = len(self.arguments.args) self.fn_ndef = len(self.arguments.defaults) if self.arguments.defaults else 0 @@ -390,24 +382,33 @@ def run(self): # Inputs are un-pickled and treated as the arguments of the function # Their order must be preserved, so we use an OrderedDict - if DropParser(self.input_parser) is DropParser.PICKLE: - all_contents = lambda x: pickle.loads(x) - elif DropParser(self.input_parser) is DropParser.EVAL: - all_contents = lambda x: ast.literal_eval(droputils.allDropContents(x).decode('utf-8')) - elif DropParser(self.input_parser) is DropParser.PATH: + if self.input_parser is DropParser.PICKLE: + #all_contents = lambda x: pickle.loads(droputils.allDropContents(x)) + all_contents = droputils.load_pickle + elif self.input_parser is DropParser.EVAL: + def optionalEval(x): + # Null and Empty Drops will return an empty byte string + # which should propogate back to None + content: str = droputils.allDropContents(x).decode('utf-8') + return ast.literal_eval(content) if len(content) > 0 else None + all_contents = optionalEval + elif self.input_parser is DropParser.NPY: + all_contents = droputils.load_npy + elif self.input_parser is DropParser.PATH: all_contents = lambda x: x.path - elif DropParser(self.input_parser) is DropParser.DATAURL: + elif self.input_parser is DropParser.DATAURL: all_contents = lambda x: x.dataurl else: raise ValueError(self.input_parser.__repr__()) inputs = collections.OrderedDict() for uid, drop in self._inputs.items(): - contents = droputils.allDropContents(drop) - # allow for Null DROPs to be passed around - inputs[uid] = all_contents(contents) if contents else None + inputs[uid] = all_contents(drop) + + outputs = collections.OrderedDict() + for uid, drop in self._outputs.items(): + outputs[uid] = all_contents(drop) if self.output_parser is DropParser.PATH else None - self.funcargs = {} # Keyword arguments are made up of the default values plus the inputs # that match one of the keyword argument names @@ -423,7 +424,7 @@ def run(self): if name in self.func_defaults or name not in argnames } logger.debug(f"updating funcargs with {kwargs}") - self.funcargs = kwargs + funcargs = kwargs # Fill arguments with rest of inputs logger.debug(f"available inputs: {inputs}") @@ -432,44 +433,88 @@ def run(self): # the correct UIDs logger.debug(f"Parameters found: {self.parameters}") posargs = self.arguments.args[:self.fn_npos] + keyargs = self.arguments.args[self.fn_npos:] kwargs = {} - self.pargs = [] - pargsDict = collections.OrderedDict(zip(posargs,[None]*len(posargs))) # Initialize pargs dictionary - if ('inputs' in self.parameters and isinstance(self.parameters['inputs'][0], dict)): - logger.debug(f"Using named ports to identify inputs: "+\ - f"{self.parameters['inputs']}") - for i in range(min(len(inputs),self.fn_nargs +\ - len(self.arguments.kwonlyargs))): - # key for final dict is value in named ports dict - key = list(self.parameters['inputs'][i].values())[0] - # value for final dict is value in inputs dict - value = inputs[list(self.parameters['inputs'][i].keys())[0]] - if key in posargs: - pargsDict.update({key:value}) - else: - kwargs.update({key:value}) + pargs = [] + # Initialize pargs dictionary and update with provided argument values + pargsDict = collections.OrderedDict(zip(posargs,[None]*len(posargs))) + if self.arguments.defaults: + keyargsDict = dict(zip(keyargs, self.arguments.defaults[self.fn_npos:])) + else: + keyargsDict = {} + logger.debug("Initial keyargs dictionary: %s", keyargsDict) + if "applicationArgs" in self.parameters: + appArgs = self.parameters["applicationArgs"] # we'll pop the identified ones + _dum = [appArgs.pop(k) for k in self.func_def_keywords if k in appArgs] + logger.debug("Identified keyword arguments removed: %s", + [i['text'] for i in _dum]) + pargsDict.update({k:self.parameters[k] for k in pargsDict if k in + self.parameters}) + # if defined in both we use AppArgs values + pargsDict.update({k:appArgs[k]['value'] for k in pargsDict if k + in appArgs}) + logger.debug("Initial posargs dictionary: %s", pargsDict) + else: + appArgs = {} + + if ('inputs' in self.parameters and + droputils.check_ports_dict(self.parameters['inputs'])): + check_len = min(len(inputs),self.fn_nargs+ + len(self.arguments.kwonlyargs)) + inputs_dict = collections.OrderedDict() + for inport in self.parameters['inputs']: + key = list(inport.keys())[0] + inputs_dict[key] = { + 'name':inport[key], + 'path':inputs[key]} + kwargs.update(droputils.identify_named_ports( + inputs_dict, + posargs, + pargsDict, + keyargsDict, + check_len=check_len, + mode="inputs")) else: for i in range(min(len(inputs),self.fn_nargs)): kwargs.update({self.arguments.args[i]: list(inputs.values())[i]}) - logger.debug(f"updating funcargs with input ports {kwargs}") - self.funcargs.update(kwargs) + if ('outputs' in self.parameters and + droputils.check_ports_dict(self.parameters['outputs'])): + check_len = min(len(outputs),self.fn_nargs+ + len(self.arguments.kwonlyargs)) + outputs_dict = collections.OrderedDict() + for outport in self.parameters['outputs']: + key = list(outport.keys())[0] + outputs_dict[key] = { + 'name':outport[key], + 'path': outputs[key] + } + + kwargs.update(droputils.identify_named_ports( + outputs_dict, + posargs, + pargsDict, + keyargsDict, + check_len=check_len, + mode="outputs")) + logger.debug(f"Updating funcargs with input ports {kwargs}") + funcargs.update(kwargs) + # Try to get values for still missing positional arguments from Application Args if "applicationArgs" in self.parameters: - appArgs = self.parameters["applicationArgs"] # we'll pop them - _dum = [appArgs.pop(k) for k in self.func_def_keywords if k in appArgs] for pa in posargs: - if pa not in self.funcargs: - if pa in appArgs and pa != 'self': + if pa != 'self' and pa not in funcargs: + if pa in appArgs: arg = appArgs.pop(pa) value = arg['value'] ptype = arg['type'] if ptype in ["Complex", "Json"]: try: value = ast.literal_eval(value) - except ValueError: - pass + except Exception as e: + # just go on if this did not work + logger.warning("Eval raised an error: %s",e) elif ptype in ["Python"]: try: import numpy @@ -477,16 +522,19 @@ def run(self): except: pass pargsDict.update({pa: value}) - elif pa != 'self': + elif pa != 'self' and pa not in pargsDict: logger.warning(f"Required positional argument '{pa}' not found!") - logger.debug(f"updating posargs with {list(kwargs.values())}") - self.pargs.extend(list(pargsDict.values())) + _dum = [appArgs.pop(k) for k in pargsDict if k in appArgs] + logger.debug("Identified positional arguments removed: %s", + [i['text'] for i in _dum]) + logger.debug(f"updating posargs with {list(pargsDict.keys())}") + pargs.extend(list(pargsDict.values())) # Try to get values for still missing kwargs arguments from Application kws kwargs = {} kws = self.arguments.args[self.fn_npos:] for ka in kws: - if ka not in self.funcargs: + if ka not in funcargs: if ka in appArgs: arg = appArgs.pop(ka) value = arg['value'] @@ -503,7 +551,32 @@ def run(self): else: logger.warning(f"Keyword argument '{ka}' not found!") logger.debug(f"updating funcargs with {kwargs}") - self.funcargs.update(kwargs) + funcargs.update(kwargs) + + # deal with kwonlyargs + kwargs = {} + kws = self.arguments.kwonlyargs + for ka in kws: + if ka not in funcargs: + if ka in appArgs: + arg = appArgs.pop(ka) + value = arg['value'] + ptype = arg['type'] + if ptype in ["Complex", "Json"]: + try: + value = ast.literal_eval(value) + except: + pass + kwargs.update({ + ka: + value + }) + else: + logger.warning(f"Keyword only argument '{ka}' not found!") + logger.debug(f"updating funcargs with kwonlyargs: {kwargs}") + funcargs.update(kwargs) + + # any remaining application arguments will be used for vargs and vkwargs vparg = [] vkarg = {} logger.debug(f"Remaining AppArguments {appArgs}") @@ -517,27 +590,26 @@ def run(self): else: vkarg.update({arg:value}) - # any remaining application arguments will be used for vargs and vkwargs if self.arguments.varargs: - self.pargs.extend(vparg) + pargs.extend(vparg) if self.arguments.varkw: - self.funcargs.update(vkarg) + funcargs.update(vkarg) # Fill rest with default arguments if there are any more kwargs = {} for kw in self.func_defaults.keys(): value = self.func_defaults[kw] - if kw not in self.funcargs: + if kw not in funcargs: kwargs.update({kw: value}) logger.debug(f"updating funcargs with {kwargs}") - self.funcargs.update(kwargs) - self._recompute_data["args"] = self.funcargs.copy() - logger.debug(f"Running {self.func_name} with *{self.pargs} **{self.funcargs}") + funcargs.update(kwargs) + self._recompute_data["args"] = funcargs.copy() + logger.debug(f"Running {self.func_name} with *{pargs} **{funcargs}") # we capture and log whatever is produced on STDOUT capture = StringIO() with redirect_stdout(capture): - result = self.f(*self.pargs, **self.funcargs) + result = self.f(*pargs, **funcargs) logger.info(f"Captured output from function app '{self.func_name}': {capture.getvalue()}") logger.debug(f"Finished execution of {self.func_name}.") @@ -552,13 +624,21 @@ def write_results(self, result): if len(outputs) == 1: result = [result] for r, o in zip(result, outputs): - if DropParser(self.output_parser) is DropParser.PICKLE: + if self.output_parser is DropParser.PICKLE: logger.debug(f"Writing pickeled result {type(r)} to {o}") o.write(pickle.dumps(r)) - elif DropParser(self.output_parser) is DropParser.EVAL: + elif self.output_parser is DropParser.EVAL: o.write(repr(r).encode('utf-8')) + elif self.output_parser is DropParser.NPY: + droputils.save_npy(o, r) else: ValueError(self.output_parser.__repr__()) def generate_recompute_data(self): + for name, val in self._recompute_data.items(): + try: + json.dumps(val) + except TypeError as e: + logger.debug(e) + self._recompute_data[name] = repr(val) return self._recompute_data diff --git a/daliuge-engine/dlg/apps/scp.py b/daliuge-engine/dlg/apps/scp.py index d894c9a68..bc3c542ef 100644 --- a/daliuge-engine/dlg/apps/scp.py +++ b/daliuge-engine/dlg/apps/scp.py @@ -41,33 +41,21 @@ ## # @brief ScpApp -# @details A BarrierAppDROP that copies the content of its single input onto its -# single output via SSH's scp protocol. +# @details A BarrierAppDROP that copies the content of its single input onto its single output via SSH's scp protocol. # @par EAGLE_START # @param category PythonApp # @param tag daliuge -# @param[in] cparam/appclass Application Class/dlg.apps.scp.ScpApp/String/readonly/False//False/ -# \~English Application class -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? -# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/ -# \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed -# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/ -# \~English Specifies the number of times the 'run' method will be executed before finally giving up -# @param[in] aparam/remoteUser Remote User//String/readwrite/False//False/ -# \~English Remote user address -# @param[in] aparam/pkeyPath Private Key Path//String/readwrite/False//False/ -# \~English Private key path -# @param[in] aparam/timeout Timeout//Float/readwrite/False//False/ -# \~English Connection timeout in seconds -# @param[in] port/file File/PathBasedDrop/ -# \~English Input file path -# @param[out] port/file File/PathBasedDrop/ -# \~English Output file path +# @param appclass Application Class/dlg.apps.scp.ScpApp/String/ComponentParameter/readonly//False/False/Application class +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? +# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up +# @param remoteUser Remote User//String/ApplicationArgument/readwrite//False/False/Remote user address +# @param pkeyPath Private Key Path//String/ApplicationArgument/readwrite//False/False/Private key path +# @param timeout Timeout/60/Float/ApplicationArgument/readwrite//False/False/Connection timeout in seconds +# @param file File//Object.PathBasedDrop/InputPort/readwrite//False/False/Input file path +# @param file File//Object.PathBasedDrop/OutputPort/readwrite//False/False/Output file path # @par EAGLE_END class ScpApp(BarrierAppDROP): """ diff --git a/daliuge-engine/dlg/apps/simple.py b/daliuge-engine/dlg/apps/simple.py index 0256e29d0..ccf5dc34e 100644 --- a/daliuge-engine/dlg/apps/simple.py +++ b/daliuge-engine/dlg/apps/simple.py @@ -73,14 +73,10 @@ def run(self): # @par EAGLE_START # @param category PythonApp # @param tag template -# @param[in] cparam/appclass Application Class//String/readonly/False//False/ -# \~English Application class -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? +# @param appclass Application Class//String/ComponentParameter/readonly//False/False/Application class +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? # @par EAGLE_END class PythonApp(BarrierAppDROP): """A placeholder BarrierAppDrop that just aids the generation of the palette component""" @@ -94,14 +90,11 @@ class PythonApp(BarrierAppDROP): # @par EAGLE_START # @param category PythonApp # @param tag template -# @param[in] aparam/sleepTime Sleep Time/5/Integer/readwrite/False//False/ -# \~English The number of seconds to sleep -# @param[in] cparam/appclass Application Class/dlg.apps.simple.SleepApp/String/readonly/False//False/ -# \~English Application class -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? +# @param sleepTime Sleep Time/5/Integer/ApplicationArgument/readwrite//False/False/The number of seconds to sleep +# @param appclass Application Class/dlg.apps.simple.SleepApp/String/ComponentParameter/readonly//False/False/Application class +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? # @par EAGLE_END class SleepApp(BarrierAppDROP): """A BarrierAppDrop that sleeps the specified amount of time (0 by default)""" @@ -132,20 +125,15 @@ def run(self): # @par EAGLE_START # @param category PythonApp # @param tag daliuge -# @param[in] cparam/appclass Application Class/dlg.apps.simple.CopyApp/String/readonly/False//False/ -# \~English Application class -# @param[in] cparam/bufsize buffer size/65536/Integer/readwrite/False//False/ -# \~English Application class -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? -# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/ -# \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed -# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/ -# \~English Specifies the number of times the 'run' method will be executed before finally giving up +# @param appclass Application Class/dlg.apps.simple.CopyApp/String/ComponentParameter/readonly//False/False/Application class +# @param bufsize buffer size/65536/Integer/ComponentParameter/readwrite//False/False/Buffer size +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? +# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up +# @param dummy Dummy//Object/InputPort/readwrite//False/False/Dummy input port +# @param dummy Dummy//Object/OutputPort/readwrite//False/False/Dummy output port # @par EAGLE_END class CopyApp(BarrierAppDROP): """ @@ -185,20 +173,13 @@ def copyRecursive(self, inputDrop): # @par EAGLE_START # @param category PythonApp # @param tag daliuge -# @param[in] aparam/sleepTime Sleep Time/5/Integer/readwrite/False//False/ -# \~English The number of seconds to sleep -# @param[in] cparam/appclass Application Class/dlg.apps.simple.SleepAndCopyApp/String/readonly/False//False/ -# \~English Application class -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? -# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/ -# \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed -# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/ -# \~English Specifies the number of times the 'run' method will be executed before finally giving up +# @param sleepTime Sleep Time/5/Integer/ApplicationArgument/readwrite//False/False/The number of seconds to sleep +# @param appclass Application Class/dlg.apps.simple.SleepAndCopyApp/String/ComponentParameter/readonly//False/False/Application class +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? +# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up # @par EAGLE_END class SleepAndCopyApp(SleepApp, CopyApp): """A combination of the SleepApp and the CopyApp. It sleeps, then copies""" @@ -217,28 +198,17 @@ def run(self): # @par EAGLE_START # @param category PythonApp # @param tag daliuge -# @param[in] aparam/size Size/100/Integer/readwrite/False//False/ -# \~English The size of the array -# @param[in] aparam/integer Integer/True/Boolean/readwrite/False//False/ -# \~English Generate integer array? -# @param[in] aparam/low Low/0/Float/readwrite/False//False/ -# \~English Low value of range in array [inclusive] -# @param[in] aparam/high High/1/Float/readwrite/False//False/ -# \~English High value of range of array [exclusive] -# @param[in] cparam/appclass Application class/dlg.apps.simple.RandomArrayApp/String/readonly/False//False/ -# \~English Application class -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? -# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/ -# \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed -# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/ -# \~English Specifies the number of times the 'run' method will be executed before finally giving up -# @param[out] port/array Array/Array/ -# \~English Port carrying the averaged array +# @param size Size/100/Integer/ApplicationArgument/readwrite//False/False/The size of the array +# @param integer Integer/True/Boolean/ApplicationArgument/readwrite//False/False/Generate integer array? +# @param low Low/0/Float/ApplicationArgument/readwrite//False/False/Low value of range in array [inclusive] +# @param high High/1/Float/ApplicationArgument/readwrite//False/False/High value of range of array [exclusive] +# @param appclass Application class/dlg.apps.simple.RandomArrayApp/String/ComponentParameter/readonly//False/False/Application class +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? +# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up +# @param array Array//Object.Array/OutputPort/readwrite//False/False/Port carrying the averaged array # @par EAGLE_END class RandomArrayApp(BarrierAppDROP): """ @@ -269,19 +239,22 @@ class RandomArrayApp(BarrierAppDROP): size = dlg_int_param("size", 100) marray = [] - def initialize(self, **kwargs): + def initialize(self, keep_array=False, **kwargs): super(RandomArrayApp, self).initialize(**kwargs) + self._keep_array = keep_array def run(self): # At least one output should have been added outs = self.outputs if len(outs) < 1: raise Exception("At least one output should have been added to %r" % self) - self.generateRandomArray() + marray = self.generateRandomArray() + if self._keep_array: + self.marray = marray for o in outs: - d = pickle.dumps(self.marray) + d = pickle.dumps(marray) o.len = len(d) - o.write(pickle.dumps(self.marray)) + o.write(d) def generateRandomArray(self): if self.integer: @@ -292,7 +265,7 @@ def generateRandomArray(self): # generate an array of self.size floats with numbers between # self.low and self.high marray = (np.random.random(size=self.size) + self.low) * self.high - self.marray = marray + return marray def _getArray(self): return self.marray @@ -307,24 +280,15 @@ def _getArray(self): # @par EAGLE_START # @param category PythonApp # @param tag daliuge -# @param[in] aparam/method Method/mean/Select/readwrite/False/mean,median/False/ -# \~English The method used for averaging -# @param[in] cparam/appclass Application Class/dlg.apps.simple.AverageArraysApp/String/readonly/False//False/ -# \~English Application class -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? -# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/ -# \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed -# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/ -# \~English Specifies the number of times the 'run' method will be executed before finally giving up -# @param[in] port/array Array/array/ -# \~English Port for the input array(s) -# @param[out] port/array Array/Array/ -# \~English Port carrying the averaged array +# @param method Method/mean/Select/ApplicationArgument/readwrite/mean,median/False/False/The method used for averaging +# @param appclass Application Class/dlg.apps.simple.AverageArraysApp/String/ComponentParameter/readonly//False/False/Application class +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? +# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up +# @param array Array//Object.Array/InputPort/readwrite//False/False/Port for the input array(s) +# @param array Array//Object.Array/OutputPort/readwrite//False/False/Port carrying the averaged array # @par EAGLE_END class AverageArraysApp(BarrierAppDROP): """ @@ -407,26 +371,16 @@ def averageArray(self): # @param category PythonApp # @param construct Gather # @param tag daliuge -# @param[in] cparam/appclass Application Class/dlg.apps.simple.GenericNpyGatherApp/String/readonly/False//False/ -# \~English Application class -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? -# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/ -# \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed -# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/ -# \~English Specifies the number of times the 'run' method will be executed before finally giving up -# @param[in] cparam/function Function/sum/Select/readwrite/False/sum,prod,min,max,add,multiply,maximum,minimum/False/ -# \~English The function used for gathering -# @param[in] cparam/reduce_axes "Reduce Axes"/None/String/readonly/False//False/ -# \~English The ndarray axes to reduce, None reduces all axes for sum, prod, max, min functions -# @param[in] port/array Array/npy/ -# \~English Port for the input array(s) -# @param[out] port/array Array/npy/ -# \~English Port carrying the reduced array +# @param appclass Application Class/dlg.apps.simple.GenericNpyGatherApp/String/ComponentParameter/readonly//False/False/Application class +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? +# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up +# @param function Function/sum/Select/ComponentParameter/readwrite/sum,prod,min,max,add,multiply,maximum,minimum/False/False/The function used for gathering +# @param reduce_axes "Reduce Axes"/None/String/ComponentParameter/readonly//False/False/The ndarray axes to reduce, None reduces all axes for sum, prod, max, min functions +# @param array Array//Object.Array/InputPort/readwrite//False/False/Port for the input array(s) +# @param array Array//Object.Array/OutputPort/readwrite//False/False/Port carrying the reduced array # @par EAGLE_END class GenericNpyGatherApp(BarrierAppDROP): """ @@ -513,22 +467,14 @@ def gather_inputs(self): # @par EAGLE_START # @param category PythonApp # @param tag daliuge -# @param[in] aparam/greet Greet/World/String/readwrite/False//False/ -# \~English What appears after 'Hello ' -# @param[in] cparam/appclass Application Class/dlg.apps.simple.HelloWorldApp/String/readonly/False//False/ -# \~English Application class -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? -# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/ -# \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed -# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/ -# \~English Specifies the number of times the 'run' method will be executed before finally giving up -# @param[out] port/hello Hello/String/ -# \~English The port carrying the message produced by the app. +# @param greet Greet/World/String/ApplicationArgument/readwrite//False/False/What appears after 'Hello ' +# @param appclass Application Class/dlg.apps.simple.HelloWorldApp/String/ComponentParameter/readonly//False/False/Application class +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? +# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up +# @param hello Hello/"world"/String/OutputPort/readwrite//False/False/The port carrying the message produced by the app. # @par EAGLE_END class HelloWorldApp(BarrierAppDROP): """ @@ -578,22 +524,14 @@ def run(self): # @par EAGLE_START # @param category PythonApp # @param tag daliuge -# @param[in] aparam/url URL/"https://eagle.icrar.org"/String/readwrite/False//False/ -# \~English The URL to retrieve -# @param[in] cparam/appclass Application Class/dlg.apps.simple.UrlRetrieveApp/String/readonly/False//False/ -# \~English Application class -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? -# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/ -# \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed -# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/ -# \~English Specifies the number of times the 'run' method will be executed before finally giving up -# @param[out] port/content Content/String/ -# \~English The port carrying the content read from the URL. +# @param url URL/"https://eagle.icrar.org"/String/ApplicationArgument/readwrite//False/False/The URL to retrieve +# @param appclass Application Class/dlg.apps.simple.UrlRetrieveApp/String/ComponentParameter/readonly//False/False/Application class +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? +# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up +# @param content Content//String/OutputPort/readwrite//False/False/The port carrying the content read from the URL # @par EAGLE_END class UrlRetrieveApp(BarrierAppDROP): """ @@ -641,20 +579,13 @@ def run(self): # @param category PythonApp # @param construct Scatter # @param tag daliuge -# @param[in] cparam/appclass Application Class/dlg.apps.simple.GenericScatterApp/String/readonly/False//False/ -# \~English Application class -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? -# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/ -# \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed -# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/ -# \~English Specifies the number of times the 'run' method will be executed before finally giving up -# @param[out] port/array Array/Array/ -# \~English A numpy array of arrays, where the first axis is of length +# @param appclass Application Class/dlg.apps.simple.GenericScatterApp/String/ComponentParameter/readonly//False/False/Application class +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? +# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up +# @param array Array//Object.Array/OutputPort/readwrite//False/False/A numpy array of arrays, where the first axis is of length # @par EAGLE_END class GenericScatterApp(BarrierAppDROP): """ @@ -711,23 +642,14 @@ def run(self): # @param category PythonApp # @param construct Scatter # @param tag daliuge -# @param[in] cparam/appclass Application Class/dlg.apps.simple.GenericNpyScatterApp/String/readonly/False//False/ -# \~English Application class -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? -# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/ -# \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed -# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/ -# \~English Specifies the number of times the 'run' method will be executed before finally giving up -# @param[in] aparam/scatter_axes Scatter Axes//String/readwrite/False//False/ -# \~English The axes to split input ndarrays on, e.g. [0,0,0], length must -# match the number of input ports -# @param[out] port/array Array/npy/ -# \~English A numpy array of arrays +# @param appclass Application Class/dlg.apps.simple.GenericNpyScatterApp/String/ComponentParameter/readonly//False/False/Application class +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? +# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up +# @param scatter_axes Scatter Axes//String/ApplicationArgument/readwrite//False/False/The axes to split input ndarrays on, e.g. [0,0,0], length must match the number of input ports +# @param array Object.Array//Object.Array/InputPort/readwrite//False/False/A numpy array of arrays # @par EAGLE_END class GenericNpyScatterApp(BarrierAppDROP): """ @@ -798,22 +720,14 @@ def condition(self): # @par EAGLE_START # @param category PythonApp # @param tag daliuge -# @param[in] aparam/size Size/100/Integer/readwrite/False//False/ -# \~English the size of the array -# @param[in] cparam/appclass Application Class/dlg.apps.simple.ListAppendThrashingApp/String/readonly/False//False/ -# \~English Application class -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? -# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/ -# \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed -# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/ -# \~English Specifies the number of times the 'run' method will be executed before finally giving up -# @param[out] port/array Array/array/ -# \~English Port carrying the random array. +# @param size Size/100/Integer/ApplicationArgument/readwrite//False/False/the size of the array +# @param appclass Application Class/dlg.apps.simple.ListAppendThrashingApp/String/ComponentParameter/readonly//False/False/Application class +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? +# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up +# @param array Array//Object.Array/OutputPort/readwrite//False/False/Port carrying the random array. # @par EAGLE_END class ListAppendThrashingApp(BarrierAppDROP): """ diff --git a/daliuge-engine/dlg/apps/socket_listener.py b/daliuge-engine/dlg/apps/socket_listener.py index d7d22d448..e2abf82f4 100644 --- a/daliuge-engine/dlg/apps/socket_listener.py +++ b/daliuge-engine/dlg/apps/socket_listener.py @@ -56,27 +56,17 @@ # @par EAGLE_START # @param category PythonApp # @param tag daliuge -# @param[in] cparam/appclass Application Class/dlg.apps.socket_listener.SocketListener/String/readonly/False//False/ -# \~English Application class -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? -# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/ -# \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed -# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/ -# \~English Specifies the number of times the 'run' method will be executed before finally giving up -# @param[in] aparam/host Host/127.0.0.1/String/readwrite/False//False/ -# \~English Host address -# @param[in] aparam/port Port/1111/Integer/readwrite/False//False/ -# \~English Host port -# @param[in] aparam/bufsize Buffer Size/4096/String/readwrite/False//False/ -# \~English Receive buffer size -# @param[in] aparam/reuseAddr Reuse Address/False/Boolean/readwrite/False//False/ -# \~English -# @param[out] port/data Data/String/ +# @param appclass Application Class/dlg.apps.socket_listener.SocketListener/String/ComponentParameter/readonly//False/False/Application class +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? +# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up +# @param host Host/127.0.0.1/String/ApplicationArgument/readwrite//False/False/Host address +# @param port Port/1111/Integer/ApplicationArgument/readwrite//False/False/Host port +# @param bufsize Buffer Size/4096/String/ApplicationArgument/readwrite//False/False/Receive buffer size +# @param reuseAddr Reuse Address/False/Boolean/ApplicationArgument/readwrite//False/False/ +# @param data Data//String/OutputPort/readwrite//False/False/ # @par EAGLE_END class SocketListenerApp(BarrierAppDROP): """ diff --git a/daliuge-engine/dlg/dask_emulation.py b/daliuge-engine/dlg/dask_emulation.py index 7238e1db7..e096e6909 100644 --- a/daliuge-engine/dlg/dask_emulation.py +++ b/daliuge-engine/dlg/dask_emulation.py @@ -284,7 +284,7 @@ def __init__(self, f, nout): self.fcode, self.fdefaults = pyfunc.serialize_func(f) self.original_kwarg_names = [] self.nout = nout - logger.debug("Created %r" % self) + logger.debug("Created %r", self) def make_dropdict(self): diff --git a/daliuge-engine/dlg/deploy/common.py b/daliuge-engine/dlg/deploy/common.py index 8b26e40e2..0300f6a9e 100644 --- a/daliuge-engine/dlg/deploy/common.py +++ b/daliuge-engine/dlg/deploy/common.py @@ -185,7 +185,7 @@ def submit( """ client = _get_client(host, port, timeout) session_id = session_id or "%f" % (time.time()) - completed_uids = droputils.get_roots(pg[:-1]) + completed_uids = droputils.get_roots(pg) with client: client.create_session(session_id) logger.info("Session %s created", session_id) diff --git a/daliuge-engine/dlg/deploy/dlg_proxy.py b/daliuge-engine/dlg/deploy/dlg_proxy.py index 9d9f9453f..533e30c6c 100644 --- a/daliuge-engine/dlg/deploy/dlg_proxy.py +++ b/daliuge-engine/dlg/deploy/dlg_proxy.py @@ -110,7 +110,7 @@ def connect_to_host(self, server, port): try: the_socket = socket.create_connection((server, port)) the_socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) - logger.info("Connected to %s on port %d" % (server, port)) + logger.info("Connected to %s on port %d", server, port) return the_socket except Exception: logger.exception("Failed to connect to %s:%d", server, port) @@ -173,7 +173,7 @@ def loop(self): continue else: tag = data[0:at] - logger.debug("Received {0} from Monitor".format(tag)) + logger.debug("Received %s from Monitor", b2s(tag)) dlg_sock = self._dlg_sock_dict.get(tag, None) to_send = data[at + dl :] if dlg_sock is None: @@ -191,21 +191,21 @@ def loop(self): if send_to_dlg: try: dlg_sock.sendall(to_send) - logger.debug("Sent {0} to DALiuGE manager".format(tag)) + logger.debug("Sent %s to DALiuGE manager", b2s(tag)) except socket.error: self.close_dlg_socket(dlg_sock, tag) else: # from one of the DALiuGE sockets data = the_socket.recv(BUFF_SIZE) tag = self._dlg_sock_tag_dict.get(the_socket, None) - logger.debug("Received {0} from DALiuGE manager".format(b2s(tag))) + logger.debug("Received %s from DALiuGE manager", b2s(tag)) if tag is None: logger.error( - "Tag for DALiuGE socket {0} is gone".format(the_socket) + "Tag for DALiuGE socket %r is gone", the_socket ) else: send_to_monitor(self.monitor_socket, delimit.join([tag, data])) - logger.debug("Sent {0} to Monitor".format(b2s(tag))) + logger.debug("Sent %s to Monitor", b2s(tag)) if len(data) == 0: self.close_dlg_socket(the_socket, tag) diff --git a/daliuge-engine/dlg/deploy/utils/monitor_replayer.py b/daliuge-engine/dlg/deploy/utils/monitor_replayer.py index 99fb66c69..04fe0c4d8 100644 --- a/daliuge-engine/dlg/deploy/utils/monitor_replayer.py +++ b/daliuge-engine/dlg/deploy/utils/monitor_replayer.py @@ -88,7 +88,7 @@ def __init__(self, graph_path, status_path): self.gnid_ip_dict = dict() self.status_path = status_path with open(graph_path) as f: - logger.info("Loading graph from file {0}".format(graph_path)) + logger.info("Loading graph from file %s", graph_path) self.pg_spec = json.load(f)["g"] for i, dropspec in enumerate(self.pg_spec.values()): gnid = str(i) @@ -142,7 +142,7 @@ def parse_status(self, gexf_file, out_dir=None, remove_gexf=False): out_dir = os.path.dirname(gexf_file) with open(gexf_file) as gf: gexf_list = gf.readlines() - logger.info("Gexf file '{0}' loaded".format(gexf_file)) + logger.info("Gexf file '%s' loaded", gexf_file) with open(self.status_path) as f: for i, line in enumerate(f): colour_dict = dict() @@ -183,14 +183,14 @@ def parse_status(self, gexf_file, out_dir=None, remove_gexf=False): # fo.write('{0}{1}'.format(new_line, os.linesep)) # fo.write('{0}{1}'.format(new_line, '\n')) fo.write(new_line) - logger.info("GEXF file '{0}' is generated".format(new_gexf)) + logger.info("GEXF file '%s' is generated", new_gexf) new_png = new_gexf.split(".gexf")[0] + ".png" cmd = "{0} {1} {2}".format(java_cmd, new_gexf, new_png) ret = commands.getstatusoutput(cmd) if ret[0] != 0: logger.error( - "Fail to print png from %s to %s: %s" - % (new_gexf, new_png, ret[1]) + "Fail to print png from %s to %s: %s", + new_gexf, new_png, ret[1] ) del colour_dict if remove_gexf: @@ -259,7 +259,7 @@ def get_state_changes(self, gexf_file, grep_log_file, steps=400, out_dir=None): state = line.split()[-1] fo.write("{0},{1},{2},{3}".format(ts, oid, state, os.linesep)) else: - logger.info("csv file already exists: {0}".format(csv_file)) + logger.info("csv file already exists: %s", csv_file) if not os.path.exists(sqlite_file): sql = sql_create_status.format(csv_file) @@ -269,10 +269,10 @@ def get_state_changes(self, gexf_file, grep_log_file, steps=400, out_dir=None): cmd = "sqlite3 {0} < {1}".format(sqlite_file, sql_file) ret = commands.getstatusoutput(cmd) if ret[0] != 0: - logger.error("fail to create sqlite: {0}".format(ret[1])) + logger.error("fail to create sqlite: %s", ret[1]) return else: - logger.info("sqlite file already exists: {0}".format(sqlite_file)) + logger.info("sqlite file already exists: %s", sqlite_file) dbconn = dbdrv.connect(sqlite_file) q = "SELECT min(ts) from ac" @@ -299,10 +299,10 @@ def get_state_changes(self, gexf_file, grep_log_file, steps=400, out_dir=None): a = el b = lr[i + 1] step_name = "{0}-{1}".format(a, b) - logger.debug("stepname: %s" % step_name) + logger.debug("stepname: %s", step_name) new_gexf = "{0}/{1}.gexf".format(out_dir, step_name) if os.path.exists(new_gexf): - logger.info("{0} already exists, ignore".format(new_gexf)) + logger.info("%s already exists, ignore", new_gexf) last_gexf = new_gexf continue sql = sql_query.format(a, b) @@ -332,18 +332,18 @@ def get_state_changes(self, gexf_file, grep_log_file, steps=400, out_dir=None): colour.attrib["g"] = "{0}".format(g) colour.attrib["b"] = "{0}".format(b) tree.write(new_gexf) - logger.info("GEXF file '{0}' is generated".format(new_gexf)) + logger.info("GEXF file '%s' is generated", new_gexf) del drop_dict if not filecmp.cmp(last_gexf, new_gexf, False): new_png = new_gexf.split(".gexf")[0] + ".png" cmd = "{0} {1} {2}".format(java_cmd, new_gexf, new_png) ret = commands.getstatusoutput(cmd) else: - logger.info("Identical {0} == {1}".format(new_gexf, last_gexf)) + logger.info("Identical %s == %s", new_gexf, last_gexf) last_gexf = new_gexf if ret[0] != 0: logger.error( - "Fail to print png from %s to %s: %s" % (last_gexf, new_png, ret[1]) + "Fail to print png from %s to %s: %s", last_gexf, new_png, ret[1] ) def build_drop_subgraphs(self, node_range="[0:20]"): @@ -519,7 +519,7 @@ def build_drop_fullgraphs(self, do_subgraph=False, graph_lib="pygraphviz"): logging.basicConfig(filename=options.log_file, level=logging.DEBUG, format=FORMAT) if options.edgelist and options.dot_file is not None: - logger.info("Loading networx graph from file {0}".format(options.graph_path)) + logger.info("Loading networx graph from file %s", options.graph_path) gp = GraphPlayer(options.graph_path, options.status_path) g = gp.build_drop_fullgraphs(graph_lib="networkx") nx.write_edgelist(g, options.dot_file) diff --git a/daliuge-engine/dlg/drop.py b/daliuge-engine/dlg/drop.py index dd6864ccc..4fd5c1a09 100644 --- a/daliuge-engine/dlg/drop.py +++ b/daliuge-engine/dlg/drop.py @@ -43,13 +43,10 @@ import time import re import sys -import inspect import binascii -from typing import List, Optional, Union +from typing import List, Union import numpy as np -import pyarrow.plasma as plasma -import six from dlg.common.reproducibility.constants import ( ReproducibilityFlags, REPRO_DEFAULT, @@ -58,7 +55,6 @@ ) from dlg.common.reproducibility.reproducibility import common_hash from merklelib import MerkleTree -from six import BytesIO from .ddap_protocol import ( ExecutionMode, @@ -383,17 +379,25 @@ def __init__(self, oid, uid, **kwargs): DROPStates.INITIALIZED ) # no need to use synchronised self.status here + _members_cache = {} + + def _get_members(self): + cls = self.__class__ + if cls not in AbstractDROP._members_cache: + members = [ + (name, val) + for c in cls.__mro__[:-1] + for name, val in vars(c).items() + if not (inspect.isfunction(val) or isinstance(val, property)) + ] + AbstractDROP._members_cache[cls] = members + return AbstractDROP._members_cache[cls] + def _extract_attributes(self, **kwargs): """ Extracts component and app params then assigns them to class instance attributes. Component params take pro """ - def getmembers(object, predicate=None): - for cls in object.__class__.__mro__[:-1]: - for k, v in vars(cls).items(): - if not predicate or predicate(v): - yield k, v - def get_param_value(attr_name, default_value): has_component_param = attr_name in kwargs has_app_param = 'applicationArgs' in kwargs \ @@ -410,9 +414,7 @@ def get_param_value(attr_name, default_value): return param # Take a class dlg defined parameter class attribute and create an instanced attribute on object - for attr_name, member in getmembers( - self, lambda a: not (inspect.isfunction(a) or isinstance(a, property)) - ): + for attr_name, member in self._get_members(): if isinstance(member, dlg_float_param): value = get_param_value(attr_name, member.default_value) if value is not None and value != "": @@ -692,7 +694,7 @@ def commit(self): # Set as committed self._committed = True else: - logger.debug("Trying to re-commit DROP %s, cannot overwrite." % self) + logger.debug("Trying to re-commit DROP %s, cannot overwrite.", self) @property def oid(self): @@ -849,7 +851,7 @@ def parent(self): def parent(self, parent): if self._parent and parent: logger.warning( - "A parent is already set in %r, overwriting with new value" % (self,) + "A parent is already set in %r, overwriting with new value", self ) if parent: prevParent = self._parent @@ -1045,8 +1047,8 @@ def addStreamingConsumer(self, streamingConsumer, back=True): if scuid in self._streamingConsumers_uids: return logger.debug( - "Adding new streaming streaming consumer for %r: %s" - % (self, streamingConsumer) + "Adding new streaming streaming consumer for %r: %s", + self, streamingConsumer ) self._streamingConsumers.append(streamingConsumer) @@ -1137,12 +1139,14 @@ def cancel(self): if self.status in [DROPStates.INITIALIZED, DROPStates.WRITING]: self._closeWriters() self.status = DROPStates.CANCELLED + self.completedrop() def skip(self): """Moves this drop to the SKIPPED state closing any writers we opened""" if self.status in [DROPStates.INITIALIZED, DROPStates.WRITING]: self._closeWriters() self.status = DROPStates.SKIPPED + self.completedrop() @property def node(self): @@ -1239,7 +1243,7 @@ def open(self, **kwargs): ) io = self.getIO() - logger.debug("Opening drop %s" % (self.oid)) + logger.debug("Opening drop %s", self.oid) io.open(OpenMode.OPEN_READ, **kwargs) # Save the IO object in the dictionary and return its descriptor instead @@ -1341,8 +1345,8 @@ def write(self, data: Union[bytes, memoryview], **kwargs): if nbytes != dataLen: # TODO: Maybe this should be an actual error? logger.warning( - "Not all data was correctly written by %s (%d/%d bytes written)" - % (self, nbytes, dataLen) + "Not all data was correctly written by %s (%d/%d bytes written)", + self, nbytes, dataLen ) # see __init__ for the initialization to None @@ -1368,12 +1372,12 @@ def write(self, data: Union[bytes, memoryview], **kwargs): else: if remaining < 0: logger.warning( - "Received and wrote more bytes than expected: " - + str(-remaining) + "Received and wrote more bytes than expected: %d", + -remaining ) logger.debug( - "Automatically moving %r to COMPLETED, all expected data arrived" - % (self,) + "Automatically moving %r to COMPLETED, all expected data arrived", + self ) self.setCompleted() else: @@ -1487,16 +1491,13 @@ def dataURL(self) -> str: # @par EAGLE_START # @param category File # @param tag daliuge -# @param[in] cparam/data_volume Data volume/5/Float/readwrite/False//False/ -# \~English Estimated size of the data contained in this node -# @param[in] cparam/group_end Group end/False/Boolean/readwrite/False//False/ -# \~English Is this node the end of a group? -# @param[in] cparam/check_filepath_exists Check file path exists/True/Boolean/readwrite/False//False/ -# \~English Perform a check to make sure the file path exists before proceeding with the application -# @param[in] cparam/filepath File Path//String/readwrite/False//False/ -# \~English Path to the file for this node -# @param[in] cparam/dirname Directory name//String/readwrite/False//False/ -# \~English Path to the file for this node +# @param data_volume Data volume/5/Float/ComponentParameter/readwrite//False/False/Estimated size of the data contained in this node +# @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group? +# @param check_filepath_exists Check file path exists/True/Boolean/ComponentParameter/readwrite//False/False/Perform a check to make sure the file path exists before proceeding with the application +# @param filepath File Path//String/ComponentParameter/readwrite//False/False/Path to the file for this node +# @param dirname Directory name//String/ComponentParameter/readwrite//False/False/Path to the file for this node +# @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port +# @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port # @par EAGLE_END class FileDROP(DataDROP, PathBasedDrop): """ @@ -1653,7 +1654,7 @@ def setCompleted(self): pass except: self.status = DROPStates.ERROR - logger.error("Path not accessible: %s" % self.path) + logger.error("Path not accessible: %s", self.path) self._size = 0 # Signal our subscribers that the show is over self._fire("dropCompleted", status=DROPStates.COMPLETED) @@ -1681,22 +1682,16 @@ def generate_reproduce_data(self): # @par EAGLE_START # @param category NGAS # @param tag daliuge -# @param[in] cparam/data_volume Data volume/5/Float/readwrite/False//False/ -# \~English Estimated size of the data contained in this node -# @param[in] cparam/group_end Group end/False/Boolean/readwrite/False//False/ -# \~English Is this node the end of a group? -# @param[in] cparam/ngsSrv NGAS Server/localhost/String/readwrite/False//False/ -# \~English The URL of the NGAS Server -# @param[in] cparam/ngasPort NGAS Port/7777/Integer/readwrite/False//False/ -# \~English The port of the NGAS Server -# @param[in] cparam/ngasFileId File ID//String/readwrite/False//False/ -# \~English File ID on NGAS (for retrieval only) -# @param[in] cparam/ngasConnectTimeout Connection timeout/2/Integer/readwrite/False//False/ -# \~English Timeout for connecting to the NGAS server -# @param[in] cparam/ngasMime NGAS mime-type/"text/ascii"/String/readwrite/False//False/ -# \~English Mime-type to be used for archiving -# @param[in] cparam/ngasTimeout NGAS timeout/2/Integer/readwrite/False//False/ -# \~English Timeout for receiving responses for NGAS +# @param data_volume Data volume/5/Float/ComponentParameter/readwrite//False/False/Estimated size of the data contained in this node +# @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group? +# @param ngasSrv NGAS Server/localhost/String/ComponentParameter/readwrite//False/False/The URL of the NGAS Server +# @param ngasPort NGAS Port/7777/Integer/ComponentParameter/readwrite//False/False/The port of the NGAS Server +# @param ngasFileId File ID//String/ComponentParameter/readwrite//False/False/File ID on NGAS (for retrieval only) +# @param ngasConnectTimeout Connection timeout/2/Integer/ComponentParameter/readwrite//False/False/Timeout for connecting to the NGAS server +# @param ngasMime NGAS mime-type/"text/ascii"/String/ComponentParameter/readwrite//False/False/Mime-type to be used for archiving +# @param ngasTimeout NGAS timeout/2/Integer/ComponentParameter/readwrite//False/False/Timeout for receiving responses for NGAS +# @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port +# @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port # @par EAGLE_END class NgasDROP(DataDROP): """ @@ -1710,6 +1705,7 @@ class NgasDROP(DataDROP): ngasConnectTimeout = dlg_int_param("ngasConnectTimeout", 2) ngasMime = dlg_string_param("ngasMime", "application/octet-stream") len = dlg_int_param("len", -1) + ngas_checksum = None def initialize(self, **kwargs): if self.len == -1: @@ -1772,9 +1768,10 @@ def setCompleted(self): try: stat = self.getIO().fileStatus() logger.debug( - "Setting size of NGASDrop %s to %s" % (self.fileId, stat["FileSize"]) + "Setting size of NGASDrop %s to %s", self.fileId, stat["FileSize"] ) self._size = int(stat["FileSize"]) + self.ngas_checksum = str(stat["Checksum"]) except: # we''ll try this again in case there is some other issue # try: @@ -1798,11 +1795,9 @@ def dataURL(self) -> str: # Override def generate_reproduce_data(self): - # TODO: This is a bad implementation. Will need to sort something better out - from .droputils import allDropContents - - data = allDropContents(self, self.size) - return {"data_hash": common_hash(data)} + if self.ngas_checksum is None or self.ngas_checksum == '': + return {"fileid": self.ngasFileId, "size": self._size} + return {"data_hash": self.ngas_checksum} ## @@ -1811,16 +1806,24 @@ def generate_reproduce_data(self): # @par EAGLE_START # @param category Memory # @param tag daliuge -# @param[in] cparam/data_volume Data volume/5/Float/readwrite/False//False/ -# \~English Estimated size of the data contained in this node -# @param[in] cparam/group_end Group end/False/Boolean/readwrite/False//False/ -# \~English Is this node the end of a group? +# @param data_volume Data volume/5/Float/ComponentParameter/readwrite//False/False/Estimated size of the data contained in this node +# @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group? +# @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port +# @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port # @par EAGLE_END class InMemoryDROP(DataDROP): """ A DROP that points data stored in memory. """ + # Allow in-memory drops to be automatically removed by default + def __init__(self, *args, **kwargs): + if 'precious' not in kwargs: + kwargs['precious'] = False + if 'expireAfterUse' not in kwargs: + kwargs['expireAfterUse'] = True + super().__init__(*args, **kwargs) + def initialize(self, **kwargs): args = [] if "pydata" in kwargs: @@ -1849,7 +1852,11 @@ def dataURL(self) -> str: def generate_reproduce_data(self): from .droputils import allDropContents - data = allDropContents(self, self.size) + data = b"" + try: + data = allDropContents(self, self.size) + except Exception: + logger.debug("Could not read drop reproduce data") return {"data_hash": common_hash(data)} @@ -1859,10 +1866,10 @@ def generate_reproduce_data(self): # @par EAGLE_START # @param category SharedMemory # @param tag template -# @param[in] cparam/data_volume Data volume/5/Float/readwrite/False//False/ -# \~English Estimated size of the data contained in this node -# @param[in] cparam/group_end Group end/False/Boolean/readwrite/False//False/ -# \~English Is this node the end of a group? +# @param data_volume Data volume/5/Float/ComponentParameter/readwrite//False/False/Estimated size of the data contained in this node +# @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group? +# @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port +# @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port # @par EAGLE_END class SharedMemoryDROP(DataDROP): """ @@ -1909,10 +1916,10 @@ def dataURL(self) -> str: # @par EAGLE_START # @param category Memory # @param tag daliuge -# @param[in] cparam/data_volume Data volume/0/Float/readonly/False//False/ -# \~English This never stores any data -# @param[in] cparam/group_end Group end/False/Boolean/readwrite/False//False/ -# \~English Is this node the end of a group? +# @param data_volume Data volume/0/Float/ComponentParameter/readonly//False/False/This never stores any data +# @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group? +# @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port +# @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port # @par EAGLE_END class NullDROP(DataDROP): """ @@ -1939,20 +1946,15 @@ class EndDROP(NullDROP): # @par EAGLE_START # @param category File # @param tag template -# @param[in] cparam/data_volume Data volume/5/Float/readwrite/False//False/ -# \~English Estimated size of the data contained in this node -# @param[in] cparam/group_end Group end/False/Boolean/readwrite/False//False/ -# \~English Is this node the end of a group? -# @param[in] cparam/dbmodule Python DB module//String/readwrite/False//False/ -# \~English Load path for python DB module -# @param[in] cparam/dbtable DB table name//String/readwrite/False//False/ -# \~English The name of the table to use -# @param[in] cparam/vals Values dictionary//Json/readwrite/False//False/ -# \~English Json encoded values dictionary used for INSERT. The keys of ``vals`` are used as the column names. -# @param[in] cparam/condition Whats used after WHERE//String/readwrite/False//False/ -# \~English Condition for SELECT. For this the WHERE statement must be written using the "{X}" or "{}" placeholders -# @param[in] cparam/selectVals values for WHERE//Json/readwrite/False//False/ -# \~English Values for the WHERE statement +# @param data_volume Data volume/5/Float/ComponentParameter/readwrite//False/False/Estimated size of the data contained in this node +# @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group? +# @param dbmodule Python DB module//String/ComponentParameter/readwrite//False/False/Load path for python DB module +# @param dbtable DB table name//String/ComponentParameter/readwrite//False/False/The name of the table to use +# @param vals Values dictionary/{}/Json/ComponentParameter/readwrite//False/False/Json encoded values dictionary used for INSERT. The keys of ``vals`` are used as the column names. +# @param condition Whats used after WHERE//String/ComponentParameter/readwrite//False/False/Condition for SELECT. For this the WHERE statement must be written using the "{X}" or "{}" placeholders +# @param selectVals values for WHERE/{}/Json/ComponentParameter/readwrite//False/False/Values for the WHERE statement +# @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port +# @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port # @par EAGLE_END class RDBMSDrop(DataDROP): """ @@ -2170,16 +2172,13 @@ def exists(self): # @par EAGLE_START # @param category Plasma # @param tag daliuge -# @param[in] cparam/data_volume Data volume/5/Float/readwrite/False//False/ -# \~English Estimated size of the data contained in this node -# @param[in] cparam/group_end Group end/False/Boolean/readwrite/False//False/ -# \~English Is this node the end of a group? -# @param[in] cparam/plasma_path Plasma Path//String/readwrite/False//False/ -# \~English Path to the local plasma store -# @param[in] cparam/object_id Object Id//String/readwrite/False//False/ -# \~English PlasmaId of the object for all compute nodes -# @param[in] cparam/use_staging Use Staging/False/Boolean/readwrite/False//False/ -# \~English Enables writing to a dynamically resizeable staging buffer +# @param data_volume Data volume/5/Float/ComponentParameter/readwrite//False/False/Estimated size of the data contained in this node +# @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group? +# @param plasma_path Plasma Path//String/ComponentParameter/readwrite//False/False/Path to the local plasma store +# @param object_id Object Id//String/ComponentParameter/readwrite//False/False/PlasmaId of the object for all compute nodes +# @param use_staging Use Staging/False/Boolean/ComponentParameter/readwrite//False/False/Enables writing to a dynamically resizeable staging buffer +# @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port +# @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port # @par EAGLE_END class PlasmaDROP(DataDROP): """ @@ -2220,16 +2219,13 @@ def dataURL(self) -> str: # @par EAGLE_START # @param category PlasmaFlight # @param tag daliuge -# @param[in] cparam/data_volume Data volume/5/Float/readwrite/False//False/ -# \~English Estimated size of the data contained in this node -# @param[in] cparam/group_end Group end/False/Boolean/readwrite/False//False/ -# \~English Is this node the end of a group? -# @param[in] cparam/plasma_path Plasma Path//String/readwrite/False//False/ -# \~English Path to the local plasma store -# @param[in] cparam/object_id Object Id//String/readwrite/False//False/ -# \~English PlasmaId of the object for all compute nodes -# @param[in] cparam/flight_path Flight Path//String/readwrite/False//False/ -# \~English IP and flight port of the drop owner +# @param data_volume Data volume/5/Float/ComponentParameter/readwrite//False/False/Estimated size of the data contained in this node +# @param group_end Group end/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the end of a group? +# @param plasma_path Plasma Path//String/ComponentParameter/readwrite//False/False/Path to the local plasma store +# @param object_id Object Id//String/ComponentParameter/readwrite//False/False/PlasmaId of the object for all compute nodes +# @param flight_path Flight Path//String/ComponentParameter/readwrite//False/False/IP and flight port of the drop owner +# @param dummy dummy//Object/InputPort/readwrite//False/False/Dummy input port +# @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port # @par EAGLE_END class PlasmaFlightDROP(DataDROP): """ @@ -2388,7 +2384,7 @@ def _generateNamedOutputs(self): """ Generates a named mapping of output data drops. Can only be called during run(). """ - named_outputs: OrderedDict[str, DataDROP] = OrderedDict() + named_outputs: OrderedDict[str, DataDROP] = OrderedDict() if 'outputs' in self.parameters and isinstance(self.parameters['outputs'][0], dict): for i in range(len(self._outputs)): key = list(self.parameters['outputs'][i].values())[0] @@ -2612,6 +2608,8 @@ def async_execute(self): t.daemon = 1 t.start() + _dlg_proc_lock = threading.Lock() + @track_current_drop def execute(self, _send_notifications=True): """ @@ -2633,8 +2631,13 @@ def execute(self, _send_notifications=True): try: if hasattr(self, "_tp"): proc = DlgProcess(target=self.run, daemon=True) - proc.start() - proc.join() + # see YAN-975 for why this is happening + lock = InputFiredAppDROP._dlg_proc_lock + with lock: + proc.start() + with lock: + proc.join() + proc.close() if proc.exception: raise proc.exception else: @@ -2648,7 +2651,7 @@ def execute(self, _send_notifications=True): return tries += 1 logger.exception( - "Error while executing %r (try %d/%d)" % (self, tries, self.n_tries) + "Error while executing %r (try %d/%d)", self, tries, self.n_tries ) # We gave up running the application, go to error @@ -2689,18 +2692,14 @@ def initialize(self, **kwargs): # @par EAGLE_START # @param category Branch # @param tag template -# @param[in] cparam/appclass Application Class/dlg.apps.simple.SimpleBranch/String/readonly/False//False/ -# \~English Application class -# @param[in] cparam/execution_time Execution Time/5/Float/readonly/False//False/ -# \~English Estimated execution time -# @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False//False/ -# \~English Number of cores used -# @param[in] cparam/group_start Group start/False/Boolean/readwrite/False//False/ -# \~English Is this node the start of a group? -# @param[in] cparam/input_error_threshold "Input error rate (%)"/0/Integer/readwrite/False//False/ -# \~English the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed -# @param[in] cparam/n_tries Number of tries/1/Integer/readwrite/False//False/ -# \~English Specifies the number of times the 'run' method will be executed before finally giving up +# @param appclass Application Class/dlg.apps.simple.SimpleBranch/String/ComponentParameter/readonly//False/False/Application class +# @param execution_time Execution Time/5/Float/ComponentParameter/readonly//False/False/Estimated execution time +# @param num_cpus No. of CPUs/1/Integer/ComponentParameter/readonly//False/False/Number of cores used +# @param group_start Group start/False/Boolean/ComponentParameter/readwrite//False/False/Is this node the start of a group? +# @param input_error_threshold "Input error rate (%)"/0/Integer/ComponentParameter/readwrite//False/False/the allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed +# @param n_tries Number of tries/1/Integer/ComponentParameter/readwrite//False/False/Specifies the number of times the 'run' method will be executed before finally giving up +# @param dummy0 dummy0//Object/OutputPort/readwrite//False/False/Dummy output port +# @param dummy1 dummy1//Object/OutputPort/readwrite//False/False/Dummy output port # @par EAGLE_END class BranchAppDrop(BarrierAppDROP): """ diff --git a/daliuge-engine/dlg/droputils.py b/daliuge-engine/dlg/droputils.py index 2c21042fb..6e4651428 100644 --- a/daliuge-engine/dlg/droputils.py +++ b/daliuge-engine/dlg/droputils.py @@ -25,13 +25,13 @@ import collections import io -import json +import time import logging import pickle import re import threading import traceback -from typing import IO, Any, AsyncIterable, BinaryIO, Dict, Iterable, overload +from typing import IO, Any, AsyncIterable, BinaryIO, Dict, Iterable, OrderedDict, Tuple, overload import numpy as np from dlg.ddap_protocol import DROPStates @@ -117,7 +117,7 @@ def __exit__(self, typ, value, tb): ) -def allDropContents(drop, bufsize=4096): +def allDropContents(drop, bufsize=4096) -> bytes: """ Returns all the data contained in a given DROP """ @@ -137,16 +137,32 @@ def copyDropContents(source: DataDROP, target: DataDROP, bufsize=4096): """ Manually copies data from one DROP into another, in bufsize steps """ - logger.debug(f"Copying from {repr(source)} to {repr(target)}") + logger.debug( + "Copying from %s to %s", repr(source), repr(target)) desc = source.open() buf = source.read(desc, bufsize) - logger.debug(f"Read {len(buf)} bytes from {repr(source)}") + logger.debug("Read %d bytes from %s", len(buf), + repr(source)) + st = time.time() + tot_w = len(buf) + ofl = True while buf: target.write(buf) - logger.debug(f"Wrote {len(buf)} bytes to {repr(target)}") + tot_w += len(buf) + dur = time.time() - st + if int(dur) % 5 == 0 and ofl: + logger.debug("Wrote %.1f MB to %s; rate %.2f MB/s", + tot_w/1024**2, repr(target), tot_w/(1024**2*dur)) + ofl = False + elif int(dur) % 5 == 4: + ofl = True buf = source.read(desc, bufsize) - if buf is not None: - logger.debug(f"Read {len(buf)} bytes from {repr(source)}") + # if buf is not None: + # logger.debug(f"Read {len(buf)} bytes from {repr(source)}") + dur = time.time() - st + logger.debug("Wrote %.1f MB to %s; rate %.2f MB/s", + tot_w/1024**2, repr(target), tot_w/(1024**2*dur)) + source.close(desc) @@ -267,24 +283,24 @@ def listify(o): return [o] -# def save_pickle(drop: DataDROP, data: Any): -# """Saves a python object in pkl format""" -# pickle.dump(data, drop) +def save_pickle(drop: DataDROP, data: Any): + """Saves a python object in pkl format""" + pickle.dump(data, drop) -# def load_pickle(drop: DataDROP) -> Any: -# """Loads a pkl formatted data object stored in a DataDROP. -# Note: does not support streaming mode. -# """ -# buf = io.BytesIO() -# desc = drop.open() -# while True: -# data = drop.read(desc) -# if not data: -# break -# buf.write(data) -# drop.close(desc) -# return pickle.loads(buf.getbuffer()) +def load_pickle(drop: DataDROP) -> Any: + """Loads a pkl formatted data object stored in a DataDROP. + Note: does not support streaming mode. + """ + buf = io.BytesIO() + desc = drop.open() + while True: + data = drop.read(desc) + if not data: + break + buf.write(data) + drop.close(desc) + return pickle.loads(buf.getbuffer()) # async def save_pickle_iter(drop: DataDROP, data: Iterable[Any]): @@ -298,7 +314,7 @@ def listify(o): # yield pickle.load(p) -def save_numpy(drop: DataDROP, ndarray: np.ndarray, allow_pickle=False): +def save_npy(drop: DataDROP, ndarray: np.ndarray, allow_pickle=False): """ Saves a numpy ndarray to a drop in npy format """ @@ -312,7 +328,11 @@ def save_numpy(drop: DataDROP, ndarray: np.ndarray, allow_pickle=False): dropio.close() -def load_numpy(drop: DataDROP, allow_pickle=False) -> np.ndarray: +def save_numpy(drop: DataDROP, ndarray: np.ndarray): + save_npy(drop, ndarray) + + +def load_npy(drop: DataDROP, allow_pickle=False) -> np.ndarray: """ Loads a numpy ndarray from a drop in npy format """ @@ -323,6 +343,10 @@ def load_numpy(drop: DataDROP, allow_pickle=False) -> np.ndarray: return res +def load_numpy(drop: DataDROP): + return load_npy(drop) + + # def save_jsonp(drop: PathBasedDrop, data: Dict[str, object]): # with open(drop.path, 'r') as f: # json.dump(data, f) @@ -505,44 +529,232 @@ def replace_dataurl_placeholders(cmd, inputs, outputs): return cmd +def serialize_kwargs(keyargs, prefix="--", separator=" "): + kwargs = [] + for (name, value) in iter(keyargs.items()): + if prefix == "--" and len(name) == 1: + kwargs += [f"-{name} {value}"] + else: + kwargs += [f"{prefix.strip()}{name.strip()}{separator}{str(value).strip()}"] + logger.debug("kwargs after serialization: %s",kwargs) + return kwargs -def serialize_applicationArgs(applicationArgs, prefix="--", separator=" "): +def clean_applicationArgs(applicationArgs:dict) -> dict: """ - Unpacks the applicationArgs dictionary and returns a string - that can be used as command line parameters. + Removes arguments with None and False values, if not precious. This + is in particular used for Bash and Docker app command lines, else + we would have empty values for command line arguments. + + Args: + applicationsArgs (dict): the complete set of arguments + + Returns: + dict: a dictionary with the relevant arguments only. """ + cleanedArgs = {} if not isinstance(applicationArgs, dict): logger.info("applicationArgs are not passed as a dict. Ignored!") else: logger.info("ApplicationArgs found %s", applicationArgs) - # construct the actual command line from all application parameters - args = [] - pargs = [] - positional = False - precious = False for (name, vdict) in applicationArgs.items(): if vdict in [None, False, ""]: continue elif isinstance(vdict, bool): - value = "" + vdict = {"precious": False, "value": "", "positional": False} elif isinstance(vdict, dict): precious = vdict["precious"] - value = vdict["value"] - if value in [None, False, ""] and not precious: + if vdict["value"] in [None, False, ""] and not precious: continue - positional = vdict["positional"] - # short and long version of keywords + cleanedArgs.update({name: vdict}) + return cleanedArgs + +def serialize_applicationArgs(applicationArgs, prefix="--", separator=" "): + """ + Unpacks the applicationArgs dictionary and returns two strings, one for + positional arguments and one for kw arguments that can be used to construct + the final command line. + """ + applicationArgs = clean_applicationArgs(applicationArgs, + prefix=prefix, separator=separator) + pargs = [] + kwargs = {} + for (name, vdict) in applicationArgs.items(): + value = vdict["value"] + positional = vdict["positional"] if positional: pargs.append(str(value).strip()) else: - if prefix == "--" and len(name) == 1: - arg = [f"-{name} {value}"] - else: - arg = [f"{prefix}{name}{separator}{value}".strip()] - args += arg - logger.info('Arguments of bash command: %s %s', args, pargs) - return f"{' '.join(args + pargs)}" # add positional arguments to end of args + kwargs.update({name:value}) + skwargs = serialize_kwargs(kwargs, prefix=prefix, separator=separator) + logger.info('Constructed command line arguments: %s %s', pargs, kwargs) + return (pargs, skwargs) + +def identify_named_ports( + port_dict:dict, + posargs:list, + pargsDict:dict, + keyargs: dict, + check_len: int=0, + mode: str="inputs" + ) -> dict: + """ + Checks port names for matches with arguments and returns mapped ports. + + Args: + port_dict (dict): ports {uid:name,...} + posargs (list): available positional arguments (will be modified) + pargsDict (dict): mapped arguments (will be modified) + + Returns: + dict: port arguments + + Side effect: + modifies the pargsDict OrderedDict + """ + logger.debug("Using named ports to remove %s from arguments port_dict, check_len): %s %d", + mode, port_dict, check_len) + portargs = {} + posargs = list(posargs) + keys = list(port_dict.keys()) + for i in range(check_len): + try: + key = port_dict[keys[i]]['name'] + value = port_dict[keys[i]]['path'] + except KeyError: + logger.debug("portDict: %s", port_dict) + raise KeyError + if not value: value = '' # make sure we are passing NULL drop events + if key in posargs: + pargsDict.update({key:value}) + logger.debug("Using %s '%s' for parg %s", mode, value, key) + posargs.pop(posargs.index(key)) + elif key in keyargs: + # if not found in appArgs we don't put them into portargs either + portargs.update({key:value}) + logger.debug("Using %s '%s' for kwarg %s", mode, value, key) + _dum = keyargs.pop(key) # remove from original arg list + else: + logger.debug("No matching argument found for %s key %s", mode, key) + logger.debug("Returning kw mapped ports: %s", portargs) + return portargs + +def check_ports_dict(ports:list) -> bool: + """ + Checks whether all ports in ports list are of type dict. This is + for backwards compatibility. + + Args: + ports (list): + Returns: + bool: True if all ports are dict, else False + """ + # all returns true if list is empty! + if len(ports) > 0: + return all(isinstance(p, dict) for p in ports) + else: + return False + + +def replace_named_ports( + iitems:dict, + oitems:dict, + inport_names:dict, + outport_names:dict, + appArgs:dict, argumentPrefix="--", + separator=" " + ) -> Tuple[str, str]: + """ + Function attempts to identify component arguments that match port names. + + Inputs: + iitems: itemized input port dictionary + oitems: itemized output port dictionary + inport_names: dictionary of input port names (key: uid) + outport_names: dictionary of output port names (key: uid) + appArgs: dictionary of all arguments + argumentPrefix: prefix for keyword arguments + separator: character used between keyword and value + + Returns: + tuple of serialized keyword arguments and positional arguments + """ + logger.debug("iitems: %s; inport_names: %s; outport_names: %s", + iitems, inport_names, outport_names) + inputs_dict = collections.OrderedDict() + for uid, drop in iitems: + inputs_dict[uid] = {'path': drop.path if hasattr(drop, 'path') else ''} + + outputs_dict = collections.OrderedDict() + for uid, drop in oitems: + outputs_dict[uid] = {'path': drop.path if hasattr(drop, 'path') else ''} + logger.debug("appArgs: %s", appArgs) + # get positional args + posargs = [arg for arg in appArgs if appArgs[arg]["positional"]] + # get kwargs + keyargs = {arg:appArgs[arg]["value"] for arg in appArgs + if not appArgs[arg]["positional"]} + # we will need an ordered dict for all positional arguments + # thus we create it here and fill it with values + portPosargsDict = collections.OrderedDict(zip(posargs,[None]*len(posargs))) + portkeyargs = {} + logger.debug("posargs: %s; keyargs: %s",posargs, keyargs) + if check_ports_dict(inport_names): + for inport in inport_names: + key = list(inport.keys())[0] + inputs_dict[key].update({'name':inport[key]}) + + ipkeyargs = identify_named_ports( + inputs_dict, + posargs, + portPosargsDict, + keyargs, + check_len=len(iitems), + mode="inputs") + portkeyargs.update(ipkeyargs) + else: + for i in range(min(len(iitems), len(posargs))): + portkeyargs.update({posargs[i]: iitems[i][1]}) + + if check_ports_dict(outport_names): + for outport in outport_names: + key = list(outport.keys())[0] + outputs_dict[key].update({'name':outport[key]}) + opkeyargs = identify_named_ports( + outputs_dict, + posargs, + portPosargsDict, + keyargs, + check_len=len(oitems), + mode="outputs") + portkeyargs.update(opkeyargs) + else: + for i in range(min(len(oitems), len(posargs))): + portkeyargs.update({posargs[i]: oitems[i][1]}) + # now that we have the mapped ports we can cleanup the appArgs + # and construct the final keyargs and pargs + logger.debug("Arguments from ports: %s %s", portkeyargs, portPosargsDict) + appArgs = clean_applicationArgs(appArgs) + # get cleaned positional args + posargs = {arg:appArgs[arg]["value"] for arg in appArgs + if appArgs[arg]["positional"]} + # get cleaned kwargs + keyargs = {arg:appArgs[arg]["value"] for arg in appArgs + if not appArgs[arg]["positional"]} + # update port dictionaries + # portkeyargs.update({key:arg for key, arg in keyargs.items() + # if key not in portkeyargs}) + # portPosargsDict.update({key:arg for key, arg in posargs.items() + # if key not in portPosargsDict}) + keyargs.update(portkeyargs) + posargs.update(portPosargsDict) + keyargs = serialize_kwargs(keyargs, + prefix=argumentPrefix, + separator=separator) if len(keyargs) > 0 else [''] + pargs = list(portPosargsDict.values()) + pargs = [''] if len(pargs) == 0 or None in pargs else pargs + logger.debug("After port replacement: pargs: %s; keyargs: %s",pargs, keyargs) + return keyargs, pargs # Easing the transition from single- to multi-package get_leaves = common.get_leaves diff --git a/daliuge-engine/dlg/environmentvar_drop.py b/daliuge-engine/dlg/environmentvar_drop.py index d0f5948c8..964f976bf 100644 --- a/daliuge-engine/dlg/environmentvar_drop.py +++ b/daliuge-engine/dlg/environmentvar_drop.py @@ -61,11 +61,12 @@ def _filter_parameters(parameters: dict): ## -# @brief Environment variables +# @brief EnvironmentVariables # @details A set of environment variables, wholly specified in EAGLE and accessible to all drops. # @par EAGLE_START # @param category EnvironmentVariables # @param tag daliuge +# @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port # @par EAGLE_END class EnvironmentVarDROP(AbstractDROP, KeyValueDROP): """ diff --git a/daliuge-engine/dlg/graph_loader.py b/daliuge-engine/dlg/graph_loader.py index ffd231e23..958afefe8 100644 --- a/daliuge-engine/dlg/graph_loader.py +++ b/daliuge-engine/dlg/graph_loader.py @@ -30,8 +30,6 @@ from dlg.common.reproducibility.constants import ReproducibilityFlags -from numpy import isin - from . import droputils from .apps.socket_listener import SocketListenerApp from .common import Categories @@ -53,7 +51,7 @@ from dlg.parset_drop import ParameterSetDROP from .exceptions import InvalidGraphException from .json_drop import JsonDROP -from .common import Categories, DropType +from .common import DropType STORAGE_TYPES = { @@ -136,19 +134,19 @@ def addLink(linkType, lhDropSpec, rhOID, force=False): def removeUnmetRelationships(dropSpecList): unmetRelationships = [] + normalise_oid = lambda oid: next(iter(oid)) if isinstance(oid, dict) else oid + # Step #1: Get all OIDs - oids = [] + oids = set() for dropSpec in dropSpecList: - oid = dropSpec["oid"] - oid = list(oid.keys())[0] if isinstance(oid, dict) else oid - oids.append(oid) + oid = normalise_oid(dropSpec["oid"]) + oids.add(oid) # Step #2: find unmet relationships and remove them from the original # DROP spec, keeping track of them for dropSpec in dropSpecList: - this_oid = dropSpec["oid"] - this_oid = list(this_oid.keys())[0] if isinstance(this_oid, dict) else this_oid + this_oid = normalise_oid(dropSpec["oid"]) to_delete = [] for rel in dropSpec: @@ -162,7 +160,8 @@ def removeUnmetRelationships(dropSpecList): # removing them from the current DROP spec ds = dropSpec[rel] if isinstance(ds[0], dict): - ds = [list(d.keys())[0] for d in ds] + ds = [next(iter(d)) for d in ds] +# ds = [normalise_oid(d) for d in ds] missingOids = [oid for oid in ds if oid not in oids] for oid in missingOids: unmetRelationships.append(DROPRel(oid, link, this_oid)) @@ -178,8 +177,7 @@ def removeUnmetRelationships(dropSpecList): link = __TOONE[rel] # Check if OID is missing - oid = dropSpec[rel] - oid = list(oid.keys())[0] if isinstance(oid, dict) else oid + oid = normalise_oid(dropSpec[rel]) if oid in oids: continue @@ -190,9 +188,7 @@ def removeUnmetRelationships(dropSpecList): to_delete.append(rel) for rel in to_delete: - ds = dropSpec[rel] - ds = list(ds.keys())[0] if isinstance(ds, dict) else ds - del ds + del dropSpec[rel] return unmetRelationships diff --git a/daliuge-engine/dlg/io.py b/daliuge-engine/dlg/io.py index 1201226a7..230eb6993 100644 --- a/daliuge-engine/dlg/io.py +++ b/daliuge-engine/dlg/io.py @@ -597,7 +597,7 @@ def _close(self, **kwargs): # If length wasn't known up-front we first send Content-Length and then the buffer here. conn.putheader("Content-Length", len(self._buf)) conn.endheaders() - logger.debug("Sending data for file %s to NGAS" % (self._fileId)) + logger.debug("Sending data for file %s to NGAS", self._fileId) conn.send(self._buf) self._buf = None else: @@ -618,7 +618,7 @@ def _write(self, data, **kwargs) -> int: self._buf += data else: self._desc.send(data) - logger.debug("Wrote %s bytes" % len(data)) + logger.debug("Wrote %s bytes", len(data)) return len(data) def exists(self) -> bool: diff --git a/daliuge-engine/dlg/lifecycle/dlm.py b/daliuge-engine/dlg/lifecycle/dlm.py index 36721315c..bf751e2ea 100644 --- a/daliuge-engine/dlg/lifecycle/dlm.py +++ b/daliuge-engine/dlg/lifecycle/dlm.py @@ -139,14 +139,14 @@ class DataLifecycleManagerBackgroundTask(threading.Thread): signaled to stop """ - def __init__(self, dlm, period, finishedEvent): + def __init__(self, name, dlm, period): threading.Thread.__init__(self, name="DLMBackgroundTask") self._dlm = dlm self._period = period - self._finishedEvent = finishedEvent + logger.info("Starting %s running every %.3f [s]", name, self._period) def run(self): - ev = self._finishedEvent + ev = self._dlm._finishedEvent dlm = self._dlm p = self._period while True: @@ -204,11 +204,19 @@ def handleEvent(self, event): self._dlm.handleCompletedDrop(event.uid) -class DataLifecycleManager(object): - def __init__(self, **kwargs): - self._hsm = manager.HierarchicalStorageManager() +class DataLifecycleManager: + """ + An object that deals with automatic data drop replication and deletion. + """ + + def __init__(self, check_period=0, cleanup_period=0, enable_drop_replication=False): self._reg = registry.InMemoryRegistry() self._listener = DropEventListener(self) + self._enable_drop_replication = enable_drop_replication + if enable_drop_replication: + self._hsm = manager.HierarchicalStorageManager() + else: + self._hsm = None # TODO: When iteration over the values of _drops we always do _drops.values() # instead of _drops.itervalues() to get a full, thread-safe copy of the @@ -216,35 +224,38 @@ def __init__(self, **kwargs): # here self._drops = {} - self._checkPeriod = 10 - if "checkPeriod" in kwargs: - self._checkPeriod = float(kwargs["checkPeriod"]) - - self._cleanupPeriod = 10 * self._checkPeriod - if "cleanupPeriod" in kwargs: - self._cleanupPeriod = float(kwargs["cleanupPeriod"]) + self._check_period = check_period + self._cleanup_period = cleanup_period + self._drop_checker = None + self._drop_garbage_collector = None + self._finishedEvent = threading.Event() def startup(self): # Spawn the background threads - finishedEvent = threading.Event() - dropChecker = DROPChecker(self, self._checkPeriod, finishedEvent) - dropChecker.start() - dropGarbageCollector = DROPGarbageCollector( - self, self._cleanupPeriod, finishedEvent - ) - dropGarbageCollector.start() - - self._dropChecker = dropChecker - self._dropGarbageCollector = dropGarbageCollector - self._finishedEvent = finishedEvent + if self._check_period: + self._drop_checker = DROPChecker( + "DropChecker", + self, + self._check_period + ) + self._drop_checker.start() + if self._cleanup_period: + self._drop_garbage_collector = DROPGarbageCollector( + "DropGarbageCollector", + self, + self._cleanup_period + ) + self._drop_garbage_collector.start() def cleanup(self): logger.info("Cleaning up the DLM") # Join the background threads self._finishedEvent.set() - self._dropChecker.join() - self._dropGarbageCollector.join() + if self._drop_checker: + self._drop_checker.join() + if self._drop_garbage_collector: + self._drop_garbage_collector.join() # Unsubscribe to all events coming from the DROPs for drop in self._drops.values(): @@ -281,10 +292,8 @@ def expireCompletedDrops(self): # are finished using this DROP if drop.expireAfterUse: allDone = all( - [ - c.execStatus in [AppDROPStates.FINISHED, AppDROPStates.ERROR] - for c in drop.consumers - ] + c.execStatus in [AppDROPStates.FINISHED, AppDROPStates.ERROR] + for c in drop.consumers ) if not allDone: continue @@ -447,6 +456,16 @@ def addDrop(self, drop): # perform this task, like using threading.Timers (probably not) or # any other that doesn't mean looping over all DROPs + def remove_drops(self, drop_oids): + """ + Remove drops from DLM's monitoring + """ + self._drops = { + oid: drop + for oid, drop in self._drops.items() + if oid not in drop_oids + } + def handleOpenedDrop(self, oid, uid): drop = self._drops[uid] if drop.status == DROPStates.COMPLETED: @@ -459,6 +478,9 @@ def handleCompletedDrop(self, uid): # Check the kind of storage used by this DROP. If it's already persisted # in a persistent storage media we don't need to save it again + if not self._enable_drop_replication: + return + drop = self._drops[uid] if drop.precious and self.isReplicable(drop): logger.debug("Replicating %r because it's precious", drop) diff --git a/daliuge-engine/dlg/lifecycle/hsm/manager.py b/daliuge-engine/dlg/lifecycle/hsm/manager.py index 1e6ac89fb..7b0cf626a 100644 --- a/daliuge-engine/dlg/lifecycle/hsm/manager.py +++ b/daliuge-engine/dlg/lifecycle/hsm/manager.py @@ -43,7 +43,7 @@ def addStore(self, newStore): """ @param newStore store.AbstractStore """ - logger.debug("Adding store to HSM: " + str(newStore)) + logger.debug("Adding store to HSM: %s", str(newStore)) self._stores.append(newStore) def getSlowestStore(self): diff --git a/daliuge-engine/dlg/lifecycle/hsm/store.py b/daliuge-engine/dlg/lifecycle/hsm/store.py index 389b3f55e..3e235a0e9 100644 --- a/daliuge-engine/dlg/lifecycle/hsm/store.py +++ b/daliuge-engine/dlg/lifecycle/hsm/store.py @@ -60,8 +60,8 @@ def updateSpaces(self): total = self.getTotalSpace() perc = avail * 100.0 / total logger.debug( - "Available/Total space on %s: %d/%d (%.2f %%)" - % (self, avail, total, perc) + "Available/Total space on %s: %d/%d (%.2f %%)", + self, avail, total, perc ) pass diff --git a/daliuge-engine/dlg/lifecycle/registry.py b/daliuge-engine/dlg/lifecycle/registry.py index 61faffef9..a8cd9e0c1 100644 --- a/daliuge-engine/dlg/lifecycle/registry.py +++ b/daliuge-engine/dlg/lifecycle/registry.py @@ -162,7 +162,7 @@ def __init__(self, dbModuleName, *connArgs): self._connArgs = connArgs except: logger.error( - "Cannot import module %s, RDBMSRegistry cannot start" % (dbModuleName) + "Cannot import module %s, RDBMSRegistry cannot start", dbModuleName ) raise diff --git a/daliuge-engine/dlg/manager/cmdline.py b/daliuge-engine/dlg/manager/cmdline.py index ad33aa71c..88d0437b1 100644 --- a/daliuge-engine/dlg/manager/cmdline.py +++ b/daliuge-engine/dlg/manager/cmdline.py @@ -69,7 +69,7 @@ def launchServer(opts): dmName = opts.dmType.__name__ logger.info("DALiuGE version %s running at %s", version.full_version, os.getcwd()) - logger.info("Creating %s" % (dmName)) + logger.info("Creating %s", dmName) try: dm = opts.dmType(*opts.dmArgs, **opts.dmKwargs) except: @@ -86,7 +86,7 @@ def handle_signal(signNo, stack_frame): if _terminating: return _terminating = True - logger.info("Exiting from %s" % (dmName)) + logger.info("Exiting from %s", dmName) server.stop_manager() @@ -350,8 +350,24 @@ def dlgNM(parser, args): "--no-dlm", action="store_true", dest="noDLM", - help="Don't start the Data Lifecycle Manager on this NodeManager", - default=True, + help="(DEPRECATED) Don't start the Data Lifecycle Manager on this NodeManager", + ) + parser.add_option( + "--dlm-check-period", + type="float", + help="Time in seconds between background DLM drop status checks (defaults to 10)", + default=10 + ) + parser.add_option( + "--dlm-cleanup-period", + type="float", + help="Time in seconds between background DLM drop automatic cleanups (defaults to 30)", + default=30 + ) + parser.add_option( + "--dlm-enable-replication", + action="store_true", + help="Turn on data drop automatic replication (off by default)", ) parser.add_option( "--dlg-path", @@ -388,13 +404,22 @@ def dlgNM(parser, args): ) (options, args) = parser.parse_args(args) + # No logging setup at this point yet + if options.noDLM: + print("WARNING: --no-dlm is deprecated, use the --dlm-* options instead") + options.dlm_check_period = 0 + options.dlm_cleanup_period = 0 + options.dlm_enable_replication = False + # Add DM-specific options # Note that the host we use to expose the NodeManager itself through Pyro is # also used to expose the Sessions it creates options.dmType = NodeManager options.dmArgs = () options.dmKwargs = { - "useDLM": not options.noDLM, + "dlm_check_period": options.dlm_check_period, + "dlm_cleanup_period": options.dlm_cleanup_period, + "dlm_enable_replication": options.dlm_enable_replication, "dlgPath": options.dlgPath, "host": options.host, "error_listener": options.errorListener, diff --git a/daliuge-engine/dlg/manager/composite_manager.py b/daliuge-engine/dlg/manager/composite_manager.py index 8e34c9d6b..198f2e6a6 100644 --- a/daliuge-engine/dlg/manager/composite_manager.py +++ b/daliuge-engine/dlg/manager/composite_manager.py @@ -127,13 +127,13 @@ class allows for multiple levels of hierarchy seamlessly. __metaclass__ = abc.ABCMeta def __init__( - self, - dmPort, - partitionAttr, - subDmId, - dmHosts=[], - pkeyPath=None, - dmCheckTimeout=10, + self, + dmPort, + partitionAttr, + subDmId, + dmHosts=[], + pkeyPath=None, + dmCheckTimeout=10, ): """ Creates a new CompositeManager. The sub-DMs it manages are to be located @@ -211,6 +211,10 @@ def dmHosts(self): def addDmHost(self, host): self._dmHosts.append(host) + def removeDmHost(self, host): + if host in self._dmHosts: + self._dmHosts.remove(host) + @property def nodes(self): return self._nodes[:] @@ -235,7 +239,7 @@ def dmAt(self, host, port=None): if not self.check_dm(host, port): raise SubManagerException( - "Manager expected but not running in %s:%d" % (host, port) + f"Manager expected but not running in {host}:{port}" ) port = port or self._dmPort @@ -284,10 +288,7 @@ def replicate(self, sessionId, f, action, collect=None, iterable=None, port=None iterable, ) if thrExs: - msg = "More than one error occurred while %s on session %s" % ( - action, - sessionId, - ) + msg = f"More than one error occurred while {action} on session {sessionId}" raise SubManagerException(msg, thrExs) # @@ -347,7 +348,7 @@ def addGraphSpec(self, sessionId, graphSpec): # belong to the same host, so we can submit that graph into the individual # DMs. For this we need to make sure that our graph has a the correct # attribute set - logger.info(f"Separating graph using partition attribute {self._partitionAttr}") + logger.info("Separating graph using partition attribute %s", self._partitionAttr) perPartition = collections.defaultdict(list) if "rmode" in graphSpec[-1]: init_pg_repro_data(graphSpec) @@ -360,19 +361,14 @@ def addGraphSpec(self, sessionId, graphSpec): graphSpec.pop() for dropSpec in graphSpec: if self._partitionAttr not in dropSpec: - msg = "Drop %s doesn't specify a %s attribute" % ( - dropSpec["oid"], - self._partitionAttr, - ) + msg = f"Drop {dropSpec.get('oid', None)} doesn't specify a {self._partitionAttr} " \ + f"attribute" raise InvalidGraphException(msg) partition = dropSpec[self._partitionAttr] if partition not in self._dmHosts: - msg = "Drop %s's %s %s does not belong to this DM" % ( - dropSpec["oid"], - self._partitionAttr, - partition, - ) + msg = f"Drop {dropSpec.get('oid', None)}'s {self._partitionAttr} {partition} " \ + f"does not belong to this DM" raise InvalidGraphException(msg) perPartition[partition].append(dropSpec) @@ -382,7 +378,7 @@ def addGraphSpec(self, sessionId, graphSpec): # At each partition the relationships between DROPs should be local at the # moment of submitting the graph; thus we record the inter-partition # relationships separately and remove them from the original graph spec - logger.info(f"Graph splitted into {perPartition.keys()}") + logger.info("Graph split into %r", perPartition.keys()) inter_partition_rels = [] for dropSpecs in perPartition.values(): inter_partition_rels += graph_loader.removeUnmetRelationships(dropSpecs) @@ -448,7 +444,7 @@ def deploySession(self, sessionId, completedDrops=[]): ) logger.info("Delivered node subscription list to node managers") logger.debug( - "Number of subscriptions: %s" % len(self._drop_rels[sessionId].items()) + "Number of subscriptions: %s", len(self._drop_rels[sessionId].items()) ) logger.info("Deploying Session %s in all hosts", sessionId) diff --git a/daliuge-engine/dlg/manager/node_manager.py b/daliuge-engine/dlg/manager/node_manager.py index 762ccae1e..732a0313a 100644 --- a/daliuge-engine/dlg/manager/node_manager.py +++ b/daliuge-engine/dlg/manager/node_manager.py @@ -35,7 +35,6 @@ import threading import time -from glob import glob from . import constants from .drop_manager import DROPManager from .session import Session @@ -128,7 +127,9 @@ class NodeManagerBase(DROPManager): def __init__( self, - useDLM=False, + dlm_check_period=0, + dlm_cleanup_period=0, + dlm_enable_replication=False, dlgPath=None, error_listener=None, event_listeners=[], @@ -136,7 +137,11 @@ def __init__( logdir=utils.getDlgLogsDir(), ): - self._dlm = DataLifecycleManager() if useDLM else None + self._dlm = DataLifecycleManager( + check_period=dlm_check_period, + cleanup_period=dlm_cleanup_period, + enable_drop_replication=dlm_enable_replication + ) self._sessions = {} self.logdir = logdir @@ -181,39 +186,16 @@ def __init__( debugging = logger.isEnabledFor(logging.DEBUG) self._logging_event_listener = LogEvtListener() if debugging else None - # Start the mix-ins - self.start() - - @abc.abstractmethod def start(self): - """ - Starts any background task required by this Node Manager - """ + super().start() + self._dlm.startup() - @abc.abstractmethod def shutdown(self): - """ - Stops any pending background task run by this Node Manager - """ - - @abc.abstractmethod - def subscribe(self, host, port): - """ - Subscribes this Node Manager to events published in from ``host``:``port`` - """ - - @abc.abstractmethod - def publish_event(self, evt): - """ - Publishes the event ``evt`` for other Node Managers to receive it - """ - - @abc.abstractmethod - def get_rpc_client(self, hostname, port): - """ - Creates an RPC client connected to the node manager running in - ``host``:``port``, and its closing method, as a 2-tuple. - """ + self._dlm.cleanup() + if self._threadpool: + self._threadpool.close() + self._threadpool.join() + super().shutdown() def deliver_event(self, evt): """ @@ -222,8 +204,8 @@ def deliver_event(self, evt): """ if not evt.session_id in self._sessions: logger.warning( - "No session %s found, event (%s) will be dropped" - % (evt.session_id, evt.type) + "No session %s found, event (%s) will be dropped", + evt.session_id, evt.type ) return self._sessions[evt.session_id].deliver_event(evt) @@ -288,8 +270,7 @@ def foreach(drop): ) drop._sessID = sessionId self._memoryManager.register_drop(drop.uid, sessionId) - if self._dlm: - self._dlm.addDrop(drop) + self._dlm.addDrop(drop) # Remote event forwarding evt_listener = NMDropEventListener(self, sessionId) @@ -326,6 +307,7 @@ def destroySession(self, sessionId): session = self._sessions.pop(sessionId) if hasattr(self, "_memoryManager"): self._memoryManager.shutdown_session(sessionId) + self._dlm.remove_drops(session.drops) session.destroy() def getSessionIds(self): @@ -374,11 +356,6 @@ def call_drop(self, sessionId, uid, method, *args): self._check_session_id(sessionId) return self._sessions[sessionId].call_drop(uid, method, *args) - def shutdown(self): - if hasattr(self, "_threadpool") and self._threadpool is not None: - self._threadpool.close() - self._threadpool.join() - class ZMQPubSubMixIn(object): """ @@ -568,28 +545,26 @@ class RpcMixIn(rpc.RPCClient, rpc.RPCServer): # Final NodeManager class -class NodeManager(EventMixIn, RpcMixIn, NodeManagerBase): +class NodeManager(NodeManagerBase, EventMixIn, RpcMixIn): def __init__( self, - useDLM=True, - dlgPath=utils.getDlgPath(), - error_listener=None, - event_listeners=[], - max_threads=0, - logdir=utils.getDlgLogsDir(), host=None, rpc_port=constants.NODE_DEFAULT_RPC_PORT, events_port=constants.NODE_DEFAULT_EVENTS_PORT, + *args, + **kwargs ): - # We "just know" that our RpcMixIn will have a create_context static - # method, which in reality means we are using the ZeroRPCServer class - self._context = RpcMixIn.create_context() host = host or "127.0.0.1" + NodeManagerBase.__init__(self, *args, **kwargs) EventMixIn.__init__(self, host, events_port) RpcMixIn.__init__(self, host, rpc_port) - NodeManagerBase.__init__( - self, useDLM, dlgPath, error_listener, event_listeners, max_threads, logdir - ) + self.start() + + def start(self): + # We "just know" that our RpcMixIn will have a create_context static + # method, which in reality means we are using the ZeroRPCServer class + self._context = RpcMixIn.create_context() + super().start() def shutdown(self): super(NodeManager, self).shutdown() diff --git a/daliuge-engine/dlg/manager/proc_daemon.py b/daliuge-engine/dlg/manager/proc_daemon.py index 5244a30a2..6f49e4f90 100644 --- a/daliuge-engine/dlg/manager/proc_daemon.py +++ b/daliuge-engine/dlg/manager/proc_daemon.py @@ -37,6 +37,7 @@ from . import constants, client from .. import utils from ..restserver import RestServer +from dlg.nm_dim_assigner import NMAssigner logger = logging.getLogger(__name__) @@ -74,7 +75,6 @@ def __init__(self, master=False, noNM=False, disable_zeroconf=False, verbosity=0 self._shutting_down = False self._verbosity = verbosity - # The three processes we run self._nm_proc = None self._dim_proc = None @@ -83,7 +83,9 @@ def __init__(self, master=False, noNM=False, disable_zeroconf=False, verbosity=0 # Zeroconf for NM and MM self._zeroconf = None if disable_zeroconf else zc.Zeroconf() self._nm_info = None - self._mm_browser = None + self._dim_info = None + self._mm_nm_browser = None + self._mm_dim_browser = None # Starting managers app = self.app @@ -125,9 +127,12 @@ def _stop_zeroconf(self): return # Stop the MM service browser, the NM registration, and ZC itself - if self._mm_browser: - self._mm_browser.cancel() - self._mm_browser.join() + if self._mm_nm_browser: + self._mm_nm_browser.cancel() + self._mm_nm_browser.join() + if self._mm_dim_browser: + self._mm_dim_browser.cancel() + self._mm_dim_browser.join() self._zeroconf.close() logger.info("Zeroconf stopped") @@ -166,6 +171,8 @@ def stopNM(self, timeout=10): return self._stop_manager("_nm_proc", timeout) def stopDIM(self, timeout=10): + if self._dim_info: + utils.deregister_service(self._zeroconf, self._dim_info) self._stop_manager("_dim_proc", timeout) def stopMM(self, timeout=10): @@ -176,15 +183,15 @@ def startNM(self): tool = get_tool() args = ["--host", "0.0.0.0"] args += self._verbosity_as_cmdline() - logger.info("Starting Node Drop Manager with args: %s" % (" ".join(args))) + logger.info("Starting Node Drop Manager with args: %s", (" ".join(args))) self._nm_proc = tool.start_process("nm", args) - logger.info("Started Node Drop Manager with PID %d" % (self._nm_proc.pid)) + logger.info("Started Node Drop Manager with PID %d", self._nm_proc.pid) # Registering the new NodeManager via zeroconf so it gets discovered # by the Master Manager if self._zeroconf: addrs = utils.get_local_ip_addr() - logger.info("Registering this NM with zeroconf: %s" % addrs) + logger.info("Registering this NM with zeroconf: %s", addrs) self._nm_info = utils.register_service( self._zeroconf, "NodeManager", @@ -201,58 +208,80 @@ def startDIM(self, nodes): if nodes: args += ["--nodes", ",".join(nodes)] logger.info( - "Starting Data Island Drop Manager with args: %s" % (" ".join(args)) + "Starting Data Island Drop Manager with args: %s", (" ".join(args)) ) self._dim_proc = tool.start_process("dim", args) logger.info( - "Started Data Island Drop Manager with PID %d" % (self._dim_proc.pid) + "Started Data Island Drop Manager with PID %d", self._dim_proc.pid ) + + # Registering the new DIM via zeroconf so it gets discovered + # by the Master Manager + if self._zeroconf: + addrs = utils.get_local_ip_addr() + logger.info("Registering this DIM with zeroconf: %s", addrs) + self._dim_info = utils.register_service( + self._zeroconf, + "DIM", + socket.gethostname(), + addrs[0][0], + constants.ISLAND_DEFAULT_REST_PORT, + ) return def startMM(self): tool = get_tool() args = ["--host", "0.0.0.0"] args += self._verbosity_as_cmdline() - logger.info("Starting Master Drop Manager with args: %s" % (" ".join(args))) + logger.info("Starting Master Drop Manager with args: %s", (" ".join(args))) self._mm_proc = tool.start_process("mm", args) - logger.info("Started Master Drop Manager with PID %d" % (self._mm_proc.pid)) + logger.info("Started Master Drop Manager with PID %d", self._mm_proc.pid) # Also subscribe to zeroconf events coming from NodeManagers and feed # the Master Manager with the new hosts we find if self._zeroconf: - mm_client = client.MasterManagerClient() - node_managers = {} + nm_assigner = NMAssigner() - def nm_callback(zeroconf, service_type, name, state_change): + def _callback(zeroconf, service_type, name, state_change, adder, remover, accessor): info = zeroconf.get_service_info(service_type, name) if state_change is zc.ServiceStateChange.Added: server = socket.inet_ntoa(_get_address(info)) port = info.port - node_managers[name] = (server, port) + adder(name, server, port) logger.info( - "Found a new Node Manager on %s:%d, will add it to the MM" - % (server, port) + "Found a new %s on %s:%d, will add it to the MM", + service_type, server, port ) - mm_client.add_node(server) elif state_change is zc.ServiceStateChange.Removed: - server, port = node_managers[name] + server, port = accessor(name) logger.info( - "Node Manager on %s:%d disappeared, removing it from the MM" - % (server, port) + "%s on %s:%d disappeared, removing it from the MM", + service_type, server, port ) # Don't bother to remove it if we're shutting down. This way # we avoid hanging in here if the MM is down already but # we are trying to remove our NM who has just disappeared if not self._shutting_down: - try: - mm_client.remove_node(server) - finally: - del node_managers[name] + remover(name) + + nm_callback = functools.partial(_callback, adder=nm_assigner.add_nm, + remover=nm_assigner.remove_nm, + accessor=nm_assigner.get_nm) - self._mm_browser = utils.browse_service( + dim_callback = functools.partial(_callback, adder=nm_assigner.add_dim, + remover=nm_assigner.remove_dim, + accessor=nm_assigner.get_dim) + + self._mm_nm_browser = utils.browse_service( self._zeroconf, "NodeManager", "tcp", nm_callback ) + self._mm_dim_browser = utils.browse_service( + self._zeroconf, + "DIM", + "tcp", + dim_callback, # DIM since name must be < 15 bytes + ) logger.info("Zeroconf started") return @@ -278,7 +307,7 @@ def _rest_stop_manager(self, proc, stop_method): def _rest_get_manager_info(self, proc): if proc: bottle.response.content_type = "application/json" - logger.info("Sending response: %s" % json.dumps({"pid": proc.pid})) + logger.info("Sending response: %s", json.dumps({"pid": proc.pid})) return json.dumps({"pid": proc.pid}) else: return json.dumps({"pid": None}) @@ -296,7 +325,7 @@ def rest_getMgrs(self): if mgrs["node"]: mgrs["node"] = self._nm_proc.pid - logger.info("Sending response: %s" % json.dumps(mgrs)) + logger.info("Sending response: %s", json.dumps(mgrs)) return json.dumps(mgrs) def rest_startNM(self): @@ -396,7 +425,7 @@ def handle_signal(signalNo, stack_frame): global terminating if terminating: return - logger.info("Received signal %d, will stop the daemon now" % (signalNo,)) + logger.info("Received signal %d, will stop the daemon now", signalNo) terminating = True daemon.stop(10) diff --git a/daliuge-engine/dlg/manager/rest.py b/daliuge-engine/dlg/manager/rest.py index 5d1af58bf..2e8454f96 100644 --- a/daliuge-engine/dlg/manager/rest.py +++ b/daliuge-engine/dlg/manager/rest.py @@ -191,8 +191,8 @@ def _stop_manager(self): self.dm.shutdown() self.stop() logger.info( - "Thanks for using our %s, come back again :-)" - % (self.dm.__class__.__name__) + "Thanks for using our %s, come back again :-)", + self.dm.__class__.__name__ ) @daliuge_aware @@ -400,7 +400,7 @@ def linkGraphParts(self, sessionId): @daliuge_aware def add_node_subscriptions(self, sessionId): - logger.debug(f"NM REST call: add_subscriptions {bottle.request.json}") + logger.debug("NM REST call: add_subscriptions %s", bottle.request.json) if bottle.request.content_type != "application/json": bottle.response.status = 415 return @@ -471,18 +471,18 @@ def getAllCMNodes(self): @daliuge_aware def addCMNode(self, node): - logger.debug("Adding node %s" % node) + logger.debug("Adding node %s", node) self.dm.add_node(node) @daliuge_aware def removeCMNode(self, node): - logger.debug("Removing node %s" % node) + logger.debug("Removing node %s", node) self.dm.remove_node(node) @daliuge_aware def getNodeSessions(self, node): if node not in self.dm.nodes: - raise Exception("%s not in current list of nodes" % (node,)) + raise Exception(f"{node} not in current list of nodes") with NodeManagerClient(host=node) as dm: return dm.sessions() @@ -527,28 +527,28 @@ def getLogFile(self, sessionId): @daliuge_aware def getNodeSessionInformation(self, node, sessionId): if node not in self.dm.nodes: - raise Exception("%s not in current list of nodes" % (node,)) + raise Exception(f"{node} not in current list of nodes") with NodeManagerClient(host=node) as dm: return dm.session(sessionId) @daliuge_aware def getNodeSessionStatus(self, node, sessionId): if node not in self.dm.nodes: - raise Exception("%s not in current list of nodes" % (node,)) + raise Exception(f"{node} not in current list of nodes") with NodeManagerClient(host=node) as dm: return dm.session_status(sessionId) @daliuge_aware def getNodeGraph(self, node, sessionId): if node not in self.dm.nodes: - raise Exception("%s not in current list of nodes" % (node,)) + raise Exception(f"{node} not in current list of nodes") with NodeManagerClient(host=node) as dm: return dm.graph(sessionId) @daliuge_aware def getNodeGraphStatus(self, node, sessionId): if node not in self.dm.nodes: - raise Exception("%s not in current list of nodes" % (node,)) + raise Exception(f"{node} not in current list of nodes") with NodeManagerClient(host=node) as dm: return dm.graph_status(sessionId) @@ -576,12 +576,15 @@ def visualizeDIM(self): class MasterManagerRestServer(CompositeManagerRestServer): def initializeSpecifics(self, app): CompositeManagerRestServer.initializeSpecifics(self, app) - + # DIM manamagement + app.post("/api/islands/", callback=self.addDIM) + app.delete("/api/islands/", callback=self.removeDIM) # Query forwarding to daemons - app.post("/api/managers//dataisland", callback=self.createDataIsland) app.post("/api/managers//node/start", callback=self.startNM) app.post("/api/managers//node/stop", callback=self.stopNM) + app.post("/api/managers//nodes/", callback=self.addNM) + app.delete("/api/managers//nodes/", callback=self.removeNM) # Querying about managers app.get("/api/islands", callback=self.getDIMs) app.get("/api/nodes", callback=self.getNMs) @@ -601,6 +604,16 @@ def createDataIsland(self, host): def getDIMs(self): return {"islands": self.dm.dmHosts} + @daliuge_aware + def addDIM(self, dim): + logger.debug("Adding DIM %s", dim) + self.dm.addDmHost(dim) + + @daliuge_aware + def removeDIM(self, dim): + logger.debug("Removing dim %s", dim) + self.dm.removeDmHost(dim) + @daliuge_aware def getNMs(self): return {"nodes": self.dm.nodes} @@ -608,21 +621,39 @@ def getNMs(self): @daliuge_aware def startNM(self, host): port = constants.DAEMON_DEFAULT_REST_PORT - logger.debug("Sending NM start request to %s:%s" % (host, port)) + logger.debug("Sending NM start request to %s:%s", host, port) with RestClient(host=host, port=port, timeout=10) as c: return json.loads(c._POST("/managers/node/start").read()) @daliuge_aware def stopNM(self, host): port = constants.DAEMON_DEFAULT_REST_PORT - logger.debug("Sending NM stop request to %s:%s" % (host, port)) + logger.debug("Sending NM stop request to %s:%s", host, port) with RestClient(host=host, port=port, timeout=10) as c: return json.loads(c._POST("/managers/node/stop").read()) + @daliuge_aware + def addNM(self, host, node): + port = constants.ISLAND_DEFAULT_REST_PORT + logger.debug("Adding NM %s to DIM %s", node, host) + with RestClient(host=host, port=port, timeout=10, url_prefix="/api") as c: + return json.loads( + c._POST( + f"/nodes/{node}", + ).read() + ) + + @daliuge_aware + def removeNM(self, host, node): + port = constants.ISLAND_DEFAULT_REST_PORT + logger.debug("Removing NM %s from DIM %s", node, host) + with RestClient(host=host, port=port, timeout=10, url_prefix="/api") as c: + return json.loads(c._DELETE(f"/nodes/{node}").read()) + @daliuge_aware def getNMInfo(self, host): port = constants.DAEMON_DEFAULT_REST_PORT - logger.debug("Sending request %s:%s/managers/node" % (host, port)) + logger.debug("Sending request %s:%s/managers/node", host, port) with RestClient(host=host, port=port, timeout=10) as c: return json.loads(c._GET("/managers/node").read()) diff --git a/daliuge-engine/dlg/manager/session.py b/daliuge-engine/dlg/manager/session.py index 2ea5e307a..0da85a838 100644 --- a/daliuge-engine/dlg/manager/session.py +++ b/daliuge-engine/dlg/manager/session.py @@ -101,10 +101,13 @@ def handleEvent(self, evt): "%d/%d drops filed reproducibility", self._completed, self._nexpected ) if self._completed == self._nexpected: - logger.debug("Building Reproducibility BlockDAG") - init_runtime_repro_data(self._session._graph, self._session._graphreprodata) - self._session.reprostatus = True - self._session.write_reprodata() + if not self._session.reprostatus: + logger.debug("Building Reproducibility BlockDAG") + new_reprodata = init_runtime_repro_data(self._session._graph, self._session._graphreprodata).get("reprodata", {}) + logger.debug("Reprodata for %s is %s", self._session.sessionId, json.dumps(new_reprodata)) + self._session._graphreprodata = new_reprodata + self._session.reprostatus = True + self._session.write_reprodata() class EndListener(object): @@ -322,7 +325,7 @@ def deploy(self, completedDrops=[], event_listeners=[], foreach=None): # in reality this particular session is managing nothing status = self.status if (self._graph and status != SessionStates.BUILDING) or ( - not self._graph and status != SessionStates.PRISTINE + not self._graph and status != SessionStates.PRISTINE ): raise InvalidSessionState( "Can't deploy this session in its current status: %d" % (status) @@ -492,7 +495,7 @@ def add_node_subscriptions(self, relationships): # We are in the event receiver side if (rel.rel in evt_consumer and rel.lhs is local_uid) or ( - rel.rel in evt_producer and rel.rhs is local_uid + rel.rel in evt_producer and rel.rhs is local_uid ): dropsubs[remote_uid].add(local_uid) @@ -515,7 +518,7 @@ def append_reprodata(self, oid, reprodata): if self._graph[oid].get("reprodata") is None: return if self._graph[oid]["reprodata"]["rmode"] == str( - ReproducibilityFlags.ALL.value + ReproducibilityFlags.ALL.value ): drop_reprodata = reprodata.get("data", {}) drop_hashes = reprodata.get("merkleroot", {}) @@ -557,9 +560,9 @@ def end(self): def getGraphStatus(self): if self.status not in ( - SessionStates.RUNNING, - SessionStates.FINISHED, - SessionStates.CANCELLED, + SessionStates.RUNNING, + SessionStates.FINISHED, + SessionStates.CANCELLED, ): raise InvalidSessionState( "The session is currently not running, cannot get graph status" @@ -594,9 +597,9 @@ def cancel(self): dsDrop for dsDrop in downStreamDrops if isinstance(dsDrop, AbstractDROP) ] if drop.status not in ( - DROPStates.ERROR, - DROPStates.COMPLETED, - DROPStates.CANCELLED, + DROPStates.ERROR, + DROPStates.COMPLETED, + DROPStates.CANCELLED, ): drop.cancel() self.status = SessionStates.CANCELLED diff --git a/daliuge-engine/dlg/manager/web/dim.html b/daliuge-engine/dlg/manager/web/dim.html index 92deb9280..0ceb9cbdb 100644 --- a/daliuge-engine/dlg/manager/web/dim.html +++ b/daliuge-engine/dlg/manager/web/dim.html @@ -90,18 +90,9 @@

Nodes

selectedNode = null; } var serverUrl = '{{serverUrl}}'; - var dmPort = { - { - dmPort - } - } - - var nodes = { - { - !nodes - } - } - + var dmPort = {{dmPort}} + var nodes = {{!nodes}} + var refreshSessionListBtn = d3.select('#refreshSessionListBtn'); var addSessionBtn = d3.select('#addSessionBtn'); var sessionsTbodyEl = d3.select('#sessionsTable tbody'); diff --git a/daliuge-engine/dlg/manager/web/dm.html b/daliuge-engine/dlg/manager/web/dm.html index 9a69535e6..b0d3f497c 100644 --- a/daliuge-engine/dlg/manager/web/dm.html +++ b/daliuge-engine/dlg/manager/web/dm.html @@ -66,11 +66,7 @@

Sessions