diff --git a/.travis.yml b/.travis.yml index cf9ce3d0d..2f8761912 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,7 +79,7 @@ cache: pip before_install: - pip install -U coveralls pytest pytest-cov - - pip install -U setuptools pip wheel + - pip install -U setuptools pip wheel dask install: - pip install -e daliuge-common/ diff --git a/daliuge-engine/dlg/apps/dockerapp.py b/daliuge-engine/dlg/apps/dockerapp.py index f15a7d533..bd2a65aa4 100644 --- a/daliuge-engine/dlg/apps/dockerapp.py +++ b/daliuge-engine/dlg/apps/dockerapp.py @@ -42,7 +42,8 @@ # DLG_ROOT = '/dlg_root' DLG_ROOT = "" if "DLG_ROOT" not in os.environ else os.environ["DLG_ROOT"] -#os.environ["DLG_ROOT"] = DLG_ROOT # make sure the environ variable is set +# TODO: the following is commented out because it breaks the dask tests for some reason +# os.environ["DLG_ROOT"] = DLG_ROOT # make sure the environ variable is set DockerPath = collections.namedtuple("DockerPath", "path") diff --git a/daliuge-engine/dlg/apps/simple.py b/daliuge-engine/dlg/apps/simple.py index 55502350e..f02776671 100644 --- a/daliuge-engine/dlg/apps/simple.py +++ b/daliuge-engine/dlg/apps/simple.py @@ -111,6 +111,8 @@ def run(self): # @param tag daliuge # @param[in] cparam/appclass Application Class/dlg.apps.simple.CopyApp/String/readonly/False/ # \~English Application class +# @param[in] param/bufsize buffer size/65536/Integer/readwrite/ +# \~English Application class # @param[in] cparam/execution_time Execution Time/5/Float/readonly/False/ # \~English Estimated execution time # @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False/ @@ -133,6 +135,8 @@ class CopyApp(BarrierAppDROP): [dlg_streaming_input("binary/*")], ) + _bufsize = dlg_int_param("bufsize", 65536) + def run(self): self.copyAll() @@ -146,7 +150,7 @@ def copyRecursive(self, inputDrop): self.copyRecursive(child) else: for outputDrop in self.outputs: - droputils.copyDropContents(inputDrop, outputDrop) + droputils.copyDropContents(inputDrop, outputDrop, bufsize=self._bufsize) ## diff --git a/daliuge-engine/docker/prepare_user.py b/daliuge-engine/docker/prepare_user.py index 72cb8e674..1a6b7d529 100644 --- a/daliuge-engine/docker/prepare_user.py +++ b/daliuge-engine/docker/prepare_user.py @@ -39,6 +39,7 @@ # get current user info pw = pwd.getpwuid(os.getuid()) gr = grp.getgrgid(pw.pw_gid) +dgr = grp.getgrnam('docker') with open(os.path.join(workdir, "passwd"), "wt") as file: file.write(open(os.path.join(template_dir, "passwd.template"), "rt").read()) file.write(f"{pw.pw_name}:x:{pw.pw_uid}:{pw.pw_gid}:{pw.pw_gecos}:/:/bin/bash") @@ -46,3 +47,4 @@ file.write(open(os.path.join(template_dir, "group.template"), "rt").read()) file.write(f"{gr.gr_name}:x:{gr.gr_gid}:") +print(dgr.gr_gid) diff --git a/daliuge-engine/run_engine.sh b/daliuge-engine/run_engine.sh index 8841971cc..c281bd585 100755 --- a/daliuge-engine/run_engine.sh +++ b/daliuge-engine/run_engine.sh @@ -14,7 +14,8 @@ common_prep () mkdir -p ${DLG_ROOT}/testdata mkdir -p ${DLG_ROOT}/code # get current user and group id and prepare passwd and group files - python docker/prepare_user.py + DOCKER_GID=`python docker/prepare_user.py` + DOCKER_OPTS=${DOCKER_OPTS}" --group-add ${DOCKER_GID}" DOCKER_OPTS=${DOCKER_OPTS}" -v ${DLG_ROOT}/workspace/settings/passwd:/etc/passwd" DOCKER_OPTS=${DOCKER_OPTS}" -v ${DLG_ROOT}/workspace/settings/group:/etc/group" DOCKER_OPTS=${DOCKER_OPTS}" -v ${PWD}/dlg/manager:/dlg/lib/python3.8/site-packages/dlg/manager"