Merge 6c33a5b into a610e4a

.travis.yml

@@ -79,7 +79,7 @@ cache: pip
 
 before_install:
  - pip install -U coveralls pytest pytest-cov
- - pip install -U setuptools pip wheel
+ - pip install -U setuptools pip wheel dask
 
 install:
  - pip install -e daliuge-common/

daliuge-engine/dlg/apps/dockerapp.py

@@ -42,7 +42,8 @@
 
 # DLG_ROOT = '/dlg_root'
 DLG_ROOT = "" if "DLG_ROOT" not in os.environ else os.environ["DLG_ROOT"]
-#os.environ["DLG_ROOT"] = DLG_ROOT # make sure the environ variable is set
+# TODO: the following is commented out because it breaks the dask tests for some reason
+# os.environ["DLG_ROOT"] = DLG_ROOT # make sure the environ variable is set
 
 DockerPath = collections.namedtuple("DockerPath", "path")
 

daliuge-engine/dlg/apps/simple.py

@@ -111,6 +111,8 @@ def run(self):
 # @param tag daliuge
 # @param[in] cparam/appclass Application Class/dlg.apps.simple.CopyApp/String/readonly/False/
 #     \~English Application class
+# @param[in] param/bufsize buffer size/65536/Integer/readwrite/
+#     \~English Application class
 # @param[in] cparam/execution_time Execution Time/5/Float/readonly/False/
 #     \~English Estimated execution time
 # @param[in] cparam/num_cpus No. of CPUs/1/Integer/readonly/False/
@@ -133,6 +135,8 @@ class CopyApp(BarrierAppDROP):
         [dlg_streaming_input("binary/*")],
     )
 
+    _bufsize = dlg_int_param("bufsize", 65536)
+
     def run(self):
         self.copyAll()
 
@@ -146,7 +150,7 @@ def copyRecursive(self, inputDrop):
                 self.copyRecursive(child)
         else:
             for outputDrop in self.outputs:
-                droputils.copyDropContents(inputDrop, outputDrop)
+                droputils.copyDropContents(inputDrop, outputDrop, bufsize=self._bufsize)
 
 
 ##

daliuge-engine/docker/prepare_user.py

@@ -39,10 +39,12 @@
 # get current user info
 pw = pwd.getpwuid(os.getuid())
 gr = grp.getgrgid(pw.pw_gid)
+dgr = grp.getgrnam('docker')
 with open(os.path.join(workdir, "passwd"), "wt") as file:
     file.write(open(os.path.join(template_dir, "passwd.template"), "rt").read())
     file.write(f"{pw.pw_name}:x:{pw.pw_uid}:{pw.pw_gid}:{pw.pw_gecos}:/:/bin/bash")
 with open(os.path.join(workdir, "group"), "wt") as file:
     file.write(open(os.path.join(template_dir, "group.template"), "rt").read())
     file.write(f"{gr.gr_name}:x:{gr.gr_gid}:")
 
+print(dgr.gr_gid)

daliuge-engine/run_engine.sh

@@ -14,7 +14,8 @@ common_prep ()
     mkdir -p ${DLG_ROOT}/testdata
     mkdir -p ${DLG_ROOT}/code
     # get current user and group id and prepare passwd and group files
-    python docker/prepare_user.py
+    DOCKER_GID=`python docker/prepare_user.py`
+    DOCKER_OPTS=${DOCKER_OPTS}" --group-add ${DOCKER_GID}"
     DOCKER_OPTS=${DOCKER_OPTS}" -v ${DLG_ROOT}/workspace/settings/passwd:/etc/passwd"
     DOCKER_OPTS=${DOCKER_OPTS}" -v ${DLG_ROOT}/workspace/settings/group:/etc/group"
     DOCKER_OPTS=${DOCKER_OPTS}" -v ${PWD}/dlg/manager:/dlg/lib/python3.8/site-packages/dlg/manager"