runner CLI: permit reading input JSON straight from URI

WDL/CLI.py

@@ -12,6 +12,7 @@
 import argcomplete
 import logging
 import urllib
+import asyncio
 import docker
 from shlex import quote as shellquote
 from datetime import datetime
@@ -656,8 +657,11 @@ def runner_input_json_file(available_inputs, namespace, input_file):
         elif input_file == "-":
             input_json = json.load(sys.stdin)
         else:
-            with open(input_file) as infile:
-                input_json = json.load(infile)
+            input_json = json.loads(
+                asyncio.get_event_loop()
+                .run_until_complete(read_source(input_file, [], None))
+                .source_text
+            )
         ans = values_from_json(input_json, available_inputs, namespace=namespace)
 
         # join relative file paths to the cwd

WDL/runtime/task.py

@@ -290,7 +290,7 @@ def _run(
         resources, user, groups = self.misc_config(logger, cpu, memory)
 
         # connect to dockerd
-        client = docker.from_env()
+        client = docker.from_env(timeout=900)
         svc = None
         try:
             # run container as a transient docker swarm service, letting docker handle the resource
@@ -479,6 +479,7 @@ def chown(self, logger: logging.Logger, client: docker.DockerClient) -> None:
         alternatives and their problems.
         """
         if not self.as_me and (os.geteuid() or os.getegid()):
+            t_0 = time.monotonic()
             script = f"""
             chown -RP {os.geteuid()}:{os.getegid()} {shlex.quote(os.path.join(self.container_dir, 'work'))}
             """.strip()
@@ -503,6 +504,15 @@ def chown(self, logger: logging.Logger, client: docker.DockerClient) -> None:
                         chowner.remove()
             except:
                 logger.exception("post-task chown failed")
+            finally:
+                t_delta = time.monotonic() - t_0
+                if t_delta >= 60:
+                    logger.warning(
+                        _(
+                            "post-task chown was slow (symptomatic of excessive file count and/or IOPS saturation)",
+                            seconds=int(t_delta),
+                        )
+                    )
 
 
 def run_local_task(

stubs/docker/__init__.py

@@ -78,6 +78,6 @@ def swarm(self) -> Swarm:
     def services(self) -> Services:
         ...
 
-def from_env() -> DockerClient:
+def from_env(timeout: Optional[int] = None) -> DockerClient:
     ...
 

tests/applied/DVGLx.t

@@ -17,5 +17,5 @@ plan tests 1
 set +e
 
 $miniwdl run https://raw.githubusercontent.com/mlin/DeepVariant-GLnexus-WDL/master/test/range1KGP.wdl --verbose \
-    -i <(wget -O - https://raw.githubusercontent.com/mlin/DeepVariant-GLnexus-WDL/master/test/range1KGP.test26.ALDH2.json)
+    -i https://raw.githubusercontent.com/mlin/DeepVariant-GLnexus-WDL/master/test/range1KGP.test26.ALDH2.json
 is "$?" "0" "pipeline success"

tests/applied/encode_atac_seq.t

@@ -18,7 +18,7 @@ plan tests 1
 set +e
 
 $miniwdl run https://raw.githubusercontent.com/ENCODE-DCC/atac-seq-pipeline/v1.5.4/atac.wdl \
-    -i <(wget -O - https://storage.googleapis.com/encode-pipeline-test-samples/encode-atac-seq-pipeline/ENCSR356KRQ_subsampled_caper.json) \
+    -i https://storage.googleapis.com/encode-pipeline-test-samples/encode-atac-seq-pipeline/ENCSR356KRQ_subsampled_caper.json \
     --runtime-defaults '{"docker":"quay.io/encode-dcc/atac-seq-pipeline:v1.5.4"}' \
     --no-quant-check --verbose --runtime-memory-max 4G --runtime-cpu-max 2
 is "$?" "0" "pipeline success"