config runtime defaults for download tasks specifically (#560)

chanzuckerberg · Mar 29, 2022 · 4bf4f93 · 4bf4f93
1 parent 43b3626
commit 4bf4f93
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 18 deletions.
diff --git a/WDL/runtime/config_templates/default.cfg b/WDL/runtime/config_templates/default.cfg
@@ -98,6 +98,11 @@ memory_limit_multiplier = 0.0
 defaults = {
         "docker": "ubuntu:20.04"
     }
+# Defaults applying to URI download tasks specifically. (New in v1.5.1)
+download_defaults = {
+        "cpu": 2,
+        "memory": "1G"
+    }
 # Run the command script as the invoking user's uid:gid instead of usually running as root. More
 # secure, but interferes with commands that assume root access e.g. apt-get. --as-me
 as_user = false

diff --git a/WDL/runtime/download.py b/WDL/runtime/download.py
@@ -188,8 +188,6 @@ def aria2c_downloader(
             File file = glob("__out/*")[0]
         }
         runtime {
-            cpu: 2
-            memory: "1G"
             docker: docker
         }
     }
@@ -242,8 +240,6 @@ def awscli_downloader(
             }
 
             runtime {
-                cpu: 2
-                memory: "1G"
                 docker: docker
             }
         }
@@ -295,8 +291,6 @@ def awscli_directory_downloader(
             }
 
             runtime {
-                cpu: 2
-                memory: "1G"
                 docker: docker
             }
         }
@@ -378,8 +372,6 @@ def gsutil_downloader(
             File file = glob("__out/*")[0]
         }
         runtime {
-            cpu: 2
-            memory: "1G"
             docker: docker
         }
     }

diff --git a/WDL/runtime/task.py b/WDL/runtime/task.py
@@ -172,7 +172,7 @@ def run_local_task(
                 # evaluate runtime fields
                 stdlib = InputStdLib(task.effective_wdl_version, logger, container)
                 container.runtime_values = _eval_task_runtime(
-                    cfg, logger, task, posix_inputs, container, container_env, stdlib
+                    cfg, logger, run_id, task, posix_inputs, container, container_env, stdlib
                 )
 
                 # interpolate command
@@ -446,22 +446,19 @@ def collector(v: Value.Base) -> None:
 def _eval_task_runtime(
     cfg: config.Loader,
     logger: logging.Logger,
+    run_id: str,
     task: Tree.Task,
     inputs: Env.Bindings[Value.Base],
     container: "runtime.task_container.TaskContainer",
     env: Env.Bindings[Value.Base],
     stdlib: StdLib.Base,
 ) -> Dict[str, Union[int, str, List[int], List[str]]]:
+    runtime_defaults = cfg.get_dict("task_runtime", "defaults")
+    if run_id.startswith("download-"):
+        runtime_defaults.update(cfg.get_dict("task_runtime", "download_defaults"))
     runtime_values = {}
-    for key, v in cfg["task_runtime"].get_dict("defaults").items():
-        if isinstance(v, str):
-            runtime_values[key] = Value.String(v)
-        elif isinstance(v, int):
-            runtime_values[key] = Value.Int(v)
-        elif isinstance(v, bool):
-            runtime_values[key] = Value.Boolean(v)
-        else:
-            raise Error.InputError(f"invalid default runtime setting {key} = {v}")
+    for key, v in runtime_defaults.items():
+        runtime_values[key] = Value.from_json(Type.Any(), v)
     for key, expr in task.runtime.items():  # evaluate expressions in source code
         runtime_values[key] = expr.eval(env, stdlib)
     for b in inputs.enter_namespace("runtime"):