log file download sizes

chanzuckerberg · Nov 19, 2019 · a436a12 · a436a12
1 parent cedfe77
commit a436a12
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 5 deletions.
diff --git a/WDL/runtime/task.py b/WDL/runtime/task.py
@@ -554,9 +554,10 @@ def _download_input_files(
     """
 
     downloads = 0
+    total_bytes = 0
 
     def map_files(v: Value.Base) -> Value.Base:
-        nonlocal downloads
+        nonlocal downloads, total_bytes
         if isinstance(v, Value.File):
             if downloadable(v.value):
                 logger.info(_("download input file", uri=v.value))
@@ -565,8 +566,10 @@ def map_files(v: Value.Base) -> Value.Base:
                     run_dir=os.path.join(run_dir, "download", str(downloads)),
                     logger_prefix=logger_prefix + [f"download{downloads}"],
                 )
-                logger.info(_("downloaded input file", uri=v.value, file=v.value))
+                sz = os.path.getsize(v.value)
+                logger.info(_("downloaded input file", uri=v.value, file=v.value, bytes=sz))
                 downloads += 1
+                total_bytes += sz
         for ch in v.children:
             map_files(ch)
         return v
@@ -575,7 +578,9 @@ def map_files(v: Value.Base) -> Value.Base:
         lambda binding: Env.Binding(binding.name, map_files(copy.deepcopy(binding.value)))
     )
     if downloads:
-        logger.notice(_("downloaded input files", count=downloads))  # pyre-fixme
+        logger.notice(  # pyre-fixme
+            _("downloaded input files", count=downloads, total_bytes=total_bytes)
+        )
     return ans
 
 

diff --git a/WDL/runtime/workflow.py b/WDL/runtime/workflow.py
@@ -793,16 +793,21 @@ def schedule_downloads(v: Value.Base) -> None:
 
     # collect the results
     downloaded = {}
+    total_bytes = 0
     try:
         for future in futures.as_completed(ops):
             uri = ops[future]
             downloaded[uri] = future.result()
-            logger.info(_("downloaded input file", uri=uri, file=downloaded[uri]))
+            sz = os.path.getsize(downloaded[uri])
+            logger.info(_("downloaded input file", uri=uri, file=downloaded[uri], bytes=sz))
+            total_bytes += sz
     except:
         for future in ops:
             future.cancel()
         raise
-    logger.notice(_("downloaded input files", count=len(downloaded)))  # pyre-fixme
+    logger.notice(  # pyre-fixme
+        _("downloaded input files", count=len(downloaded), total_bytes=total_bytes)
+    )
 
     # rewrite the input URIs to the downloaded filenames
     def rewrite_downloaded(v: Value.Base) -> Value.Base: