wdl demux changes to support novaseq

docker/mem_in_mb_80.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+head -n1 /proc/meminfo | awk '{print int($2*0.80/1024)}'

docker/mem_in_mb_85.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+head -n1 /proc/meminfo | awk '{print int($2*0.85/1024)}'

illumina.py

@@ -11,6 +11,7 @@
 import os
 import os.path
 import re
+import gc
 import csv
 import shutil
 import subprocess
@@ -211,6 +212,8 @@ def main_illumina_demux(args):
     if not picardOpts.get('sequencing_center') and runinfo:
         picardOpts['sequencing_center'] = runinfo.get_machine()
 
+    # manually garbage collect to make sure we have as much RAM free as possible
+    gc.collect()
     if multiplexed_samples:
         tools.picard.IlluminaBasecallsToSamTool().execute(
             illumina.get_BCLdir(),

pipes/WDL/workflows/tasks/demux.wdl

@@ -42,6 +42,7 @@ task illumina_demux {
   Int?    maxNoCalls
   String? readStructure
   Int?    minimumQuality = 10
+  Int?    threads = 32
   String? runStartDate
 
   parameter_meta {
@@ -60,8 +61,8 @@ task illumina_demux {
       ${flowcell_tgz} $FLOWCELL_DIR \
       --loglevel=DEBUG
 
-    # find 95% memory
-    mem_in_mb=`/opt/viral-ngs/source/docker/mem_in_mb_95.sh`
+    # find N% memory
+    mem_in_mb=`/opt/viral-ngs/source/docker/mem_in_mb_90.sh`
 
     # note that we are intentionally setting --threads to about 2x the core
     # count. seems to still provide speed benefit (over 1x) when doing so.
@@ -83,7 +84,7 @@ task illumina_demux {
       ${'--minimum_quality=' + minimumQuality} \
       ${'--run_start_date=' + runStartDate} \
       --JVMmemory="$mem_in_mb"m \
-      --threads=64 \
+      ${'--threads=' + threads} \
       --compression_level=5 \
       --loglevel=DEBUG