Merge pull request #298 from BioPsyk/issue-256

Issue #256

CHANGELOG.md

@@ -15,6 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Support for extra paths to search for files in the metafile using `-p path1,path2`
 - Support for variant specific information of number of metastudies: `col_StudyN`
 - Support for two metafile minimum requiriments options, default and library
+- New user options to set tmp and workdirs -b -w
+- New user option to set dev mode -l
+- Automatic cleanup of workdir when not in dev move
 
 ### Changed
 - DSL-1 converted to DSL-2

cleansumstats.sh

@@ -23,16 +23,20 @@ function general_usage(){
  echo "-o <dir> 	 path to output directory"
  echo "-d <dir> 	 path to dbsnp processed reference"
  echo "-k <dir> 	 path to 1000 genomes processed reference"
+ echo "-b <dir> 	 path to system tmp or scratch (default: /tmp)"
+ echo "-w <dir> 	 path to workdir/intermediate files (default: work)"
  echo "-p path1:path2 	 path to metadata associated folders"
  echo "-t  	 	 quick test for all paths and params"
  echo "-e  	 	 quick example run using shrinked dbsnp and 1000 genomes references"
+ echo "-l  	 	 dev mode, saving intermediate files, no cleanup of workdir(default: not active)"
  echo "-v  	 	 get the version number"
 }
 
 ################################################################################
 # Prepare path parsing
 ################################################################################
 # All paths we see will start from the project root, even if the command is called from somewhere else
+present_dir="${PWD}"
 project_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 ################################################################################
@@ -56,7 +60,7 @@ fi
 
 
 # starting getops with :, puts the checking in silent mode for errors.
-getoptsstring=":hvi:o:d:k:p:te"
+getoptsstring=":hvi:o:d:k:b:w:p:tle"
 
 # Set default dbsnpdir to where the files are automatically placed when
 # following the instrucitons in the README.md
@@ -72,6 +76,7 @@ infile_given=false
 outdir_given=false
 dbsnpdir_given=false
 kgpdir_given=false
+tmpdir_given=false
 extrapaths_given=false
 pathquicktest=false
 runexampledata=false
@@ -80,6 +85,11 @@ runexampledata=false
 unset extrapaths
 unset extrapaths2
 
+# default system tmp
+tmpdir="/tmp"
+workdir="${present_dir}/work"
+devmode=""
+
 while getopts "${getoptsstring}" opt "${paramarray[@]}"; do
   case ${opt} in
     h )
@@ -107,13 +117,24 @@ while getopts "${getoptsstring}" opt "${paramarray[@]}"; do
       kgpdir="$OPTARG"
       kgpdir_given=true
       ;;
+    b )
+      tmpdir="$OPTARG"
+      tmpdir_given=true
+      ;;
+    w )
+      workdir="$OPTARG"
+      workdir_given=true
+      ;;
     p )
       extrapaths="$OPTARG"
       extrapaths_given=true
       ;;
     e )
       runexampledata=true
       ;;
+    l )
+      devmode="--dev"
+      ;;
     t )
       pathquicktest=true
       ;;
@@ -192,6 +213,9 @@ fi
 # make outdir if it doesn't already exist
 mkdir -p ${outdir}
 
+# make workdir if it doesn't already exist
+mkdir -p ${workdir}
+
 infile_host=$(realpath "${infile}")
 outdir_host=$(realpath "${outdir}")
 
@@ -208,6 +232,8 @@ else
 fi
 dbsnpdir_host=$(realpath "${dbsnpdir}")
 kgpdir_host=$(realpath "${kgpdir}")
+tmpdir_host=$(realpath "${tmpdir}")
+workdir_host=$(realpath "${workdir}")
 
 # Test that file and folder exists, all of these will always get mounted
 if [ ! -f $infile_host ]; then
@@ -226,6 +252,14 @@ if [ ! -d $kgpdir_host ]; then
   >&2 echo "kgpdir doesn't exist"
   exit 1
 fi
+if [ ! -d $tmpdir_host ]; then
+  >&2 echo "tmpdir doesn't exist"
+  exit 1
+fi
+if [ ! -d $workdir_host ]; then
+  >&2 echo "workdir doesn't exist"
+  exit 1
+fi
 
 # Add metadata env variable folders (fix realpath)
 if ${extrapaths_given} ;
@@ -261,6 +295,12 @@ outdir_container="/cleansumstats/outdir"
 # dbsnpdir
 dbsnpdir_container="/cleansumstats/dbsnp"
 
+# tmpdir
+tmpdir_container="/tmp"
+
+# workdir
+workdir_container="/cleansumstats/work"
+
 # kgpdir
 kgpfile_name="1kg_af_ref.sorted.joined"
 kgpdir_container="/cleansumstats/kgpdir"
@@ -271,14 +311,14 @@ FAKE_HOME="tmp/fake-home"
 export SINGULARITY_HOME="/cleansumstats/${FAKE_HOME}"
 mkdir -p "${FAKE_HOME}"
 
-
 if ${pathquicktest}; then
  echo "cleansumstats.sh to-mount"
  echo "------------------"
  echo "infile: ${infile}"
  echo "outdir: ${outdir}"
  echo "dbsnpdir: ${dbsnpdir}"
  echo "kgpdir: ${kgpdir}"
+ echo "tmpdir: ${tmpdir}"
  echo ""
  echo "cleansumstats.sh logic"
  echo "------------------"
@@ -317,11 +357,12 @@ else
      -B "${outdir_host}:${outdir_container}" \
      -B "${dbsnpdir_host}:${dbsnpdir_container}" \
      -B "${kgpdir_host}:${kgpdir_container}" \
-     -B "/tmp:/tmp" \
+     -B "${tmpdir_host}:${tmpdir_container}" \
+     -B "${workdir_host}:${workdir_container}" \
      "tmp/${singularity_image_tag}" \
      nextflow run /cleansumstats ${runtype} \
        --extrapaths ${extrapaths3} \
-       --dev \
+       ${devmode} \
        --input "${infile_container}" \
        --outdir "${outdir_container}" \
        --libdirdbsnp "${dbsnpdir_container}" \

nextflow.config

@@ -12,8 +12,16 @@ today = new Date().format("yyyy-MM-dd-HHmm")
 // manually switch to scratch for production version
 //workDir = "/scratch/${USER}-${today}-nf-workdir"
 //tmpDir = "/scratch/${USER}-${today}-nf-tmpdir"
-workDir = "work"
-tmpDir = "tmp"
+//workDir = "work"
+//tmpDir = "tmp"
+
+if(params.dev){
+  cleanup = false
+}else{
+  cleanup = true
+}
+workDir = "/cleansumstats/work"
+tmpDir = "/tmp"
 
 // Global default params, used in configs
 params {