[WIP] STAR Workflow v2.0.0

.gitignore

@@ -1,4 +1,9 @@
 # Blacklist common bioinformatics formats used in these workflows.
 **/*.fastq.gz
 **/*.fa
-**/*.gtf
+**/*.gtf
+**/*.bam
+**/*.log
+
+# Blacklist Cromwell files
+**/cromwell*

LICENSE.txt

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2019 St. Jude Children's Research Hospital
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice (including the next paragraph) shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -3,4 +3,10 @@
 This repository is intended to contain all source code related to bioinformatics
 workflows in St. Jude Cloud. This includes but is not limited to Docker container
 definition, WDL/CWL tools and workflows, and relevant documentation. The repo is 
-currently under construction and should be considered in an experimental state. 
+currently under construction and should be considered in an experimental state. 
+
+## RNA-seq workflow V2
+The RNA-seq workflow used by St. Jude Cloud to generate BAM files is implemented 
+in WDL. It uses the bioinformatics-base Docker image found in this repository. 
+Configuration files for the Cromwell runner have been included to facilitate running 
+on LSF in addition to local execution. 

conf/aws.conf

@@ -0,0 +1,51 @@
+include required(classpath("application"))
+
+aws {
+
+  application-name = "cromwell"
+  auths = [
+    {
+      name = "default"
+      scheme = "default"
+    }
+  ]
+  region = "<your-region>"
+}
+
+engine {
+  filesystems {
+    s3.auth = "default"
+  }
+}
+
+backend {
+  default = "AWSBatch"
+  providers {
+    AWSBatch {
+      actor-factory = "cromwell.backend.impl.aws.AwsBatchBackendLifecycleActorFactory"
+      config {
+
+        numSubmitAttempts = 6
+        numCreateDefinitionAttempts = 6
+
+        // Base bucket for workflow executions
+        root = "s3://<your-s3-bucket-name>/cromwell-execution"
+
+        // A reference to an auth defined in the `aws` stanza at the top.  This auth is used to create
+        // Jobs and manipulate auth JSONs.
+        auth = "default"
+
+        default-runtime-attributes {
+          queueArn: "<your arn here>"
+        }
+
+        filesystems {
+          s3 {
+            // A reference to a potentially different auth for manipulating files via engine functions.
+            auth = "default"
+          }
+        }
+      }
+    }
+  }
+}

conf/lsf.conf

@@ -0,0 +1,59 @@
+include required(classpath("application"))
+
+call-caching {
+  enabled = true
+}
+
+backend {
+  default = LSF
+  providers {
+    LSF {
+      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
+      config {
+        runtime-attributes = """
+        Int cpu = 1
+        Int hosts = 1
+        Float? memory_mb = 3800
+        String lsf_queue = "standard"
+        String? lsf_job_group
+        String? docker 
+        """
+
+        #submit = """bsub -J ${job_name} -cwd ${cwd} -o ${out} -e ${err} -R rusage[mem=${memory_mb + "MB"}] -R "span[hosts=1]" -n ${cpu} ${"-q " + lsf_queue}  /usr/bin/env bash ${script}"""
+
+        submit = """
+          bsub \
+            -q ${lsf_queue} \
+            -n ${cpu} \
+            ${"-g " + lsf_job_group} \
+            -R "rusage[mem=${memory_mb}] span[hosts=${hosts}]" \
+            -J ${job_name} \
+            -cwd ${cwd} \
+            -o ${out} \
+            -e ${err} \
+            /usr/bin/env bash ${script}
+        """
+
+        submit-docker = """
+          bsub \
+            -q ${lsf_queue} \
+            -n ${cpu} \
+            ${"-g " + lsf_job_group} \
+            -R "select[singularity] rusage[mem=${memory_mb}] span[hosts=${hosts}]" \
+            -J ${job_name} \
+            -cwd ${cwd} \
+            -o ${cwd}/execution/stdout \
+            -e ${cwd}/execution/stderr \
+            "singularity exec --bind ${cwd}:${docker_cwd} docker://${docker} ${job_shell} ${script}"
+        """
+
+
+        kill = "bkill ${job_id}"
+        check-alive = "bjobs ${job_id}"
+        job-id-regex = "Job <(\\d+)>.*"
+
+        exit-code-timeout-seconds = 120
+      }
+    }
+  }
+}

docker/bioinformatics-base/Dockerfile

@@ -0,0 +1,39 @@
+FROM rust:1.36.0 as fqlib-builder
+RUN cargo install \
+      --git https://github.com/stjude/fqlib.git \
+      --tag v0.3.1 \
+      --root /opt/fqlib/
+
+FROM ubuntu:18.04 as builder
+
+ENV PATH /opt/conda/bin:$PATH
+
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install wget -y && \
+    apt-get install gcc -y && \
+    rm -r /var/lib/apt/lists/*
+
+RUN wget "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh" -O miniconda.sh && \
+    bash miniconda.sh -b -p /opt/conda/ && \
+    rm miniconda.sh
+
+RUN conda update -n base -c defaults conda -y && \
+    conda install \
+    -c conda-forge \
+    -c bioconda \
+    coreutils==8.31 \
+    picard==2.20.2 \
+    samtools==1.9 \
+    bwa==0.7.17 \
+    star==2.7.1a \
+    fastqc==0.11.8 \
+    qualimap==2.2.2c \
+    multiqc==1.7 \
+    rseqc==3.0.0 \
+    htseq==0.11.2 \
+    deeptools==3.3.1 \
+    -y && \
+    conda clean --all -y
+
+COPY --from=fqlib-builder /opt/fqlib/bin/fq /usr/local/bin/

docker/star/Dockerfile

@@ -0,0 +1 @@
+FROM stjudecloud/bioinformatics-base:bleeding-edge

tools/deeptools.wdl

@@ -0,0 +1,27 @@
+## Description: 
+##
+## This WDL tool wraps the DeepTools tool (https://deeptools.readthedocs.io/en/develop/index.html).
+## DeepTools is a suite of Python tools for analysis of high throughput sequencing analysis.
+
+task bamCoverage {
+    File bam
+    File bai
+    String prefix = basename(bam, ".bam")   
+
+    command {
+        if [ ! -e ${bam}.bai ] 
+        then 
+           ln -s ${bai} ${bam}.bai
+        fi
+
+        bamCoverage --bam ${bam} --outFileName ${prefix}.bw --outFileFormat bigwig --numberOfProcessors "max"
+    }
+
+    runtime {
+        docker: 'stjudecloud/bioinformatics-base:bleeding-edge'
+    }
+
+    output {
+        File bigwig = "${prefix}.bw"
+    }
+} 

tools/fastqc.wdl

@@ -0,0 +1,26 @@
+## Description: 
+##
+## This WDL tool wraps the FastQC tool (https://www.bioinformatics.babraham.ac.uk/projects/fastqc/).
+## FastQC generates quality control metrics for sequencing pipelines. 
+
+task fastqc {
+    File bam
+    Int ncpu
+    String prefix = basename(bam, ".bam")
+
+    command {
+        mkdir ${prefix}_fastqc_results
+        fastqc -f bam \
+            -o ${prefix}_fastqc_results \
+            -t ${ncpu} \
+            ${bam}
+    }
+
+    runtime {
+        docker: 'stjudecloud/bioinformatics-base:bleeding-edge'
+    }
+
+    output {
+        Array[File] out_files = glob("${prefix}_fastqc_results/*")
+    }
+}

tools/fq.wdl

@@ -0,0 +1,33 @@
+## Description: 
+##
+## This WDL tool wraps the fq tool (https://github.com/stjude/fqlib).
+## The fq library provides methods for manipulating Illumina generated 
+## FastQ files.
+
+task print_version {
+    command {
+        fq --version
+    }
+
+    runtime {
+        docker: 'stjudecloud/bioinformatics-base:bleeding-edge'
+    }
+
+    output {
+        String out = read_string(stdout())
+    }
+
+}
+
+task fqlint {
+    File read1
+    File read2
+
+    runtime {
+        docker: 'stjudecloud/bioinformatics-base:bleeding-edge'
+    }
+
+    command {
+        fq lint ${read1} ${read2}
+    }
+}

tools/htseq.wdl

@@ -0,0 +1,24 @@
+## Description: 
+##
+## This WDL tool wraps the htseq tool (https://github.com/simon-anders/htseq).
+## HTSeq is a Python library for analyzing sequencing data. 
+
+task count {
+    File bam
+    File gtf
+    String strand = "reverse"
+    String outfile = basename(bam, ".bam") + ".counts.txt"
+
+    command {
+        htseq-count -f bam -r pos -s ${strand} -m union -i gene_name --secondary-alignments ignore --supplementary-alignments ignore ${bam} ${gtf} > ${outfile}
+    }
+
+    runtime {
+        memory: "8G"
+        docker: 'stjudecloud/bioinformatics-base:bleeding-edge'
+    }
+
+    output {
+       File out = "${outfile}"
+    }
+}

tools/md5sum.wdl

@@ -0,0 +1,50 @@
+## Description: 
+##
+## This WDL tool wraps the md5sum tool from the GNU core 
+## utilities (https://github.com/coreutils/coreutils).
+## md5sum is a utility for generating and verifying MD5
+## hashes.  
+
+task print_version {
+    command {
+        md5sum --version
+    }
+
+    runtime {
+        docker: 'stjudecloud/bioinformatics-base:bleeding-edge'
+    }
+
+    output {
+        String out = read_string(stdout())
+    }
+}
+task compute_checksum {
+    File infile 
+
+    command {
+        md5sum ${infile} > stdout.txt
+    }
+
+    runtime {
+        docker: 'stjudecloud/bioinformatics-base:bleeding-edge'
+    }
+
+    output {
+        String out = read_string("stdout.txt")
+    }
+}
+task check_checksum {
+    File infile
+
+    command { 
+        md5sum -c ${infile} > stdout.txt
+    } 
+
+    runtime {
+        docker: 'stjudecloud/bioinformatics-base:bleeding-edge'
+    }
+
+    output {
+        String out = read_string("stdout.txt")
+    }
+}