Merge pull request #361 from ENCODE-DCC/dev

v2.1.0

.circleci/config.yml

@@ -16,17 +16,6 @@ make_tag: &make_tag
     echo "export TAG=encodedcc/atac-seq-pipeline:${CIRCLE_BRANCH}_${CIRCLE_WORKFLOW_ID}" > ${BASH_ENV}
 
 commands:
-  download_task_test_data:
-    description: "Download task test data. This is based on py2 so run this before installing py3."
-    steps:
-      - run:
-          command: |
-            cd dev/test/test_task/
-            rm -rf atac-seq-pipeline-test-data
-            export BOTO_CONFIG=/dev/null
-            gsutil -m cp -r gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data .
-            ./download_hg38_fasta_for_test_ataqc.sh
-
 
   install_python3_caper_gcs:
     description: "Install py3, caper and gcs. Set py3 as default python."
@@ -88,7 +77,7 @@ jobs:
           name: build image
           command: |
             source ${BASH_ENV}
-            export DOCKER_CACHE_TAG=v1.9.3
+            export DOCKER_CACHE_TAG=v2.0.0
             echo "pulling ${DOCKER_CACHE_TAG}!"
             docker pull encodedcc/atac-seq-pipeline:${DOCKER_CACHE_TAG}
             docker login -u=${DOCKERHUB_USER} -p=${DOCKERHUB_PASS}
@@ -99,7 +88,6 @@ jobs:
     <<: *machine_defaults
     steps:
       - checkout
-      - download_task_test_data
       - install_python3_caper_gcs
       - run: *make_tag
       - run:
@@ -108,6 +96,9 @@ jobs:
             source ${BASH_ENV}
             cd dev/test/test_task/
 
+            echo ${GCLOUD_SERVICE_ACCOUNT_SECRET_JSON} > tmp_secret_key.json
+            export GOOGLE_APPLICATION_CREDENTIALS=$PWD/tmp_secret_key.json
+
             for wdl in test_*.wdl
             do
               prefix=${wdl%.*}
@@ -118,12 +109,17 @@ jobs:
               # add docker image to input JSON
               cat ${input} | jq ".+{\"${prefix}.docker\": \"${TAG}\"}" > ${input_with_docker}
 
-              caper run ${wdl} -i ${input_with_docker} -m ${metadata}
-              if [[ "${wdl}" != "test_choose_ctl.wdl" ]]; then
-                echo "Validating outputs of ${prefix}"
-                res=$(jq '.outputs["'${prefix}'.compare_md5sum.match_overall"]' "${metadata}")
-                [[ "$res" != true ]] && exit 100
+              if [[ "${wdl}" == test_macs2* ]]; then
+                # run heavy MACS2 tasks on GCP
+                caper run ${wdl} -i ${input_with_docker} -m ${metadata} \
+                  --backend gcp --gcp-prj ${GOOGLE_PROJECT_ID} --docker ${TAG} \
+                  --gcp-service-account-key-json $PWD/tmp_secret_key.json \
+                  --out-gcs-bucket ${CAPER_OUT_DIR} --tmp-gcs-bucket ${CAPER_TMP_DIR}
+              else
+                # run other tasks locally
+                caper run ${wdl} -i ${input_with_docker} -m ${metadata}
               fi
+
               rm -f ${metadata}
             done
 

.gitignore

@@ -1,4 +1,4 @@
-# Byte-compiled / optimized / DLL files
+# Byte-compiled / optimized / DLL files/
 __pycache__/
 *.py[cod]
 *$py.class
@@ -121,4 +121,5 @@ tmp_db*
 *.local.json
 temp_db*
 cromwell.out
+cromwell.out.*
 .dev

README.md

@@ -3,6 +3,10 @@
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.156534.svg)](https://doi.org/10.5281/zenodo.156534)[![CircleCI](https://circleci.com/gh/ENCODE-DCC/atac-seq-pipeline/tree/master.svg?style=svg)](https://circleci.com/gh/ENCODE-DCC/atac-seq-pipeline/tree/master)
 
 
+## Updated genome TSV files (v3 -> v4)
+
+
+
 ## Download new Caper>=2.1
 
 New Caper is out. You need to update your Caper to work with the latest ENCODE ATAC-seq pipeline.
@@ -138,8 +142,8 @@ You cannot use these input JSON files directly. Go to the destination directory
 We have a separate project on DNANexus to provide example FASTQs and `genome_tsv` for `hg38` and `mm10`. We recommend to make copies of these directories on your own project.
 
 `genome_tsv`
-- AWS: https://platform.dnanexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/pipeline-genome-data/genome_tsv/v3
-- Azure: https://platform.dnanexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/pipeline-genome-data/genome_tsv/v3
+- AWS: https://platform.dnanexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/pipeline-genome-data/genome_tsv/v4
+- Azure: https://platform.dnanexus.com/projects/F6K911Q9xyfgJ36JFzv03Z5J/data/pipeline-genome-data/genome_tsv/v4
 
 Example FASTQs
 - AWS: https://platform.dnanexus.com/projects/BKpvFg00VBPV975PgJ6Q03v6/data/pipeline-test-samples/encode-atac-seq-pipeline/ENCSR356KRQ/fastq_subsampled

atac.wdl

@@ -7,10 +7,10 @@ struct RuntimeEnvironment {
 }
 
 workflow atac {
-    String pipeline_ver = 'v2.0.3'
+    String pipeline_ver = 'v2.1.0'
 
     meta {
-        version: 'v2.0.3'
+        version: 'v2.1.0'
 
         author: 'Jin wook Lee'
         email: 'leepc12@gmail.com'
@@ -19,8 +19,8 @@ workflow atac {
 
         specification_document: 'https://docs.google.com/document/d/1f0Cm4vRyDQDu0bMehHD7P7KOMxTOP-HiNoIvL1VcBt8/edit?usp=sharing'
 
-        default_docker: 'encodedcc/atac-seq-pipeline:v2.0.3'
-        default_singularity: 'https://encode-pipeline-singularity-image.s3.us-west-2.amazonaws.com/atac-seq-pipeline_v2.0.3.sif'
+        default_docker: 'encodedcc/atac-seq-pipeline:v2.1.0'
+        default_singularity: 'https://encode-pipeline-singularity-image.s3.us-west-2.amazonaws.com/atac-seq-pipeline_v2.1.0.sif'
         default_conda: 'encode-atac-seq-pipeline'
         croo_out_def: 'https://storage.googleapis.com/encode-pipeline-output-definition/atac.croo.v5.json'
 
@@ -72,8 +72,8 @@ workflow atac {
     }
     input {
         # group: runtime_environment
-        String docker = 'encodedcc/atac-seq-pipeline:v2.0.3'
-        String singularity = 'https://encode-pipeline-singularity-image.s3.us-west-2.amazonaws.com/atac-seq-pipeline_v2.0.3.sif'
+        String docker = 'encodedcc/atac-seq-pipeline:v2.1.0'
+        String singularity = 'https://encode-pipeline-singularity-image.s3.us-west-2.amazonaws.com/atac-seq-pipeline_v2.1.0.sif'
         String conda = 'encode-atac-seq-pipeline'
         String conda_macs2 = 'encode-atac-seq-pipeline-macs2'
         String conda_spp = 'encode-atac-seq-pipeline-spp'

dev/test/test_task/compare_md5sum.wdl

@@ -12,6 +12,13 @@ task compare_md5sum {
         import os
         import json
         import hashlib
+        import struct
+
+        def getuncompressedsize(filename):
+            # https://stackoverflow.com/a/22348071
+            with open(filename, 'rb') as f:
+                f.seek(-4, 2)
+                return struct.unpack('I', f.read(4))[0]
 
         def md5sum(filename, blocksize=65536):
             hash = hashlib.md5()
@@ -39,16 +46,33 @@ task compare_md5sum {
             ref_f = ref_files[i]
             md5 = md5sum(f)
             ref_md5 = md5sum(ref_f)
+
+            filesize = os.path.getsize(f)
+            ref_filesize = os.path.getsize(ref_f)
+
             # if text file, read in contents
             if f.endswith('.qc') or f.endswith('.txt') or \
                 f.endswith('.log') or f.endswith('.out'):
                 with open(f,'r') as fp:
                     contents = fp.read()
                 with open(ref_f,'r') as fp:
                     ref_contents = fp.read()
+            elif f.endswith('.gz'):
+                uncompressed_filesize = getuncompressedsize(f)
+                ref_uncompressed_filesize = getuncompressedsize(ref_f)
+
+                contents = 'filesize={filesize}, uncompressed_filesize={uncompressed_filesize}'.format(
+                    filesize=filesize,
+                    uncompressed_filesize=uncompressed_filesize,
+                )
+                ref_contents = 'filesize={filesize}, uncompressed_filesize={uncompressed_filesize}'.format(
+                    filesize=ref_filesize,
+                    uncompressed_filesize=ref_uncompressed_filesize,
+                )
             else:
-                contents = ''
-                ref_contents = ''
+                contents = 'filesize={filesize}'.format(filesize=filesize)
+                ref_contents = 'filesize={filesize}'.format(filesize=ref_filesize)
+
             matched = md5==ref_md5
             result['tasks'].append(OrderedDict([
                 ('label', label),

dev/test/test_task/test_annot_enrich.json

@@ -1,10 +1,10 @@
 {
-    "test_annot_enrich.blacklist" : "atac-seq-pipeline-test-data/genome_data/hg38_chr19_chrM/hg38.blacklist.bed.gz",
-    "test_annot_enrich.dnase" : "atac-seq-pipeline-test-data/genome_data/hg38/ataqc/reg2map_honeybadger2_dnase_all_p10_ucsc.hg19_to_hg38.bed.gz",
-    "test_annot_enrich.prom" : "atac-seq-pipeline-test-data/genome_data/hg38/ataqc/reg2map_honeybadger2_dnase_prom_p2.hg19_to_hg38.bed.gz",
-    "test_annot_enrich.enh" : "atac-seq-pipeline-test-data/genome_data/hg38/ataqc/reg2map_honeybadger2_dnase_enh_p2.hg19_to_hg38.bed.gz",
+    "test_annot_enrich.blacklist" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/genome_data/hg38_chr19_chrM/hg38.blacklist.bed.gz",
+    "test_annot_enrich.dnase" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/genome_data/hg38/ataqc/reg2map_honeybadger2_dnase_all_p10_ucsc.hg19_to_hg38.bed.gz",
+    "test_annot_enrich.prom" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/genome_data/hg38/ataqc/reg2map_honeybadger2_dnase_prom_p2.hg19_to_hg38.bed.gz",
+    "test_annot_enrich.enh" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/genome_data/hg38/ataqc/reg2map_honeybadger2_dnase_enh_p2.hg19_to_hg38.bed.gz",
 
-    "test_annot_enrich.ta" : "atac-seq-pipeline-test-data/input/pe/ataqc/ENCFF341MYG.subsampled.400.trim.merged.nodup.tn5.tagAlign.gz",
+    "test_annot_enrich.ta" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/input/pe/ataqc/ENCFF341MYG.subsampled.400.trim.merged.nodup.tn5.tagAlign.gz",
 
-    "test_annot_enrich.ref_annot_enrich_qc" : "atac-seq-pipeline-test-data/ref_output/test_annot_enrich/ENCFF341MYG.subsampled.400.trim.merged.nodup.tn5.annot_enrich.qc"
+    "test_annot_enrich.ref_annot_enrich_qc" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/ref_output/test_annot_enrich/ENCFF341MYG.subsampled.400.trim.merged.nodup.tn5.annot_enrich.qc"
 }

dev/test/test_task/test_bam2ta.json

@@ -1,14 +1,14 @@
 {
-    "test_bam2ta.pe_nodup_bam" : "atac-seq-pipeline-test-data/input/pe/nodup_bams/rep1/ENCFF341MYG.subsampled.400.trim.merged.nodup.bam",
-    "test_bam2ta.se_nodup_bam" : "atac-seq-pipeline-test-data/input/se/nodup_bams/rep1/ENCFF439VSY.subsampled.400.trim.merged.nodup.bam",
+    "test_bam2ta.pe_nodup_bam" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/input/pe/nodup_bams/rep1/ENCFF341MYG.subsampled.400.trim.merged.nodup.bam",
+    "test_bam2ta.se_nodup_bam" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/input/se/nodup_bams/rep1/ENCFF439VSY.subsampled.400.trim.merged.nodup.bam",
 
-    "test_bam2ta.ref_pe_ta" : "atac-seq-pipeline-test-data/ref_output/test_bam2ta/pe/ENCFF341MYG.subsampled.400.trim.merged.nodup.tn5.tagAlign.gz",
-    "test_bam2ta.ref_pe_ta_disable_tn5_shift" : "atac-seq-pipeline-test-data/ref_output/test_bam2ta/pe/disable_tn5_shift/ENCFF341MYG.subsampled.400.trim.merged.nodup.tagAlign.gz",
-    "test_bam2ta.ref_pe_ta_subsample" : "atac-seq-pipeline-test-data/ref_output/test_bam2ta/pe/subsample/fix_PIP-917/ENCFF341MYG.subsampled.400.trim.merged.nodup.5K.tn5.tagAlign.gz",
+    "test_bam2ta.ref_pe_ta" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/ref_output/test_bam2ta/pe/ENCFF341MYG.subsampled.400.trim.merged.nodup.tn5.tagAlign.gz",
+    "test_bam2ta.ref_pe_ta_disable_tn5_shift" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/ref_output/test_bam2ta/pe/disable_tn5_shift/ENCFF341MYG.subsampled.400.trim.merged.nodup.tagAlign.gz",
+    "test_bam2ta.ref_pe_ta_subsample" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/ref_output/test_bam2ta/pe/subsample/fix_PIP-917/ENCFF341MYG.subsampled.400.trim.merged.nodup.5K.tn5.tagAlign.gz",
 
-    "test_bam2ta.ref_se_ta" : "atac-seq-pipeline-test-data/ref_output/test_bam2ta/se/ENCFF439VSY.subsampled.400.trim.merged.nodup.tn5.tagAlign.gz",
-    "test_bam2ta.ref_se_ta_disable_tn5_shift" : "atac-seq-pipeline-test-data/ref_output/test_bam2ta/se/disable_tn5_shift/ENCFF439VSY.subsampled.400.trim.merged.nodup.tagAlign.gz",
-    "test_bam2ta.ref_se_ta_subsample" : "atac-seq-pipeline-test-data/ref_output/test_bam2ta/se/subsample/ENCFF439VSY.subsampled.400.trim.merged.nodup.5K.tn5.tagAlign.gz",
+    "test_bam2ta.ref_se_ta" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/ref_output/test_bam2ta/se/ENCFF439VSY.subsampled.400.trim.merged.nodup.tn5.tagAlign.gz",
+    "test_bam2ta.ref_se_ta_disable_tn5_shift" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/ref_output/test_bam2ta/se/disable_tn5_shift/ENCFF439VSY.subsampled.400.trim.merged.nodup.tagAlign.gz",
+    "test_bam2ta.ref_se_ta_subsample" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/ref_output/test_bam2ta/se/subsample/ENCFF439VSY.subsampled.400.trim.merged.nodup.5K.tn5.tagAlign.gz",
 
     "test_bam2ta.bam2ta_subsample" : 5000
 }

dev/test/test_task/test_bowtie2.json

@@ -1,28 +1,28 @@
 {
     "test_bowtie2.pe_fastqs_R1" : [
-        "atac-seq-pipeline-test-data/input/pe/fastqs/rep1/pair1/ENCFF341MYG.subsampled.400.fastq.gz",
-        "atac-seq-pipeline-test-data/input/pe/fastqs/rep1/pair1/ENCFF106QGY.subsampled.400.fastq.gz"
+        "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/input/pe/fastqs/rep1/pair1/ENCFF341MYG.subsampled.400.fastq.gz",
+        "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/input/pe/fastqs/rep1/pair1/ENCFF106QGY.subsampled.400.fastq.gz"
     ],
     "test_bowtie2.pe_fastqs_R2" : [
-        "atac-seq-pipeline-test-data/input/pe/fastqs/rep1/pair2/ENCFF248EJF.subsampled.400.fastq.gz",
-        "atac-seq-pipeline-test-data/input/pe/fastqs/rep1/pair2/ENCFF368TYI.subsampled.400.fastq.gz"
+        "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/input/pe/fastqs/rep1/pair2/ENCFF248EJF.subsampled.400.fastq.gz",
+        "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/input/pe/fastqs/rep1/pair2/ENCFF368TYI.subsampled.400.fastq.gz"
     ],
     "test_bowtie2.se_fastqs_R1" : [
-        "atac-seq-pipeline-test-data/input/se/fastqs/rep1/ENCFF439VSY.subsampled.400.fastq.gz",
-        "atac-seq-pipeline-test-data/input/se/fastqs/rep1/ENCFF325FCQ.subsampled.400.fastq.gz",
-        "atac-seq-pipeline-test-data/input/se/fastqs/rep1/ENCFF683IQS.subsampled.400.fastq.gz",
-        "atac-seq-pipeline-test-data/input/se/fastqs/rep1/ENCFF744CHW.subsampled.400.fastq.gz"
+        "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/input/se/fastqs/rep1/ENCFF439VSY.subsampled.400.fastq.gz",
+        "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/input/se/fastqs/rep1/ENCFF325FCQ.subsampled.400.fastq.gz",
+        "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/input/se/fastqs/rep1/ENCFF683IQS.subsampled.400.fastq.gz",
+        "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/input/se/fastqs/rep1/ENCFF744CHW.subsampled.400.fastq.gz"
     ],    
-    "test_bowtie2.pe_bowtie2_idx_tar" : "atac-seq-pipeline-test-data/genome_data/hg38_chr19_chrM/bowtie2_index/GRCh38_no_alt_analysis_set_GCA_000001405.15.chr19_chrM.fasta.tar",
-    "test_bowtie2.se_bowtie2_idx_tar" : "atac-seq-pipeline-test-data/genome_data/mm10_chr19_chrM/bowtie2_index/mm10_no_alt_analysis_set_ENCODE.chr19_chrM.fasta.tar",
+    "test_bowtie2.pe_bowtie2_idx_tar" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/genome_data/hg38_chr19_chrM/bowtie2_index/GRCh38_no_alt_analysis_set_GCA_000001405.15.chr19_chrM.fasta.tar",
+    "test_bowtie2.se_bowtie2_idx_tar" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/genome_data/mm10_chr19_chrM/bowtie2_index/mm10_no_alt_analysis_set_ENCODE.chr19_chrM.fasta.tar",
 
-    "test_bowtie2.se_chrsz" : "atac-seq-pipeline-test-data/genome_data/mm10_chr19_chrM/mm10_chr19_chrM.chrom.sizes",
-    "test_bowtie2.pe_chrsz" : "atac-seq-pipeline-test-data/genome_data/hg38_chr19_chrM/hg38_chr19_chrM.chrom.sizes",
+    "test_bowtie2.se_chrsz" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/genome_data/mm10_chr19_chrM/mm10_chr19_chrM.chrom.sizes",
+    "test_bowtie2.pe_chrsz" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/genome_data/hg38_chr19_chrM/hg38_chr19_chrM.chrom.sizes",
 
-    "test_bowtie2.ref_pe_flagstat" : "atac-seq-pipeline-test-data/ref_output/test_bowtie2/pe/multimapping/merge_fastqs_R1_ENCFF341MYG.subsampled.400.trim.merged.samstats.qc",
-    "test_bowtie2.ref_pe_flagstat_no_multimapping" : "atac-seq-pipeline-test-data/ref_output/test_bowtie2/pe/no_multimapping/merge_fastqs_R1_ENCFF341MYG.subsampled.400.trim.merged.samstats.qc",
-    "test_bowtie2.ref_se_flagstat" : "atac-seq-pipeline-test-data/ref_output/test_bowtie2/se/multimapping/merge_fastqs_R1_ENCFF439VSY.subsampled.400.trim.merged.samstats.qc",
-    "test_bowtie2.ref_se_flagstat_no_multimapping" : "atac-seq-pipeline-test-data/ref_output/test_bowtie2/se/no_multimapping/merge_fastqs_R1_ENCFF439VSY.subsampled.400.trim.merged.samstats.qc",
+    "test_bowtie2.ref_pe_flagstat" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/ref_output/test_bowtie2/pe/multimapping/merge_fastqs_R1_ENCFF341MYG.subsampled.400.trim.merged.samstats.qc",
+    "test_bowtie2.ref_pe_flagstat_no_multimapping" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/ref_output/test_bowtie2/pe/no_multimapping/merge_fastqs_R1_ENCFF341MYG.subsampled.400.trim.merged.samstats.qc",
+    "test_bowtie2.ref_se_flagstat" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/ref_output/test_bowtie2/se/multimapping/merge_fastqs_R1_ENCFF439VSY.subsampled.400.trim.merged.samstats.qc",
+    "test_bowtie2.ref_se_flagstat_no_multimapping" : "gs://encode-pipeline-test-samples/encode-atac-seq-pipeline/atac-seq-pipeline-test-data/ref_output/test_bowtie2/se/no_multimapping/merge_fastqs_R1_ENCFF439VSY.subsampled.400.trim.merged.samstats.qc",
 
     "test_bowtie2.multimapping" : 4
 }