Merge pull request #10 from ajmaurais/improve_sky_reports

Improve Skyline reports

docs/source/workflow_parameters.rst

@@ -112,7 +112,9 @@ The ``params`` Section
      - If set to ``true``, the generated Skyline document will be imported into PanoramaWeb's relational database for inline visualization. The import will appear in the parent folder for the ``panorama.upload_url`` parameter, and will have the named used for the ``skyline_document_name`` parameter. Default: ``false``. Note: ``panorama_upload`` must be set to ``true`` and ``skip_skyline`` must be set to ``false`` to use this feature.
    * -
      - ``skyline.skyr_file``
-     - The path (local file system or Panorama WebDAV) to a ``.skyr`` file, which is a Skyline file that specifies reports. Any reports specified in the ``.skyr`` file will be run automatically as the last step of the workflow and the results saved in your ``results`` directory and (if requested) uploaded to Panorama.
+     - Path(s) (local file system or Panorama WebDAV) to a ``.skyr`` file, which is a Skyline report template. Any reports specified in the ``.skyr`` file will be run automatically as the last step of the workflow and the results saved in your ``results`` directory and (if requested) uploaded to Panorama. The report template(s) can be a single string, or for multiple ``.skyr`` files can be given as a list of strings.
+       For example: ``'/path/to/report.skyr'`` for a single file, or
+       ``['/path/to/report_1.skyr', '/path/to/report_2.skyr']`` for multiple files.
    * -
      - ``skyline.template_file``
      - The Skyline template file used to generate the final Skyline file. By default a

modules/panorama.nf

@@ -69,75 +69,7 @@ process PANORAMA_GET_RAW_FILE_LIST {
     """
 }
 
-process PANORAMA_GET_SKYLINE_TEMPLATE {
-    label 'process_low_constant'
-    label 'error_retry'
-    container params.images.panorama_client
-    publishDir "${params.result_dir}/panorama", failOnError: true, mode: 'copy', pattern: "*.stdout"
-    publishDir "${params.result_dir}/panorama", failOnError: true, mode: 'copy', pattern: "*.stderr"
-
-    input:
-        val web_dav_dir_url
-
-    output:
-        path("${file(web_dav_dir_url).name}"), emit: panorama_file
-        path("*.stdout"), emit: stdout
-        path("*.stderr"), emit: stderr
-
-    script:
-        file_name = file(web_dav_dir_url).name
-        """
-        echo "Downloading ${file_name} from Panorama..."
-            ${exec_java_command(task.memory)} \
-            -d \
-            -w "${web_dav_dir_url}" \
-            -k \$PANORAMA_API_KEY \
-            > >(tee "panorama-get-${file_name}.stdout") 2> >(tee "panorama-get-${file_name}.stderr" >&2)
-        echo "Done!" # Needed for proper exit
-        """
-
-    stub:
-    """
-    touch "${file(web_dav_dir_url).name}"
-    touch stub.stderr stub.stdout
-    """
-}
-
-process PANORAMA_GET_FASTA {
-    label 'process_low_constant'
-    label 'error_retry'
-    container params.images.panorama_client
-    publishDir "${params.result_dir}/panorama", failOnError: true, mode: 'copy', pattern: "*.stdout"
-    publishDir "${params.result_dir}/panorama", failOnError: true, mode: 'copy', pattern: "*.stderr"
-
-    input:
-        val web_dav_dir_url
-
-    output:
-        path("${file(web_dav_dir_url).name}"), emit: panorama_file
-        path("*.stdout"), emit: stdout
-        path("*.stderr"), emit: stderr
-
-    script:
-        file_name = file(web_dav_dir_url).name
-        """
-        echo "Downloading ${file_name} from Panorama..."
-            ${exec_java_command(task.memory)} \
-            -d \
-            -w "${web_dav_dir_url}" \
-            -k \$PANORAMA_API_KEY \
-            > >(tee "panorama-get-${file_name}.stdout") 2> >(tee "panorama-get-${file_name}.stderr" >&2)
-        echo "Done!" # Needed for proper exit
-        """
-
-    stub:
-    """
-    touch "${file(web_dav_dir_url).name}"
-    touch stub.stderr stub.stdout
-    """
-}
-
-process PANORAMA_GET_SPECTRAL_LIBRARY {
+process PANORAMA_GET_FILE {
     label 'process_low_constant'
     label 'error_retry'
     container params.images.panorama_client

modules/skyline.nf

@@ -244,47 +244,46 @@ process SKYLINE_RUN_REPORTS {
 
     output:
         path("*.report.tsv"), emit: skyline_report_files
-        path("*.log"), emit: log
+        path("*.stdout"), emit: stdout
+        path("*.stderr"), emit: stderr
 
-    script:
-    """
-    unzip ${skyline_zipfile}
+    shell:
+    '''
+    unzip !{skyline_zipfile}
 
-    # add reports to skyline file
-    for skyrfile in *.skyr; do
-        wine SkylineCmd \
-            --in="${skyline_zipfile.baseName}" \
-            --log-file="skyline-import-\$skyrfile.log" \
-            --report-add="\$skyrfile" \
-            --save
-    done
+    # generate skyline batch file to export reports
+    echo "--in=\\"!{skyline_zipfile.baseName}\\"" > export_reports.bat
+
+    for skyrfile in ./*.skyr; do
+        # Add report to document
+        echo "--report-add=\\"${skyrfile}\\"" >> export_reports.bat
 
-    # run the reports
-    for xmlfile in ./*.skyr; do
-        awk -F'"' '/<view name=/ { print \$2 }' "\$xmlfile" | while read reportname; do
-            wine SkylineCmd \
-                --in="${skyline_zipfile.baseName}" \
-                --log-file="\$reportname.report-generation.log" \
-                --report-name="\$reportname" \
-                --report-file="./\$reportname.report.tsv" \
-                --report-format="TSV" \
-                --report-invariant
+        # Export report
+        awk -F'"' '/<view name=/ { print $2 }' "$skyrfile" | while read reportname; do
+            echo "--report-name=\\"${reportname}\\" \
+                  --report-file=\\"${reportname}.report.tsv\\" \
+                  --report-format=TSV --report-invariant" \
+                  >> export_reports.bat
         done
     done
-    """
 
-    stub:
+    # Run batch commands
+    wine SkylineCmd --batch-commands=export_reports.bat \
+        > >(tee 'export_reports.stdout') 2> >(tee 'export_reports.stderr' >&2)
     '''
-    touch stub.log
 
-    for xmlfile in ./*.skyr; do
-        awk -F'"' '/<view name=/ { print $2 }' "$xmlfile" | while read reportname; do
+    stub:
+    '''
+    for skyrfile in ./*.skyr; do
+        awk -F'"' '/<view name=/ { print $2 }' "$skyrfile" | while read reportname; do
             touch "${reportname}.report.tsv"
         done
     done
 
     if [ $(ls *.report.tsv|wc -l) -eq 0 ] ; then
         touch stub.report.tsv
     fi
+
+    touch stub.stdout stub.stderr
     '''
 }

workflows/generate_qc_report.nf

@@ -20,7 +20,7 @@ workflow generate_dia_qc_report {
 
         // export skyline reports
         skyr_files = Channel.fromList([params.qc_report.replicate_report_template,
-                                       params.qc_report.precursor_report_template]).map{ file(it) }
+                                       params.qc_report.precursor_report_template]).map{ file(it, checkIfExists: true) }
         SKYLINE_RUN_REPORTS(sky_zip_file, skyr_files.collect())
         sky_reports = SKYLINE_RUN_REPORTS.out.skyline_report_files.flatten().map{ it -> tuple(it.name, it) }
         precursor_report = sky_reports.filter{ it[0] =~ /^precursor_quality\.report\.tsv$/ }.map{ it -> it[1] }

workflows/get_input_files.nf

@@ -1,11 +1,32 @@
 // modules
-include { PANORAMA_GET_FASTA } from "../modules/panorama"
-include { PANORAMA_GET_SPECTRAL_LIBRARY } from "../modules/panorama"
-include { PANORAMA_GET_SKYLINE_TEMPLATE } from "../modules/panorama"
+include { PANORAMA_GET_FILE as PANORAMA_GET_FASTA } from "../modules/panorama"
+include { PANORAMA_GET_FILE as PANORAMA_GET_SPECTRAL_LIBRARY } from "../modules/panorama"
+include { PANORAMA_GET_FILE as PANORAMA_GET_SKYLINE_TEMPLATE } from "../modules/panorama"
 include { PANORAMA_GET_SKYR_FILE } from "../modules/panorama"
-include { PANORAMA_GET_FASTA as PANORAMA_GET_METADATA } from "../modules/panorama"
+include { PANORAMA_GET_FILE as PANORAMA_GET_METADATA } from "../modules/panorama"
 include { MAKE_EMPTY_FILE as METADATA_PLACEHOLDER } from "../modules/qc_report"
 
+PANORAMA_URL = 'https://panoramaweb.org'
+
+/**
+* Process a parameter variable which is specified as either a single value or List.
+* If param_variable has multiple lines, each line with text is returned as an
+* element in a List.
+*
+* @param param_variable A parameter variable which can either be a single value or List.
+* @return param_variable as a List with 1 or more values.
+*/
+def param_to_list(param_variable) {
+    if(param_variable instanceof List) {
+        return param_variable
+    }
+    if(param_variable instanceof String) {
+        // Split string by new line, remove whitespace, and skip empty lines
+        return param_variable.split('\n').collect{ it.trim() }.findAll{ it }
+    }
+    return [param_variable]
+}
+
 workflow get_input_files {
 
    emit:
@@ -18,15 +39,15 @@ workflow get_input_files {
     main:
 
         // get files from Panorama as necessary
-        if(params.fasta.startsWith("https://")) {
+        if(params.fasta.startsWith(PANORAMA_URL)) {
             PANORAMA_GET_FASTA(params.fasta)
             fasta = PANORAMA_GET_FASTA.out.panorama_file
         } else {
             fasta = file(params.fasta, checkIfExists: true)
         }
 
         if(params.spectral_library) {
-            if(params.spectral_library.startsWith("https://")) {
+            if(params.spectral_library.startsWith(PANORAMA_URL)) {
                 PANORAMA_GET_SPECTRAL_LIBRARY(params.spectral_library)
                 spectral_library = PANORAMA_GET_SPECTRAL_LIBRARY.out.panorama_file
             } else {
@@ -37,7 +58,7 @@ workflow get_input_files {
         }
 
         if(params.skyline.template_file != null) {
-            if(params.skyline.template_file.startsWith("https://")) {
+            if(params.skyline.template_file.startsWith(PANORAMA_URL)) {
                 PANORAMA_GET_SKYLINE_TEMPLATE(params.skyline.template_file)
                 skyline_template_zipfile = PANORAMA_GET_SKYLINE_TEMPLATE.out.panorama_file
             } else {
@@ -48,39 +69,24 @@ workflow get_input_files {
         }
 
         if(params.skyline.skyr_file != null) {
-            if(params.skyline.skyr_file.trim().startsWith("https://")) {
-
-                // get file(s) from Panorama
-                skyr_location_ch = Channel.from(params.skyline.skyr_file)
-                                        .splitText()               // split multiline input
-                                        .map{ it.trim() }          // removing surrounding whitespace
-                                        .filter{ it.length() > 0 } // skip empty lines
 
-                // get raw files from panorama
-                PANORAMA_GET_SKYR_FILE(skyr_location_ch)
-                skyr_files = PANORAMA_GET_SKYR_FILE.out.panorama_file
+            // Split skyr files stored on Panorama and locally into separate channels.
+            Channel.fromList(param_to_list(params.skyline.skyr_file)).branch{
+                panorama_files: it.startsWith(PANORAMA_URL)
+                local_files: true
+                    return file(it, checkIfExists: true)
+                }.set{skyr_paths}
 
-            } else {
-                // files are local
-                skyr_files = Channel.from(params.skyline.skyr_file)
-                                        .splitText()               // split multiline input
-                                        .map{ it.trim() }          // removing surrounding whitespace
-                                        .filter{ it.length() > 0 } // skip empty lines
-                                        .map { path ->             // convert to files, check all exist
-                                            def fileObj = file(path)
-                                            if (!fileObj.exists()) {
-                                                error "File does not exist: $path"
-                                            }
-                                            return fileObj
-                                        }
+            skyr_files = skyr_paths.local_files
+            skyr_paths.panorama_files | PANORAMA_GET_SKYR_FILE
+            skyr_files = skyr_files.concat(PANORAMA_GET_SKYR_FILE.out.panorama_file)
 
-            }
         } else {
             skyr_files = Channel.empty()
         }
 
         if(params.replicate_metadata != null) {
-            if(params.replicate_metadata.trim().startsWith("https://panoramaweb.org")) {
+            if(params.replicate_metadata.trim().startsWith(PANORAMA_URL)) {
                 PANORAMA_GET_METADATA(params.replicate_metadata)
                 replicate_metadata = PANORAMA_GET_METADATA.out.panorama_file
             } else {