From 2ad8e73f49242de29727252ae12ca7c6b5d00496 Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Mon, 11 Dec 2023 17:14:44 +0000
Subject: [PATCH 01/36] Added a SAMTOOLS_PIPELINE to run multiple samtools
 commands at once

---
 .../nf-core/samtools/pipeline/environment.yml |  7 ++
 modules/nf-core/samtools/pipeline/main.nf     | 92 +++++++++++++++++++
 modules/nf-core/samtools/pipeline/meta.yml    | 64 +++++++++++++
 tests/config/pytest_modules.yml               |  3 +
 .../modules/nf-core/samtools/pipeline/main.nf | 37 ++++++++
 .../nf-core/samtools/pipeline/nextflow.config | 22 +++++
 .../nf-core/samtools/pipeline/test.yml        | 29 ++++++
 7 files changed, 254 insertions(+)
 create mode 100644 modules/nf-core/samtools/pipeline/environment.yml
 create mode 100644 modules/nf-core/samtools/pipeline/main.nf
 create mode 100644 modules/nf-core/samtools/pipeline/meta.yml
 create mode 100644 tests/modules/nf-core/samtools/pipeline/main.nf
 create mode 100644 tests/modules/nf-core/samtools/pipeline/nextflow.config
 create mode 100644 tests/modules/nf-core/samtools/pipeline/test.yml

diff --git a/modules/nf-core/samtools/pipeline/environment.yml b/modules/nf-core/samtools/pipeline/environment.yml
new file mode 100644
index 000000000000..1a5847d7920a
--- /dev/null
+++ b/modules/nf-core/samtools/pipeline/environment.yml
@@ -0,0 +1,7 @@
+name: samtools_pipeline
+channels:
+  - conda-forge
+  - bioconda
+  - defaults
+dependencies:
+  - bioconda::samtools=1.18
diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
new file mode 100644
index 000000000000..a7ddae1373eb
--- /dev/null
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -0,0 +1,92 @@
+process SAMTOOLS_PIPELINE {
+    tag "$meta.id"
+    label 'process_medium'
+
+    conda "${moduleDir}/environment.yml"
+    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
+        'https://depot.galaxyproject.org/singularity/samtools:1.18--h50ea8bc_1':
+        'biocontainers/samtools:1.18--h50ea8bc_1' }"
+
+    input:
+    tuple val(meta), path(input)
+    tuple val(meta2), path(fasta)
+    val commands
+
+    output:
+    tuple val(meta), path("*.{bam,cram,sam}"), emit: output
+    path "versions.yml",                       emit: versions
+
+    when:
+    task.ext.when == null || task.ext.when
+
+    script:
+
+    // Check that we are asked to run a pipeline
+    def n_commands = commands.size()
+    if (n_commands < 2) error "SAMTOOLS_PIPELINE is used to chain 2 or more samtools commands"
+
+    // Fetch the arguments
+    def all_args = []
+    for (int index = 0; index < n_commands; index++) {
+        all_args.add( task.ext.args?[ commands[index] ] ?: '' )
+    }
+    def last_args = all_args[-1]
+
+    // Output file
+    def prefix = task.ext.prefix ?: "${meta.id}"
+    def reference = fasta ? "--reference ${fasta}" : ""
+    def extension = last_args.contains("--output-fmt sam") ? "sam" :
+                    last_args.contains("--output-fmt bam") ? "bam" :
+                    last_args.contains("--output-fmt cram") ? "cram" :
+                    "bam"
+    if ("$input" == "${prefix}.${extension}") error "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!"
+
+    def pipeline_command = ""
+    for (int index = 0; index < commands.size(); index++) {
+        def this_command = """
+        samtools \\
+            ${commands[index]} \\
+            ${all_args[index]} \\
+            -@ $task.cpus \\
+        """
+        if (commands[index] != "fixmate") {
+            this_command += "    -T ${prefix}.tmp.${commands[index]} \\\n"
+        }
+        this_command += "    "
+        if (index < (n_commands - 1)) {
+            this_command += "-u "
+        }
+        this_input = (index ? "-" : "$input")
+        if (["collate", "sort"].contains(commands[index])) {
+            if (index == (n_commands - 1)) {
+                this_command += "-o ${prefix}.${extension} "
+            } else {
+                if (commands[index] != "sort") {
+                    this_command += "-O "
+                }
+            }
+            this_command += this_input
+        } else {
+            // e.g. fixmate, markdup
+            this_command += this_input
+            if (index == (n_commands - 1)) {
+                this_command += " ${prefix}.${extension}"
+            } else {
+                this_command += " -"
+            }
+        }
+        pipeline_command += this_command
+        if (index < (n_commands - 1)) {
+            pipeline_command += " | \\"
+        }
+    }
+
+    """
+    $pipeline_command
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//')
+    END_VERSIONS
+    """
+}
diff --git a/modules/nf-core/samtools/pipeline/meta.yml b/modules/nf-core/samtools/pipeline/meta.yml
new file mode 100644
index 000000000000..1907477f7bc6
--- /dev/null
+++ b/modules/nf-core/samtools/pipeline/meta.yml
@@ -0,0 +1,64 @@
+name: "samtools_pipeline"
+description: custom bash pipeline made of samtools commands
+keywords:
+  - pipeline
+  - bam
+  - sam
+  - cram
+tools:
+  - "samtools":
+      description: "Tools for dealing with SAM, BAM and CRAM files"
+      homepage: "http://www.htslib.org"
+      documentation: "https://www.htslib.org/doc/samtools.html"
+      tool_dev_url: "https://github.com/samtools/samtools"
+      doi: "10.1093/bioinformatics/btp352"
+      licence: ["MIT"]
+input:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - input:
+      type: file
+      description: BAM/CRAM/SAM file
+      pattern: "*.{bam,cram,sam}"
+  - meta2:
+      type: map
+      description: |
+        Groovy Map containing reference information
+        e.g. [ id:'test' ]
+  - fasta:
+      type: file
+      description: FASTA reference file
+      pattern: "*.{fasta,fa}"
+output:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - bam:
+      type: file
+      description: optional output BAM file
+      pattern: "*.{bam}"
+  - cram:
+      type: file
+      description: optional output BAM file
+      pattern: "*.{cram}"
+  - sam:
+      type: file
+      description: optional output BAM file
+      pattern: "*.{sam}"
+  - versions:
+      type: file
+      description: File containing software versions
+      pattern: "versions.yml"
+  - output:
+      type: file
+      description: Collated BAM/CRAM/SAM file
+      pattern: "*.{bam,cram,sam}"
+authors:
+  - "@muffato"
+maintainers:
+  - "@muffato"
diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml
index e02d84c919c9..2adf81284ef2 100644
--- a/tests/config/pytest_modules.yml
+++ b/tests/config/pytest_modules.yml
@@ -2132,6 +2132,9 @@ samtools/import:
 samtools/markdup:
   - modules/nf-core/samtools/markdup/**
   - tests/modules/nf-core/samtools/markdup/**
+samtools/pipeline:
+  - modules/nf-core/samtools/pipeline/**
+  - tests/modules/nf-core/samtools/pipeline/**
 samtools/reheader:
   - modules/nf-core/samtools/reheader/**
   - tests/modules/nf-core/samtools/reheader/**
diff --git a/tests/modules/nf-core/samtools/pipeline/main.nf b/tests/modules/nf-core/samtools/pipeline/main.nf
new file mode 100644
index 000000000000..ebde1cf4f436
--- /dev/null
+++ b/tests/modules/nf-core/samtools/pipeline/main.nf
@@ -0,0 +1,37 @@
+#!/usr/bin/env nextflow
+
+nextflow.enable.dsl = 2
+
+include { SAMTOOLS_PIPELINE as SAMTOOLS_PIPELINE_SORMADUP     } from '../../../../../modules/nf-core/samtools/pipeline/main.nf'
+include { SAMTOOLS_PIPELINE as SAMTOOLS_PIPELINE_COLLFIXMSORT } from '../../../../../modules/nf-core/samtools/pipeline/main.nf'
+include { SAMTOOLS_PIPELINE as SAMTOOLS_PIPELINE_COLLFIXM     } from '../../../../../modules/nf-core/samtools/pipeline/main.nf'
+
+workflow test_samtools_pipeline_complete_sormadup {
+    
+    input = [
+        [ id:'test', single_end:false ], // meta map
+        file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true)
+    ]
+    commands = ['collate', 'fixmate', 'sort', 'markdup']
+    SAMTOOLS_PIPELINE_SORMADUP ( input, [[],[]], commands )
+}
+
+workflow test_samtools_pipeline_collate_fixmate_sort {
+    
+    input = [
+        [ id:'test', single_end:false ], // meta map
+        file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true)
+    ]
+    commands = ['collate', 'fixmate', 'sort']
+    SAMTOOLS_PIPELINE_COLLFIXMSORT ( input, [[],[]], commands )
+}
+
+workflow test_samtools_pipeline_collate_fixmate {
+    
+    input = [
+        [ id:'test', single_end:false ], // meta map
+        file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true)
+    ]
+    commands = ['collate', 'fixmate']
+    SAMTOOLS_PIPELINE_COLLFIXM ( input, [[],[]], commands )
+}
diff --git a/tests/modules/nf-core/samtools/pipeline/nextflow.config b/tests/modules/nf-core/samtools/pipeline/nextflow.config
new file mode 100644
index 000000000000..f5d629a042ba
--- /dev/null
+++ b/tests/modules/nf-core/samtools/pipeline/nextflow.config
@@ -0,0 +1,22 @@
+process {
+
+    publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }
+
+    withName: SAMTOOLS_PIPELINE_SORMADUP {
+        ext.args = [
+            fixmate: '-m'
+        ]
+    }
+
+    withName: SAMTOOLS_PIPELINE_COLLFIXMSORT {
+        ext.args = [
+            fixmate: '-m'
+        ]
+    }
+
+    withName: SAMTOOLS_PIPELINE_COLLFIXM {
+        ext.args = [
+            fixmate: '-m'
+        ]
+    }
+}
diff --git a/tests/modules/nf-core/samtools/pipeline/test.yml b/tests/modules/nf-core/samtools/pipeline/test.yml
new file mode 100644
index 000000000000..2dca5392c7e7
--- /dev/null
+++ b/tests/modules/nf-core/samtools/pipeline/test.yml
@@ -0,0 +1,29 @@
+- name: samtools pipeline test_samtools_pipeline_complete_sormadup
+  command: nextflow run ./tests/modules/nf-core/samtools/pipeline -entry test_samtools_pipeline_complete_sormadup -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/samtools/pipeline/nextflow.config
+  tags:
+    - samtools/pipeline
+    - samtools
+  files:
+    - path: output/samtools/test.bam
+      md5sum: d505badd610ef5a32d28db3ec8e6aea1
+    - path: output/samtools/versions.yml
+
+- name: samtools pipeline test_samtools_pipeline_collate_fixmate_sort
+  command: nextflow run ./tests/modules/nf-core/samtools/pipeline -entry test_samtools_pipeline_collate_fixmate_sort -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/samtools/pipeline/nextflow.config
+  tags:
+    - samtools/pipeline
+    - samtools
+  files:
+    - path: output/samtools/test.bam
+      md5sum: 543221828943029864db75bd617b79d1
+    - path: output/samtools/versions.yml
+
+- name: samtools pipeline test_samtools_pipeline_collate_fixmate
+  command: nextflow run ./tests/modules/nf-core/samtools/pipeline -entry test_samtools_pipeline_collate_fixmate -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/samtools/pipeline/nextflow.config
+  tags:
+    - samtools/pipeline
+    - samtools
+  files:
+    - path: output/samtools/test.bam
+      md5sum: 37f2c0f510aff13bc73c7ddd6989c1f7
+    - path: output/samtools/versions.yml

From 9c3ec9dca20b6de8c4716f2460fcc290c3deafac Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Mon, 11 Dec 2023 18:05:52 +0000
Subject: [PATCH 02/36] Just call it sormadup

---
 tests/modules/nf-core/samtools/pipeline/main.nf  | 2 +-
 tests/modules/nf-core/samtools/pipeline/test.yml | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/modules/nf-core/samtools/pipeline/main.nf b/tests/modules/nf-core/samtools/pipeline/main.nf
index ebde1cf4f436..96b62e195ae5 100644
--- a/tests/modules/nf-core/samtools/pipeline/main.nf
+++ b/tests/modules/nf-core/samtools/pipeline/main.nf
@@ -6,7 +6,7 @@ include { SAMTOOLS_PIPELINE as SAMTOOLS_PIPELINE_SORMADUP     } from '../../../.
 include { SAMTOOLS_PIPELINE as SAMTOOLS_PIPELINE_COLLFIXMSORT } from '../../../../../modules/nf-core/samtools/pipeline/main.nf'
 include { SAMTOOLS_PIPELINE as SAMTOOLS_PIPELINE_COLLFIXM     } from '../../../../../modules/nf-core/samtools/pipeline/main.nf'
 
-workflow test_samtools_pipeline_complete_sormadup {
+workflow test_samtools_pipeline_sormadup {
     
     input = [
         [ id:'test', single_end:false ], // meta map
diff --git a/tests/modules/nf-core/samtools/pipeline/test.yml b/tests/modules/nf-core/samtools/pipeline/test.yml
index 2dca5392c7e7..06f3c836d97f 100644
--- a/tests/modules/nf-core/samtools/pipeline/test.yml
+++ b/tests/modules/nf-core/samtools/pipeline/test.yml
@@ -1,5 +1,5 @@
-- name: samtools pipeline test_samtools_pipeline_complete_sormadup
-  command: nextflow run ./tests/modules/nf-core/samtools/pipeline -entry test_samtools_pipeline_complete_sormadup -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/samtools/pipeline/nextflow.config
+- name: samtools pipeline test_samtools_pipeline_sormadup
+  command: nextflow run ./tests/modules/nf-core/samtools/pipeline -entry test_samtools_pipeline_sormadup -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/samtools/pipeline/nextflow.config
   tags:
     - samtools/pipeline
     - samtools

From a2f9f581b0f6530b92c5923d4620e00e85c60495 Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Mon, 11 Dec 2023 18:06:35 +0000
Subject: [PATCH 03/36] Better text

---
 modules/nf-core/samtools/pipeline/main.nf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
index a7ddae1373eb..42ab08203f12 100644
--- a/modules/nf-core/samtools/pipeline/main.nf
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -21,7 +21,7 @@ process SAMTOOLS_PIPELINE {
 
     script:
 
-    // Check that we are asked to run a pipeline
+    // Check that we are asked to run more than 1 command
     def n_commands = commands.size()
     if (n_commands < 2) error "SAMTOOLS_PIPELINE is used to chain 2 or more samtools commands"
 

From 12085a4c2252704f7ad76601f74a22ab1d90d73e Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Mon, 11 Dec 2023 18:06:55 +0000
Subject: [PATCH 04/36] Added a stub

---
 modules/nf-core/samtools/pipeline/main.nf | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
index 42ab08203f12..46f69e8a9da9 100644
--- a/modules/nf-core/samtools/pipeline/main.nf
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -89,4 +89,15 @@ process SAMTOOLS_PIPELINE {
         samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//')
     END_VERSIONS
     """
+
+    stub:
+    def prefix = task.ext.prefix ?: "${meta.id}"
+    """
+    touch ${prefix}.${extension}
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//')
+    END_VERSIONS
+    """
 }

From 56f81e80e1e588165f91a694d91c0bbe5106836e Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Mon, 11 Dec 2023 18:17:38 +0000
Subject: [PATCH 05/36] Introduced helper variables

---
 modules/nf-core/samtools/pipeline/main.nf | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
index 46f69e8a9da9..650f656e8057 100644
--- a/modules/nf-core/samtools/pipeline/main.nf
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -42,7 +42,9 @@ process SAMTOOLS_PIPELINE {
     if ("$input" == "${prefix}.${extension}") error "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!"
 
     def pipeline_command = ""
-    for (int index = 0; index < commands.size(); index++) {
+    for (int index = 0; index < n_commands; index++) {
+        def is_first_command = (index == 0)
+        def is_last_command = (index == (n_commands - 1))
         def this_command = """
         samtools \\
             ${commands[index]} \\
@@ -53,12 +55,12 @@ process SAMTOOLS_PIPELINE {
             this_command += "    -T ${prefix}.tmp.${commands[index]} \\\n"
         }
         this_command += "    "
-        if (index < (n_commands - 1)) {
+        if (!is_last_command) {
             this_command += "-u "
         }
-        this_input = (index ? "-" : "$input")
+        this_input = (is_first_command ? "$input" : "-")
         if (["collate", "sort"].contains(commands[index])) {
-            if (index == (n_commands - 1)) {
+            if (is_last_command) {
                 this_command += "-o ${prefix}.${extension} "
             } else {
                 if (commands[index] != "sort") {
@@ -69,14 +71,14 @@ process SAMTOOLS_PIPELINE {
         } else {
             // e.g. fixmate, markdup
             this_command += this_input
-            if (index == (n_commands - 1)) {
+            if (is_last_command) {
                 this_command += " ${prefix}.${extension}"
             } else {
                 this_command += " -"
             }
         }
         pipeline_command += this_command
-        if (index < (n_commands - 1)) {
+        if (!is_last_command) {
             pipeline_command += " | \\"
         }
     }

From c8332a52f43760d4d8c6aef8b3e5fa7da77f0adf Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Mon, 11 Dec 2023 18:28:54 +0000
Subject: [PATCH 06/36] Directly modify the main variable

---
 modules/nf-core/samtools/pipeline/main.nf | 30 +++++++++++------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
index 650f656e8057..e9dcd202c8f8 100644
--- a/modules/nf-core/samtools/pipeline/main.nf
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -43,41 +43,41 @@ process SAMTOOLS_PIPELINE {
 
     def pipeline_command = ""
     for (int index = 0; index < n_commands; index++) {
+        def this_command = commands[index]
         def is_first_command = (index == 0)
         def is_last_command = (index == (n_commands - 1))
-        def this_command = """
+        pipeline_command += """
         samtools \\
-            ${commands[index]} \\
+            ${this_command} \\
             ${all_args[index]} \\
             -@ $task.cpus \\
         """
-        if (commands[index] != "fixmate") {
-            this_command += "    -T ${prefix}.tmp.${commands[index]} \\\n"
+        if (this_command != "fixmate") {
+            pipeline_command += "    -T ${prefix}.tmp.${this_command} \\\n"
         }
-        this_command += "    "
+        pipeline_command += "    "
         if (!is_last_command) {
-            this_command += "-u "
+            pipeline_command += "-u "
         }
         this_input = (is_first_command ? "$input" : "-")
-        if (["collate", "sort"].contains(commands[index])) {
+        if (["collate", "sort"].contains(this_command)) {
             if (is_last_command) {
-                this_command += "-o ${prefix}.${extension} "
+                pipeline_command += "-o ${prefix}.${extension} "
             } else {
-                if (commands[index] != "sort") {
-                    this_command += "-O "
+                if (this_command != "sort") {
+                    pipeline_command += "-O "
                 }
             }
-            this_command += this_input
+            pipeline_command += this_input
         } else {
             // e.g. fixmate, markdup
-            this_command += this_input
+            pipeline_command += this_input
             if (is_last_command) {
-                this_command += " ${prefix}.${extension}"
+                pipeline_command += " ${prefix}.${extension}"
             } else {
-                this_command += " -"
+                pipeline_command += " -"
             }
         }
-        pipeline_command += this_command
         if (!is_last_command) {
             pipeline_command += " | \\"
         }

From a85140a8a187172ed90a95c417b9563879ce4a4d Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Mon, 11 Dec 2023 18:45:55 +0000
Subject: [PATCH 07/36] More commands (and syntaxes)

---
 modules/nf-core/samtools/pipeline/main.nf | 39 +++++++++++++----------
 1 file changed, 23 insertions(+), 16 deletions(-)

diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
index e9dcd202c8f8..d20e08502b5b 100644
--- a/modules/nf-core/samtools/pipeline/main.nf
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -59,25 +59,32 @@ process SAMTOOLS_PIPELINE {
         if (!is_last_command) {
             pipeline_command += "-u "
         }
-        this_input = (is_first_command ? "$input" : "-")
-        if (["collate", "sort"].contains(this_command)) {
-            if (is_last_command) {
-                pipeline_command += "-o ${prefix}.${extension} "
-            } else {
-                if (this_command != "sort") {
-                    pipeline_command += "-O "
-                }
-            }
+        this_input = (is_first_command ? " $input" : " -")
+
+        // samtools commands have slightly different syntax
+        if (["collate"].contains(this_command)) {
+            // [-o OUTPUT|-O] [INPUT|-]
+            pipeline_command += is_last_command ? " -o ${prefix}.${extension}" : " -O"
             pipeline_command += this_input
-        } else {
-            // e.g. fixmate, markdup
+
+        } else if (["addreplacerg", "sort", "view"].contains(this_command)) {
+            // [-o OUTPUT] [INPUT|-]
+            pipeline_command += is_last_command ? " -o ${prefix}.${extension}" : ""
+            pipeline_command += this_input
+
+        } else if (["reheader"].contains(this_command)) {
+            // [INPUT|-]
             pipeline_command += this_input
-            if (is_last_command) {
-                pipeline_command += " ${prefix}.${extension}"
-            } else {
-                pipeline_command += " -"
-            }
+
+        } else if (["fixmate", "markdup"].contains(this_command)) {
+            // [INPUT|-] [OUTPUT|-]
+            pipeline_command += this_input
+            pipeline_command += is_last_command ? " ${prefix}.${extension}" : " -"
+
+        } else {
+            error "${this_command} is not supported"
         }
+
         if (!is_last_command) {
             pipeline_command += " | \\"
         }

From 41d9fba5deb63e7eb496a8530eb5a744be4bee25 Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Mon, 11 Dec 2023 18:51:43 +0000
Subject: [PATCH 08/36] Expanded support

---
 modules/nf-core/samtools/pipeline/main.nf | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
index d20e08502b5b..270eaa173d8c 100644
--- a/modules/nf-core/samtools/pipeline/main.nf
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -34,7 +34,6 @@ process SAMTOOLS_PIPELINE {
 
     // Output file
     def prefix = task.ext.prefix ?: "${meta.id}"
-    def reference = fasta ? "--reference ${fasta}" : ""
     def extension = last_args.contains("--output-fmt sam") ? "sam" :
                     last_args.contains("--output-fmt bam") ? "bam" :
                     last_args.contains("--output-fmt cram") ? "cram" :
@@ -52,12 +51,17 @@ process SAMTOOLS_PIPELINE {
             ${all_args[index]} \\
             -@ $task.cpus \\
         """
-        if (this_command != "fixmate") {
-            pipeline_command += "    -T ${prefix}.tmp.${this_command} \\\n"
-        }
-        pipeline_command += "    "
-        if (!is_last_command) {
-            pipeline_command += "-u "
+
+        if (is_last_command) {
+            // All commands support --reference, except reheader
+            if (! ["reheader"].contains(this_command)) {
+                pipeline_command += fasta ? " --reference ${fasta}" : ""
+            }
+        } else {
+            // All commands support uncompressed output, except reheader
+            if (! ["reheader"].contains(this_command)) {
+                pipeline_command += " -u"
+            }
         }
         this_input = (is_first_command ? " $input" : " -")
 

From d409456f0d6eb08eb4bec80425710a24b7ae4e2e Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Mon, 11 Dec 2023 19:19:42 +0000
Subject: [PATCH 09/36] Made .command.sh more legible

---
 modules/nf-core/samtools/pipeline/main.nf | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
index 270eaa173d8c..dad5c1e38c10 100644
--- a/modules/nf-core/samtools/pipeline/main.nf
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -45,11 +45,9 @@ process SAMTOOLS_PIPELINE {
         def this_command = commands[index]
         def is_first_command = (index == 0)
         def is_last_command = (index == (n_commands - 1))
+
         pipeline_command += """
-        samtools \\
-            ${this_command} \\
-            ${all_args[index]} \\
-            -@ $task.cpus \\
+        samtools ${this_command} ${all_args[index]} -@ $task.cpus \\
         """
 
         if (is_last_command) {
@@ -90,7 +88,7 @@ process SAMTOOLS_PIPELINE {
         }
 
         if (!is_last_command) {
-            pipeline_command += " | \\"
+            pipeline_command += " |"
         }
     }
 

From f33f4418743e2cf2268e7b2c60d1534b6bbb24fb Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Mon, 11 Dec 2023 19:58:06 +0000
Subject: [PATCH 10/36] Documentation update

---
 modules/nf-core/samtools/pipeline/meta.yml | 24 ++++++++--------------
 1 file changed, 9 insertions(+), 15 deletions(-)

diff --git a/modules/nf-core/samtools/pipeline/meta.yml b/modules/nf-core/samtools/pipeline/meta.yml
index 1907477f7bc6..071ae8b33c43 100644
--- a/modules/nf-core/samtools/pipeline/meta.yml
+++ b/modules/nf-core/samtools/pipeline/meta.yml
@@ -32,32 +32,26 @@ input:
       type: file
       description: FASTA reference file
       pattern: "*.{fasta,fa}"
+  - commands:
+      type: list
+      description: |
+        List of the samtools command names to run (in order). Currently
+        supported: addreplacerg, collate, fixmate, markdup, reheader, sort,
+        view
 output:
   - meta:
       type: map
       description: |
         Groovy Map containing sample information
         e.g. [ id:'test', single_end:false ]
-  - bam:
-      type: file
-      description: optional output BAM file
-      pattern: "*.{bam}"
-  - cram:
-      type: file
-      description: optional output BAM file
-      pattern: "*.{cram}"
-  - sam:
+  - output:
       type: file
-      description: optional output BAM file
-      pattern: "*.{sam}"
+      description: Collated BAM/CRAM/SAM file
+      pattern: "*.{bam,cram,sam}"
   - versions:
       type: file
       description: File containing software versions
       pattern: "versions.yml"
-  - output:
-      type: file
-      description: Collated BAM/CRAM/SAM file
-      pattern: "*.{bam,cram,sam}"
 authors:
   - "@muffato"
 maintainers:

From 6d7722d64a4091c86894123b1cdd08e64763fa4a Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Mon, 11 Dec 2023 20:22:19 +0000
Subject: [PATCH 11/36] bugfix: reheader cannot use multiple CPUs

---
 modules/nf-core/samtools/pipeline/main.nf | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
index dad5c1e38c10..f7e85a302c05 100644
--- a/modules/nf-core/samtools/pipeline/main.nf
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -47,17 +47,15 @@ process SAMTOOLS_PIPELINE {
         def is_last_command = (index == (n_commands - 1))
 
         pipeline_command += """
-        samtools ${this_command} ${all_args[index]} -@ $task.cpus \\
+        samtools ${this_command} ${all_args[index]} \\
         """
 
-        if (is_last_command) {
-            // All commands support --reference, except reheader
-            if (! ["reheader"].contains(this_command)) {
+        // The reheader has no useful option
+        if (! ["reheader"].contains(this_command)) {
+            pipeline_command += " -@ $task.cpus"
+            if (is_last_command) {
                 pipeline_command += fasta ? " --reference ${fasta}" : ""
-            }
-        } else {
-            // All commands support uncompressed output, except reheader
-            if (! ["reheader"].contains(this_command)) {
+            } else {
                 pipeline_command += " -u"
             }
         }

From c102c938d1f0df6f1056fc4cd6ead460c80e8b27 Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Mon, 11 Dec 2023 20:28:07 +0000
Subject: [PATCH 12/36] Updated the tests and added one that uses all possible
 commands

---
 .../modules/nf-core/samtools/pipeline/main.nf | 17 +++++++++---
 .../nf-core/samtools/pipeline/nextflow.config | 15 ++++++++---
 .../nf-core/samtools/pipeline/test.yml        | 26 +++++++++++++------
 3 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/tests/modules/nf-core/samtools/pipeline/main.nf b/tests/modules/nf-core/samtools/pipeline/main.nf
index 96b62e195ae5..bf488472157a 100644
--- a/tests/modules/nf-core/samtools/pipeline/main.nf
+++ b/tests/modules/nf-core/samtools/pipeline/main.nf
@@ -5,9 +5,10 @@ nextflow.enable.dsl = 2
 include { SAMTOOLS_PIPELINE as SAMTOOLS_PIPELINE_SORMADUP     } from '../../../../../modules/nf-core/samtools/pipeline/main.nf'
 include { SAMTOOLS_PIPELINE as SAMTOOLS_PIPELINE_COLLFIXMSORT } from '../../../../../modules/nf-core/samtools/pipeline/main.nf'
 include { SAMTOOLS_PIPELINE as SAMTOOLS_PIPELINE_COLLFIXM     } from '../../../../../modules/nf-core/samtools/pipeline/main.nf'
+include { SAMTOOLS_PIPELINE as SAMTOOLS_PIPELINE_ALL          } from '../../../../../modules/nf-core/samtools/pipeline/main.nf'
 
 workflow test_samtools_pipeline_sormadup {
-    
+
     input = [
         [ id:'test', single_end:false ], // meta map
         file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true)
@@ -17,7 +18,7 @@ workflow test_samtools_pipeline_sormadup {
 }
 
 workflow test_samtools_pipeline_collate_fixmate_sort {
-    
+
     input = [
         [ id:'test', single_end:false ], // meta map
         file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true)
@@ -27,7 +28,7 @@ workflow test_samtools_pipeline_collate_fixmate_sort {
 }
 
 workflow test_samtools_pipeline_collate_fixmate {
-    
+
     input = [
         [ id:'test', single_end:false ], // meta map
         file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true)
@@ -35,3 +36,13 @@ workflow test_samtools_pipeline_collate_fixmate {
     commands = ['collate', 'fixmate']
     SAMTOOLS_PIPELINE_COLLFIXM ( input, [[],[]], commands )
 }
+
+workflow test_samtools_pipeline_all {
+
+    input = [
+        [ id:'test', single_end:false ], // meta map
+        file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true)
+    ]
+    commands = ['collate', 'addreplacerg', 'fixmate', 'reheader', 'sort', 'markdup', 'view']
+    SAMTOOLS_PIPELINE_ALL ( input, [[],[]], commands )
+}
diff --git a/tests/modules/nf-core/samtools/pipeline/nextflow.config b/tests/modules/nf-core/samtools/pipeline/nextflow.config
index f5d629a042ba..f40bbca62fe4 100644
--- a/tests/modules/nf-core/samtools/pipeline/nextflow.config
+++ b/tests/modules/nf-core/samtools/pipeline/nextflow.config
@@ -4,19 +4,28 @@ process {
 
     withName: SAMTOOLS_PIPELINE_SORMADUP {
         ext.args = [
-            fixmate: '-m'
+            fixmate: '-m',
         ]
     }
 
     withName: SAMTOOLS_PIPELINE_COLLFIXMSORT {
         ext.args = [
-            fixmate: '-m'
+            fixmate: '-m',
         ]
     }
 
     withName: SAMTOOLS_PIPELINE_COLLFIXM {
         ext.args = [
-            fixmate: '-m'
+            fixmate: '-m',
+        ]
+    }
+
+    withName: SAMTOOLS_PIPELINE_ALL {
+        ext.args = [
+            addreplacerg: '-r ID:FOO',
+            fixmate: '-m',
+            reheader: '-c "sed s/FOO/BAR/"',
+            view: '-F 0x08',
         ]
     }
 }
diff --git a/tests/modules/nf-core/samtools/pipeline/test.yml b/tests/modules/nf-core/samtools/pipeline/test.yml
index 06f3c836d97f..54dd3b80bc83 100644
--- a/tests/modules/nf-core/samtools/pipeline/test.yml
+++ b/tests/modules/nf-core/samtools/pipeline/test.yml
@@ -1,29 +1,39 @@
 - name: samtools pipeline test_samtools_pipeline_sormadup
-  command: nextflow run ./tests/modules/nf-core/samtools/pipeline -entry test_samtools_pipeline_sormadup -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/samtools/pipeline/nextflow.config
+  command: nextflow run ./tests/modules/nf-core/samtools/pipeline -entry test_samtools_pipeline_sormadup -c ./tests/config/nextflow.config
   tags:
-    - samtools/pipeline
     - samtools
+    - samtools/pipeline
   files:
     - path: output/samtools/test.bam
-      md5sum: d505badd610ef5a32d28db3ec8e6aea1
+      md5sum: 6ffaaac1f4a02cc67eb68027c9bcd997
     - path: output/samtools/versions.yml
 
 - name: samtools pipeline test_samtools_pipeline_collate_fixmate_sort
-  command: nextflow run ./tests/modules/nf-core/samtools/pipeline -entry test_samtools_pipeline_collate_fixmate_sort -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/samtools/pipeline/nextflow.config
+  command: nextflow run ./tests/modules/nf-core/samtools/pipeline -entry test_samtools_pipeline_collate_fixmate_sort -c ./tests/config/nextflow.config
   tags:
-    - samtools/pipeline
     - samtools
+    - samtools/pipeline
   files:
     - path: output/samtools/test.bam
-      md5sum: 543221828943029864db75bd617b79d1
+      md5sum: e1c8f4b07b535c037694b4ed5dbbacb2
     - path: output/samtools/versions.yml
 
 - name: samtools pipeline test_samtools_pipeline_collate_fixmate
-  command: nextflow run ./tests/modules/nf-core/samtools/pipeline -entry test_samtools_pipeline_collate_fixmate -c ./tests/config/nextflow.config -c ./tests/modules/nf-core/samtools/pipeline/nextflow.config
+  command: nextflow run ./tests/modules/nf-core/samtools/pipeline -entry test_samtools_pipeline_collate_fixmate -c ./tests/config/nextflow.config
   tags:
+    - samtools
     - samtools/pipeline
+  files:
+    - path: output/samtools/test.bam
+      md5sum: af385ec8b3038bb9ae54b5ea1e87d58f
+    - path: output/samtools/versions.yml
+
+- name: samtools pipeline test_samtools_pipeline_all
+  command: nextflow run ./tests/modules/nf-core/samtools/pipeline -entry test_samtools_pipeline_all -c ./tests/config/nextflow.config
+  tags:
     - samtools
+    - samtools/pipeline
   files:
     - path: output/samtools/test.bam
-      md5sum: 37f2c0f510aff13bc73c7ddd6989c1f7
+      md5sum: ca506ea8953682f8409667aa549a9fe3
     - path: output/samtools/versions.yml

From 915e5af0a93aaef5501c0585c3ce7cf25251db3b Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Tue, 12 Dec 2023 22:51:57 +0000
Subject: [PATCH 13/36] Can't use TMPDIR as it's set to a non-writable location
 in Singularity

---
 tests/modules/nf-core/samtools/pipeline/nextflow.config | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tests/modules/nf-core/samtools/pipeline/nextflow.config b/tests/modules/nf-core/samtools/pipeline/nextflow.config
index f40bbca62fe4..e788279880af 100644
--- a/tests/modules/nf-core/samtools/pipeline/nextflow.config
+++ b/tests/modules/nf-core/samtools/pipeline/nextflow.config
@@ -2,26 +2,35 @@ process {
 
     publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }
 
+    // NOTE: collate needs to creata temporary files. CI sets TMPDIR to ~,
+    // which is not mounted any more in Singularity since Nextflow 23.10. We
+    // add "-T ." so that the temporary files are created in a writable
+    // location.
+
     withName: SAMTOOLS_PIPELINE_SORMADUP {
         ext.args = [
+            collate: '-T .',
             fixmate: '-m',
         ]
     }
 
     withName: SAMTOOLS_PIPELINE_COLLFIXMSORT {
         ext.args = [
+            collate: '-T .',
             fixmate: '-m',
         ]
     }
 
     withName: SAMTOOLS_PIPELINE_COLLFIXM {
         ext.args = [
+            collate: '-T .',
             fixmate: '-m',
         ]
     }
 
     withName: SAMTOOLS_PIPELINE_ALL {
         ext.args = [
+            collate: '-T .',
             addreplacerg: '-r ID:FOO',
             fixmate: '-m',
             reheader: '-c "sed s/FOO/BAR/"',

From 4f10688fc2557840c4c3620339b6c427aa79366e Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Tue, 12 Dec 2023 23:00:17 +0000
Subject: [PATCH 14/36] Need to update the checksum to include the new -T
 option that is in the @PG header

---
 tests/modules/nf-core/samtools/pipeline/test.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/modules/nf-core/samtools/pipeline/test.yml b/tests/modules/nf-core/samtools/pipeline/test.yml
index 54dd3b80bc83..784949bed67d 100644
--- a/tests/modules/nf-core/samtools/pipeline/test.yml
+++ b/tests/modules/nf-core/samtools/pipeline/test.yml
@@ -5,7 +5,7 @@
     - samtools/pipeline
   files:
     - path: output/samtools/test.bam
-      md5sum: 6ffaaac1f4a02cc67eb68027c9bcd997
+      md5sum: de5d6a0d54d580b71c946c13b58c5d66
     - path: output/samtools/versions.yml
 
 - name: samtools pipeline test_samtools_pipeline_collate_fixmate_sort
@@ -15,7 +15,7 @@
     - samtools/pipeline
   files:
     - path: output/samtools/test.bam
-      md5sum: e1c8f4b07b535c037694b4ed5dbbacb2
+      md5sum: aab086c3bfc7e32cef9e8e62c6f1f774
     - path: output/samtools/versions.yml
 
 - name: samtools pipeline test_samtools_pipeline_collate_fixmate
@@ -25,7 +25,7 @@
     - samtools/pipeline
   files:
     - path: output/samtools/test.bam
-      md5sum: af385ec8b3038bb9ae54b5ea1e87d58f
+      md5sum: e574ac78aef3617888697733a01de8e3
     - path: output/samtools/versions.yml
 
 - name: samtools pipeline test_samtools_pipeline_all
@@ -35,5 +35,5 @@
     - samtools/pipeline
   files:
     - path: output/samtools/test.bam
-      md5sum: ca506ea8953682f8409667aa549a9fe3
+      md5sum: 9815772ce3b23468858f1e76a6123dca
     - path: output/samtools/versions.yml

From a8859c707a64371013f80d714f9ab5fb141f9d22 Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Tue, 12 Dec 2023 23:16:30 +0000
Subject: [PATCH 15/36] Documentation update

---
 modules/nf-core/samtools/pipeline/main.nf  | 2 +-
 modules/nf-core/samtools/pipeline/meta.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
index f7e85a302c05..0effcb70f5b7 100644
--- a/modules/nf-core/samtools/pipeline/main.nf
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -50,7 +50,7 @@ process SAMTOOLS_PIPELINE {
         samtools ${this_command} ${all_args[index]} \\
         """
 
-        // The reheader has no useful option
+        // The reheader has none of these options
         if (! ["reheader"].contains(this_command)) {
             pipeline_command += " -@ $task.cpus"
             if (is_last_command) {
diff --git a/modules/nf-core/samtools/pipeline/meta.yml b/modules/nf-core/samtools/pipeline/meta.yml
index 071ae8b33c43..230d673f6839 100644
--- a/modules/nf-core/samtools/pipeline/meta.yml
+++ b/modules/nf-core/samtools/pipeline/meta.yml
@@ -1,5 +1,5 @@
 name: "samtools_pipeline"
-description: custom bash pipeline made of samtools commands
+description: custom bash pipeline made only of samtools commands
 keywords:
   - pipeline
   - bam

From 09d86940f246fd703db8e86654702b5b66c07b02 Mon Sep 17 00:00:00 2001
From: Mahesh Binzer-Panchal <mahesh.binzer-panchal@nbis.se>
Date: Wed, 13 Dec 2023 22:58:07 +0000
Subject: [PATCH 16/36] Use collect() + switch + join()

---
 modules/nf-core/samtools/pipeline/main.nf | 101 +++++++++-------------
 1 file changed, 40 insertions(+), 61 deletions(-)

diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
index 0effcb70f5b7..b661ed817dfe 100644
--- a/modules/nf-core/samtools/pipeline/main.nf
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -22,76 +22,55 @@ process SAMTOOLS_PIPELINE {
     script:
 
     // Check that we are asked to run more than 1 command
-    def n_commands = commands.size()
-    if (n_commands < 2) error "SAMTOOLS_PIPELINE is used to chain 2 or more samtools commands"
-
-    // Fetch the arguments
-    def all_args = []
-    for (int index = 0; index < n_commands; index++) {
-        all_args.add( task.ext.args?[ commands[index] ] ?: '' )
-    }
-    def last_args = all_args[-1]
-
-    // Output file
+    assert commands.size() > 1
     def prefix = task.ext.prefix ?: "${meta.id}"
+    def cmd_size  = commands.size()
+    def last_args = task.ext."args$cmd_size" ?: ''
     def extension = last_args.contains("--output-fmt sam") ? "sam" :
                     last_args.contains("--output-fmt bam") ? "bam" :
                     last_args.contains("--output-fmt cram") ? "cram" :
                     "bam"
-    if ("$input" == "${prefix}.${extension}") error "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!"
-
-    def pipeline_command = ""
-    for (int index = 0; index < n_commands; index++) {
-        def this_command = commands[index]
-        def is_first_command = (index == 0)
-        def is_last_command = (index == (n_commands - 1))
-
-        pipeline_command += """
-        samtools ${this_command} ${all_args[index]} \\
-        """
-
-        // The reheader has none of these options
-        if (! ["reheader"].contains(this_command)) {
-            pipeline_command += " -@ $task.cpus"
-            if (is_last_command) {
-                pipeline_command += fasta ? " --reference ${fasta}" : ""
-            } else {
-                pipeline_command += " -u"
-            }
-        }
-        this_input = (is_first_command ? " $input" : " -")
-
-        // samtools commands have slightly different syntax
-        if (["collate"].contains(this_command)) {
-            // [-o OUTPUT|-O] [INPUT|-]
-            pipeline_command += is_last_command ? " -o ${prefix}.${extension}" : " -O"
-            pipeline_command += this_input
-
-        } else if (["addreplacerg", "sort", "view"].contains(this_command)) {
-            // [-o OUTPUT] [INPUT|-]
-            pipeline_command += is_last_command ? " -o ${prefix}.${extension}" : ""
-            pipeline_command += this_input
-
-        } else if (["reheader"].contains(this_command)) {
-            // [INPUT|-]
-            pipeline_command += this_input
-
-        } else if (["fixmate", "markdup"].contains(this_command)) {
-            // [INPUT|-] [OUTPUT|-]
-            pipeline_command += this_input
-            pipeline_command += is_last_command ? " ${prefix}.${extension}" : " -"
-
-        } else {
-            error "${this_command} is not supported"
-        }
-
-        if (!is_last_command) {
-            pipeline_command += " |"
+    assert "$input" != "${prefix}.${extension}" : "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!"
+    // Compose pipe
+    def cmds = commands.indexed().collect { index, cmd ->
+        def command = [
+            "samtools $cmd",
+            task.ext."args${index!=0 ? index+1 : ''}" ?: ''
+        ]
+        switch(cmd){
+            case !"reheader":
+                // The reheader has no useful option
+                command << "-@ $task.cpus"
+                command << fasta && index == cmd_size ? "--reference ${fasta}" : ''
+                command << index != cmd_size ? '-u' : ''
+            // samtools commands have slightly different syntax
+            case "collate":
+                // [-o OUTPUT|-O] [INPUT|-]
+                command << index == cmd_size ? " -o ${prefix}.${extension}" : " -O"
+                command << index == 0 ? index : '-'
+                break
+            case ["addreplacerg", "sort", "view"]:
+                // [-o OUTPUT] [INPUT|-]
+                command << index == cmd_size ? "-o ${prefix}.${extension}" : ""
+                command << index == 0 ? index : '-'
+                break
+            case "reheader":
+                // [INPUT|-]
+                command << index == 0 ? index : '-'
+                break
+            case ["fixmate", "markdup"]:
+                // [INPUT|-] [OUTPUT|-]
+                command << index == 0 ? index : '-'
+                command << index == cmd_size ? "${prefix}.${extension}" : "-"
+                break
+            default:
+                assert false: "$cmd is not supported"
         }
+        command.join(' ')
     }
 
     """
-    $pipeline_command
+    ${cmds.join(' | ')}
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":

From e6460556dfd44ead5759b4c6ea46b0ab2a472e97 Mon Sep 17 00:00:00 2001
From: Mahesh Binzer-Panchal <mahesh.binzer-panchal@nbis.se>
Date: Fri, 15 Dec 2023 22:38:50 +0000
Subject: [PATCH 17/36] off-by-one bugfix

---
 modules/nf-core/samtools/pipeline/main.nf | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
index b661ed817dfe..a5e435c7a38f 100644
--- a/modules/nf-core/samtools/pipeline/main.nf
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -33,6 +33,8 @@ process SAMTOOLS_PIPELINE {
     assert "$input" != "${prefix}.${extension}" : "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!"
     // Compose pipe
     def cmds = commands.indexed().collect { index, cmd ->
+        def first = index == 0
+        def last = index == cmd_size-1
         def command = [
             "samtools $cmd",
             task.ext."args${index!=0 ? index+1 : ''}" ?: ''
@@ -41,27 +43,27 @@ process SAMTOOLS_PIPELINE {
             case !"reheader":
                 // The reheader has no useful option
                 command << "-@ $task.cpus"
-                command << fasta && index == cmd_size ? "--reference ${fasta}" : ''
-                command << index != cmd_size ? '-u' : ''
+                command << fasta && last ? "--reference ${fasta}" : ''
+                command << !last ? '-u' : ''
             // samtools commands have slightly different syntax
             case "collate":
                 // [-o OUTPUT|-O] [INPUT|-]
-                command << index == cmd_size ? " -o ${prefix}.${extension}" : " -O"
-                command << index == 0 ? index : '-'
+                command << last ? "-o ${prefix}.${extension}" : "-O"
+                command << first ? index : '-'
                 break
             case ["addreplacerg", "sort", "view"]:
                 // [-o OUTPUT] [INPUT|-]
-                command << index == cmd_size ? "-o ${prefix}.${extension}" : ""
-                command << index == 0 ? index : '-'
+                command << last ? "-o ${prefix}.${extension}" : ""
+                command << first ? index : '-'
                 break
             case "reheader":
                 // [INPUT|-]
-                command << index == 0 ? index : '-'
+                command << first ? index : '-'
                 break
             case ["fixmate", "markdup"]:
                 // [INPUT|-] [OUTPUT|-]
-                command << index == 0 ? index : '-'
-                command << index == cmd_size ? "${prefix}.${extension}" : "-"
+                command << first ? index : '-'
+                command << last ? "${prefix}.${extension}" : "-"
                 break
             default:
                 assert false: "$cmd is not supported"

From 1dd6933c0ae76c4b493371bdd22f0e6042ca8575 Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Fri, 15 Dec 2023 22:39:32 +0000
Subject: [PATCH 18/36] These need to be the input file

---
 modules/nf-core/samtools/pipeline/main.nf | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
index a5e435c7a38f..6037e2424927 100644
--- a/modules/nf-core/samtools/pipeline/main.nf
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -49,20 +49,20 @@ process SAMTOOLS_PIPELINE {
             case "collate":
                 // [-o OUTPUT|-O] [INPUT|-]
                 command << last ? "-o ${prefix}.${extension}" : "-O"
-                command << first ? index : '-'
+                command << first ? input : '-'
                 break
             case ["addreplacerg", "sort", "view"]:
                 // [-o OUTPUT] [INPUT|-]
                 command << last ? "-o ${prefix}.${extension}" : ""
-                command << first ? index : '-'
+                command << first ? input : '-'
                 break
             case "reheader":
                 // [INPUT|-]
-                command << first ? index : '-'
+                command << first ? input : '-'
                 break
             case ["fixmate", "markdup"]:
                 // [INPUT|-] [OUTPUT|-]
-                command << first ? index : '-'
+                command << first ? input : '-'
                 command << last ? "${prefix}.${extension}" : "-"
                 break
             default:

From 64fc726f99c3f3c19f11ae3fcd4360e340f27473 Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Fri, 15 Dec 2023 22:40:11 +0000
Subject: [PATCH 19/36] Use the boolean here too

---
 modules/nf-core/samtools/pipeline/main.nf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
index 6037e2424927..01891fddf303 100644
--- a/modules/nf-core/samtools/pipeline/main.nf
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -37,7 +37,7 @@ process SAMTOOLS_PIPELINE {
         def last = index == cmd_size-1
         def command = [
             "samtools $cmd",
-            task.ext."args${index!=0 ? index+1 : ''}" ?: ''
+            task.ext."args${first ? '' : index+1}" ?: ''
         ]
         switch(cmd){
             case !"reheader":

From c925779b11a68437d0186ab40c10010c7bdc1386 Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Fri, 15 Dec 2023 22:40:40 +0000
Subject: [PATCH 20/36] Updated the test suite

---
 .../nf-core/samtools/pipeline/nextflow.config | 30 +++++++------------
 1 file changed, 11 insertions(+), 19 deletions(-)

diff --git a/tests/modules/nf-core/samtools/pipeline/nextflow.config b/tests/modules/nf-core/samtools/pipeline/nextflow.config
index e788279880af..b88da04c6024 100644
--- a/tests/modules/nf-core/samtools/pipeline/nextflow.config
+++ b/tests/modules/nf-core/samtools/pipeline/nextflow.config
@@ -8,33 +8,25 @@ process {
     // location.
 
     withName: SAMTOOLS_PIPELINE_SORMADUP {
-        ext.args = [
-            collate: '-T .',
-            fixmate: '-m',
-        ]
+        ext.args  = '-T .'     // collate
+        ext.args2 = '-m'       // fixmate
     }
 
     withName: SAMTOOLS_PIPELINE_COLLFIXMSORT {
-        ext.args = [
-            collate: '-T .',
-            fixmate: '-m',
-        ]
+        ext.args  = '-T .'     // collate
+        ext.args2 = '-m'       // fixmate
     }
 
     withName: SAMTOOLS_PIPELINE_COLLFIXM {
-        ext.args = [
-            collate: '-T .',
-            fixmate: '-m',
-        ]
+        ext.args  = '-T .'     // collate
+        ext.args2 = '-m'       // fixmate
     }
 
     withName: SAMTOOLS_PIPELINE_ALL {
-        ext.args = [
-            collate: '-T .',
-            addreplacerg: '-r ID:FOO',
-            fixmate: '-m',
-            reheader: '-c "sed s/FOO/BAR/"',
-            view: '-F 0x08',
-        ]
+        ext.args  = '-T .'                 // collate
+        ext.args2 = '-r ID:FOO'            // addreplacerg
+        ext.args3 = '-m'                   // fixmate
+        ext.args4 = '-c "sed s/FOO/BAR/"'  // reheader
+        ext.args7 = '-F 0x08'              // view
     }
 }

From 620ed0ee4336f2022a64bf8a99a31294e5ed3548 Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Fri, 15 Dec 2023 22:43:55 +0000
Subject: [PATCH 21/36] Documentation update

---
 modules/nf-core/samtools/pipeline/main.nf | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
index 01891fddf303..67390137fe1d 100644
--- a/modules/nf-core/samtools/pipeline/main.nf
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -40,12 +40,15 @@ process SAMTOOLS_PIPELINE {
             task.ext."args${first ? '' : index+1}" ?: ''
         ]
         switch(cmd){
+            //// First the common options
             case !"reheader":
-                // The reheader has no useful option
+                // "reheader" is the only command not to offer these
                 command << "-@ $task.cpus"
                 command << fasta && last ? "--reference ${fasta}" : ''
                 command << !last ? '-u' : ''
-            // samtools commands have slightly different syntax
+                // NOTE: no "break" here because we want to run the next batch of "case"
+
+            //// Then the input/ouput parameters, which differ between commands
             case "collate":
                 // [-o OUTPUT|-O] [INPUT|-]
                 command << last ? "-o ${prefix}.${extension}" : "-O"

From 9f82a3a84efde060acb16f06e76c2e7461b8ba3f Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Fri, 15 Dec 2023 22:58:16 +0000
Subject: [PATCH 22/36] bugfix: Groovy was complaining that "commands" is
 defined multiple times

```
- file : /lustre/scratch123/tol/teams/tolit/users/mm49/nextflow/src/nf-core/modules/tests/modules/nf-core/samtools/pipeline/../../../../../modules/nf-core/samtools/pipeline/main.nf
- cause: Variable `commands` already defined in the process scope @ line 28, column 21.
       def cmd_size  = commands.size()
                       ^

Module compilation error
- file : /lustre/scratch123/tol/teams/tolit/users/mm49/nextflow/src/nf-core/modules/tests/modules/nf-core/samtools/pipeline/../../../../../modules/nf-core/samtools/pipeline/main.nf
- cause: Variable `commands` already defined in the process scope @ line 42, column 28.
           def this_command = commands[index]
                              ^

Module compilation error
- file : /lustre/scratch123/tol/teams/tolit/users/mm49/nextflow/src/nf-core/modules/tests/modules/nf-core/samtools/pipeline/../../../../../modules/nf-core/samtools/pipeline/main.nf
- cause: Variable `commands` already defined in the process scope @ line 44, column 42.
    is_last_command = (index == (commands.s
                                 ^
```
---
 modules/nf-core/samtools/pipeline/main.nf | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
index 67390137fe1d..cc5ca9b1229b 100644
--- a/modules/nf-core/samtools/pipeline/main.nf
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -20,17 +20,19 @@ process SAMTOOLS_PIPELINE {
     task.ext.when == null || task.ext.when
 
     script:
+    def prefix   = task.ext.prefix ?: "${meta.id}"
 
     // Check that we are asked to run more than 1 command
-    assert commands.size() > 1
-    def prefix = task.ext.prefix ?: "${meta.id}"
-    def cmd_size  = commands.size()
+    def cmd_size = commands.size()
+    assert cmd_size > 1
+
     def last_args = task.ext."args$cmd_size" ?: ''
     def extension = last_args.contains("--output-fmt sam") ? "sam" :
                     last_args.contains("--output-fmt bam") ? "bam" :
                     last_args.contains("--output-fmt cram") ? "cram" :
                     "bam"
     assert "$input" != "${prefix}.${extension}" : "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!"
+
     // Compose pipe
     def cmds = commands.indexed().collect { index, cmd ->
         def first = index == 0

From d9d33519be0932f823d85c3515571c29ae87332d Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Fri, 15 Dec 2023 23:25:18 +0000
Subject: [PATCH 23/36] bugfix: operator precedence means we need brackets

---
 modules/nf-core/samtools/pipeline/main.nf | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
index cc5ca9b1229b..8055697bc0cf 100644
--- a/modules/nf-core/samtools/pipeline/main.nf
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -46,29 +46,29 @@ process SAMTOOLS_PIPELINE {
             case !"reheader":
                 // "reheader" is the only command not to offer these
                 command << "-@ $task.cpus"
-                command << fasta && last ? "--reference ${fasta}" : ''
-                command << !last ? '-u' : ''
+                command << (fasta && last ? "--reference ${fasta}" : '')
+                command << (!last ? '-u' : '')
                 // NOTE: no "break" here because we want to run the next batch of "case"
 
             //// Then the input/ouput parameters, which differ between commands
             case "collate":
                 // [-o OUTPUT|-O] [INPUT|-]
-                command << last ? "-o ${prefix}.${extension}" : "-O"
-                command << first ? input : '-'
+                command << (last ? "-o ${prefix}.${extension}" : "-O")
+                command << (first ? input : '-')
                 break
             case ["addreplacerg", "sort", "view"]:
                 // [-o OUTPUT] [INPUT|-]
-                command << last ? "-o ${prefix}.${extension}" : ""
-                command << first ? input : '-'
+                command << (last ? "-o ${prefix}.${extension}" : "")
+                command << (first ? input : '-')
                 break
             case "reheader":
                 // [INPUT|-]
-                command << first ? input : '-'
+                command << (first ? input : '-')
                 break
             case ["fixmate", "markdup"]:
                 // [INPUT|-] [OUTPUT|-]
-                command << first ? input : '-'
-                command << last ? "${prefix}.${extension}" : "-"
+                command << (first ? input : '-')
+                command << (last ? "${prefix}.${extension}" : "-")
                 break
             default:
                 assert false: "$cmd is not supported"

From e4b3e9e16e4830364d3246edf6b84f7554ce112c Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Fri, 15 Dec 2023 23:27:28 +0000
Subject: [PATCH 24/36] The exclamation mark doesn't work

---
 modules/nf-core/samtools/pipeline/main.nf | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
index 8055697bc0cf..188a282499f4 100644
--- a/modules/nf-core/samtools/pipeline/main.nf
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -41,16 +41,15 @@ process SAMTOOLS_PIPELINE {
             "samtools $cmd",
             task.ext."args${first ? '' : index+1}" ?: ''
         ]
+        // First the common options
+        if (cmd != "reheader") {
+            // "reheader" is the only command not to offer these
+            command << "-@ $task.cpus"
+            command << (fasta && last ? "--reference ${fasta}" : '')
+            command << (!last ? '-u' : '')
+        }
+        // Then the input/ouput parameters, which differ between commands
         switch(cmd){
-            //// First the common options
-            case !"reheader":
-                // "reheader" is the only command not to offer these
-                command << "-@ $task.cpus"
-                command << (fasta && last ? "--reference ${fasta}" : '')
-                command << (!last ? '-u' : '')
-                // NOTE: no "break" here because we want to run the next batch of "case"
-
-            //// Then the input/ouput parameters, which differ between commands
             case "collate":
                 // [-o OUTPUT|-O] [INPUT|-]
                 command << (last ? "-o ${prefix}.${extension}" : "-O")

From 22f27cc6ad9db3c80d1a1b5f68d53aaa709b42ea Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Sat, 16 Dec 2023 00:01:08 +0000
Subject: [PATCH 25/36] Should default to the same format as the input

---
 modules/nf-core/samtools/pipeline/main.nf  | 2 +-
 modules/nf-core/samtools/pipeline/meta.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
index 188a282499f4..f7f46fd42083 100644
--- a/modules/nf-core/samtools/pipeline/main.nf
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -30,7 +30,7 @@ process SAMTOOLS_PIPELINE {
     def extension = last_args.contains("--output-fmt sam") ? "sam" :
                     last_args.contains("--output-fmt bam") ? "bam" :
                     last_args.contains("--output-fmt cram") ? "cram" :
-                    "bam"
+                    input.extension
     assert "$input" != "${prefix}.${extension}" : "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!"
 
     // Compose pipe
diff --git a/modules/nf-core/samtools/pipeline/meta.yml b/modules/nf-core/samtools/pipeline/meta.yml
index 230d673f6839..b391e331c3c2 100644
--- a/modules/nf-core/samtools/pipeline/meta.yml
+++ b/modules/nf-core/samtools/pipeline/meta.yml
@@ -46,7 +46,7 @@ output:
         e.g. [ id:'test', single_end:false ]
   - output:
       type: file
-      description: Collated BAM/CRAM/SAM file
+      description: Output BAM/CRAM/SAM file (defaults to the same extension as the input)
       pattern: "*.{bam,cram,sam}"
   - versions:
       type: file

From 11fea2c89f04081c3944c3d8defdea1370deb8ec Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Fri, 15 Dec 2023 23:59:21 +0000
Subject: [PATCH 26/36] Added a test for CRAM

---
 tests/modules/nf-core/samtools/pipeline/main.nf  | 11 +++++++++++
 tests/modules/nf-core/samtools/pipeline/test.yml |  9 +++++++++
 2 files changed, 20 insertions(+)

diff --git a/tests/modules/nf-core/samtools/pipeline/main.nf b/tests/modules/nf-core/samtools/pipeline/main.nf
index bf488472157a..83be13f8cb7e 100644
--- a/tests/modules/nf-core/samtools/pipeline/main.nf
+++ b/tests/modules/nf-core/samtools/pipeline/main.nf
@@ -46,3 +46,14 @@ workflow test_samtools_pipeline_all {
     commands = ['collate', 'addreplacerg', 'fixmate', 'reheader', 'sort', 'markdup', 'view']
     SAMTOOLS_PIPELINE_ALL ( input, [[],[]], commands )
 }
+
+workflow test_samtools_pipeline_collate_fixmate_cram {
+
+    input = [
+        [ id:'test', single_end:false ], // meta map
+        file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_cram'], checkIfExists: true)
+    ]
+    commands = ['collate', 'fixmate']
+    SAMTOOLS_PIPELINE_COLLFIXM ( input, [[],[]], commands )
+}
+
diff --git a/tests/modules/nf-core/samtools/pipeline/test.yml b/tests/modules/nf-core/samtools/pipeline/test.yml
index 784949bed67d..6baa77ad8237 100644
--- a/tests/modules/nf-core/samtools/pipeline/test.yml
+++ b/tests/modules/nf-core/samtools/pipeline/test.yml
@@ -37,3 +37,12 @@
     - path: output/samtools/test.bam
       md5sum: 9815772ce3b23468858f1e76a6123dca
     - path: output/samtools/versions.yml
+- name: samtools pipeline test_samtools_pipeline_collate_fixmate_cram
+  command: nextflow run ./tests/modules/nf-core/samtools/pipeline -entry test_samtools_pipeline_collate_fixmate_cram -c ./tests/config/nextflow.config
+  tags:
+    - samtools
+    - samtools/pipeline
+  files:
+    - path: output/samtools/test.cram
+      md5sum: b71fc2e9b7c04e32b13e6071f955434d
+    - path: output/samtools/versions.yml

From e977015b9a5b88dfb1051adc8c1c98684366fe77 Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Sat, 16 Dec 2023 00:20:51 +0000
Subject: [PATCH 27/36] Added a test for the conversion from BAM to CRAM

---
 tests/modules/nf-core/samtools/pipeline/main.nf    | 14 ++++++++++++++
 .../nf-core/samtools/pipeline/nextflow.config      |  5 +++++
 tests/modules/nf-core/samtools/pipeline/test.yml   |  9 +++++++++
 3 files changed, 28 insertions(+)

diff --git a/tests/modules/nf-core/samtools/pipeline/main.nf b/tests/modules/nf-core/samtools/pipeline/main.nf
index 83be13f8cb7e..d8f5804bc764 100644
--- a/tests/modules/nf-core/samtools/pipeline/main.nf
+++ b/tests/modules/nf-core/samtools/pipeline/main.nf
@@ -5,6 +5,7 @@ nextflow.enable.dsl = 2
 include { SAMTOOLS_PIPELINE as SAMTOOLS_PIPELINE_SORMADUP     } from '../../../../../modules/nf-core/samtools/pipeline/main.nf'
 include { SAMTOOLS_PIPELINE as SAMTOOLS_PIPELINE_COLLFIXMSORT } from '../../../../../modules/nf-core/samtools/pipeline/main.nf'
 include { SAMTOOLS_PIPELINE as SAMTOOLS_PIPELINE_COLLFIXM     } from '../../../../../modules/nf-core/samtools/pipeline/main.nf'
+include { SAMTOOLS_PIPELINE as SAMTOOLS_PIPELINE_COLLFIXM_B2C } from '../../../../../modules/nf-core/samtools/pipeline/main.nf'
 include { SAMTOOLS_PIPELINE as SAMTOOLS_PIPELINE_ALL          } from '../../../../../modules/nf-core/samtools/pipeline/main.nf'
 
 workflow test_samtools_pipeline_sormadup {
@@ -57,3 +58,16 @@ workflow test_samtools_pipeline_collate_fixmate_cram {
     SAMTOOLS_PIPELINE_COLLFIXM ( input, [[],[]], commands )
 }
 
+workflow test_samtools_pipeline_collate_fixmate_bam2cram {
+
+    input = [
+        [ id:'test', single_end:false ], // meta map
+        file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true)
+    ]
+    fasta = [
+        [ id:'test' ], // meta map
+        file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)
+    ]
+    commands = ['collate', 'fixmate']
+    SAMTOOLS_PIPELINE_COLLFIXM_B2C ( input, fasta, commands )
+}
diff --git a/tests/modules/nf-core/samtools/pipeline/nextflow.config b/tests/modules/nf-core/samtools/pipeline/nextflow.config
index b88da04c6024..723fff6b228a 100644
--- a/tests/modules/nf-core/samtools/pipeline/nextflow.config
+++ b/tests/modules/nf-core/samtools/pipeline/nextflow.config
@@ -22,6 +22,11 @@ process {
         ext.args2 = '-m'       // fixmate
     }
 
+    withName: SAMTOOLS_PIPELINE_COLLFIXM_B2C {
+        ext.args  = '-T .'                 // collate
+        ext.args2 = '-m --output-fmt cram' // fixmate
+    }
+
     withName: SAMTOOLS_PIPELINE_ALL {
         ext.args  = '-T .'                 // collate
         ext.args2 = '-r ID:FOO'            // addreplacerg
diff --git a/tests/modules/nf-core/samtools/pipeline/test.yml b/tests/modules/nf-core/samtools/pipeline/test.yml
index 6baa77ad8237..90c5eb693ea1 100644
--- a/tests/modules/nf-core/samtools/pipeline/test.yml
+++ b/tests/modules/nf-core/samtools/pipeline/test.yml
@@ -46,3 +46,12 @@
     - path: output/samtools/test.cram
       md5sum: b71fc2e9b7c04e32b13e6071f955434d
     - path: output/samtools/versions.yml
+- name: samtools pipeline test_samtools_pipeline_collate_fixmate_bam2cram
+  command: nextflow run ./tests/modules/nf-core/samtools/pipeline -entry test_samtools_pipeline_collate_fixmate_bam2cram -c ./tests/config/nextflow.config
+  tags:
+    - samtools
+    - samtools/pipeline
+  files:
+    - path: output/samtools/test.cram
+      md5sum: b71fc2e9b7c04e32b13e6071f955434d
+    - path: output/samtools/versions.yml

From a3cb48356401031990501efaca2111cdfd7e8c0f Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Sat, 16 Dec 2023 00:40:35 +0000
Subject: [PATCH 28/36] Added support for output the BAM/CRAM index too

---
 modules/nf-core/samtools/pipeline/main.nf               | 1 +
 modules/nf-core/samtools/pipeline/meta.yml              | 4 ++++
 tests/modules/nf-core/samtools/pipeline/nextflow.config | 5 +++--
 tests/modules/nf-core/samtools/pipeline/test.yml        | 2 ++
 4 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
index f7f46fd42083..9612833b0f7c 100644
--- a/modules/nf-core/samtools/pipeline/main.nf
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -14,6 +14,7 @@ process SAMTOOLS_PIPELINE {
 
     output:
     tuple val(meta), path("*.{bam,cram,sam}"), emit: output
+    tuple val(meta), path("*.{bai,csi,crai}"), emit: index, optional: true
     path "versions.yml",                       emit: versions
 
     when:
diff --git a/modules/nf-core/samtools/pipeline/meta.yml b/modules/nf-core/samtools/pipeline/meta.yml
index b391e331c3c2..66f39658dfed 100644
--- a/modules/nf-core/samtools/pipeline/meta.yml
+++ b/modules/nf-core/samtools/pipeline/meta.yml
@@ -48,6 +48,10 @@ output:
       type: file
       description: Output BAM/CRAM/SAM file (defaults to the same extension as the input)
       pattern: "*.{bam,cram,sam}"
+  - index:
+      type: file
+      description: BAM.BAI/BAM.CSI/CRAM.CRAI file (optional)
+      pattern: "*.{.bai,.csi,.crai}"
   - versions:
       type: file
       description: File containing software versions
diff --git a/tests/modules/nf-core/samtools/pipeline/nextflow.config b/tests/modules/nf-core/samtools/pipeline/nextflow.config
index 723fff6b228a..c764903bbb29 100644
--- a/tests/modules/nf-core/samtools/pipeline/nextflow.config
+++ b/tests/modules/nf-core/samtools/pipeline/nextflow.config
@@ -13,8 +13,9 @@ process {
     }
 
     withName: SAMTOOLS_PIPELINE_COLLFIXMSORT {
-        ext.args  = '-T .'     // collate
-        ext.args2 = '-m'       // fixmate
+        ext.args  = '-T .'          // collate
+        ext.args2 = '-m'            // fixmate
+        ext.args3 = '--write-index' // sort
     }
 
     withName: SAMTOOLS_PIPELINE_COLLFIXM {
diff --git a/tests/modules/nf-core/samtools/pipeline/test.yml b/tests/modules/nf-core/samtools/pipeline/test.yml
index 90c5eb693ea1..0580858a1ea7 100644
--- a/tests/modules/nf-core/samtools/pipeline/test.yml
+++ b/tests/modules/nf-core/samtools/pipeline/test.yml
@@ -16,6 +16,8 @@
   files:
     - path: output/samtools/test.bam
       md5sum: aab086c3bfc7e32cef9e8e62c6f1f774
+    - path: output/samtools/test.bami.csi
+      md5sum: b9378d1e55b1dfecc5610a61cfd80724
     - path: output/samtools/versions.yml
 
 - name: samtools pipeline test_samtools_pipeline_collate_fixmate

From 33ac78e7cad69d1a50fb78c69eb1fe050931dada Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Sat, 16 Dec 2023 00:56:25 +0000
Subject: [PATCH 29/36] Added an option to support the input faidx file

---
 modules/nf-core/samtools/pipeline/main.nf       |  2 +-
 modules/nf-core/samtools/pipeline/meta.yml      |  4 ++++
 tests/modules/nf-core/samtools/pipeline/main.nf | 13 +++++++------
 3 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
index 9612833b0f7c..be6cc4099c98 100644
--- a/modules/nf-core/samtools/pipeline/main.nf
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -9,7 +9,7 @@ process SAMTOOLS_PIPELINE {
 
     input:
     tuple val(meta), path(input)
-    tuple val(meta2), path(fasta)
+    tuple val(meta2), path(fasta), path(fai)
     val commands
 
     output:
diff --git a/modules/nf-core/samtools/pipeline/meta.yml b/modules/nf-core/samtools/pipeline/meta.yml
index 66f39658dfed..25e33ee7aeca 100644
--- a/modules/nf-core/samtools/pipeline/meta.yml
+++ b/modules/nf-core/samtools/pipeline/meta.yml
@@ -32,6 +32,10 @@ input:
       type: file
       description: FASTA reference file
       pattern: "*.{fasta,fa}"
+  - fai:
+      type: file
+      description: Index of the reference file for the CRAM
+      pattern: "*.fai"
   - commands:
       type: list
       description: |
diff --git a/tests/modules/nf-core/samtools/pipeline/main.nf b/tests/modules/nf-core/samtools/pipeline/main.nf
index d8f5804bc764..58b44294202e 100644
--- a/tests/modules/nf-core/samtools/pipeline/main.nf
+++ b/tests/modules/nf-core/samtools/pipeline/main.nf
@@ -15,7 +15,7 @@ workflow test_samtools_pipeline_sormadup {
         file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true)
     ]
     commands = ['collate', 'fixmate', 'sort', 'markdup']
-    SAMTOOLS_PIPELINE_SORMADUP ( input, [[],[]], commands )
+    SAMTOOLS_PIPELINE_SORMADUP ( input, [[],[],[]], commands )
 }
 
 workflow test_samtools_pipeline_collate_fixmate_sort {
@@ -25,7 +25,7 @@ workflow test_samtools_pipeline_collate_fixmate_sort {
         file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true)
     ]
     commands = ['collate', 'fixmate', 'sort']
-    SAMTOOLS_PIPELINE_COLLFIXMSORT ( input, [[],[]], commands )
+    SAMTOOLS_PIPELINE_COLLFIXMSORT ( input, [[],[],[]], commands )
 }
 
 workflow test_samtools_pipeline_collate_fixmate {
@@ -35,7 +35,7 @@ workflow test_samtools_pipeline_collate_fixmate {
         file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true)
     ]
     commands = ['collate', 'fixmate']
-    SAMTOOLS_PIPELINE_COLLFIXM ( input, [[],[]], commands )
+    SAMTOOLS_PIPELINE_COLLFIXM ( input, [[],[],[]], commands )
 }
 
 workflow test_samtools_pipeline_all {
@@ -45,7 +45,7 @@ workflow test_samtools_pipeline_all {
         file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true)
     ]
     commands = ['collate', 'addreplacerg', 'fixmate', 'reheader', 'sort', 'markdup', 'view']
-    SAMTOOLS_PIPELINE_ALL ( input, [[],[]], commands )
+    SAMTOOLS_PIPELINE_ALL ( input, [[],[],[]], commands )
 }
 
 workflow test_samtools_pipeline_collate_fixmate_cram {
@@ -55,7 +55,7 @@ workflow test_samtools_pipeline_collate_fixmate_cram {
         file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_cram'], checkIfExists: true)
     ]
     commands = ['collate', 'fixmate']
-    SAMTOOLS_PIPELINE_COLLFIXM ( input, [[],[]], commands )
+    SAMTOOLS_PIPELINE_COLLFIXM ( input, [[],[],[]], commands )
 }
 
 workflow test_samtools_pipeline_collate_fixmate_bam2cram {
@@ -66,7 +66,8 @@ workflow test_samtools_pipeline_collate_fixmate_bam2cram {
     ]
     fasta = [
         [ id:'test' ], // meta map
-        file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)
+        file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true),
+        file(params.test_data['sarscov2']['genome']['genome_fasta_fai'], checkIfExists: true)
     ]
     commands = ['collate', 'fixmate']
     SAMTOOLS_PIPELINE_COLLFIXM_B2C ( input, fasta, commands )

From 5aa27cd8d40e73aff8d015cb59fdf690f33c1c58 Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Sat, 16 Dec 2023 01:17:51 +0000
Subject: [PATCH 30/36] typo

---
 tests/modules/nf-core/samtools/pipeline/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/modules/nf-core/samtools/pipeline/test.yml b/tests/modules/nf-core/samtools/pipeline/test.yml
index 0580858a1ea7..24113a3a695f 100644
--- a/tests/modules/nf-core/samtools/pipeline/test.yml
+++ b/tests/modules/nf-core/samtools/pipeline/test.yml
@@ -16,7 +16,7 @@
   files:
     - path: output/samtools/test.bam
       md5sum: aab086c3bfc7e32cef9e8e62c6f1f774
-    - path: output/samtools/test.bami.csi
+    - path: output/samtools/test.bam.csi
       md5sum: b9378d1e55b1dfecc5610a61cfd80724
     - path: output/samtools/versions.yml
 

From f6008fe15d2a9af864baa827dce01825699c4db1 Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Sat, 16 Dec 2023 01:21:08 +0000
Subject: [PATCH 31/36] Updated MD5

---
 tests/modules/nf-core/samtools/pipeline/test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/modules/nf-core/samtools/pipeline/test.yml b/tests/modules/nf-core/samtools/pipeline/test.yml
index 24113a3a695f..6459e739cb01 100644
--- a/tests/modules/nf-core/samtools/pipeline/test.yml
+++ b/tests/modules/nf-core/samtools/pipeline/test.yml
@@ -15,7 +15,7 @@
     - samtools/pipeline
   files:
     - path: output/samtools/test.bam
-      md5sum: aab086c3bfc7e32cef9e8e62c6f1f774
+      md5sum: 4ba760209400274dee8d67961e66a28b
     - path: output/samtools/test.bam.csi
       md5sum: b9378d1e55b1dfecc5610a61cfd80724
     - path: output/samtools/versions.yml
@@ -46,7 +46,7 @@
     - samtools/pipeline
   files:
     - path: output/samtools/test.cram
-      md5sum: b71fc2e9b7c04e32b13e6071f955434d
+      md5sum: 61ca3ec27be08646525d083cdc7b3655
     - path: output/samtools/versions.yml
 - name: samtools pipeline test_samtools_pipeline_collate_fixmate_bam2cram
   command: nextflow run ./tests/modules/nf-core/samtools/pipeline -entry test_samtools_pipeline_collate_fixmate_bam2cram -c ./tests/config/nextflow.config

From 4318da10e7a8447e0102bb65a59649caf0eb88fe Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Sat, 16 Dec 2023 09:07:18 +0000
Subject: [PATCH 32/36] Removed the test with the fasta file because it leads
 to CRAM files with different checksums

---
 tests/modules/nf-core/samtools/pipeline/main.nf | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/tests/modules/nf-core/samtools/pipeline/main.nf b/tests/modules/nf-core/samtools/pipeline/main.nf
index 58b44294202e..06e36b205bae 100644
--- a/tests/modules/nf-core/samtools/pipeline/main.nf
+++ b/tests/modules/nf-core/samtools/pipeline/main.nf
@@ -64,11 +64,6 @@ workflow test_samtools_pipeline_collate_fixmate_bam2cram {
         [ id:'test', single_end:false ], // meta map
         file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true)
     ]
-    fasta = [
-        [ id:'test' ], // meta map
-        file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true),
-        file(params.test_data['sarscov2']['genome']['genome_fasta_fai'], checkIfExists: true)
-    ]
     commands = ['collate', 'fixmate']
-    SAMTOOLS_PIPELINE_COLLFIXM_B2C ( input, fasta, commands )
+    SAMTOOLS_PIPELINE_COLLFIXM_B2C ( input, [[],[],[]], commands )
 }

From bdc3cfc113a1e92addb71aa76b4bf471ee409634 Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Sat, 16 Dec 2023 09:08:58 +0000
Subject: [PATCH 33/36] Updated MD5

---
 tests/modules/nf-core/samtools/pipeline/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/modules/nf-core/samtools/pipeline/test.yml b/tests/modules/nf-core/samtools/pipeline/test.yml
index 6459e739cb01..ab64fd014639 100644
--- a/tests/modules/nf-core/samtools/pipeline/test.yml
+++ b/tests/modules/nf-core/samtools/pipeline/test.yml
@@ -46,7 +46,7 @@
     - samtools/pipeline
   files:
     - path: output/samtools/test.cram
-      md5sum: 61ca3ec27be08646525d083cdc7b3655
+      md5sum: d091d4cd8ee1fbc3c0937296588962a3
     - path: output/samtools/versions.yml
 - name: samtools pipeline test_samtools_pipeline_collate_fixmate_bam2cram
   command: nextflow run ./tests/modules/nf-core/samtools/pipeline -entry test_samtools_pipeline_collate_fixmate_bam2cram -c ./tests/config/nextflow.config

From 8c4176574d50562ff076109eb8c3fc8ef66359d2 Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Sat, 16 Dec 2023 09:11:58 +0000
Subject: [PATCH 34/36] Fixed the stub

---
 modules/nf-core/samtools/pipeline/main.nf | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/modules/nf-core/samtools/pipeline/main.nf b/modules/nf-core/samtools/pipeline/main.nf
index be6cc4099c98..dff3509e5592 100644
--- a/modules/nf-core/samtools/pipeline/main.nf
+++ b/modules/nf-core/samtools/pipeline/main.nf
@@ -86,7 +86,15 @@ process SAMTOOLS_PIPELINE {
     """
 
     stub:
-    def prefix = task.ext.prefix ?: "${meta.id}"
+    def prefix    = task.ext.prefix ?: "${meta.id}"
+    def cmd_size  = commands.size()
+    def last_args = task.ext."args$cmd_size" ?: ''
+    def extension = last_args.contains("--output-fmt sam") ? "sam" :
+                    last_args.contains("--output-fmt bam") ? "bam" :
+                    last_args.contains("--output-fmt cram") ? "cram" :
+                    input.extension
+    assert "$input" != "${prefix}.${extension}" : "Input and output names are the same, use \"task.ext.prefix\" to disambiguate!"
+
     """
     touch ${prefix}.${extension}
 

From bcc94293e84a20c9fa9b40186745aa9e5049c488 Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Sat, 16 Dec 2023 09:16:00 +0000
Subject: [PATCH 35/36] Updated MD5

---
 tests/modules/nf-core/samtools/pipeline/test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/modules/nf-core/samtools/pipeline/test.yml b/tests/modules/nf-core/samtools/pipeline/test.yml
index ab64fd014639..d43fb9d57510 100644
--- a/tests/modules/nf-core/samtools/pipeline/test.yml
+++ b/tests/modules/nf-core/samtools/pipeline/test.yml
@@ -46,7 +46,7 @@
     - samtools/pipeline
   files:
     - path: output/samtools/test.cram
-      md5sum: d091d4cd8ee1fbc3c0937296588962a3
+      md5sum: b71fc2e9b7c04e32b13e6071f955434d
     - path: output/samtools/versions.yml
 - name: samtools pipeline test_samtools_pipeline_collate_fixmate_bam2cram
   command: nextflow run ./tests/modules/nf-core/samtools/pipeline -entry test_samtools_pipeline_collate_fixmate_bam2cram -c ./tests/config/nextflow.config
@@ -55,5 +55,5 @@
     - samtools/pipeline
   files:
     - path: output/samtools/test.cram
-      md5sum: b71fc2e9b7c04e32b13e6071f955434d
+      md5sum: d091d4cd8ee1fbc3c0937296588962a3
     - path: output/samtools/versions.yml

From 7f8d203bb1bcf56f7572c6bef2f5d1edf0a5e664 Mon Sep 17 00:00:00 2001
From: Matthieu Muffato <mm49@sanger.ac.uk>
Date: Sat, 16 Dec 2023 09:22:12 +0000
Subject: [PATCH 36/36] Removed the bam2cram test as it makes different files
 on Conda

---
 tests/modules/nf-core/samtools/pipeline/main.nf       | 11 -----------
 .../modules/nf-core/samtools/pipeline/nextflow.config |  5 -----
 tests/modules/nf-core/samtools/pipeline/test.yml      |  9 ---------
 3 files changed, 25 deletions(-)

diff --git a/tests/modules/nf-core/samtools/pipeline/main.nf b/tests/modules/nf-core/samtools/pipeline/main.nf
index 06e36b205bae..000774bfff75 100644
--- a/tests/modules/nf-core/samtools/pipeline/main.nf
+++ b/tests/modules/nf-core/samtools/pipeline/main.nf
@@ -5,7 +5,6 @@ nextflow.enable.dsl = 2
 include { SAMTOOLS_PIPELINE as SAMTOOLS_PIPELINE_SORMADUP     } from '../../../../../modules/nf-core/samtools/pipeline/main.nf'
 include { SAMTOOLS_PIPELINE as SAMTOOLS_PIPELINE_COLLFIXMSORT } from '../../../../../modules/nf-core/samtools/pipeline/main.nf'
 include { SAMTOOLS_PIPELINE as SAMTOOLS_PIPELINE_COLLFIXM     } from '../../../../../modules/nf-core/samtools/pipeline/main.nf'
-include { SAMTOOLS_PIPELINE as SAMTOOLS_PIPELINE_COLLFIXM_B2C } from '../../../../../modules/nf-core/samtools/pipeline/main.nf'
 include { SAMTOOLS_PIPELINE as SAMTOOLS_PIPELINE_ALL          } from '../../../../../modules/nf-core/samtools/pipeline/main.nf'
 
 workflow test_samtools_pipeline_sormadup {
@@ -57,13 +56,3 @@ workflow test_samtools_pipeline_collate_fixmate_cram {
     commands = ['collate', 'fixmate']
     SAMTOOLS_PIPELINE_COLLFIXM ( input, [[],[],[]], commands )
 }
-
-workflow test_samtools_pipeline_collate_fixmate_bam2cram {
-
-    input = [
-        [ id:'test', single_end:false ], // meta map
-        file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true)
-    ]
-    commands = ['collate', 'fixmate']
-    SAMTOOLS_PIPELINE_COLLFIXM_B2C ( input, [[],[],[]], commands )
-}
diff --git a/tests/modules/nf-core/samtools/pipeline/nextflow.config b/tests/modules/nf-core/samtools/pipeline/nextflow.config
index c764903bbb29..6292b7c15c5c 100644
--- a/tests/modules/nf-core/samtools/pipeline/nextflow.config
+++ b/tests/modules/nf-core/samtools/pipeline/nextflow.config
@@ -23,11 +23,6 @@ process {
         ext.args2 = '-m'       // fixmate
     }
 
-    withName: SAMTOOLS_PIPELINE_COLLFIXM_B2C {
-        ext.args  = '-T .'                 // collate
-        ext.args2 = '-m --output-fmt cram' // fixmate
-    }
-
     withName: SAMTOOLS_PIPELINE_ALL {
         ext.args  = '-T .'                 // collate
         ext.args2 = '-r ID:FOO'            // addreplacerg
diff --git a/tests/modules/nf-core/samtools/pipeline/test.yml b/tests/modules/nf-core/samtools/pipeline/test.yml
index d43fb9d57510..577c1f997dee 100644
--- a/tests/modules/nf-core/samtools/pipeline/test.yml
+++ b/tests/modules/nf-core/samtools/pipeline/test.yml
@@ -48,12 +48,3 @@
     - path: output/samtools/test.cram
       md5sum: b71fc2e9b7c04e32b13e6071f955434d
     - path: output/samtools/versions.yml
-- name: samtools pipeline test_samtools_pipeline_collate_fixmate_bam2cram
-  command: nextflow run ./tests/modules/nf-core/samtools/pipeline -entry test_samtools_pipeline_collate_fixmate_bam2cram -c ./tests/config/nextflow.config
-  tags:
-    - samtools
-    - samtools/pipeline
-  files:
-    - path: output/samtools/test.cram
-      md5sum: d091d4cd8ee1fbc3c0937296588962a3
-    - path: output/samtools/versions.yml