improve support for nf-core configs

CHANGELOG.md

@@ -7,8 +7,12 @@ description: A full list of Bactopia releases and a description of the changes.
 ## v3.0.2 bactopia/bactopia "" 2024/??/??
 
 ### `Added`
+
 - Bactopia Tools (`bactopia --wf <NAME>`)
     - `defensefinder` - Systematic search of all known anti-phage systems
+- full support of config files from nf-core/configs
+    - no longer prints efficiency for `standard`, `docker`, and `singularity` profiles
+    - now required non-integer values for `--max_time` (e.g. `4.h`) and `--max_memory` (e.g. `8.GB`)
 
 ### `Fixed`
 

conf/base.config

@@ -1,11 +1,13 @@
 process {
     // Defaults
-    container = "quay.io/bactopia/bactopia:${manifest.version}"
-    cpus = {check_max('request', RESOURCES.MAX_CPUS, 'cpus' )}
-    memory = {check_max(2.GB * task.attempt, RESOURCES.MAX_MEMORY, 'memory' )}
-    time = {check_max( (params.min_time).m * task.attempt, (params.max_time).m, 'time' )}
+    container     = "quay.io/bactopia/bactopia:${manifest.version}"
     errorStrategy = 'retry'
-    maxRetries = params.max_retry
+    maxRetries    = params.max_retry
+
+    // Resources
+    cpus   = { check_max( 1    * task.attempt, 'cpus'   ) }
+    memory = { check_max( 4.GB * task.attempt, 'memory' ) }
+    time   = { check_max( 1.h  * task.attempt, 'time'   ) }
 
     publishDir = [
         path: params.outdir,
@@ -16,38 +18,32 @@ process {
         }
     ]
 
-    // Process specific
-    withLabel: 'assemble_genome' {
-        memory = { meta.runtype == 'hybrid' || params.use_unicycler ? check_max((32.GB * task.attempt), RESOURCES.MAX_MEMORY, 'memory') : check_max((8.GB * task.attempt), RESOURCES.MAX_MEMORY, 'memory')}
-        time   = { meta.runtype == 'hybrid' || params.use_unicycler ? check_max( 2.h * task.attempt, (params.max_time).m, 'time' ) : check_max( (params.min_time).m * task.attempt, (params.max_time).m, 'time' )}
-    }
-
     // Modules imported from nf-core
-    withLabel:process_single {
-        cpus   = { check_max('request'          , RESOURCES.MAX_CPUS  , 'cpus'   )}
-        memory = { check_max(4.GB * task.attempt, RESOURCES.MAX_MEMORY, 'memory' )}
-        time   = { check_max( 1.h * task.attempt, 2.h * task.attempt  , 'time'   )}
+    withLabel: process_single {
+        cpus   = { check_max( 1                  , 'cpus'    ) }
+        memory = { check_max( 4.GB * task.attempt, 'memory'  ) }
+        time   = { check_max( 2.h  * task.attempt, 'time'    ) }
     }
     withLabel: process_low {
-        cpus   = {check_max('request'          , RESOURCES.MAX_CPUS  , 'cpus'   )}
-        memory = {check_max(8.GB * task.attempt, RESOURCES.MAX_MEMORY, 'memory' )}
-        time   = {check_max( 2.h * task.attempt, 2.h * task.attempt  , 'time'   )}
+        cpus   = { check_max( 4     * task.attempt, 'cpus'   ) }
+        memory = { check_max( 8.GB  * task.attempt, 'memory' ) }
+        time   = { check_max( 4.h   * task.attempt, 'time'   ) }
     }
     withLabel: process_medium {
-        cpus   = {check_max('request'           , RESOURCES.MAX_CPUS  , 'cpus'   )}
-        memory = {check_max(16.GB * task.attempt, RESOURCES.MAX_MEMORY, 'memory' )}
-        time   = {check_max( 12.h * task.attempt, 12.h * task.attempt , 'time'   )}
+        cpus   = { check_max( 8     * task.attempt, 'cpus'   ) }
+        memory = { check_max( 32.GB * task.attempt, 'memory' ) }
+        time   = { check_max( 12.h  * task.attempt, 'time'   ) }
     }
     withLabel: process_high {
-        cpus   = {check_max('request'           , RESOURCES.MAX_CPUS  , 'cpus'   )}
-        memory = {check_max(64.GB * task.attempt, RESOURCES.MAX_MEMORY, 'memory' )}
-        time   = {check_max( 24.h * task.attempt, 24.h * task.attempt , 'time'   )}
+        cpus   = { check_max( 12    * task.attempt, 'cpus'   ) }
+        memory = { check_max( 64.GB * task.attempt, 'memory' ) }
+        time   = { check_max( 24.h  * task.attempt, 'time'   ) }
     }
     withLabel: process_long {
-        time = {check_max( 96.h * task.attempt, 96.h * task.attempt, 'time' )}
+        time   = { check_max( 96.h  * task.attempt, 'time'   ) }
     }
     withLabel: process_high_memory {
-        memory = {check_max(128.GB * task.attempt, RESOURCES.MAX_MEMORY, 'memory' )}
+        memory = { check_max( 128.GB * task.attempt, 'memory' ) }
     }
     withLabel: error_ignore {
         errorStrategy = 'ignore'

conf/params.config

@@ -12,14 +12,13 @@ params {
     merge_folder = "merged-results"
 
     //nf-core
-    schema_ignore_params = 'no_check_certificate,run_name,prokka_debug,is_ci,sourmash_url,mash_url,mlst_url,amrfinder_url,merge_folder,wf_has_subdir,build_all,use_mamba,empty_proteins,empty_tf,empty_adapters,empty_phix,help_all,include_tools,min_time,publish_dir,silent,test_data,test_data_dir,verbose,schema_inputs,empty_extra,empty_r2,empty_r1,workflows,available_workflows,is_subworkflow,enable_conda,force_rebuild,config_profile_url,config_profile_contact,config_profile_description,config_profile_name'
+    schema_ignore_params = 'no_check_certificate,run_name,prokka_debug,is_ci,sourmash_url,mash_url,mlst_url,amrfinder_url,merge_folder,wf_has_subdir,build_all,use_mamba,empty_proteins,empty_tf,empty_adapters,empty_phix,help_all,include_tools,publish_dir,silent,test_data,test_data_dir,verbose,schema_inputs,empty_extra,empty_r2,empty_r1,workflows,available_workflows,is_subworkflow,enable_conda,force_rebuild,config_profile_url,config_profile_contact,config_profile_description,config_profile_name'
     schema_inputs = ['fastqs', 'r1,r2,se,sample,hybrid', 'r1,r2,sample', 'se,sample', 'assembly,sample', 'accessions', 'accession']
 
     // Max Job Request Parameters
     max_retry = 3
-    min_time = 60
-    max_time = 120
-    max_memory = 32
+    max_time = 240.h
+    max_memory = 128.GB
     max_cpus = 4
     max_downloads = 3
 

conf/params/bactopia.config

@@ -41,8 +41,7 @@ params {
 
     // Max Job Request Parameters
     max_retry = 3
-    min_time = 60
-    max_time = 120
-    max_memory = 32
+    max_time = 2.h
+    max_memory = 32.GB
     max_cpus = 4
 }

conf/params/clean-yer-reads.config

@@ -4,7 +4,7 @@ This file includes default values for shared parameters.
 
 params {
     // Max Job Request Parameters
-    max_time = 240
+    max_time = 4.h
 
     // Optional Parameters
     coverage = 0

conf/profiles/test.config

@@ -6,7 +6,7 @@ params {
 
     // Limit resources so that this can run on GitHub Actions
     max_cpus = 2
-    max_memory = 6
+    max_memory = 6.GB
 
     // Inputs
     r1 = 'https://github.com/bactopia/bactopia-tests/raw/main/data/species/portiera/illumina/SRR2838702_R1.fastq.gz'

conf/schema/generic.json

@@ -67,20 +67,22 @@
                     "help_text": "Use to set an upper-limit for the CPU requirement for each process. Should be an integer e.g. `--max_cpus 1`"
                 },
                 "max_memory": {
-                    "type": "integer",
-                    "description": "Maximum amount of memory (in GB) that can be requested for any single job.",
-                    "default": 32,
+                    "type": "string",
+                    "description": "Maximum amount of memory that can be requested for any single job.",
+                    "default": "128.GB",
                     "fa_icon": "fas fa-memory",
+                    "pattern": "^\\d+(\\.\\d+)?\\.?\\s*(K|M|G|T)?B$",
                     "hidden": true,
-                    "help_text": "Use to set an upper-limit for the memory requirement for each process. Should be a string in the format integer-unit e.g. `--max_memory 8`"
+                    "help_text": "Use to set an upper-limit for the memory requirement for each process. Should be a string in the format integer-unit e.g. `--max_memory '8.GB'`"
                 },
                 "max_time": {
-                    "type": "integer",
-                    "description": "Maximum amount of time (in minutes) that can be requested for any single job.",
-                    "default": 120,
+                    "type": "string",
+                    "description": "Maximum amount of time that can be requested for any single job.",
+                    "default": "240.h",
                     "fa_icon": "far fa-clock",
+                    "pattern": "^(\\d+\\.?\\s*(s|m|h|d|day)\\s*)+$",
                     "hidden": true,
-                    "help_text": "Use to set an upper-limit for the time requirement for each process. Should be a string in the format integer-unit e.g. `--max_time 60`"
+                    "help_text": "Use to set an upper-limit for the time requirement for each process. Should be a string in the format integer-unit e.g. `--max_time '1.h'`"
                 },
                 "max_downloads": {
                     "type": "integer",

lib/WorkflowBactopia.groovy

@@ -98,11 +98,6 @@ class WorkflowBactopia {
             error += Utils.isPositiveInteger(params.genome_size, 'genome_size', log)
         }
 
-        if (params.min_time > params.max_time) {
-            log.error "The value for min_time (${params.min_time}) exceeds max_time (${params.max_time}), Please correct to continue."
-            error += 1
-        }
-
         if (params.containsKey('adapters')){
             if (params.adapters) {
                 if (Utils.isLocal(params.adapters)) {

lib/nf/functions.nf

@@ -110,9 +110,9 @@ def _get_module_schemas(modules) {
 def get_resources(profile, max_memory, max_cpus) {
     /* Adjust memory/cpu requests for standard profile only */
     def Map resources = [:]
-    resources.MAX_MEMORY = ['standard', 'docker', 'singularity'].contains(profile) ? _get_max_memory(max_memory).GB : (max_memory).GB
+    resources.MAX_MEMORY = max_memory
     resources.MAX_MEMORY_INT = resources.MAX_MEMORY.toString().split(" ")[0]
-    resources.MAX_CPUS = ['standard', 'docker', 'singularity'].contains(profile) ? _get_max_cpus(max_cpus.toInteger()) : max_cpus.toInteger()
+    resources.MAX_CPUS = max_cpus.toInteger()
     resources.MAX_CPUS_75 = Math.round(resources.MAX_CPUS * 0.75)
     resources.MAX_CPUS_50 = Math.round(resources.MAX_CPUS * 0.50)
     resources.MAX_CPUS_1 = 1
@@ -122,7 +122,7 @@ def get_resources(profile, max_memory, max_cpus) {
 def _get_max_memory(requested) {
     /* Get the maximum available memory for the given system */
     def available = Math.floor(Double.parseDouble(SysHelper.getAvailMemory().toGiga().toString().split(" ")[0])).toInteger()
-    if (available < requested) {
+    if (available < requested.toInteger()) {
         log.warn "Maximum memory (${requested}) was adjusted to fit your system (${available})"
         return available
     }
@@ -141,18 +141,18 @@ def _get_max_cpus(requested) {
     return requested
 }
 
-def print_efficiency(cpus) {
+def print_efficiency() {
     /* Inform user how local bactiopia run will use resources */
     if (['standard', 'docker', 'singularity'].contains(workflow.profile)) {
         // This is a local run on a single machine
         available = SysHelper.getAvailCpus()
-        tasks = available / cpus
+        tasks = Math.round(available / params.max_cpus)
         log.info """
-            Each task will use ${cpus} CPUs out of the available ${available} CPUs. At most 
-            ${tasks} task(s) will be run at a time, this can affect the efficiency 
-            of Bactopia. You can use the '-qs' parameter to alter the number of 
-            tasks to run at a time (e.g. '-qs 2', means only 2 tasks or a maximum 
-            of ${2 * cpus} CPUs will be used at once)
+            Each task will use a maximum of ${params.max_cpus} CPUs out of the available ${available}
+            CPUs. At most ${tasks} task(s) will be run at a time, this can affect
+            the efficiency of Bactopia. You can use the '-qs' parameter to
+            alter the number of tasks to run at a time (e.g. '-qs 2', means
+            only 2 tasks or a maximum of ${2 * params.max_cpus} CPUs will be used at once)
         """.stripIndent()
         log.info ""
     }

modules/local/bactopia/assembler/main.nf

@@ -1,6 +1,5 @@
 // Import generic module functions
-include { get_resources; initOptions; saveFiles } from '../../../../lib/nf/functions'
-RESOURCES      = get_resources(workflow.profile, params.max_memory, params.max_cpus)
+include { initOptions; saveFiles } from '../../../../lib/nf/functions'
 options        = initOptions(params.options ? params.options : [:], 'assembler')
 options.ignore = [".fastq.gz"]
 options.btype  = options.btype ?: "main"
@@ -10,8 +9,7 @@ conda_env      = file("${params.condadir}/${conda_name}").exists() ? "${params.c
 
 process ASSEMBLER {
     tag "${meta.id}"
-    label "process_low"
-    label "assemble_genome"
+    label (params.use_unicycler ? "process_medium" : "process_low")
 
     conda (params.enable_conda ? conda_env : null)
     container "${ workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container ?

modules/local/bactopia/datasets/main.nf

@@ -1,6 +1,4 @@
 // Import generic module functions
-include { get_resources } from '../../../../lib/nf/functions'
-RESOURCES   = get_resources(workflow.profile, params.max_memory, params.max_cpus)
 conda_tools = "bioconda::gnu-wget=1.18"
 conda_name  = conda_tools.replace("=", "-").replace(":", "-").replace(" ", "-")
 conda_env   = file("${params.condadir}/${conda_name}").exists() ? "${params.condadir}/${conda_name}" : conda_tools

modules/local/bactopia/gather/main.nf

@@ -1,6 +1,5 @@
 // Import generic module functions
-include { get_resources; initOptions; saveFiles } from '../../../../lib/nf/functions'
-RESOURCES      = get_resources(workflow.profile, params.max_memory, params.max_cpus)
+include { initOptions; saveFiles } from '../../../../lib/nf/functions'
 options        = initOptions(params.options ? params.options : [:], 'gather')
 options.ignore = [".fastq.gz", ".fna.gz"]
 options.btype  = options.btype ?: "main"

modules/local/bactopia/qc/main.nf

@@ -1,6 +1,5 @@
 // Import generic module functions
-include { get_resources; initOptions; saveFiles } from '../../../../lib/nf/functions'
-RESOURCES      = get_resources(workflow.profile, params.max_memory, params.max_cpus)
+include { initOptions; saveFiles } from '../../../../lib/nf/functions'
 options        = initOptions(params.options ? params.options : [:], 'qc')
 options.ignore = ['.fna.gz']
 options.btype  = options.btype ?: "main"

modules/local/bactopia/sketcher/main.nf

@@ -1,6 +1,5 @@
 // Import generic module functions
-include { get_resources; initOptions; saveFiles } from '../../../../lib/nf/functions'
-RESOURCES      = get_resources(workflow.profile, params.max_memory, params.max_cpus)
+include { initOptions; saveFiles } from '../../../../lib/nf/functions'
 options        = initOptions(params.options ? params.options : [:], 'sketcher')
 options.ignore = [".fastq.gz"]
 options.btype  = options.btype ?: "main"

modules/nf-core/abricate/run/main.nf

@@ -1,6 +1,5 @@
 // Import generic module functions
-include { get_resources; initOptions; saveFiles } from '../../../../lib/nf/functions'
-RESOURCES     = get_resources(workflow.profile, params.max_memory, params.max_cpus)
+include { initOptions; saveFiles } from '../../../../lib/nf/functions'
 options       = initOptions(params.containsKey("options") ? params.options : [:], 'abricate')
 options.btype = options.btype ?: "tools"
 conda_tools   = "bioconda::abricate=1.0.1"

modules/nf-core/abricate/summary/main.nf

@@ -1,6 +1,5 @@
 // Import generic module functions
-include { get_resources; initOptions; saveFiles } from '../../../../lib/nf/functions'
-RESOURCES     = get_resources(workflow.profile, params.max_memory, params.max_cpus)
+include { initOptions; saveFiles } from '../../../../lib/nf/functions'
 options       = initOptions(params.containsKey("options") ? params.options : [:], 'abricate')
 options.btype = options.btype ?: "comparative"
 conda_tools   = "bioconda::abricate=1.0.1"

modules/nf-core/abritamr/run/main.nf

@@ -1,6 +1,5 @@
 // Import generic module functions
-include { get_resources; initOptions; saveFiles } from '../../../../lib/nf/functions'
-RESOURCES     = get_resources(workflow.profile, params.max_memory, params.max_cpus)
+include { initOptions; saveFiles } from '../../../../lib/nf/functions'
 options       = initOptions(params.containsKey("options") ? params.options : [:], 'abritamr')
 options.btype = options.btype ?: "tools"
 conda_tools   = "bioconda::abritamr=1.0.17"

modules/nf-core/agrvate/main.nf

@@ -1,6 +1,5 @@
 // Import generic module functions
-include { get_resources; initOptions; saveFiles } from '../../../lib/nf/functions'
-RESOURCES     = get_resources(workflow.profile, params.max_memory, params.max_cpus)
+include { initOptions; saveFiles } from '../../../lib/nf/functions'
 options       = initOptions(params.containsKey("options") ? params.options : [:], 'agrvate')
 options.btype = options.btype ?: "tools"
 conda_tools   = "bioconda::agrvate=1.0.2"

modules/nf-core/amrfinderplus/run/main.nf

@@ -1,6 +1,5 @@
 // Import generic module functions
-include { get_resources; initOptions; saveFiles } from '../../../../lib/nf/functions'
-RESOURCES     = get_resources(workflow.profile, params.max_memory, params.max_cpus)
+include { initOptions; saveFiles } from '../../../../lib/nf/functions'
 options       = initOptions(params.containsKey("options") ? params.options : [:], 'amrfinderplus')
 options.btype = options.btype ?: "tools"
 conda_tools   = "bioconda::ncbi-amrfinderplus=3.12.8"

modules/nf-core/amrfinderplus/update/main.nf

@@ -1,6 +1,5 @@
 // Import generic module functions
-include { get_resources; initOptions; saveFiles } from '../../../../lib/nf/functions'
-RESOURCES   = get_resources(workflow.profile, params.max_memory, params.max_cpus)
+include { initOptions; saveFiles } from '../../../../lib/nf/functions'
 options     = initOptions(params.containsKey("options") ? params.options : [:], 'amrfinderplus_update')
 conda_tools = "bioconda::ncbi-amrfinderplus=3.12.8"
 conda_name  = conda_tools.replace("=", "-").replace(":", "-").replace(" ", "-")

modules/nf-core/ariba/getref/main.nf

@@ -1,6 +1,5 @@
 // Import generic module functions
-include { get_resources; initOptions; saveFiles } from '../../../../lib/nf/functions'
-RESOURCES   = get_resources(workflow.profile, params.max_memory, params.max_cpus)
+include { initOptions; saveFiles } from '../../../../lib/nf/functions'
 options     = initOptions(params.containsKey("options") ? params.options : [:], 'ariba')
 conda_tools = "bioconda::ariba=2.14.6 bioconda::bcftools=1.14 bioconda::pysam=0.18.0"
 conda_name  = conda_tools.replace("=", "-").replace(":", "-").replace(" ", "-")

modules/nf-core/ariba/run/main.nf

@@ -1,6 +1,5 @@
 // Import generic module functions
-include { get_resources; initOptions; saveFiles } from '../../../../lib/nf/functions'
-RESOURCES     = get_resources(workflow.profile, params.max_memory, params.max_cpus)
+include { initOptions; saveFiles } from '../../../../lib/nf/functions'
 options       = initOptions(params.containsKey("options") ? params.options : [:], 'ariba')
 options.btype = options.btype ?: "tools"
 conda_tools   = "bioconda::ariba=2.14.6"

modules/nf-core/bakta/download/main.nf

@@ -1,6 +1,5 @@
 // Import generic module functions
-include { get_resources; initOptions; saveFiles } from '../../../../lib/nf/functions'
-RESOURCES   = get_resources(workflow.profile, params.max_memory, params.max_cpus)
+include { initOptions; saveFiles } from '../../../../lib/nf/functions'
 options     = initOptions(params.containsKey("options") ? params.options : [:], 'bakta')
 conda_tools = "bioconda::bakta=1.9.3"
 conda_name  = conda_tools.replace("=", "-").replace(":", "-").replace(" ", "-")

modules/nf-core/bakta/run/main.nf

@@ -1,6 +1,5 @@
 // Import generic module functions
-include { get_resources; initOptions; saveFiles } from '../../../../lib/nf/functions'
-RESOURCES     = get_resources(workflow.profile, params.max_memory, params.max_cpus)
+include { initOptions; saveFiles } from '../../../../lib/nf/functions'
 options       = initOptions(params.containsKey("options") ? params.options : [:], 'bakta')
 options.btype = options.btype ?: "main"
 conda_tools   = "bioconda::bakta=1.9.3"

modules/nf-core/blast/blastn/main.nf

@@ -1,6 +1,5 @@
 // Import generic module functions
-include { get_resources; initOptions; saveFiles } from '../../../../lib/nf/functions'
-RESOURCES     = get_resources(workflow.profile, params.max_memory, params.max_cpus)
+include { initOptions; saveFiles } from '../../../../lib/nf/functions'
 options       = initOptions(params.containsKey("options") ? params.options : [:], 'blastn')
 options.btype = options.btype ?: "tools"
 conda_tools   = "bioconda::blast=2.15.0"

modules/nf-core/blast/blastp/main.nf

@@ -1,6 +1,5 @@
 // Import generic module functions
-include { get_resources; initOptions; saveFiles } from '../../../../lib/nf/functions'
-RESOURCES     = get_resources(workflow.profile, params.max_memory, params.max_cpus)
+include { initOptions; saveFiles } from '../../../../lib/nf/functions'
 options       = initOptions(params.containsKey("options") ? params.options : [:], 'blastp')
 options.btype = options.btype ?: "tools"
 conda_tools   = "bioconda::blast=2.15.0"