diff --git a/CHANGELOG.md b/CHANGELOG.md index ff0e6bdb2cf..72092f67013 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,8 @@ Warning: These experimental features are subject to change in future releases. * `workflows/multiomics/process_samples`: Add optional `--skip_scrublet_doublet_detection` flag to bypass Scrublet doublet detection. Scrublet doublet detection runs by default and can now be optionally disabled (PR #1049). +* Nextflow runner: use `resourceLimits` directive in the labels config to set a global limit on the memory (PR #1060). + ## BUG FIXES * `cellranger_multi`: Fix error when running Cell Ranger without any computational resources specified (PR #1056) diff --git a/src/workflows/utils/labels.config b/src/workflows/utils/labels.config index 8aca1a02968..94570ec62a3 100644 --- a/src/workflows/utils/labels.config +++ b/src/workflows/utils/labels.config @@ -6,7 +6,10 @@ process { // Retry for exit codes that have something to do with memory issues errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } maxRetries = 3 - maxMemory = null + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] // CPU resources withLabel: singlecpu { cpus = 1 } @@ -15,10 +18,10 @@ process { withLabel: highcpu { cpus = 20 } // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } // Disk space withLabel: lowdisk { @@ -43,24 +46,3 @@ process { // withLabel: highdisk { disk = { 200.GB * task.attempt } } // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } } - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -}