Generalise nextflow.config and avoid Nextflow job submission issues

Ensembl · Jan 11, 2023 · 21f0f64 · 21f0f64
1 parent 09b638a
commit 21f0f64
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 35 deletions.
diff --git a/nextflow/ProteinFunction/nextflow.config b/nextflow/ProteinFunction/nextflow.config
@@ -1,42 +1,37 @@
 profiles {
-  standard {
-    process.executor = 'local'
-    singularity {
-      enabled = true
-      autoMounts = true
-    }
-  }
+  standard { process.executor = 'local' }
 
   lsf {
     executor {
-      name = 'lsf'
-      queueSize = 500
-      submitRateLimit = '100/1sec'
+      name            = 'lsf'
+      queueSize       = 500
+      submitRateLimit = '50/10sec'
     }
+  }
 
-    singularity {
-      enabled = true
-      autoMounts = true
-    }
+  //untested
+  slurm { process.executor = 'slurm' }
+}
 
-    process {
-      queue = 'production'
-      memory = '2GB'
-      withLabel: medmem { memory =  '8 GB' }
-      withLabel: urgent { memory =  '2 GB' }
-      withLabel: higmem { memory = '24 GB' }
-      withLabel: long   { memory =  '2 GB' }
-    }
-  }
+singularity {
+  enabled    = true
+  autoMounts = true
+}
 
-  //untested 
-  slurm {
-    process.executor = 'slurm'
-    singularity {
-      enabled = true
-      autoMounts = true
-    }
-  }  
+process {
+  queue  = 'production'
+  memory = '2 GB'
+  withLabel: medmem { memory =  '8 GB' }
+  withLabel: urgent { memory =  '2 GB' }
+  withLabel: higmem { memory = '24 GB' }
+  withLabel: long   { memory =  '2 GB' }
+
+  // avoid Nextflow job submission issues, e.g., "Request from non-LSF host rejected"
+  errorStrategy = { task.exitStatus = 255 ? 'retry' : 'finish' }
+  withLabel: error-ignore {
+    errorStrategy = { task.exitStatus = 255 ? 'retry' : 'ignore' }
+  }
+  maxRetries = 3
 }
 
 trace {

diff --git a/nextflow/ProteinFunction/nf_modules/polyphen2.nf b/nextflow/ProteinFunction/nf_modules/polyphen2.nf
@@ -30,7 +30,7 @@ process run_pph2_on_all_aminoacid_substitutions {
   container "ensemblorg/polyphen-2:2.2.3"
   containerOptions "--bind ${params.pph_data}:/opt/pph2/data"
   label 'highmem'
-  errorStrategy 'ignore'
+  label 'error-ignore'
 
   input:
     val peptide
@@ -72,7 +72,7 @@ process run_weka {
 
   //tag "${pph2_out.baseName} $model"
   container "ensemblorg/polyphen-2:2.2.3"
-  errorStrategy 'ignore'
+  label 'error-ignore'
 
   input:
     each model

diff --git a/nextflow/ProteinFunction/nf_modules/sift.nf b/nextflow/ProteinFunction/nf_modules/sift.nf
@@ -32,7 +32,7 @@ process align_peptides {
   tag "${peptide.md5}"
   container "ensemblorg/sift:6.2.1"
   label 'medmem'
-  errorStrategy 'ignore'
+  label 'error-ignore'
 
   input:
     val peptide
@@ -70,7 +70,7 @@ process run_sift_on_all_aminoacid_substitutions {
   tag "${peptide.md5}"
   container "ensemblorg/sift:6.2.1"
   label 'medmem'
-  errorStrategy 'ignore'
+  label 'error-ignore'
   publishDir "${params.outdir}/sift"
 
   input: