Merge e7e5556 into b340c93

Ensembl · May 19, 2023 · c8d37bf · c8d37bf
2 parents b340c93 + e7e5556
commit c8d37bf
Show file tree

Hide file tree

Showing 6 changed files with 80 additions and 41 deletions.
diff --git a/modules/Bio/EnsEMBL/Variation/Pipeline/AncestralAlleles/AncestralAlleles_conf.pm b/modules/Bio/EnsEMBL/Variation/Pipeline/AncestralAlleles/AncestralAlleles_conf.pm
@@ -82,6 +82,9 @@ sub default_options {
         default_lsf_options => '-q production -R"select[mem>2000] rusage[mem=2000]" -M2000',
         medmem_lsf_options  => '-q production -R"select[mem>5000] rusage[mem=5000]" -M5000',
 
+        default_slurm_options => '--partition=standard --time=12:00:00 --mem=2G',
+        medmem_slurm_options  => '--partition=standard --time=12:00:00 --mem=5G',
+
         hive_db_host    => 'mysql-ens-var-prod-1',
         hive_db_port    => 4449,
         hive_db_user    => 'ensadmin',
@@ -101,8 +104,10 @@ sub default_options {
 sub resource_classes {
     my ($self) = @_;
     return {
-          'default' => { 'LSF' => $self->o('default_lsf_options') },
-          'medmem'  => { 'LSF' => $self->o('medmem_lsf_options') },
+          'default' => { 'LSF' => $self->o('default_lsf_options'),
+                         'SLURM' => $self->o('default_slurm_options') },
+          'medmem'  => { 'LSF' => $self->o('medmem_lsf_options'),
+                         'SLURM' => $self->o('medmem_slurm_options') },
     };
 }
 
@@ -157,4 +162,3 @@ sub pipeline_analyses {
 }
 
 1;
-
diff --git a/modules/Bio/EnsEMBL/Variation/Pipeline/EquivalentAlleles/EquivalentAlleles_conf.pm b/modules/Bio/EnsEMBL/Variation/Pipeline/EquivalentAlleles/EquivalentAlleles_conf.pm
@@ -74,10 +74,13 @@ sub default_options {
     # configuration for the various resource options used in the pipeline
     # EBI farm users should either change these here, or override them on the
     # command line to suit the EBI farm.
-
     default_lsf_options => '-qproduction -R"select[mem>2000] rusage[mem=2000]" -M2000',
     medium_lsf_options  => '-qproduction -R"select[mem>4000] rusage[mem=4000]" -M4000',
 
+    default_slurm_options      => '--partition=standard --time=12:00:00 --mem=2G',
+    default_long_slurm_options => '--partition=standard --time=36:00:00 --mem=2G',
+    medium_slurm_options       => '--partition=standard --time=12:00:00 --mem=4G',
+
 
     # size of region to be checked in a single job
     region_size =>  100000000,
@@ -118,8 +121,12 @@ sub default_options {
 sub resource_classes {
   my ($self) = @_;
   return {
-      'default' => { 'LSF' => $self->o('default_lsf_options') },
-      'medium'  => { 'LSF' => $self->o('medium_lsf_options')  },
+      'default'      => { 'LSF' => $self->o('default_lsf_options'),
+                          'SLURM' => $self->o('default_slurm_options') },
+      'default_long' => { 'LSF' => $self->o('default_lsf_options'),
+                          'SLURM' => $self->o('default_long_slurm_options') },
+      'medium'       => { 'LSF' => $self->o('medium_lsf_options'),
+                          'SLURM' => $self->o('medium_slurm_options')  },
   };
 }
 
@@ -162,7 +169,7 @@ sub pipeline_analyses {
        -input_ids        => [],
        -hive_capacity    => $self->o('capacity'),
        -max_retry_count  => 0,
-       -rc_name          => 'default',
+       -rc_name          => 'default_long',
        -wait_for         => [ 'init_equivalent_alleles' ],
        -flow_into        => {},
     },

diff --git a/modules/Bio/EnsEMBL/Variation/Pipeline/GetCAR/GetCAR_conf.pm b/modules/Bio/EnsEMBL/Variation/Pipeline/GetCAR/GetCAR_conf.pm
@@ -87,10 +87,14 @@ sub resource_classes {
     my ($self) = @_;
     return {
         %{$self->SUPER::resource_classes},  # inherit 'default' from the parent class
-            'test_mem'    => { 'LSF' => '-q production -R"select[mem>100] rusage[mem=100]" -M100'},
-            'default_mem' => { 'LSF' => '-q production -R"select[mem>1000] rusage[mem=1000]" -M1000'},
-            'medium_mem'  => { 'LSF' => '-q production -R"select[mem>2000] rusage[mem=2000]" -M2000'},
-            'high_mem'    => { 'LSF' => '-q production -R"select[mem>4000] rusage[mem=4000]" -M4000'},
+            'test_mem'    => { 'LSF' => '-q production -R"select[mem>100] rusage[mem=100]" -M100',
+                               'SLURM' => '--partition=standard --time=12:00:00 --mem=100MB' },
+            'default_mem' => { 'LSF' => '-q production -R"select[mem>1000] rusage[mem=1000]" -M1000',
+                               'SLURM' => '--partition=standard --time=12:00:00 --mem=1G' },
+            'medium_mem'  => { 'LSF' => '-q production -R"select[mem>2000] rusage[mem=2000]" -M2000',
+                               'SLURM' => '--partition=standard --time=12:00:00 --mem=2G'},
+            'high_mem'    => { 'LSF' => '-q production -R"select[mem>4000] rusage[mem=4000]" -M4000',
+                               'SLURM' => '--partition=standard --time=12:00:00 --mem=4G' },
     };
 }
 
@@ -127,6 +131,7 @@ sub pipeline_analyses {
     {
       -logic_name => 'finish_get_car',
       -module     => 'Bio::EnsEMBL::Variation::Pipeline::GetCAR::FinishGetCAR',
+      -rc_name           => 'default_mem',
       -max_retry_count   => 0,
     },
   );

diff --git a/modules/Bio/EnsEMBL/Variation/Pipeline/PhenotypeAnnotation/PhenotypeAnnotation_conf.pm b/modules/Bio/EnsEMBL/Variation/Pipeline/PhenotypeAnnotation/PhenotypeAnnotation_conf.pm
@@ -143,9 +143,13 @@ sub default_options {
         default_lsf_options => '-qproduction -R"select[mem>2000] rusage[mem=2000]" -M2000',
         medmem_lsf_options  => '-qproduction -R"select[mem>4000] rusage[mem=4000]" -M4000',
         urgent_lsf_options  => '-qproduction -R"select[mem>2000] rusage[mem=2000]" -M2000',
-        highmem_lsf_options => '-qproduction -R"select[mem>15000] rusage[mem=15000] span[hosts=1]" -M15000 -n4', # this is LSF speak for "give me 15GB of memory"
+        highmem_lsf_options => '-qproduction -R"select[mem>15000] rusage[mem=15000] span[hosts=1]" -M15000 -n4',
         long_lsf_options    => '-qproduction -R"select[mem>2000] rusage[mem=2000]" -M2000',
 
+        default_slurm_options => '--partition=standard --time=12:00:00 --mem=2G',
+        highmem_slurm_options => '--partition=standard --time=12:00:00 --mem=15G',
+        long_slurm_options    => '--partition=standard --time=24:00:00 --mem=2G',
+
         # connection parameters for the hive database, you should supply the hive_db_password
         # option on the command line to init_pipeline.pl (parameters for the target database
         # should be set in the registry file defined above)
@@ -176,10 +180,13 @@ sub default_options {
 sub resource_classes {
     my ($self) = @_;
     return {
-          'default' => { 'LSF' => $self->o('default_lsf_options') },
+          'default' => { 'LSF' => $self->o('default_lsf_options'),
+                         'SLURM' => $self->o('default_slurm_options') },
           'urgent'  => { 'LSF' => $self->o('urgent_lsf_options')  },
-          'highmem' => { 'LSF' => $self->o('highmem_lsf_options') },
-          'long'    => { 'LSF' => $self->o('long_lsf_options')    },
+          'highmem' => { 'LSF' => $self->o('highmem_lsf_options'),
+                         'SLURM' => $self->o('highmem_slurm_options') },
+          'long'    => { 'LSF' => $self->o('long_lsf_options'),
+                         'SLURM' => $self->o('long_slurm_options') },
           'medmem'  => { 'LSF' => $self->o('medmem_lsf_options') },
     };
 }
@@ -210,7 +217,7 @@ sub pipeline_analyses {
 
                 '5' => [ 'import_rgd' ],
                 '6' => [ 'import_zfin' ],
-		'14' => [ 'import_wormbase' ],
+                '14' => [ 'import_wormbase' ],
             },
         },
 
@@ -242,7 +249,7 @@ sub pipeline_analyses {
             },
             -input_ids      => [], #default
             -hive_capacity  => 1,
-            -rc_name    => 'default',
+            -rc_name    => 'long',
             -flow_into  => {
                 2 => [ 'check_gwas']
             },
@@ -582,18 +589,18 @@ sub pipeline_analyses {
             },
         },
         {   -logic_name => 'import_wormbase',
-      	    -module     => 'Bio::EnsEMBL::Variation::Pipeline::PhenotypeAnnotation::ImportWORMBASE',
-      	    -parameters => {
-      	    	source_version => $self->o('wormbase_release'),
+            -module     => 'Bio::EnsEMBL::Variation::Pipeline::PhenotypeAnnotation::ImportWORMBASE',
+            -parameters => {
+              source_version => $self->o('wormbase_release'),
                       @common_params,
                   },
-      	    -input_ids  => [],
-      	    -hive_capacity  => 1,
-      	    -rc_name    => 'default',
-      	    -flow_into  => {
-            		1 => [ 'check_phenotypes']
-      	    },
-        },    
+            -input_ids  => [],
+            -hive_capacity  => 1,
+            -rc_name    => 'default',
+            -flow_into  => {
+                1 => [ 'check_phenotypes']
+            },
+        },
         {   -logic_name => 'import_ontology_mapping',
             -module     => 'Bio::EnsEMBL::Variation::Pipeline::PhenotypeAnnotation::OntologyMapping',
             -parameters => {

diff --git a/modules/Bio/EnsEMBL/Variation/Pipeline/RegulationEffect_conf.pm b/modules/Bio/EnsEMBL/Variation/Pipeline/RegulationEffect_conf.pm
@@ -88,8 +88,12 @@ sub resource_classes {
     # requirements, queue parameters etc.) to suit your own data
 
     return {
-      'default' => { 'LSF' => '-qproduction -R"select[mem>2000] rusage[mem=2000]" -M2000'},
-      'highmem' => { 'LSF' => '-qproduction -R"select[mem>15000] rusage[mem=15000]" -M15000'},
+      'default' => { 'LSF' => '-qproduction -R"select[mem>2000] rusage[mem=2000]" -M2000',
+                     'SLURM' => "--partition=standard --time=4:00:00 --mem=2G" },
+      'highmem' => { 'LSF' => '-qproduction -R"select[mem>15000] rusage[mem=15000]" -M15000',
+                     'SLURM' => "--partition=standard --time=28:00:00 --mem=15G"},
+      'long'    => { 'LSF' => '-qproduction -R"select[mem>2000] rusage[mem=2000]" -M2000',
+                     'SLURM' => "--partition=standard --time=28:00:00 --mem=2G"},
     };
 }
 

diff --git a/modules/Bio/EnsEMBL/Variation/Pipeline/VariationConsequence_conf.pm b/modules/Bio/EnsEMBL/Variation/Pipeline/VariationConsequence_conf.pm
@@ -96,7 +96,7 @@ sub default_options {
         disambiguate_single_nucleotide_alleles => 0,
 
         # shifting variants within repeated regions in the 3' direction is switched off by default.
-	
+
         prevent_shifting => 1,
 
         # configuration for the various resource options used in the pipeline
@@ -109,11 +109,18 @@ sub default_options {
         medmem_lsf_options  => '-qproduction -R"select[mem>5000] rusage[mem=5000]" -M5000',
         highmem_lsf_options => '-qproduction -R"select[mem>15000] rusage[mem=15000] span[hosts=1]" -M15000 -n4', # this is LSF speak for "give me 15GB of memory"
 
+        default_slurm_options      => '--partition=standard --time=24:00:00 --mem=2G',
+        default_long_slurm_options => '--partition=standard --time=140:00:00 --mem=2G',
+        medmem_slurm_options       => '--partition=standard --time=24:00:00 --mem=5G',
+        medmem_long_slurm_options  => '--partition=standard --time=140:00:00 --mem=5G',
+        highmem_slurm_options      => '--partition=standard --time=24:00:00 --mem=15G',
+        highmem_long_slurm_options => '--partition=standard --time=140:00:00 --mem=15G',
+
         # options controlling the number of workers used for the parallelisable analyses
         # these default values seem to work for most species
 
         set_variation_class_capacity    => 10,
-        
+
         # set this flag to 1 to include LRG transcripts in the transcript effect analysis
 
         include_lrg => 1, 
@@ -150,7 +157,7 @@ sub default_options {
 
         # Human runs switch off run_var_class and set max_distance to 0 by default. To override
         # this behaviour, set this flag to 1
-        human_default_override		=> 0,
+        human_default_override  => 0,
 
         # connection parameters for the hive database, you should supply the hive_db_password
         # option on the command line to init_pipeline.pl (parameters for the target database
@@ -192,7 +199,7 @@ sub default_options {
             -host   => $self->o('hive_db_host'),
             -port   => $self->o('hive_db_port'),
             -user   => $self->o('hive_db_user'),
-            -pass   => $self->o('hive_db_password'),            
+            -pass   => $self->o('hive_db_password'),
             -dbname => $self->o('hive_db_name'),
             -driver => 'mysql',
             -reconnect_when_lost => 1
@@ -212,9 +219,14 @@ sub pipeline_wide_parameters {  # these parameter values are visible to all anal
 sub resource_classes {
     my ($self) = @_;
     return {
-          'default' => { 'LSF' => $self->o('default_lsf_options') },
-          'highmem' => { 'LSF' => $self->o('highmem_lsf_options') },
-          'medmem'  => { 'LSF' => $self->o('medmem_lsf_options') },
+          'default' => { 'LSF'   => $self->o('default_lsf_options'),
+                         'SLURM' => $self->o('default_slurm_options') },
+          'default_long' => { 'LSF'   => $self->o('default_lsf_options'),
+                              'SLURM' => $self->o('default_long_slurm_options') },
+          'highmem'      => { 'LSF'   => $self->o('highmem_lsf_options'),
+                              'SLURM' => $self->o('highmem_slurm_options') },
+          'medmem'       => { 'LSF'   => $self->o('medmem_lsf_options'),
+                              'SLURM' => $self->o('medmem_slurm_options') },
     };
 }
 
@@ -317,7 +329,7 @@ sub pipeline_analyses {
               update_diff => $self->o('update_diff'),
               @common_params,
             },
-            -rc_name   => 'default',
+            -rc_name   => ($self->o('species') !~ /homo_sapiens|human/) ? 'default' : 'default_long',
           },
           { -logic_name => 'transcript_effect',
             -module => 'Bio::EnsEMBL::Variation::Pipeline::TranscriptEffect',
@@ -365,14 +377,14 @@ sub pipeline_analyses {
               update_diff => $self->o('update_diff'),
               @common_params,
             },
-            -rc_name   => 'default',
+            -rc_name   => ($self->o('species') !~ /homo_sapiens|human/) ? 'default' : 'default_long',
           },
           { -logic_name => 'rebuild_tv_indexes',
             -module => 'Bio::EnsEMBL::Variation::Pipeline::RebuildIndexes',
             -parameters => {
               @common_params,
             },
-            -rc_name   => 'default',
+            -rc_name   => ($self->o('species') !~ /homo_sapiens|human/) ? 'default' : 'default_long',
             -wait_for => 'web_index_load',
             -flow_into => {
               1 => ['update_variation_feature'],
@@ -384,7 +396,7 @@ sub pipeline_analyses {
             -parameters => {
               @common_params,
             },
-            -rc_name   => 'default',
+            -rc_name   => ($self->o('species') !~ /homo_sapiens|human/) ? 'default' : 'default_long',
             -flow_into => {
               1 => ['check_transcript_variation']
             },
@@ -450,7 +462,7 @@ sub pipeline_analyses {
                 },
                 -input_ids      => [],
                 -hive_capacity  => 1,
-                -rc_name        => 'default',
+                -rc_name        => ($self->o('species') !~ /homo_sapiens|human/) ? 'default' : 'default_long',
                 -wait_for       => [ 'set_variation_class' ],
                 -flow_into      => {},
             },