Merge pull request #305 from Ensembl/feature/non_core_dc_pipeline

Handful of fixes and new "core_sync" group
Ensembl · Sep 18, 2020 · 0714850 · 0714850
2 parents 5b1ce5a + 51acb40
commit 0714850
Show file tree

Hide file tree

Showing 10 changed files with 43 additions and 16 deletions.
diff --git a/lib/Bio/EnsEMBL/DataCheck/Checks/CompareMetaKeys.pm b/lib/Bio/EnsEMBL/DataCheck/Checks/CompareMetaKeys.pm
@@ -91,6 +91,8 @@ sub tests {
       my $desc = "Meta key '$geneset_meta_key' is different between $compare_dbs";
       isnt($geneset, $old_geneset, $desc);
     }
+
+    $old_dba->dbc->disconnect_if_idle();
   }
 }
 

diff --git a/lib/Bio/EnsEMBL/DataCheck/Checks/CoreTables.pm b/lib/Bio/EnsEMBL/DataCheck/Checks/CoreTables.pm
@@ -30,7 +30,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME        => 'CoreTables',
   DESCRIPTION => 'Requisite core-like tables are identical to those in the core database',
-  GROUPS      => ['corelike'],
+  GROUPS      => ['core_sync', 'corelike'],
   DB_TYPES    => ['cdna', 'otherfeatures', 'rnaseq'],
   TABLES      => ['assembly', 'coord_system', 'seq_region'],
   FORCE       => 1

diff --git a/lib/Bio/EnsEMBL/DataCheck/Checks/DNAFragCore.pm b/lib/Bio/EnsEMBL/DataCheck/Checks/DNAFragCore.pm
@@ -32,7 +32,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME        => 'DNAFragCore',
   DESCRIPTION => 'Top-level sequences in the core database match dnafrags in compara database',
-  GROUPS      => ['compara', 'compara_genome_alignments', 'compara_gene_trees', 'compara_syntenies', 'compara_master'],
+  GROUPS      => ['compara', 'compara_gene_trees', 'compara_genome_alignments', 'compara_master', 'compara_syntenies', 'core_sync'],
   DATACHECK_TYPE => 'critical',
   DB_TYPES    => ['compara'],
   TABLES      => ['dnafrag', 'genome_db']

diff --git a/lib/Bio/EnsEMBL/DataCheck/Checks/ForeignKeysMultiDB.pm b/lib/Bio/EnsEMBL/DataCheck/Checks/ForeignKeysMultiDB.pm
@@ -30,7 +30,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME        => 'ForeignKeysMultiDB',
   DESCRIPTION => 'Foreign key relationships between tables from different databases are not violated',
-  GROUPS      => ['funcgen', 'schema', 'variation'],
+  GROUPS      => ['core_sync', 'funcgen', 'schema', 'variation'],
   DB_TYPES    => ['funcgen', 'variation'],
   FORCE       => 1
 };

diff --git a/lib/Bio/EnsEMBL/DataCheck/Checks/GenomeDBCore.pm b/lib/Bio/EnsEMBL/DataCheck/Checks/GenomeDBCore.pm
@@ -29,7 +29,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME        => 'GenomeDBCore',
   DESCRIPTION => 'Species, assembly, and geneset metadata are the same in core and compara databases',
-  GROUPS      => ['compara', 'compara_genome_alignments', 'compara_gene_trees', 'compara_syntenies', 'compara_master'],
+  GROUPS      => ['compara', 'compara_gene_trees', 'compara_genome_alignments', 'compara_master', 'compara_syntenies', 'core_sync'],
   DATACHECK_TYPE => 'critical',
   DB_TYPES    => ['compara'],
   TABLES      => ['genome_db'],

diff --git a/lib/Bio/EnsEMBL/DataCheck/Checks/MetaCoord.pm b/lib/Bio/EnsEMBL/DataCheck/Checks/MetaCoord.pm
@@ -29,7 +29,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME        => 'MetaCoord',
   DESCRIPTION => 'The meta_coord table is correctly populated',
-  GROUPS      => ['annotation', 'core', 'brc4_core', 'corelike', 'funcgen', 'geneset', 'protein_features', 'variation'],
+  GROUPS      => ['annotation', 'brc4_core', 'core', 'core_sync', 'corelike', 'funcgen', 'geneset', 'protein_features', 'variation'],
   DB_TYPES    => ['cdna', 'core', 'funcgen', 'otherfeatures', 'rnaseq', 'variation'],
   FORCE       => 1
 };

diff --git a/lib/Bio/EnsEMBL/DataCheck/Checks/MetaKeyConsistent.pm b/lib/Bio/EnsEMBL/DataCheck/Checks/MetaKeyConsistent.pm
@@ -30,7 +30,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME        => 'MetaKeyConsistent',
   DESCRIPTION => 'Assembly and species meta keys are identical between core and core-like databases',
-  GROUPS      => ['corelike', 'meta'],
+  GROUPS      => ['core_sync', 'corelike', 'meta'],
   DB_TYPES    => ['cdna', 'otherfeatures', 'rnaseq'],
   TABLES      => ['meta'],
   FORCE       => 1

diff --git a/lib/Bio/EnsEMBL/DataCheck/Manager.pm b/lib/Bio/EnsEMBL/DataCheck/Manager.pm
@@ -168,7 +168,11 @@ sub load_config {
     die "Config file does not exist" unless -e $self->config_file;
 
     my $json = path($self->config_file)->slurp;
-    my %config = %{ JSON->new->decode($json) };
+    my %config;
+    eval {
+      %config = %{ JSON->new->decode($json) };
+    };
+    die $self->config_file . " is not a valid json file:\n$@" if $@;
 
     foreach my $key (keys %{$config{'datacheck_params'}}) {
       if (!exists $params{$key}) {
@@ -364,9 +368,21 @@ sub read_history {
   my %history = ();
 
   if (-s $self->history_file) {
-    # slurp gets an exclusive lock on the file before reading it.
-    my $json = path($self->history_file)->slurp;
-    %history = %{ JSON->new->decode($json) };
+    # 'slurp' gets an exclusive lock on the file before reading it.
+    # But sometimes we get flock problems, if a bunch of datachecks
+    # are all completing very quickly, so have a brief pause to
+    # calm things down a bit. That doesn't always work, so give
+    # it a second go if the first attempt fails.
+    eval {
+      sleep(2);
+      my $json = path($self->history_file)->slurp;
+      %history = %{ JSON->new->decode($json) };
+    };
+    if ($@) {
+      sleep(2);
+      my $json = path($self->history_file)->slurp;
+      %history = %{ JSON->new->decode($json) };
+    }
 
     foreach (@$datachecks) {
       my $name = $_->name;

diff --git a/lib/Bio/EnsEMBL/DataCheck/Pipeline/DbDataChecks_conf.pm b/lib/Bio/EnsEMBL/DataCheck/Pipeline/DbDataChecks_conf.pm
@@ -205,7 +205,10 @@ sub pipeline_analyses {
                                 ELSE 
                                   ['RunDataChecks']
                                 ),
-                              'A->1' => ['DataCheckResults'],
+                              'A->1' =>
+                                WHEN('scalar @{#all_dbs#}' =>
+                                  ['DataCheckResults']
+                                ),
 
                             },
       -rc_name           => 'default',

diff --git a/lib/Bio/EnsEMBL/DataCheck/index.json b/lib/Bio/EnsEMBL/DataCheck/index.json
@@ -972,6 +972,7 @@
       "datacheck_type" : "critical",
       "description" : "Requisite core-like tables are identical to those in the core database",
       "groups" : [
+         "core_sync",
          "corelike"
       ],
       "name" : "CoreTables",
@@ -1001,10 +1002,11 @@
       "description" : "Top-level sequences in the core database match dnafrags in compara database",
       "groups" : [
          "compara",
-         "compara_genome_alignments",
          "compara_gene_trees",
+         "compara_genome_alignments",
+         "compara_master",
          "compara_syntenies",
-         "compara_master"
+         "core_sync"
       ],
       "name" : "DNAFragCore",
       "package_name" : "Bio::EnsEMBL::DataCheck::Checks::DNAFragCore"
@@ -1289,6 +1291,7 @@
       "datacheck_type" : "critical",
       "description" : "Foreign key relationships between tables from different databases are not violated",
       "groups" : [
+         "core_sync",
          "funcgen",
          "schema",
          "variation"
@@ -1408,10 +1411,11 @@
       "description" : "Species, assembly, and geneset metadata are the same in core and compara databases",
       "groups" : [
          "compara",
-         "compara_genome_alignments",
          "compara_gene_trees",
+         "compara_genome_alignments",
+         "compara_master",
          "compara_syntenies",
-         "compara_master"
+         "core_sync"
       ],
       "name" : "GenomeDBCore",
       "package_name" : "Bio::EnsEMBL::DataCheck::Checks::GenomeDBCore"
@@ -1610,8 +1614,9 @@
       "description" : "The meta_coord table is correctly populated",
       "groups" : [
          "annotation",
-         "core",
          "brc4_core",
+         "core",
+         "core_sync",
          "corelike",
          "funcgen",
          "geneset",
@@ -1670,6 +1675,7 @@
       "datacheck_type" : "critical",
       "description" : "Assembly and species meta keys are identical between core and core-like databases",
       "groups" : [
+         "core_sync",
          "corelike",
          "meta"
       ],