Release/101

lib/Bio/EnsEMBL/DataCheck/BaseCheck.pm

@@ -131,7 +131,11 @@ sub skip_datacheck {
 sub run_datacheck {
   # Method can be overridden by a subclass, if required.
   my $self = shift;
-  $self->tests(@_);
+  eval { $self->tests(@_) };
+  if ($@) {
+    fail("Datacheck ran without errors");
+    diag($@);
+  }
 }
 
 sub skip_tests {

lib/Bio/EnsEMBL/DataCheck/Checks/AlignmentCoordinates.pm

@@ -31,7 +31,7 @@ use constant {
   NAME           => 'AlignmentCoordinates',
   DESCRIPTION    => 'Alignment coordinates are within the length of their dnafrag',
   DATACHECK_TYPE => 'critical',
-  GROUPS         => ['compara', 'compara_pairwise_alignments', 'compara_multiple_alignments'],
+  GROUPS         => ['compara', 'compara_genome_alignments'],
   DB_TYPES       => ['compara'],
   TABLES         => ['dnafrag', 'genomic_align']
 };

lib/Bio/EnsEMBL/DataCheck/Checks/AnalysisFormat.pm

@@ -30,7 +30,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME        => 'AnalysisFormat',
   DESCRIPTION => 'Analysis logic name and date are formatted correctly',
-  GROUPS      => ['core', 'brc4_core', 'corelike'],
+  GROUPS      => ['ancestral', 'brc4_core', 'core', 'corelike'],
   DB_TYPES    => ['cdna', 'core', 'otherfeatures', 'rnaseq'],
   TABLES      => ['analysis'],
   PER_DB      => 1

lib/Bio/EnsEMBL/DataCheck/Checks/AssemblyExceptions.pm

@@ -31,7 +31,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME        => 'AssemblyExceptions',
   DESCRIPTION => 'Assembly exceptions are correctly configured',
-  GROUPS      => ['assembly', 'core', 'brc4_core'],
+  GROUPS      => ['ancestral', 'assembly', 'brc4_core', 'core'],
   DB_TYPES    => ['core'],
   TABLES      => ['analysis', 'assembly_exception', 'dna_align_feature',
                   'external_db', 'seq_region',],

lib/Bio/EnsEMBL/DataCheck/Checks/AssemblySeqregion.pm

@@ -31,7 +31,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME        => 'AssemblySeqregion',
   DESCRIPTION => 'Assembly and seq_region tables are consistent',
-  GROUPS      => ['assembly', 'core', 'brc4_core'],
+  GROUPS      => ['ancestral', 'assembly', 'brc4_core', 'core'],
   DB_TYPES    => ['core'],
   TABLES      => ['assembly', 'coord_system', 'seq_region'],
   PER_DB      => 1,
@@ -85,19 +85,15 @@ sub tests {
   is_rows_zero($self->dba, $sql_5, $desc_5);
 
   my $desc_6 = 'assembly and seq_region lengths consistent';
-  my $diag_6 = 'seq_region length != largest asm_end value';
+  my $diag_6 = 'seq_region length < largest asm_end value';
   my $sql_6  = q/
-    SELECT
-      sr.name AS seq_region_name,
-      cs.name AS coord_system_name,
-      sr.length AS seq_length,
-      MAX(a.asm_end) AS max_asm_end
+    SELECT sr.name AS seq_region_name, sr.length, cs.name AS coord_system_name
     FROM
       seq_region sr INNER JOIN
       coord_system cs ON sr.coord_system_id = cs.coord_system_id INNER JOIN
       assembly a ON a.asm_seq_region_id = sr.seq_region_id
     GROUP BY a.asm_seq_region_id
-    HAVING sr.length != MAX(a.asm_end)
+    HAVING sr.length < MAX(a.asm_end)
   /;
   is_rows_zero($self->dba, $sql_6, $desc_6, $diag_6);
 }

lib/Bio/EnsEMBL/DataCheck/Checks/BlankEnums.pm

@@ -30,7 +30,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME        => 'BlankEnums',
   DESCRIPTION => 'Enum columns do not have empty string values',
-  GROUPS      => ['compara', 'core', 'brc4_core', 'corelike', 'funcgen', 'schema', 'variation'],
+  GROUPS      => ['ancestral', 'brc4_core', 'compara', 'compara_gene_trees', 'compara_master', 'compara_syntenies', 'core', 'corelike', 'funcgen', 'schema', 'variation'],
   DB_TYPES    => ['cdna', 'compara', 'core', 'funcgen', 'otherfeatures', 'rnaseq', 'variation'],
   PER_DB      => 1
 };

lib/Bio/EnsEMBL/DataCheck/Checks/BlankNulls.pm

@@ -30,7 +30,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME        => 'BlankNulls',
   DESCRIPTION => 'Nullable columns do not have empty string values',
-  GROUPS      => ['compara', 'core', 'brc4_core', 'corelike', 'funcgen', 'schema', 'variation'],
+  GROUPS      => ['ancestral', 'brc4_core', 'compara', 'compara_gene_trees', 'compara_genome_alignments', 'compara_master', 'compara_syntenies', 'core', 'corelike', 'funcgen', 'schema', 'variation'],
   DB_TYPES    => ['cdna', 'compara', 'core', 'funcgen', 'otherfeatures', 'rnaseq', 'variation'],
   PER_DB      => 1
 };

lib/Bio/EnsEMBL/DataCheck/Checks/BlankSets.pm

@@ -0,0 +1,63 @@
+=head1 LICENSE
+
+Copyright [2018-2020] EMBL-European Bioinformatics Institute
+
+Licensed under the Apache License, Version 2.0 (the 'License');
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an 'AS IS' BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+=cut
+
+package Bio::EnsEMBL::DataCheck::Checks::BlankSets;
+
+use warnings;
+use strict;
+
+use Moose;
+use Test::More;
+use Bio::EnsEMBL::DataCheck::Test::DataCheck;
+
+extends 'Bio::EnsEMBL::DataCheck::DbCheck';
+
+use constant {
+  NAME        => 'BlankSets',
+  DESCRIPTION => 'Set columns do not have empty string values (unless default)',
+  GROUPS      => ['variation'],
+  DB_TYPES    => ['variation']
+};
+
+sub tests {
+  my ($self) = @_;
+
+  my $set_sql = q/
+    SELECT TABLE_NAME, COLUMN_NAME FROM
+      INFORMATION_SCHEMA.COLUMNS
+    WHERE
+      TABLE_SCHEMA = database() AND
+      DATA_TYPE = 'set' AND 
+      COLUMN_DEFAULT <> ''
+  /;
+
+  my $sets = $self->dba->dbc->sql_helper->execute(-SQL => $set_sql);
+
+  foreach my $set (@$sets) {
+    my ($table, $column) = @$set;
+
+    my $desc = "SET column $table.$column has no empty string values";
+    my $sql  = qq/
+      SELECT COUNT(*) FROM $table
+      WHERE $column = ''
+    /;
+    is_rows_zero($self->dba, $sql, $desc);
+  }
+}
+
+1;

lib/Bio/EnsEMBL/DataCheck/Checks/CheckCAFETable.pm

@@ -30,7 +30,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME           => 'CheckCAFETable',
   DESCRIPTION    => 'Each row should show a one-to-many relationship',
-  GROUPS         => ['compara', 'compara_protein_trees'],
+  GROUPS         => ['compara', 'compara_gene_trees'],
   DATACHECK_TYPE => 'critical',
   DB_TYPES       => ['compara'],
   TABLES         => ['cafe_species_gene']

lib/Bio/EnsEMBL/DataCheck/Checks/CheckComparaStableIDs.pm

@@ -30,7 +30,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME           => 'CheckComparaStableIDs',
   DESCRIPTION    => 'gene trees in gene_tree_root and family all have stable_ids generated',
-  GROUPS         => ['compara', 'compara_families', 'compara_protein_trees'],
+  GROUPS         => ['compara', 'compara_gene_trees'],
   DATACHECK_TYPE => 'critical',
   DB_TYPES       => ['compara'],
   TABLES         => ['family', 'gene_tree_root']

lib/Bio/EnsEMBL/DataCheck/Checks/CheckConservationScore.pm

@@ -30,7 +30,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME           => 'CheckConservationScore',
   DESCRIPTION    => 'The MLSS for GERP_CONSERVATION_SCORE should have conservation score entries',
-  GROUPS         => ['compara', 'compara_pairwise_alignments'],
+  GROUPS         => ['compara', 'compara_genome_alignments'],
   DATACHECK_TYPE => 'critical',
   DB_TYPES       => ['compara'],
   TABLES         => ['conservation_score', 'genomic_align_block', 'method_link', 'method_link_species_set', 'method_link_species_set_tag']
@@ -64,7 +64,11 @@ sub tests {
         AND method_link_species_set_id = $mlss_id
     /;
     my $desc_1 = "There is an msa_mlss_id tag for $mlss_name";
-    my $msa_mlss_id = $helper->execute_single_result( -SQL => $sql_1 );
+    my $msa_mlss_id = $helper->execute_single_result( -SQL => $sql_1, -NO_ERROR => 1 );
+    ok($msa_mlss_id, $desc_1);
+
+    # Can't test this mlss without an msa_mlss_id
+    next unless $msa_mlss_id;
 
     my $sql_2 = qq/
       SELECT COUNT(*) 

lib/Bio/EnsEMBL/DataCheck/Checks/CheckConservationScorePerBlock.pm

@@ -31,7 +31,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME        => 'CheckConservationScorePerBlock',
   DESCRIPTION => 'Multiple alignments with >3 species and >3 sequences must have a conservation score',
-  GROUPS      => ['compara', 'compara_multiple_alignments'],
+  GROUPS      => ['compara', 'compara_genome_alignments'],
   DB_TYPES    => ['compara'],
   TABLES      => ['conservation_score', 'dnafrag', 'genome_db', 'genomic_align', 'genomic_align_block', 'method_link', 'method_link_species_set', 'method_link_species_set_tag']
 };
@@ -46,8 +46,7 @@ sub tests {
     FROM method_link_species_set mlss
       JOIN method_link USING(method_link_id)
       LEFT JOIN method_link_species_set_tag mlsst ON (mlss.method_link_species_set_id = mlsst.method_link_species_set_id AND tag = "msa_mlss_id" AND value != "")
-    WHERE (type = "GERP_CONSERVATION_SCORE" 
-      OR class LIKE "ConservationScore%") 
+    WHERE type = "GERP_CONSERVATION_SCORE"
       AND tag IS NULL;
   /;
 
@@ -59,8 +58,7 @@ sub tests {
     FROM method_link_species_set 
       LEFT JOIN method_link USING(method_link_id) 
       LEFT JOIN method_link_species_set_tag USING(method_link_species_set_id) 
-    WHERE (type = "GERP_CONSERVATION_SCORE" 
-      OR class LIKE "ConservationScore%") 
+    WHERE type = "GERP_CONSERVATION_SCORE"
       AND tag = "msa_mlss_id";
   /;
 

lib/Bio/EnsEMBL/DataCheck/Checks/CheckConstrainedElementTable.pm

@@ -30,7 +30,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME           => 'CheckConstrainedElementTable',
   DESCRIPTION    => 'Each row should show a one-to-many relationship',
-  GROUPS         => ['compara', 'compara_multiple_alignments'],
+  GROUPS         => ['compara', 'compara_genome_alignments'],
   DATACHECK_TYPE => 'critical',
   DB_TYPES       => ['compara'],
   TABLES         => ['constrained_elements']

lib/Bio/EnsEMBL/DataCheck/Checks/CheckDuplicatedTaxaNames.pm

@@ -30,7 +30,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME           => 'CheckDuplicatedTaxaNames',
   DESCRIPTION    => 'Check that the ncbi_taxa_name contains only unique rows',
-  GROUPS         => ['compara'],
+  GROUPS         => ['compara', 'compara_gene_trees', 'compara_genome_alignments', 'compara_master', 'compara_syntenies'],
   DATACHECK_TYPE => 'critical',
   DB_TYPES       => ['compara'],
   TABLES         => ['ncbi_taxa_name']

lib/Bio/EnsEMBL/DataCheck/Checks/CheckEmptyLeavesTrees.pm

@@ -30,7 +30,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME           => 'CheckEmptyLeavesTrees',
   DESCRIPTION    => 'Check that none of the gene tree leaves have children',
-  GROUPS         => ['compara', 'compara_protein_trees'],
+  GROUPS         => ['compara', 'compara_gene_trees'],
   DATACHECK_TYPE => 'critical',
   DB_TYPES       => ['compara'],
   TABLES         => ['gene_tree_node']

lib/Bio/EnsEMBL/DataCheck/Checks/CheckFlatProteinTrees.pm

@@ -30,7 +30,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME           => 'CheckFlatProteinTrees',
   DESCRIPTION    => 'Check protein tree integrity ensuring number of leaves with parent node at root < 3',
-  GROUPS         => ['compara', 'compara_protein_trees'],
+  GROUPS         => ['compara', 'compara_gene_trees'],
   DATACHECK_TYPE => 'critical',
   DB_TYPES       => ['compara'],
   TABLES         => ['gene_tree_node', 'gene_tree_root']

lib/Bio/EnsEMBL/DataCheck/Checks/CheckGOCScoreStats.pm

@@ -29,15 +29,15 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME           => 'CheckGOCScoreStats',
   DESCRIPTION    => 'The number of rows for GOC have not dropped from previous release',
-  GROUPS         => ['compara', 'compara_protein_trees'],
-  DATACHECK_TYPE => 'critical',
+  GROUPS         => ['compara', 'compara_gene_trees'],
+  DATACHECK_TYPE => 'advisory',
   DB_TYPES       => ['compara'],
   TABLES         => ['homology']
 };
 
 sub tests {
   my ($self) = @_;
-  my $prev_dba = $self->registry->get_DBAdaptor('compara_prev', 'compara') || $self->get_old_dba;
+  my $prev_dba = $self->get_old_dba;
 
   my $curr_helper = $self->dba->dbc->sql_helper;
   my $prev_helper = $prev_dba->dbc->sql_helper;
@@ -54,7 +54,11 @@ sub tests {
 
   foreach my $type ( keys %$prev_results ) {
     my $desc = "There are the same number of goc_score populated rows between releases for $type";
-    cmp_ok( $curr_results->{$type}, ">=", $prev_results->{$type}, $desc );
+    cmp_ok( $curr_results->{$type} // 0, ">=", $prev_results->{$type}, $desc );
+  }
+
+  unless (%$prev_results) {
+    plan skip_all => "No MLSSs to test in this database";
   }
 }
 

lib/Bio/EnsEMBL/DataCheck/Checks/CheckGeneGainLossData.pm

@@ -30,7 +30,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME           => 'CheckGeneGainLossData',
   DESCRIPTION    => 'ncRNA and protein trees must have gene Gain/Loss trees',
-  GROUPS         => ['compara', 'compara_protein_trees'],
+  GROUPS         => ['compara', 'compara_gene_trees'],
   DATACHECK_TYPE => 'critical',
   DB_TYPES       => ['compara'],
   TABLES         => ['CAFE_gene_family', 'gene_tree_root']

lib/Bio/EnsEMBL/DataCheck/Checks/CheckGenomicAlignGenomeDBs.pm

@@ -31,7 +31,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME           => 'CheckGenomicAlignGenomeDBs',
   DESCRIPTION    => 'Check all genome_dbs for each method_link_species_set is present in genomic_aligns',
-  GROUPS         => ['compara', 'compara_multiple_alignments', 'compara_pairwise_alignments'],
+  GROUPS         => ['compara', 'compara_genome_alignments'],
   DATACHECK_TYPE => 'critical',
   DB_TYPES       => ['compara'],
   TABLES         => ['dnafrag', 'genome_db', 'genomic_align', 'genomic_align_block', 'method_link_species_set', 'species_set']
@@ -40,7 +40,7 @@ use constant {
 sub skip_tests {
     my ($self) = @_;
     my $mlss_adap = $self->dba->get_MethodLinkSpeciesSetAdaptor;
-    my @methods = qw (PECAN EPO EPO_LOW_COVERAGE LASTZ_NET LASTZ_PATCH);
+    my @methods = qw (PECAN EPO EPO_EXTENDED LASTZ_NET LASTZ_PATCH);
     my $db_name = $self->dba->dbc->dbname;
 
     my @mlsses;
@@ -60,7 +60,7 @@ sub tests {
   my $helper = $dba->dbc->sql_helper;
   my $mlss_adap = $dba->get_MethodLinkSpeciesSetAdaptor;
   my $gdb_adap = $dba->get_GenomeDBAdaptor;
-  my @mlss_types = qw ( PECAN EPO EPO_LOW_COVERAGE LASTZ_NET LASTZ_PATCH);
+  my @mlss_types = qw ( PECAN EPO EPO_EXTENDED LASTZ_NET LASTZ_PATCH);
   my $ancestral = $gdb_adap->fetch_all_by_name('ancestral_sequences');
   my @mlsses;
 

lib/Bio/EnsEMBL/DataCheck/Checks/CheckGenomicAlignMTs.pm

@@ -30,7 +30,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME        => 'CheckGenomicAlignMTs',
   DESCRIPTION => 'The multiple alignments should include all the MT sequences',
-  GROUPS      => ['compara', 'compara_multiple_alignments'],
+  GROUPS      => ['compara', 'compara_genome_alignments'],
   DATACHECK_TYPE => 'advisory',
   DB_TYPES    => ['compara'],
   TABLES      => ['dnafrag', 'genome_db', 'genomic_align', 'method_link', 'method_link_species_set', 'species_set']
@@ -39,7 +39,7 @@ use constant {
 sub skip_tests {
     my ($self) = @_;
     my $mlss_adap = $self->dba->get_MethodLinkSpeciesSetAdaptor;
-    my @methods = qw( EPO EPO_LOW_COVERAGE PECAN );
+    my @methods = qw( EPO EPO_EXTENDED PECAN );
     my $db_name = $self->dba->dbc->dbname;
 
     my @mlsses;
@@ -71,9 +71,7 @@ sub tests {
         JOIN genome_db USING(genome_db_id)
         JOIN dnafrag USING(genome_db_id)
       WHERE cellular_component = 'MT' 
-        AND (class LIKE 'GenomicAlignTree%' 
-          OR class LIKE 'GenomicAlign%multiple%') 
-        AND (type NOT LIKE 'CACTUS_HAL%')
+        AND type IN ("EPO", "EPO_EXTENDED", "PECAN")
     /;
 
   my $entries_array = $helper->execute(  

lib/Bio/EnsEMBL/DataCheck/Checks/CheckGenomicAlignTreeTable.pm

@@ -30,7 +30,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME           => 'CheckGenomicAlignTreeTable',
   DESCRIPTION    => 'Check the consistency and validity of genomic_align_tree',
-  GROUPS         => ['compara', 'compara_multiple_alignments'],
+  GROUPS         => ['compara', 'compara_genome_alignments'],
   DATACHECK_TYPE => 'critical',
   DB_TYPES       => ['compara'],
   TABLES         => ['genomic_align_tree', 'method_link_species_set']
@@ -39,7 +39,7 @@ use constant {
 sub skip_tests {
     my ($self) = @_;
     my $mlss_adap = $self->dba->get_MethodLinkSpeciesSetAdaptor;
-    my @methods = qw( EPO EPO_LOW_COVERAGE );
+    my @methods = qw( EPO EPO_EXTENDED );
     my $db_name = $self->dba->dbc->dbname;
 
     my @mlsses;
@@ -57,7 +57,7 @@ sub skip_tests {
 sub tests {
   my ($self) = @_;
   my $mlss_adap = $self->dba->get_MethodLinkSpeciesSetAdaptor;
-  my @methods = qw( EPO EPO_LOW_COVERAGE );
+  my @methods = qw( EPO EPO_EXTENDED );
   my $db_name = $self->dba->dbc->dbname;
   my $dbc = $self->dba->dbc;
   my $helper = $dbc->sql_helper;

lib/Bio/EnsEMBL/DataCheck/Checks/CheckHomology.pm

@@ -30,7 +30,7 @@ extends 'Bio::EnsEMBL::DataCheck::DbCheck';
 use constant {
   NAME           => 'CheckHomology',
   DESCRIPTION    => 'Check homology_id are all one-to-many for homology_members',
-  GROUPS         => ['compara', 'compara_protein_trees'],
+  GROUPS         => ['compara', 'compara_gene_trees'],
   DATACHECK_TYPE => 'critical',
   DB_TYPES       => ['compara'],
   TABLES         => ['homology', 'homology_member']