Skip to content

Commit

Permalink
MDEV-10604 Create a list of unstable MTR tests to be disabled in dist…
Browse files Browse the repository at this point in the history 
…ribution builds

- mysql-test/unstable-tests list is created, it includes
  = tests identified as unstable by Debian;
  = tests which failed in buildbot on 10.0 over the last ~6 months and were not fixed;
  = tests which have been recently modified or newly added
- '*' wildcard is now supported in skip lists
  • Loading branch information
@elenst
elenst committed Aug 24, 2016
1 parent ed99e2c commit 5bbe929
Show file tree
Hide file tree
Showing 2 changed files with 274 additions and 3 deletions.
23 changes: 20 additions & 3 deletions mysql-test/lib/mtr_cases.pm
View file
Expand Up @@ -313,6 +313,7 @@ sub combinations_from_file($$)
}

our %disabled;
our %disabled_wildcards;
sub parse_disabled {
my ($filename, $suitename) = @_;

Expand All @@ -321,10 +322,18 @@ sub parse_disabled {
chomp;
next if /^\s*#/ or /^\s*$/;
mtr_error("Syntax error in $filename line $.")
unless /^\s*(?:([-0-9A-Za-z_\/]+)\.)?([-0-9A-Za-z_]+)\s*:\s*(.*?)\s*$/;
mtr_error("Wrong suite name in $filename line $.")
unless /^\s*(?:([-0-9A-Za-z_\/]+)\.)?([-0-9A-Za-z_\*]+)\s*:\s*(.*?)\s*$/;
mtr_error("Wrong suite name in $filename line $.: suitename = $suitename but the file says $1")
if defined $1 and defined $suitename and $1 ne $suitename;
$disabled{($1 || $suitename || '') . ".$2"} = $3;
my ($sname, $casename, $text)= (($1 || $suitename || ''), $2, $3);

if ($casename =~ /\*/) {
# Wildcard
$disabled_wildcards{$sname . ".$casename"}= $text;
}
else {
$disabled{$sname . ".$casename"}= $text;
}
}
close DISABLED;
}
Expand Down Expand Up @@ -721,6 +730,14 @@ sub collect_one_test_case {
# Check for disabled tests
# ----------------------------------------------------------------------
my $disable = $disabled{".$tname"} || $disabled{$name};
if (not $disable) {
foreach my $w (keys %disabled_wildcards) {
if ($name =~ /^$w/) {
$disable= $disabled_wildcards{$w};
last;
}
}
}
if (not defined $disable and $suite->{parent}) {
$disable = $disabled{$suite->{parent}->{name} . ".$tname"};
}
Expand Down
254 changes: 254 additions & 0 deletions mysql-test/unstable-tests
View file
@@ -0,0 +1,254 @@
##############################################################################
#
# List the test cases which, unlike tests from disabled.def files,
# can still be run on the current tree meaningfully, but are known
# or suspected to fail sporadically on different reasons.
#
# Most common reasons are either test failures observed in buildbot,
# or recent modifications to the tests which make their stability
# unknown.
#
# Tests included due to recent modifications are later removed from the
# list, if during a certain period they do not fail (and are not
# modified again). Tests included due to intermittent failures are
# removed when corresponding bug reports are closed.
#
# Separate the test case name and the comment with ':'.
#
# <suitename>.<testcasename> : MDEV-xxxxx - <comment>
#
# '*' wildcard in testcase names is supported.
#
# To use the list, run MTR with --skip-test-list=unstable-tests option.
#
##############################################################################

main.bootstrap : Modified on 2016-06-18 (MDEV-9969)
main.create_delayed : MDEV-10605 - failed with timeout
main.create_or_replace : Modified on 2016-06-23 (MDEV-9728)
main.ctype_recoding : Modified on 2016-06-10 (MDEV-10181)
main.ctype_utf8 : Modified on 2016-06-21 (merge)
main.ctype_utf8mb4 : Modified on 2016-06-21 (merge)
main.events_1 : Modified on 2016-06-21 (MDEV-9524)
main.func_group : Modified on 2016-08-08 (MDEV-10468)
main.func_in : Modified on 2016-06-20 (MDEV-10020)
main.func_math : Modified on 2016-08-10 (merge)
main.func_misc : Modified on 2016-08-10 (merge)
main.grant2 : Modified on 2016-07-18 (MDEV-8569)
main.help : Modified on 2016-06-21 (MDEV-9524)
main.host_cache_size_functionality : MDEV-10606 - sporadic failure on shutdown
main.index_intersect_innodb : MDEV-10643 - failed with timeout
main.index_merge_innodb : MDEV-7142 - sporadic wrong execution plan
main.information_schema_stats : Modified on 2016-07-25 (MDEV-10428)
main.innodb_mysql_lock : MDEV-7861 - sporadic lock detection failure
main.insert_innodb : Modified on 2016-06-14 (merge from upstream)
main.loaddata : Modified on 2016-08-10 (merge)
main.locale : Modified on 2016-06-21 (merge)
main.mdev-504 : MDEV-10607 - sporadic "can't connect"
main.mdev375 : MDEV-10607 - sporadic "can't connect"
main.merge : MDEV-10607 - sporadic "can't connect"
main.multi_update : Modified on 2016-06-20 (MDEV-5973)
main.myisam_enable_keys-10506 : New test, added on 2016-08-10 (MDEV-10506)
main.mysqlcheck : Modified on 2016-08-10 (merge)
main.mysqldump : MDEV-10512 - sporadic assertion failure
main.mysqltest : MDEV-9269 - fails on Alpha
main.named_pipe : Modified on 2016-08-02 (MDEV-10383)
main.openssl_1 : Modified on 2016-07-11 (MDEV-10211)
main.parser : Modified on 2016-06-21 (merge)
main.pool_of_threads : MDEV-10100 - sporadic error on detecting max connections
main.ps_1general : Modified on 2016-07-12 (merge)
main.range : Modified on 2016-08-10 (merge)
main.range_mrr_icp : Modified on 2016-08-10 (merge)
main.query_cache : MDEV-10611 - sporadic mutex problem
main.shutdown : MDEV-10612 - sporadic crashes
main.sp-prelocking : Modified on 2016-08-10 (merge)
main.sp-security : MDEV-10607 - sporadic "can't connect"
main.ssl : MDEV-10211 - different ciphers on some platforms
main.ssl_ca : Modified on 2016-07-11 (MDEV-10211)
main.ssl_compress : Modified on 2016-07-11 (MDEV-10211)
main.ssl_timeout : Modified on 2016-07-11 (MDEV-10211)
main.stat_tables_par_innodb : MDEV-10515 - sporadic wrong results
main.status_user : Modified on 2016-06-20 (MDEV-8633)
main.subselect_innodb : MDEV-10614 - sporadic wrong results
main.temp_table : Modified on 2016-06-18 (MDEV-8569)
main.type_date : Modified on 2016-08-10 (merge)
main.type_datetime : Modified on 2016-06-16 (MDEV-9374)
main.view : Modified on 2016-08-10 (merge)
main.xtradb_mrr : Modified on 2016-08-04 (MDEV-9946)

#----------------------------------------------------------------

archive.archive-big : MDEV-10615 - table is marked as crashed
archive.discover : MDEV-10510 - table is marked as crashed

#----------------------------------------------------------------

binlog.binlog_commit_wait : MDEV-10150 - Error: too much time elapsed
binlog.binlog_dmls_on_tmp_tables_readonly : New test, added on 2016-05-04 (upstream)
binlog.binlog_xa_recover : MDEV-8517 - Extra checkpoint

#----------------------------------------------------------------

connect.tbl : MDEV-9844, MDEV-10179 - sporadic crashes, valgrind warnings, wrong results
connect.jdbc : New test, added on 2016-07-15
connect.jdbc-new : New test, added on 2016-07-14
connect.jdbc-oracle : New test, added on 2016-07-13
connect.jdbc-postgresql : New test, added on 2016-07-13

#----------------------------------------------------------------

federated.federatedx : MDEV-10617 - Wrong checksum, timeouts
federated.federated_innodb : MDEV-10617, MDEV-10417 - Wrong checksum, timeouts, fails on Mips
federated.federated_partition : MDEV-10417 - Fails on Mips
federated.federated_transactions : MDEV-10617, MDEV-10417 - Wrong checksum, timeouts, fails on Mips

#----------------------------------------------------------------

funcs_1.processlist_priv_no_prot : Include file modified on 2016-07-12 (merge)
funcs_1.processlist_priv_ps : Include file modified on 2016-07-12 (merge)

#----------------------------------------------------------------

innodb.binlog_consistent : MDEV-10618 - Server fails to start
innodb.innodb-alter-table : MDEV-10619 - Testcase timeout
innodb.innodb-alter-tempfile : Modified on 2016-08-09 (MDEV-10469)
innodb.innodb_corrupt_bit : Modified on 2016-06-21 (merge)
innodb.innodb_bug30423 : MDEV-7311 - Wrong number of rows in the plan
innodb.innodb-fk-warnings : Modified on 2016-07-18 (MDEV-8569)
innodb.innodb-fkcheck : Modified on 2016-06-13 (MDEV-10083)
innodb.innodb-wl5522 : rdiff file modified on 2016-08-10 (merge)
innodb.innodb-wl5522-debug-zip : MDEV-10427 - Warning: database page corruption

#----------------------------------------------------------------

mroonga/storage.column_datetime_32bit_2038 : Wrong result on Alpha
mroonga/storage.column_datetime_32bit_before_unix_epoch : Wrong result on Alpha
mroonga/storage.column_datetime_32bit_max : Wrong result on Alpha
mroonga/storage.column_datetime_32bit_out_of_range : Wrong result on Alpha
mroonga/storage.index_multiple_column_unique_date_32bit_equal : Wrong result on Alpha
mroonga/storage.index_multiple_column_unique_date_order_32bit_desc : Wrong result on Alpha

#----------------------------------------------------------------

multi_source.gtid : MDEV-10620, MDEV-10417 - Timeout in wait condition, fails on Mips
multi_source.multisource : MDEV-10417 - Fails on Mips
multi_source.simple : MDEV-4633 - Wrong slave status output
multi_source.status_vars : MDEV-4632 - failed while waiting for Slave_received_heartbeats

#----------------------------------------------------------------

parts.partition_float_myisam : MDEV-10621 - Testcase timeout
parts.partition_int_myisam : MDEV-10621 - Testcase timeout

#----------------------------------------------------------------

perfschema.digest_table_full : Modified on 2016-06-21 (merge)
perfschema.func_file_io : MDEV-5708 - fails for s390x
perfschema.func_mutex : MDEV-5708 - fails for s390x
perfschema.rpl_gtid_func : Modified on 2016-06-21 (merge)
perfschema.sizing_low : Modified on 2016-04-26 (5.6.30 merge)
perfschema.socket_summary_by_event_name_func : MDEV-10622 - Socket summary tables do not match
perfschema.start_server_low_digest : Modified on 2016-06-21 (merge)
perfschema.statement_digest : Modified on 2016-06-21 (merge)
perfschema.statement_digest_consumers : Modified on 2016-06-21 (merge)
perfschema.statement_digest_long_query : Modified on 2016-06-21 (merge)
perfschema.table_name : New test, added on 2016-04-26 (5.6.30 merge)

#----------------------------------------------------------------

plugins.feedback_plugin_send : MDEV-7932 - ssl failed for url
plugins.pam : Modified on 2016-08-03 (MDEV-7329)
plugins.pam_cleartext : Modified on 2016-08-03
plugins.server_audit : MDEV-9562 - crashes on sol10-sparc
plugins.thread_pool_server_audit : MDEV-9562 - crashes on sol10-sparc

#----------------------------------------------------------------

roles.rpl_grant_revoke_current_role-8638 : New test, added on 2016-06-20 (MDEV-8638)
roles.set_role-9614 : New test, added on 2016-05-30 (MDEV-9614)

#----------------------------------------------------------------

rpl.last_insert_id : MDEV-10625 - warnings in error log
rpl.rpl_auto_increment : MDEV-10417 - Fails on Mips
rpl.rpl_auto_increment_bug45679 : MDEV-10417 - Fails on Mips
rpl.rpl_auto_increment_update_failure : MDEV-10625 - warnings in error log
rpl.rpl_binlog_index : MDEV-9501 - Warning: failed registering on master
rpl.rpl_checksum_cache : MDEV-10626 - Testcase timeout
rpl.rpl_circular_for_4_hosts : MDEV-10627 - Testcase timeout
rpl.rpl_ddl : MDEV-10417 - Fails on Mips
rpl.rpl_gtid_crash : MDEV-9501 - Warning: failed registering on master
rpl.rpl_gtid_master_promote : MDEV-10628 - Timeout in sync_with_master
rpl.rpl_gtid_stop_start : MDEV-10629 - Crash on shutdown
rpl.rpl_gtid_until : MDEV-10625 - warnings in error log
rpl.rpl_ignore_table : Modified on 2016-06-22
rpl.rpl_innodb_bug30888 : MDEV-10417 - Fails on Mips
rpl.rpl_insert : MDEV-9329 - Fails on Ubuntu/s390x
rpl.rpl_insert_delayed : MDEV-9329 - Fails on Ubuntu/s390x
rpl.rpl_invoked_features : MDEV-10417 - Fails on Mips
rpl.rpl_mdev6020 : MDEV-10630, MDEV-10417 - Timeouts, fails on Mips
rpl.rpl_mdev6386 : MDEV-10631 - Wrong result on slave
rpl.rpl_parallel : MDEV-10632, MDEV-10653 - Failures to sync, timeouts
rpl.rpl_parallel_temptable : MDEV-10356 - Crash in close_thread_tables
rpl.rpl_partition_innodb : MDEV-10417 - Fails on Mips
rpl.rpl_row_drop_create_temp_table : MDEV-10626 - Testcase timeout
rpl.rpl_row_sp001 : MDEV-9329 - Fails on Ubuntu/s390x
rpl.rpl_semi_sync_uninstall_plugin : MDEV-7140 - Wrong plugin status
rpl.rpl_slave_grp_exec : MDEV-10514 - Unexpected deadlock
rpl.rpl_switch_stm_row_mixed : MDEV-10611 - Wrong usage of mutex
rpl.rpl_sync : MDEV-10633 - Database page corruption
rpl.rpl_temporary_error2 : MDEV-10634 - Wrong number of retries
rpl.sec_behind_master-5114 : MDEV-8518 - Wrong value of Seconds_Behind_Master
rpl.rpl_skip_replication : MDEV-9268 - Fails with timeout in sync_slave_with_master on Alpha

#----------------------------------------------------------------

spider.* : MDEV-9329 - tests are too memory-consuming

spider/bg.direct_aggregate : MDEV-7098 - Trying to unlock mutex that wasn't locked
spider/bg.direct_aggregate_part : MDEV-7098 - Trying to unlock mutex that wasn't locked
spider/bg.ha : MDEV-7914, MDEV-9329 - Crash, failures on s390x
spider/bg.ha_part : MDEV-9329 - Fails on Ubuntu/s390x
spider/bg.spider_fixes : MDEV-7098, MDEV-9329 - Mutex problem, failures on s390x
spider/bg.vp_fixes : MDEV-9329 - Fails on Ubuntu/s390x

#----------------------------------------------------------------

stress.ddl_innodb : MDEV-10635 - Testcase timeout

#----------------------------------------------------------------

sys_vars.autocommit_func2 : MDEV-9329 - Fails on Ubuntu/s390x
sys_vars.general_log_file_basic : Modified on 2016-08-09 (MDEV-10465)
sys_vars.slow_query_log_file_basic : Modified on 2016-08-09 (MDEV-10465)
sys_vars.innodb_buffer_pool_dump_pct_basic : MDEV-10651 - sporadic failure on file_exists

#----------------------------------------------------------------

tokudb.background_job_manager : MDEV-10327 - Assertion failure on server shutdown
tokudb.cluster_filter_unpack_varchar : MDEV-10636 - Wrong execution plan
tokudb.* : MDEV-9891 - massive crashes on shutdown
tokudb_alter_table.* : MDEV-9891 - massive crashes on shutdown
tokudb_bugs.checkpoint_lock : MDEV-10637 - Wrong processlist output
tokudb_bugs.checkpoint_lock_3 : MDEV-10637 - Wrong processlist output
tokudb_bugs.* : MDEV-9891 - massive crashes on shutdown
tokudb_parts.* : MDEV-9891 - massive crashes on shutdown
rpl-tokudb.* : MDEV-9891 - massive crashes on shutdown, also modified on 2016-06-10 (Merge)
tokudb/tokudb_add_index.* : MDEV-9891 - massive crashes on shutdown
tokudb/tokudb_backup.* : MDEV-9891 - massive crashes on shutdown
tokudb/tokudb_mariadb.* : MDEV-9891 - massive crashes on shutdown
tokudb/tokudb_sys_vars.* : MDEV-9891 - massive crashes on shutdown
tokudb/tokudb_rpl.* : MDEV-9891 - massive crashes on shutdown


#----------------------------------------------------------------

unit.ma_test_loghandler : MDEV-10638 - record read not ok

#----------------------------------------------------------------

vcol.charsets : Added on 2016-06-23
vcol.not_supported : MDEV-10639 - Testcase timeout
vcol.vcol_keys_innodb : MDEV-10639 - Testcase timeout

#----------------------------------------------------------------

0 comments on commit 5bbe929

Please sign in to comment.