Skip to content

Commit 86167e9

Browse files
spetruniaspetrunia
committed
MDEV-20611: MRR scan over partitioned InnoDB table produces "Out of memory" error
Fix partitioning and DS-MRR to work together - In ha_partition::index_end(): take into account that ha_innobase (and other engines using DS-MRR) will have inited=RND when initialized for DS-MRR scan. - In ha_partition::multi_range_read_next(): if the MRR scan is using HA_MRR_NO_ASSOCIATION mode, it is not guaranteed that the partition's handler will store anything into *range_info. - In DsMrr_impl::choose_mrr_impl(): ha_partition will inquire partitions about how much memory their MRR implementation needs by passing *buffer_size=0. DS-MRR code didn't know about this (actually it used uint for buffer size calculation and would have an under-flow). Returning *buffer_size=0 made ha_partition assume that partitions do not need MRR memory and pass the same buffer to each of them. Now, this is fixed. If DS-MRR gets *buffer_size=0, it will return the amount of buffer space needed, but not more than about @@mrr_buffer_size. * Fix ha_{innobase,maria,myisam}::clone. If ha_partition uses MRR on its partitions, and partition use DS-MRR, the code will call handler->clone with TABLE (*NOT partition*) name as an argument. DS-MRR has no way of knowing the partition name, so the solution was to have the ::clone() function for the affected storage engine to ignore the name argument and get it elsewhere.
1 parent 3d4a801 commit 86167e9

13 files changed

+343
-18
lines changed

‎mysql-test/include/partition_mrr.inc

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
--source include/have_partition.inc
2+
3+
--disable_warnings
4+
drop table if exists t1,t3;
5+
--enable_warnings
6+
7+
--echo #
8+
--echo # MDEV-20611: MRR scan over partitioned InnoDB table produces "Out of memory" error
9+
--echo #
10+
create table t1(a int);
11+
insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9);
12+
13+
set @tmp=@@storage_engine;
14+
eval set storage_engine=$engine_type;
15+
16+
create table t3 (
17+
ID bigint(20) NOT NULL AUTO_INCREMENT,
18+
part_id int,
19+
key_col int,
20+
col2 int,
21+
key(key_col),
22+
PRIMARY KEY (ID,part_id)
23+
) PARTITION BY RANGE (part_id)
24+
(PARTITION p1 VALUES LESS THAN (3),
25+
PARTITION p2 VALUES LESS THAN (7),
26+
PARTITION p3 VALUES LESS THAN (10)
27+
);
28+
29+
show create table t3;
30+
set storage_engine= @tmp;
31+
32+
insert into t3 select
33+
A.a+10*B.a,
34+
A.a,
35+
B.a,
36+
123456
37+
from t1 A, t1 B;
38+
39+
set optimizer_switch='mrr=on';
40+
--replace_column 9 #
41+
explain
42+
select * from t3 force index (key_col) where key_col < 3;
43+
select * from t3 force index (key_col) where key_col < 3;
44+
45+
drop table t1,t3;
46+

‎mysql-test/main/partition_mrr_aria.result

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
drop table if exists t1,t3;
2+
#
3+
# MDEV-20611: MRR scan over partitioned InnoDB table produces "Out of memory" error
4+
#
5+
create table t1(a int);
6+
insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9);
7+
set @tmp=@@storage_engine;
8+
set storage_engine=Aria;
9+
create table t3 (
10+
ID bigint(20) NOT NULL AUTO_INCREMENT,
11+
part_id int,
12+
key_col int,
13+
col2 int,
14+
key(key_col),
15+
PRIMARY KEY (ID,part_id)
16+
) PARTITION BY RANGE (part_id)
17+
(PARTITION p1 VALUES LESS THAN (3),
18+
PARTITION p2 VALUES LESS THAN (7),
19+
PARTITION p3 VALUES LESS THAN (10)
20+
);
21+
show create table t3;
22+
Table Create Table
23+
t3 CREATE TABLE `t3` (
24+
`ID` bigint(20) NOT NULL AUTO_INCREMENT,
25+
`part_id` int(11) NOT NULL,
26+
`key_col` int(11) DEFAULT NULL,
27+
`col2` int(11) DEFAULT NULL,
28+
PRIMARY KEY (`ID`,`part_id`),
29+
KEY `key_col` (`key_col`)
30+
) ENGINE=Aria DEFAULT CHARSET=latin1
31+
PARTITION BY RANGE (`part_id`)
32+
(PARTITION `p1` VALUES LESS THAN (3) ENGINE = Aria,
33+
PARTITION `p2` VALUES LESS THAN (7) ENGINE = Aria,
34+
PARTITION `p3` VALUES LESS THAN (10) ENGINE = Aria)
35+
set storage_engine= @tmp;
36+
insert into t3 select
37+
A.a+10*B.a,
38+
A.a,
39+
B.a,
40+
123456
41+
from t1 A, t1 B;
42+
set optimizer_switch='mrr=on';
43+
explain
44+
select * from t3 force index (key_col) where key_col < 3;
45+
id select_type table type possible_keys key key_len ref rows Extra
46+
1 SIMPLE t3 range key_col key_col 5 NULL # Using where; Rowid-ordered scan
47+
select * from t3 force index (key_col) where key_col < 3;
48+
ID part_id key_col col2
49+
1 0 0 123456
50+
1 1 0 123456
51+
2 2 0 123456
52+
10 0 1 123456
53+
11 1 1 123456
54+
12 2 1 123456
55+
20 0 2 123456
56+
21 1 2 123456
57+
22 2 2 123456
58+
3 3 0 123456
59+
4 4 0 123456
60+
5 5 0 123456
61+
6 6 0 123456
62+
13 3 1 123456
63+
14 4 1 123456
64+
15 5 1 123456
65+
16 6 1 123456
66+
23 3 2 123456
67+
24 4 2 123456
68+
25 5 2 123456
69+
26 6 2 123456
70+
7 7 0 123456
71+
8 8 0 123456
72+
9 9 0 123456
73+
17 7 1 123456
74+
18 8 1 123456
75+
19 9 1 123456
76+
27 7 2 123456
77+
28 8 2 123456
78+
29 9 2 123456
79+
drop table t1,t3;

‎mysql-test/main/partition_mrr_aria.test

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
let $engine_type= Aria;
2+
--source include/partition_mrr.inc

‎mysql-test/main/partition_mrr_innodb.result

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
drop table if exists t1,t3;
2+
#
3+
# MDEV-20611: MRR scan over partitioned InnoDB table produces "Out of memory" error
4+
#
5+
create table t1(a int);
6+
insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9);
7+
set @tmp=@@storage_engine;
8+
set storage_engine=InnoDB;
9+
create table t3 (
10+
ID bigint(20) NOT NULL AUTO_INCREMENT,
11+
part_id int,
12+
key_col int,
13+
col2 int,
14+
key(key_col),
15+
PRIMARY KEY (ID,part_id)
16+
) PARTITION BY RANGE (part_id)
17+
(PARTITION p1 VALUES LESS THAN (3),
18+
PARTITION p2 VALUES LESS THAN (7),
19+
PARTITION p3 VALUES LESS THAN (10)
20+
);
21+
show create table t3;
22+
Table Create Table
23+
t3 CREATE TABLE `t3` (
24+
`ID` bigint(20) NOT NULL AUTO_INCREMENT,
25+
`part_id` int(11) NOT NULL,
26+
`key_col` int(11) DEFAULT NULL,
27+
`col2` int(11) DEFAULT NULL,
28+
PRIMARY KEY (`ID`,`part_id`),
29+
KEY `key_col` (`key_col`)
30+
) ENGINE=InnoDB DEFAULT CHARSET=latin1
31+
PARTITION BY RANGE (`part_id`)
32+
(PARTITION `p1` VALUES LESS THAN (3) ENGINE = InnoDB,
33+
PARTITION `p2` VALUES LESS THAN (7) ENGINE = InnoDB,
34+
PARTITION `p3` VALUES LESS THAN (10) ENGINE = InnoDB)
35+
set storage_engine= @tmp;
36+
insert into t3 select
37+
A.a+10*B.a,
38+
A.a,
39+
B.a,
40+
123456
41+
from t1 A, t1 B;
42+
set optimizer_switch='mrr=on';
43+
explain
44+
select * from t3 force index (key_col) where key_col < 3;
45+
id select_type table type possible_keys key key_len ref rows Extra
46+
1 SIMPLE t3 range key_col key_col 5 NULL # Using where; Rowid-ordered scan
47+
select * from t3 force index (key_col) where key_col < 3;
48+
ID part_id key_col col2
49+
1 0 0 123456
50+
1 1 0 123456
51+
2 2 0 123456
52+
10 0 1 123456
53+
11 1 1 123456
54+
12 2 1 123456
55+
20 0 2 123456
56+
21 1 2 123456
57+
22 2 2 123456
58+
3 3 0 123456
59+
4 4 0 123456
60+
5 5 0 123456
61+
6 6 0 123456
62+
13 3 1 123456
63+
14 4 1 123456
64+
15 5 1 123456
65+
16 6 1 123456
66+
23 3 2 123456
67+
24 4 2 123456
68+
25 5 2 123456
69+
26 6 2 123456
70+
7 7 0 123456
71+
8 8 0 123456
72+
9 9 0 123456
73+
17 7 1 123456
74+
18 8 1 123456
75+
19 9 1 123456
76+
27 7 2 123456
77+
28 8 2 123456
78+
29 9 2 123456
79+
drop table t1,t3;

‎mysql-test/main/partition_mrr_innodb.test

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
--source include/have_innodb.inc
2+
let $engine_type= InnoDB;
3+
4+
--source include/partition_mrr.inc

‎mysql-test/main/partition_mrr_myisam.result

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
drop table if exists t1,t3;
2+
#
3+
# MDEV-20611: MRR scan over partitioned InnoDB table produces "Out of memory" error
4+
#
5+
create table t1(a int);
6+
insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9);
7+
set @tmp=@@storage_engine;
8+
set storage_engine=myisam;
9+
create table t3 (
10+
ID bigint(20) NOT NULL AUTO_INCREMENT,
11+
part_id int,
12+
key_col int,
13+
col2 int,
14+
key(key_col),
15+
PRIMARY KEY (ID,part_id)
16+
) PARTITION BY RANGE (part_id)
17+
(PARTITION p1 VALUES LESS THAN (3),
18+
PARTITION p2 VALUES LESS THAN (7),
19+
PARTITION p3 VALUES LESS THAN (10)
20+
);
21+
show create table t3;
22+
Table Create Table
23+
t3 CREATE TABLE `t3` (
24+
`ID` bigint(20) NOT NULL AUTO_INCREMENT,
25+
`part_id` int(11) NOT NULL,
26+
`key_col` int(11) DEFAULT NULL,
27+
`col2` int(11) DEFAULT NULL,
28+
PRIMARY KEY (`ID`,`part_id`),
29+
KEY `key_col` (`key_col`)
30+
) ENGINE=MyISAM DEFAULT CHARSET=latin1
31+
PARTITION BY RANGE (`part_id`)
32+
(PARTITION `p1` VALUES LESS THAN (3) ENGINE = MyISAM,
33+
PARTITION `p2` VALUES LESS THAN (7) ENGINE = MyISAM,
34+
PARTITION `p3` VALUES LESS THAN (10) ENGINE = MyISAM)
35+
set storage_engine= @tmp;
36+
insert into t3 select
37+
A.a+10*B.a,
38+
A.a,
39+
B.a,
40+
123456
41+
from t1 A, t1 B;
42+
set optimizer_switch='mrr=on';
43+
explain
44+
select * from t3 force index (key_col) where key_col < 3;
45+
id select_type table type possible_keys key key_len ref rows Extra
46+
1 SIMPLE t3 range key_col key_col 5 NULL # Using where; Rowid-ordered scan
47+
select * from t3 force index (key_col) where key_col < 3;
48+
ID part_id key_col col2
49+
1 0 0 123456
50+
1 1 0 123456
51+
2 2 0 123456
52+
10 0 1 123456
53+
11 1 1 123456
54+
12 2 1 123456
55+
20 0 2 123456
56+
21 1 2 123456
57+
22 2 2 123456
58+
3 3 0 123456
59+
4 4 0 123456
60+
5 5 0 123456
61+
6 6 0 123456
62+
13 3 1 123456
63+
14 4 1 123456
64+
15 5 1 123456
65+
16 6 1 123456
66+
23 3 2 123456
67+
24 4 2 123456
68+
25 5 2 123456
69+
26 6 2 123456
70+
7 7 0 123456
71+
8 8 0 123456
72+
9 9 0 123456
73+
17 7 1 123456
74+
18 8 1 123456
75+
19 9 1 123456
76+
27 7 2 123456
77+
28 8 2 123456
78+
29 9 2 123456
79+
drop table t1,t3;

‎mysql-test/main/partition_mrr_myisam.test

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
let $engine_type= myisam;
2+
3+
--source include/partition_mrr.inc

‎sql/ha_partition.cc

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5478,6 +5478,13 @@ int ha_partition::index_end()
54785478
if ((tmp= (*file)->ha_index_end()))
54795479
error= tmp;
54805480
}
5481+
else if ((*file)->inited == RND)
5482+
{
5483+
// Possible due to MRR
5484+
int tmp;
5485+
if ((tmp= (*file)->ha_rnd_end()))
5486+
error= tmp;
5487+
}
54815488
} while (*(++file));
54825489
destroy_record_priority_queue();
54835490
DBUG_RETURN(error);
@@ -6519,8 +6526,11 @@ int ha_partition::multi_range_read_next(range_id_t *range_info)
65196526
else if (unlikely((error= handle_unordered_next(table->record[0], FALSE))))
65206527
DBUG_RETURN(error);
65216528

6522-
*range_info=
6523-
((PARTITION_KEY_MULTI_RANGE *) m_range_info[m_last_part])->ptr;
6529+
if (!(m_mrr_mode & HA_MRR_NO_ASSOCIATION))
6530+
{
6531+
*range_info=
6532+
((PARTITION_KEY_MULTI_RANGE *) m_range_info[m_last_part])->ptr;
6533+
}
65246534
}
65256535
DBUG_RETURN(0);
65266536
}

‎sql/multi_range_read.cc

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1589,11 +1589,10 @@ bool DsMrr_impl::choose_mrr_impl(uint keyno, ha_rows rows, uint *flags,
15891589
}
15901590

15911591
uint add_len= share->key_info[keyno].key_length + primary_file->ref_length;
1592-
*bufsz -= add_len;
1593-
if (get_disk_sweep_mrr_cost(keyno, rows, *flags, bufsz, &dsmrr_cost))
1592+
if (get_disk_sweep_mrr_cost(keyno, rows, *flags, bufsz, add_len,
1593+
&dsmrr_cost))
15941594
return TRUE;
1595-
*bufsz += add_len;
1596-
1595+
15971596
bool force_dsmrr;
15981597
/*
15991598
If mrr_cost_based flag is not set, then set cost of DS-MRR to be minimum of
@@ -1682,6 +1681,11 @@ static void get_sort_and_sweep_cost(TABLE *table, ha_rows nrows, Cost_estimate *
16821681
@param rows E(Number of rows to be scanned)
16831682
@param flags Scan parameters (HA_MRR_* flags)
16841683
@param buffer_size INOUT Buffer size
1684+
IN: Buffer of size 0 means the function
1685+
will determine the best size and return it.
1686+
@param extra_mem_overhead Extra memory overhead of the MRR implementation
1687+
(the function assumes this many bytes of buffer
1688+
space will not be usable by DS-MRR)
16851689
@param cost OUT The cost
16861690
16871691
@retval FALSE OK
@@ -1690,7 +1694,9 @@ static void get_sort_and_sweep_cost(TABLE *table, ha_rows nrows, Cost_estimate *
16901694
*/
16911695

16921696
bool DsMrr_impl::get_disk_sweep_mrr_cost(uint keynr, ha_rows rows, uint flags,
1693-
uint *buffer_size, Cost_estimate *cost)
1697+
uint *buffer_size,
1698+
uint extra_mem_overhead,
1699+
Cost_estimate *cost)
16941700
{
16951701
ulong max_buff_entries, elem_size;
16961702
ha_rows rows_in_full_step;
@@ -1700,11 +1706,24 @@ bool DsMrr_impl::get_disk_sweep_mrr_cost(uint keynr, ha_rows rows, uint flags,
17001706

17011707
elem_size= primary_file->ref_length +
17021708
sizeof(void*) * (!MY_TEST(flags & HA_MRR_NO_ASSOCIATION));
1703-
max_buff_entries = *buffer_size / elem_size;
17041709

1705-
if (!max_buff_entries)
1710+
if (!*buffer_size)
1711+
{
1712+
/*
1713+
We are requested to determine how much memory we need.
1714+
Request memory to finish the scan in one pass but do not request
1715+
more than @@mrr_buff_size.
1716+
*/
1717+
*buffer_size = MY_MIN(extra_mem_overhead + rows*elem_size,
1718+
MY_MAX(table->in_use->variables.mrr_buff_size,
1719+
extra_mem_overhead));
1720+
}
1721+
1722+
if (elem_size + extra_mem_overhead > *buffer_size)
17061723
return TRUE; /* Buffer has not enough space for even 1 rowid */
17071724

1725+
max_buff_entries = (*buffer_size - extra_mem_overhead) / elem_size;
1726+
17081727
/* Number of iterations we'll make with full buffer */
17091728
n_full_steps= (uint)floor(rows2double(rows) / max_buff_entries);
17101729

0 commit comments

Comments
 (0)