Skip to content

Commit bef6b19

Browse files
Kentokuvuvova
Kentoku
authored and
vuvova
committed
MDEV-16520 Out-Of-Memory running big aggregate query on Spider Engine
Change default value of the followings quick_mode 0 -> 3 quick_page_size 100 -> 1024 Add the following parameter for limiting result page size by byte - quick_page_byte(qpb) Number of bytes in a page when acquisition one by one. When quick_mode is 1 or 2, Spider stores at least 1 record even if quick_page_byte is smaller than 1 record. When quick_mode is 3, quick_page_byte is used for judging using temporary table. That is given to priority when server parameter spider_quick_page_byte is set. The default value is 10485760 Fix "out of sync" issue at using quick_mode = 1 or 2
1 parent 3cb7c5f commit bef6b19

33 files changed

+3816
-30
lines changed

‎storage/spider/ha_spider.cc

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7378,6 +7378,8 @@ int ha_spider::rnd_init(
73787378
#endif
73797379
if (quick_targets[roop_count])
73807380
{
7381+
spider_db_free_one_quick_result(
7382+
(SPIDER_RESULT *) result_list.current);
73817383
DBUG_ASSERT(quick_targets[roop_count] ==
73827384
conns[roop_count]->quick_target);
73837385
DBUG_PRINT("info", ("spider conn[%p]->quick_target=NULL",

‎storage/spider/hs_client/hstcpcli.cpp

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ struct hstcpcli : public hstcpcli_i, private noncopyable {
6767
virtual int get_result(hstresult& result);
6868
virtual const string_ref *get_next_row();
6969
virtual const string_ref *get_next_row_from_result(hstresult& result);
70+
virtual size_t get_row_size();
71+
virtual size_t get_row_size_from_result(hstresult& result);
7072
virtual void response_buf_remove();
7173
virtual int get_error_code();
7274
virtual String& get_error();
@@ -93,6 +95,7 @@ struct hstcpcli : public hstcpcli_i, private noncopyable {
9395
string_buffer writebuf;
9496
size_t response_end_offset; /* incl newline */
9597
size_t cur_row_offset;
98+
size_t cur_row_size;
9699
size_t num_flds;
97100
size_t num_req_bufd; /* buffered but not yet sent */
98101
size_t num_req_sent; /* sent but not yet received */
@@ -104,8 +107,9 @@ struct hstcpcli : public hstcpcli_i, private noncopyable {
104107
};
105108

106109
hstcpcli::hstcpcli(const socket_args& args)
107-
: sargs(args), response_end_offset(0), cur_row_offset(0), num_flds(0),
108-
num_req_bufd(0), num_req_sent(0), num_req_rcvd(0), error_code(0), errno_buf(0)
110+
: sargs(args), response_end_offset(0), cur_row_offset(0), cur_row_size(0),
111+
num_flds(0), num_req_bufd(0), num_req_sent(0), num_req_rcvd(0),
112+
error_code(0), errno_buf(0)
109113
{
110114
String err;
111115
SPD_INIT_DYNAMIC_ARRAY2(&flds, sizeof(string_ref), NULL, 16, 16, MYF(MY_WME));
@@ -503,6 +507,7 @@ hstcpcli::response_recv(size_t& num_flds_r)
503507
}
504508
return set_error(resp_code, e);
505509
}
510+
cur_row_size = 0;
506511
cur_row_offset = start - readbuf.begin();
507512
DBG(fprintf(stderr, "[%s] ro=%zu eol=%zu\n",
508513
String(readbuf.begin(), readbuf.begin() + response_end_offset)
@@ -529,6 +534,7 @@ hstcpcli::get_result(hstresult& result)
529534
result.readbuf.space_wrote(response_end_offset);
530535
result.response_end_offset = response_end_offset;
531536
result.num_flds = num_flds;
537+
result.cur_row_size = cur_row_size;
532538
result.cur_row_offset = cur_row_offset;
533539
if (result.flds.max_element < num_flds)
534540
{
@@ -566,6 +572,7 @@ hstcpcli::get_next_row()
566572
((string_ref *) flds.buffer)[i] = string_ref(fld_begin, wp);
567573
}
568574
}
575+
cur_row_size = start - (readbuf.begin() + cur_row_offset);
569576
cur_row_offset = start - readbuf.begin();
570577
return (string_ref *) flds.buffer;
571578
}
@@ -597,10 +604,24 @@ hstcpcli::get_next_row_from_result(hstresult& result)
597604
((string_ref *) result.flds.buffer)[i] = string_ref(fld_begin, wp);
598605
}
599606
}
607+
result.cur_row_size =
608+
start - (result.readbuf.begin() + result.cur_row_offset);
600609
result.cur_row_offset = start - result.readbuf.begin();
601610
return (string_ref *) result.flds.buffer;
602611
}
603612

613+
size_t
614+
hstcpcli::get_row_size()
615+
{
616+
return cur_row_size;
617+
}
618+
619+
size_t
620+
hstcpcli::get_row_size_from_result(hstresult& result)
621+
{
622+
return result.cur_row_size;
623+
}
624+
604625
void
605626
hstcpcli::response_buf_remove()
606627
{

‎storage/spider/hs_client/hstcpcli.hpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ struct hstresult {
4646
size_t response_end_offset;
4747
size_t num_flds;
4848
size_t cur_row_offset;
49+
size_t cur_row_size;
4950
DYNAMIC_ARRAY flds;
5051
};
5152

@@ -71,6 +72,8 @@ struct hstcpcli_i {
7172
virtual int get_result(hstresult& result) = 0;
7273
virtual const string_ref *get_next_row() = 0;
7374
virtual const string_ref *get_next_row_from_result(hstresult& result) = 0;
75+
virtual size_t get_row_size() = 0;
76+
virtual size_t get_row_size_from_result(hstresult& result) = 0;
7477
virtual void response_buf_remove() = 0;
7578
virtual int get_error_code() = 0;
7679
virtual String& get_error() = 0;

‎storage/spider/mysql-test/spider/include/quick_mode_0_deinit.inc

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
--let $MASTER_1_COMMENT_2_1= $MASTER_1_COMMENT_2_1_BACKUP
2+
--let $MASTER_1_COMMENT_2_2= $MASTER_1_COMMENT_2_2_BACKUP
3+
--let $CHILD2_1_DROP_TABLES= $CHILD2_1_DROP_TABLES_BACKUP
4+
--let $CHILD2_1_CREATE_TABLES= $CHILD2_1_CREATE_TABLES_BACKUP
5+
--let $CHILD2_1_SELECT_TABLES= $CHILD2_1_SELECT_TABLES_BACKUP
6+
--let $CHILD2_2_DROP_TABLES= $CHILD2_2_DROP_TABLES_BACKUP
7+
--let $CHILD2_2_CREATE_TABLES= $CHILD2_2_CREATE_TABLES_BACKUP
8+
--let $CHILD2_2_SELECT_TABLES= $CHILD2_2_SELECT_TABLES_BACKUP
9+
--let $OUTPUT_CHILD_GROUP2= $OUTPUT_CHILD_GROUP2_BACKUP
10+
--let $USE_GENERAL_LOG= $USE_GENERAL_LOG_BACKUP
11+
--connection master_1
12+
set session spider_quick_mode= @old_spider_quick_mode;
13+
set session spider_quick_page_size= @old_spider_quick_page_size;
14+
set session spider_quick_page_byte= @old_spider_quick_page_byte;
15+
--disable_warnings
16+
--disable_query_log
17+
--disable_result_log
18+
--source ../t/test_deinit.inc
19+
--enable_result_log
20+
--enable_query_log
21+
--enable_warnings

‎storage/spider/mysql-test/spider/include/quick_mode_0_init.inc

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
--disable_warnings
2+
--disable_query_log
3+
--disable_result_log
4+
--source ../t/test_init.inc
5+
--enable_result_log
6+
--enable_query_log
7+
--enable_warnings
8+
--let $MASTER_1_COMMENT_2_1_BACKUP= $MASTER_1_COMMENT_2_1
9+
let $MASTER_1_COMMENT_2_1=
10+
COMMENT='table "tbl_a", srv "s_2_1"';
11+
--let $MASTER_1_COMMENT_2_2_BACKUP= $MASTER_1_COMMENT_2_2
12+
let $MASTER_1_COMMENT_2_2=
13+
COMMENT='table "tbl_b", srv "s_2_2"';
14+
--let $CHILD2_1_DROP_TABLES_BACKUP= $CHILD2_1_DROP_TABLES
15+
let $CHILD2_1_DROP_TABLES=
16+
DROP TABLE IF EXISTS tbl_a;
17+
--let $CHILD2_1_CREATE_TABLES_BACKUP= $CHILD2_1_CREATE_TABLES
18+
let $CHILD2_1_CREATE_TABLES=
19+
CREATE TABLE tbl_a (
20+
pkey int NOT NULL,
21+
PRIMARY KEY (pkey)
22+
) $CHILD2_1_ENGINE $CHILD2_1_CHARSET;
23+
--let $CHILD2_1_SELECT_TABLES_BACKUP= $CHILD2_1_SELECT_TABLES
24+
let $CHILD2_1_SELECT_TABLES=
25+
SELECT pkey FROM tbl_a ORDER BY pkey;
26+
let $CHILD2_1_SELECT_ARGUMENT1=
27+
SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
28+
--let $CHILD2_2_DROP_TABLES_BACKUP= $CHILD2_2_DROP_TABLES
29+
let $CHILD2_2_DROP_TABLES=
30+
DROP TABLE IF EXISTS tbl_b;
31+
--let $CHILD2_2_CREATE_TABLES_BACKUP= $CHILD2_2_CREATE_TABLES
32+
let $CHILD2_2_CREATE_TABLES=
33+
CREATE TABLE tbl_b (
34+
pkey int NOT NULL,
35+
PRIMARY KEY (pkey)
36+
) $CHILD2_2_ENGINE $CHILD2_2_CHARSET;
37+
--let $CHILD2_2_SELECT_TABLES_BACKUP= $CHILD2_2_SELECT_TABLES
38+
let $CHILD2_2_SELECT_TABLES=
39+
SELECT pkey FROM tbl_b ORDER BY pkey;
40+
let $CHILD2_2_SELECT_ARGUMENT1=
41+
SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
42+
--let $OUTPUT_CHILD_GROUP2_BACKUP= $OUTPUT_CHILD_GROUP2
43+
--let $OUTPUT_CHILD_GROUP2= 1
44+
--let $USE_GENERAL_LOG_BACKUP= $USE_GENERAL_LOG
45+
--let $USE_GENERAL_LOG= 1
46+
--connection master_1
47+
set @old_spider_quick_mode= @@spider_quick_mode;
48+
set session spider_quick_mode= 0;
49+
set @old_spider_quick_page_size= @@spider_quick_page_size;
50+
set session spider_quick_page_size= 3;
51+
set @old_spider_quick_page_byte= @@spider_quick_page_byte;
52+
let $MASTER_1_SET_QUICK_PAGE_BYTE_6=
53+
set session spider_quick_page_byte= 6;
54+
let $MASTER_1_SET_QUICK_PAGE_BYTE_0=
55+
set session spider_quick_page_byte= 0;

‎storage/spider/mysql-test/spider/include/quick_mode_1_deinit.inc

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
--let $MASTER_1_COMMENT_2_1= $MASTER_1_COMMENT_2_1_BACKUP
2+
--let $MASTER_1_COMMENT_2_2= $MASTER_1_COMMENT_2_2_BACKUP
3+
--let $CHILD2_1_DROP_TABLES= $CHILD2_1_DROP_TABLES_BACKUP
4+
--let $CHILD2_1_CREATE_TABLES= $CHILD2_1_CREATE_TABLES_BACKUP
5+
--let $CHILD2_1_SELECT_TABLES= $CHILD2_1_SELECT_TABLES_BACKUP
6+
--let $CHILD2_2_DROP_TABLES= $CHILD2_2_DROP_TABLES_BACKUP
7+
--let $CHILD2_2_CREATE_TABLES= $CHILD2_2_CREATE_TABLES_BACKUP
8+
--let $CHILD2_2_SELECT_TABLES= $CHILD2_2_SELECT_TABLES_BACKUP
9+
--let $OUTPUT_CHILD_GROUP2= $OUTPUT_CHILD_GROUP2_BACKUP
10+
--let $USE_GENERAL_LOG= $USE_GENERAL_LOG_BACKUP
11+
--connection master_1
12+
set session spider_quick_mode= @old_spider_quick_mode;
13+
set session spider_quick_page_size= @old_spider_quick_page_size;
14+
set session spider_quick_page_byte= @old_spider_quick_page_byte;
15+
--disable_warnings
16+
--disable_query_log
17+
--disable_result_log
18+
--source ../t/test_deinit.inc
19+
--enable_result_log
20+
--enable_query_log
21+
--enable_warnings

‎storage/spider/mysql-test/spider/include/quick_mode_1_init.inc

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
--disable_warnings
2+
--disable_query_log
3+
--disable_result_log
4+
--source ../t/test_init.inc
5+
--enable_result_log
6+
--enable_query_log
7+
--enable_warnings
8+
--let $MASTER_1_COMMENT_2_1_BACKUP= $MASTER_1_COMMENT_2_1
9+
let $MASTER_1_COMMENT_2_1=
10+
COMMENT='table "tbl_a", srv "s_2_1"';
11+
--let $MASTER_1_COMMENT_2_2_BACKUP= $MASTER_1_COMMENT_2_2
12+
let $MASTER_1_COMMENT_2_2=
13+
COMMENT='table "tbl_b", srv "s_2_2"';
14+
--let $CHILD2_1_DROP_TABLES_BACKUP= $CHILD2_1_DROP_TABLES
15+
let $CHILD2_1_DROP_TABLES=
16+
DROP TABLE IF EXISTS tbl_a;
17+
--let $CHILD2_1_CREATE_TABLES_BACKUP= $CHILD2_1_CREATE_TABLES
18+
let $CHILD2_1_CREATE_TABLES=
19+
CREATE TABLE tbl_a (
20+
pkey int NOT NULL,
21+
PRIMARY KEY (pkey)
22+
) $CHILD2_1_ENGINE $CHILD2_1_CHARSET;
23+
--let $CHILD2_1_SELECT_TABLES_BACKUP= $CHILD2_1_SELECT_TABLES
24+
let $CHILD2_1_SELECT_TABLES=
25+
SELECT pkey FROM tbl_a ORDER BY pkey;
26+
let $CHILD2_1_SELECT_ARGUMENT1=
27+
SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
28+
--let $CHILD2_2_DROP_TABLES_BACKUP= $CHILD2_2_DROP_TABLES
29+
let $CHILD2_2_DROP_TABLES=
30+
DROP TABLE IF EXISTS tbl_b;
31+
--let $CHILD2_2_CREATE_TABLES_BACKUP= $CHILD2_2_CREATE_TABLES
32+
let $CHILD2_2_CREATE_TABLES=
33+
CREATE TABLE tbl_b (
34+
pkey int NOT NULL,
35+
PRIMARY KEY (pkey)
36+
) $CHILD2_2_ENGINE $CHILD2_2_CHARSET;
37+
--let $CHILD2_2_SELECT_TABLES_BACKUP= $CHILD2_2_SELECT_TABLES
38+
let $CHILD2_2_SELECT_TABLES=
39+
SELECT pkey FROM tbl_b ORDER BY pkey;
40+
let $CHILD2_2_SELECT_ARGUMENT1=
41+
SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
42+
--let $OUTPUT_CHILD_GROUP2_BACKUP= $OUTPUT_CHILD_GROUP2
43+
--let $OUTPUT_CHILD_GROUP2= 1
44+
--let $USE_GENERAL_LOG_BACKUP= $USE_GENERAL_LOG
45+
--let $USE_GENERAL_LOG= 1
46+
--connection master_1
47+
set @old_spider_quick_mode= @@spider_quick_mode;
48+
set session spider_quick_mode= 1;
49+
set @old_spider_quick_page_size= @@spider_quick_page_size;
50+
set session spider_quick_page_size= 3;
51+
set @old_spider_quick_page_byte= @@spider_quick_page_byte;
52+
let $MASTER_1_SET_QUICK_PAGE_BYTE_6=
53+
set session spider_quick_page_byte= 6;
54+
let $MASTER_1_SET_QUICK_PAGE_BYTE_0=
55+
set session spider_quick_page_byte= 0;

‎storage/spider/mysql-test/spider/include/quick_mode_2_deinit.inc

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
--let $MASTER_1_COMMENT_2_1= $MASTER_1_COMMENT_2_1_BACKUP
2+
--let $MASTER_1_COMMENT_2_2= $MASTER_1_COMMENT_2_2_BACKUP
3+
--let $CHILD2_1_DROP_TABLES= $CHILD2_1_DROP_TABLES_BACKUP
4+
--let $CHILD2_1_CREATE_TABLES= $CHILD2_1_CREATE_TABLES_BACKUP
5+
--let $CHILD2_1_SELECT_TABLES= $CHILD2_1_SELECT_TABLES_BACKUP
6+
--let $CHILD2_2_DROP_TABLES= $CHILD2_2_DROP_TABLES_BACKUP
7+
--let $CHILD2_2_CREATE_TABLES= $CHILD2_2_CREATE_TABLES_BACKUP
8+
--let $CHILD2_2_SELECT_TABLES= $CHILD2_2_SELECT_TABLES_BACKUP
9+
--let $OUTPUT_CHILD_GROUP2= $OUTPUT_CHILD_GROUP2_BACKUP
10+
--let $USE_GENERAL_LOG= $USE_GENERAL_LOG_BACKUP
11+
--connection master_1
12+
set session spider_quick_mode= @old_spider_quick_mode;
13+
set session spider_quick_page_size= @old_spider_quick_page_size;
14+
set session spider_quick_page_byte= @old_spider_quick_page_byte;
15+
--disable_warnings
16+
--disable_query_log
17+
--disable_result_log
18+
--source ../t/test_deinit.inc
19+
--enable_result_log
20+
--enable_query_log
21+
--enable_warnings

‎storage/spider/mysql-test/spider/include/quick_mode_2_init.inc

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
--disable_warnings
2+
--disable_query_log
3+
--disable_result_log
4+
--source ../t/test_init.inc
5+
--enable_result_log
6+
--enable_query_log
7+
--enable_warnings
8+
--let $MASTER_1_COMMENT_2_1_BACKUP= $MASTER_1_COMMENT_2_1
9+
let $MASTER_1_COMMENT_2_1=
10+
COMMENT='table "tbl_a", srv "s_2_1"';
11+
--let $MASTER_1_COMMENT_2_2_BACKUP= $MASTER_1_COMMENT_2_2
12+
let $MASTER_1_COMMENT_2_2=
13+
COMMENT='table "tbl_b", srv "s_2_2"';
14+
--let $CHILD2_1_DROP_TABLES_BACKUP= $CHILD2_1_DROP_TABLES
15+
let $CHILD2_1_DROP_TABLES=
16+
DROP TABLE IF EXISTS tbl_a;
17+
--let $CHILD2_1_CREATE_TABLES_BACKUP= $CHILD2_1_CREATE_TABLES
18+
let $CHILD2_1_CREATE_TABLES=
19+
CREATE TABLE tbl_a (
20+
pkey int NOT NULL,
21+
PRIMARY KEY (pkey)
22+
) $CHILD2_1_ENGINE $CHILD2_1_CHARSET;
23+
--let $CHILD2_1_SELECT_TABLES_BACKUP= $CHILD2_1_SELECT_TABLES
24+
let $CHILD2_1_SELECT_TABLES=
25+
SELECT pkey FROM tbl_a ORDER BY pkey;
26+
let $CHILD2_1_SELECT_ARGUMENT1=
27+
SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
28+
--let $CHILD2_2_DROP_TABLES_BACKUP= $CHILD2_2_DROP_TABLES
29+
let $CHILD2_2_DROP_TABLES=
30+
DROP TABLE IF EXISTS tbl_b;
31+
--let $CHILD2_2_CREATE_TABLES_BACKUP= $CHILD2_2_CREATE_TABLES
32+
let $CHILD2_2_CREATE_TABLES=
33+
CREATE TABLE tbl_b (
34+
pkey int NOT NULL,
35+
PRIMARY KEY (pkey)
36+
) $CHILD2_2_ENGINE $CHILD2_2_CHARSET;
37+
--let $CHILD2_2_SELECT_TABLES_BACKUP= $CHILD2_2_SELECT_TABLES
38+
let $CHILD2_2_SELECT_TABLES=
39+
SELECT pkey FROM tbl_b ORDER BY pkey;
40+
let $CHILD2_2_SELECT_ARGUMENT1=
41+
SELECT argument FROM mysql.general_log WHERE argument LIKE '%select %';
42+
--let $OUTPUT_CHILD_GROUP2_BACKUP= $OUTPUT_CHILD_GROUP2
43+
--let $OUTPUT_CHILD_GROUP2= 1
44+
--let $USE_GENERAL_LOG_BACKUP= $USE_GENERAL_LOG
45+
--let $USE_GENERAL_LOG= 1
46+
--connection master_1
47+
set @old_spider_quick_mode= @@spider_quick_mode;
48+
set session spider_quick_mode= 2;
49+
set @old_spider_quick_page_size= @@spider_quick_page_size;
50+
set session spider_quick_page_size= 3;
51+
set @old_spider_quick_page_byte= @@spider_quick_page_byte;
52+
let $MASTER_1_SET_QUICK_PAGE_BYTE_6=
53+
set session spider_quick_page_byte= 6;
54+
let $MASTER_1_SET_QUICK_PAGE_BYTE_0=
55+
set session spider_quick_page_byte= 0;

‎storage/spider/mysql-test/spider/include/quick_mode_3_deinit.inc

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
--let $MASTER_1_COMMENT_2_1= $MASTER_1_COMMENT_2_1_BACKUP
2+
--let $MASTER_1_COMMENT_2_2= $MASTER_1_COMMENT_2_2_BACKUP
3+
--let $CHILD2_1_DROP_TABLES= $CHILD2_1_DROP_TABLES_BACKUP
4+
--let $CHILD2_1_CREATE_TABLES= $CHILD2_1_CREATE_TABLES_BACKUP
5+
--let $CHILD2_1_SELECT_TABLES= $CHILD2_1_SELECT_TABLES_BACKUP
6+
--let $CHILD2_2_DROP_TABLES= $CHILD2_2_DROP_TABLES_BACKUP
7+
--let $CHILD2_2_CREATE_TABLES= $CHILD2_2_CREATE_TABLES_BACKUP
8+
--let $CHILD2_2_SELECT_TABLES= $CHILD2_2_SELECT_TABLES_BACKUP
9+
--let $OUTPUT_CHILD_GROUP2= $OUTPUT_CHILD_GROUP2_BACKUP
10+
--let $USE_GENERAL_LOG= $USE_GENERAL_LOG_BACKUP
11+
--connection master_1
12+
set session spider_quick_mode= @old_spider_quick_mode;
13+
set session spider_quick_page_size= @old_spider_quick_page_size;
14+
set session spider_quick_page_byte= @old_spider_quick_page_byte;
15+
--disable_warnings
16+
--disable_query_log
17+
--disable_result_log
18+
--source ../t/test_deinit.inc
19+
--enable_result_log
20+
--enable_query_log
21+
--enable_warnings

0 commit comments

Comments
 (0)