Skip to content

Commit

Permalink
MDEV-5542: GROUP_CONCAT truncate output to 65.536 chars when using DI…
Browse files Browse the repository at this point in the history 
…STINCT or ORDER BY

port of mysql fix WL#6098
  • Loading branch information
@sanja-byelkin
sanja-byelkin committed Mar 1, 2016
1 parent 66832b6 commit 6c414fc
Show file tree
Hide file tree
Showing 13 changed files with 312 additions and 101 deletions.
81 changes: 81 additions & 0 deletions mysql-test/r/func_gconcat.result
View file
Expand Up @@ -1119,3 +1119,84 @@ GROUP_CONCAT(t1a.a ORDER BY 1, t1a.a=0)
1,1
2,2
DROP TABLE t1;
#
# WL#6098 Eliminate GROUP_CONCAT intermediate result limitation.
# Bug#13387020 GROUP_CONCAT WITH ORDER BY RESULTS ARE TRUNCATED.
#
SET group_concat_max_len= 9999999;
CREATE TABLE t1 (f1 LONGTEXT , f2 INTEGER);
INSERT INTO t1 VALUES (REPEAT('a', 500000), 0), (REPEAT('b', 500000), 1), (REPEAT('c', 500000), 2);
SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1;
LENGTH(GROUP_CONCAT(f1 ORDER BY f2))
1500002
SELECT LENGTH(GROUP_CONCAT(DISTINCT f1 ORDER BY f1 DESC)) FROM t1;
LENGTH(GROUP_CONCAT(DISTINCT f1 ORDER BY f1 DESC))
1500002
SELECT SUBSTRING(GROUP_CONCAT(DISTINCT f1 ORDER BY f1 DESC), 1, 5) FROM t1;
SUBSTRING(GROUP_CONCAT(DISTINCT f1 ORDER BY f1 DESC), 1, 5)
ccccc
SELECT LENGTH(GROUP_CONCAT(DISTINCT f1)) FROM t1;
LENGTH(GROUP_CONCAT(DISTINCT f1))
1500002
SELECT LENGTH(GROUP_CONCAT(UPPER(f1) ORDER BY f2)) FROM t1;
LENGTH(GROUP_CONCAT(UPPER(f1) ORDER BY f2))
1500002
SELECT LENGTH(GROUP_CONCAT(DISTINCT UPPER(f1) ORDER BY f1)) FROM t1;
LENGTH(GROUP_CONCAT(DISTINCT UPPER(f1) ORDER BY f1))
1500002
SELECT SUBSTRING(GROUP_CONCAT(DISTINCT UPPER(f1) ORDER BY f1), 1, 5) FROM t1;
SUBSTRING(GROUP_CONCAT(DISTINCT UPPER(f1) ORDER BY f1), 1, 5)
AAAAA
SELECT LENGTH(GROUP_CONCAT(DISTINCT UPPER(f1))) FROM t1;
LENGTH(GROUP_CONCAT(DISTINCT UPPER(f1)))
1500002
CREATE TABLE t2 SELECT GROUP_CONCAT(f1 order by f2) FROM t1;
SHOW CREATE TABLE t2;
Table Create Table
t2 CREATE TABLE `t2` (
`GROUP_CONCAT(f1 order by f2)` mediumtext
) ENGINE=MyISAM DEFAULT CHARSET=latin1
DROP TABLE t2;
CREATE TABLE t2 SELECT GROUP_CONCAT(UPPER(f1) ORDER BY f2) FROM t1;
SHOW CREATE TABLE t2;
Table Create Table
t2 CREATE TABLE `t2` (
`GROUP_CONCAT(UPPER(f1) ORDER BY f2)` mediumtext
) ENGINE=MyISAM DEFAULT CHARSET=latin1
DROP TABLE t2;
SET group_concat_max_len= DEFAULT;
SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1;
LENGTH(GROUP_CONCAT(f1 ORDER BY f2))
1024
Warnings:
Warning 1260 Row 2 was cut by GROUP_CONCAT()
SET group_concat_max_len= 499999;
SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1 WHERE f2 = 0;
LENGTH(GROUP_CONCAT(f1 ORDER BY f2))
499999
Warnings:
Warning 1260 Row 1 was cut by GROUP_CONCAT()
SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1 GROUP BY f2;
LENGTH(GROUP_CONCAT(f1 ORDER BY f2))
499999
499999
499999
Warnings:
Warning 1260 Row 1 was cut by GROUP_CONCAT()
Warning 1260 Row 2 was cut by GROUP_CONCAT()
Warning 1260 Row 3 was cut by GROUP_CONCAT()
INSERT INTO t1 VALUES (REPEAT('a', 499999), 3), (REPEAT('b', 500000), 4);
SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1 GROUP BY f2;
LENGTH(GROUP_CONCAT(f1 ORDER BY f2))
499999
499999
499999
499999
499999
Warnings:
Warning 1260 Row 1 was cut by GROUP_CONCAT()
Warning 1260 Row 2 was cut by GROUP_CONCAT()
Warning 1260 Row 3 was cut by GROUP_CONCAT()
Warning 1260 Row 5 was cut by GROUP_CONCAT()
DROP TABLE t1;
SET group_concat_max_len= DEFAULT;
24 changes: 24 additions & 0 deletions mysql-test/r/subselect_mat.result
View file
Expand Up @@ -762,6 +762,10 @@ select left(a1,7), left(a2,7)
from t1_512
where a1 in (select group_concat(b1) from t2_512 group by b2);
left(a1,7) left(a2,7)
Warnings:
Warning 1260 Row 1 was cut by GROUP_CONCAT()
Warning 1260 Row 2 was cut by GROUP_CONCAT()
Warning 1260 Row 3 was cut by GROUP_CONCAT()
set @@group_concat_max_len = 256;
explain extended select left(a1,7), left(a2,7)
from t1_512
Expand All @@ -775,6 +779,10 @@ select left(a1,7), left(a2,7)
from t1_512
where a1 in (select group_concat(b1) from t2_512 group by b2);
left(a1,7) left(a2,7)
Warnings:
Warning 1260 Row 1 was cut by GROUP_CONCAT()
Warning 1260 Row 2 was cut by GROUP_CONCAT()
Warning 1260 Row 3 was cut by GROUP_CONCAT()
drop table t1_512, t2_512, t3_512;
set @blob_len = 1024;
set @suffix_len = @blob_len - @prefix_len;
Expand Down Expand Up @@ -855,6 +863,10 @@ select left(a1,7), left(a2,7)
from t1_1024
where a1 in (select group_concat(b1) from t2_1024 group by b2);
left(a1,7) left(a2,7)
Warnings:
Warning 1260 Row 1 was cut by GROUP_CONCAT()
Warning 1260 Row 2 was cut by GROUP_CONCAT()
Warning 1260 Row 3 was cut by GROUP_CONCAT()
set @@group_concat_max_len = 256;
explain extended select left(a1,7), left(a2,7)
from t1_1024
Expand All @@ -868,6 +880,10 @@ select left(a1,7), left(a2,7)
from t1_1024
where a1 in (select group_concat(b1) from t2_1024 group by b2);
left(a1,7) left(a2,7)
Warnings:
Warning 1260 Row 1 was cut by GROUP_CONCAT()
Warning 1260 Row 2 was cut by GROUP_CONCAT()
Warning 1260 Row 3 was cut by GROUP_CONCAT()
drop table t1_1024, t2_1024, t3_1024;
set @blob_len = 1025;
set @suffix_len = @blob_len - @prefix_len;
Expand Down Expand Up @@ -948,6 +964,10 @@ select left(a1,7), left(a2,7)
from t1_1025
where a1 in (select group_concat(b1) from t2_1025 group by b2);
left(a1,7) left(a2,7)
Warnings:
Warning 1260 Row 1 was cut by GROUP_CONCAT()
Warning 1260 Row 2 was cut by GROUP_CONCAT()
Warning 1260 Row 3 was cut by GROUP_CONCAT()
set @@group_concat_max_len = 256;
explain extended select left(a1,7), left(a2,7)
from t1_1025
Expand All @@ -961,6 +981,10 @@ select left(a1,7), left(a2,7)
from t1_1025
where a1 in (select group_concat(b1) from t2_1025 group by b2);
left(a1,7) left(a2,7)
Warnings:
Warning 1260 Row 1 was cut by GROUP_CONCAT()
Warning 1260 Row 2 was cut by GROUP_CONCAT()
Warning 1260 Row 3 was cut by GROUP_CONCAT()
drop table t1_1025, t2_1025, t3_1025;
create table t1bit (a1 bit(3), a2 bit(3));
create table t2bit (b1 bit(3), b2 bit(3));
Expand Down
24 changes: 24 additions & 0 deletions mysql-test/r/subselect_sj_mat.result
View file
Expand Up @@ -787,6 +787,10 @@ select left(a1,7), left(a2,7)
from t1_512
where a1 in (select group_concat(b1) from t2_512 group by b2);
left(a1,7) left(a2,7)
Warnings:
Warning 1260 Row 1 was cut by GROUP_CONCAT()
Warning 1260 Row 2 was cut by GROUP_CONCAT()
Warning 1260 Row 3 was cut by GROUP_CONCAT()
set @@group_concat_max_len = 256;
explain extended select left(a1,7), left(a2,7)
from t1_512
Expand All @@ -801,6 +805,10 @@ select left(a1,7), left(a2,7)
from t1_512
where a1 in (select group_concat(b1) from t2_512 group by b2);
left(a1,7) left(a2,7)
Warnings:
Warning 1260 Row 1 was cut by GROUP_CONCAT()
Warning 1260 Row 2 was cut by GROUP_CONCAT()
Warning 1260 Row 3 was cut by GROUP_CONCAT()
drop table t1_512, t2_512, t3_512;
set @blob_len = 1024;
set @suffix_len = @blob_len - @prefix_len;
Expand Down Expand Up @@ -882,6 +890,10 @@ select left(a1,7), left(a2,7)
from t1_1024
where a1 in (select group_concat(b1) from t2_1024 group by b2);
left(a1,7) left(a2,7)
Warnings:
Warning 1260 Row 1 was cut by GROUP_CONCAT()
Warning 1260 Row 2 was cut by GROUP_CONCAT()
Warning 1260 Row 3 was cut by GROUP_CONCAT()
set @@group_concat_max_len = 256;
explain extended select left(a1,7), left(a2,7)
from t1_1024
Expand All @@ -896,6 +908,10 @@ select left(a1,7), left(a2,7)
from t1_1024
where a1 in (select group_concat(b1) from t2_1024 group by b2);
left(a1,7) left(a2,7)
Warnings:
Warning 1260 Row 1 was cut by GROUP_CONCAT()
Warning 1260 Row 2 was cut by GROUP_CONCAT()
Warning 1260 Row 3 was cut by GROUP_CONCAT()
drop table t1_1024, t2_1024, t3_1024;
set @blob_len = 1025;
set @suffix_len = @blob_len - @prefix_len;
Expand Down Expand Up @@ -977,6 +993,10 @@ select left(a1,7), left(a2,7)
from t1_1025
where a1 in (select group_concat(b1) from t2_1025 group by b2);
left(a1,7) left(a2,7)
Warnings:
Warning 1260 Row 1 was cut by GROUP_CONCAT()
Warning 1260 Row 2 was cut by GROUP_CONCAT()
Warning 1260 Row 3 was cut by GROUP_CONCAT()
set @@group_concat_max_len = 256;
explain extended select left(a1,7), left(a2,7)
from t1_1025
Expand All @@ -991,6 +1011,10 @@ select left(a1,7), left(a2,7)
from t1_1025
where a1 in (select group_concat(b1) from t2_1025 group by b2);
left(a1,7) left(a2,7)
Warnings:
Warning 1260 Row 1 was cut by GROUP_CONCAT()
Warning 1260 Row 2 was cut by GROUP_CONCAT()
Warning 1260 Row 3 was cut by GROUP_CONCAT()
drop table t1_1025, t2_1025, t3_1025;
create table t1bit (a1 bit(3), a2 bit(3));
create table t2bit (b1 bit(3), b2 bit(3));
Expand Down
41 changes: 41 additions & 0 deletions mysql-test/t/func_gconcat.test
View file
Expand Up @@ -832,3 +832,44 @@ PREPARE stmt FROM "SELECT GROUP_CONCAT(t1a.a ORDER BY 1, t1a.a=0) FROM t1 AS t1a
EXECUTE stmt;
EXECUTE stmt;
DROP TABLE t1;

--echo #
--echo # WL#6098 Eliminate GROUP_CONCAT intermediate result limitation.
--echo # Bug#13387020 GROUP_CONCAT WITH ORDER BY RESULTS ARE TRUNCATED.
--echo #


SET group_concat_max_len= 9999999;
CREATE TABLE t1 (f1 LONGTEXT , f2 INTEGER);
INSERT INTO t1 VALUES (REPEAT('a', 500000), 0), (REPEAT('b', 500000), 1), (REPEAT('c', 500000), 2);

SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1;
SELECT LENGTH(GROUP_CONCAT(DISTINCT f1 ORDER BY f1 DESC)) FROM t1;
SELECT SUBSTRING(GROUP_CONCAT(DISTINCT f1 ORDER BY f1 DESC), 1, 5) FROM t1;
SELECT LENGTH(GROUP_CONCAT(DISTINCT f1)) FROM t1;

SELECT LENGTH(GROUP_CONCAT(UPPER(f1) ORDER BY f2)) FROM t1;
SELECT LENGTH(GROUP_CONCAT(DISTINCT UPPER(f1) ORDER BY f1)) FROM t1;
SELECT SUBSTRING(GROUP_CONCAT(DISTINCT UPPER(f1) ORDER BY f1), 1, 5) FROM t1;
SELECT LENGTH(GROUP_CONCAT(DISTINCT UPPER(f1))) FROM t1;

CREATE TABLE t2 SELECT GROUP_CONCAT(f1 order by f2) FROM t1;
SHOW CREATE TABLE t2;
DROP TABLE t2;

CREATE TABLE t2 SELECT GROUP_CONCAT(UPPER(f1) ORDER BY f2) FROM t1;
SHOW CREATE TABLE t2;
DROP TABLE t2;

SET group_concat_max_len= DEFAULT;
SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1;

SET group_concat_max_len= 499999;
SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1 WHERE f2 = 0;
SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1 GROUP BY f2;

INSERT INTO t1 VALUES (REPEAT('a', 499999), 3), (REPEAT('b', 500000), 4);
SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1 GROUP BY f2;

DROP TABLE t1;
SET group_concat_max_len= DEFAULT;
33 changes: 27 additions & 6 deletions sql/field.cc
View file
Expand Up @@ -7845,7 +7845,7 @@ int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs)
ASSERT_COLUMN_MARKED_FOR_WRITE_OR_COMPUTED;
uint copy_length, new_length;
String_copier copier;
const char *tmp;
char *tmp;
char buff[STRING_BUFFER_USUAL_SIZE];
String tmpstr(buff,sizeof(buff), &my_charset_bin);

Expand All @@ -7855,6 +7855,29 @@ int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs)
return 0;
}

if (table->blob_storage) // GROUP_CONCAT with ORDER BY | DISTINCT
{
DBUG_ASSERT(!f_is_hex_escape(flags));
DBUG_ASSERT(field_charset == cs);
DBUG_ASSERT(length <= max_data_length());

new_length= length;
copy_length= table->in_use->variables.group_concat_max_len;
if (new_length > copy_length)
{
int well_formed_error;
new_length= cs->cset->well_formed_len(cs, from, from + copy_length,
new_length, &well_formed_error);
table->blob_storage->set_truncated_value(true);
}
if (!(tmp= table->blob_storage->store(from, new_length)))
goto oom_error;

Field_blob::store_length(new_length);
bmove(ptr + packlength, (uchar*) &tmp, sizeof(char*));
return 0;
}

/*
If the 'from' address is in the range of the temporary 'value'-
object we need to copy the content to a different location or it will be
Expand All @@ -7881,23 +7904,21 @@ int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs)
new_length= MY_MIN(max_data_length(), field_charset->mbmaxlen * length);
if (value.alloc(new_length))
goto oom_error;

tmp= const_cast<char*>(value.ptr());

if (f_is_hex_escape(flags))
{
copy_length= my_copy_with_hex_escaping(field_charset,
(char*) value.ptr(), new_length,
from, length);
tmp, new_length,
from, length);
Field_blob::store_length(copy_length);
tmp= value.ptr();
bmove(ptr + packlength, (uchar*) &tmp, sizeof(char*));
return 0;
}
copy_length= copier.well_formed_copy(field_charset,
(char*) value.ptr(), new_length,
cs, from, length);
Field_blob::store_length(copy_length);
tmp= value.ptr();
bmove(ptr+packlength,(uchar*) &tmp,sizeof(char*));

return check_conversion_status(&copier, from + length, cs, true);
Expand Down
3 changes: 0 additions & 3 deletions sql/field_conv.cc
View file
Expand Up @@ -629,9 +629,6 @@ void Copy_field::set(uchar *to,Field *from)
Field_blob::store. Is this in order to trigger the call to
well_formed_copy_nchars, by changing the pointer copy->tmp.ptr()?
That call will take place anyway in all known cases.
- The above causes a truncation to MAX_FIELD_WIDTH. Is this the intended
effect? Truncation is handled by well_formed_copy_nchars anyway.
*/
void Copy_field::set(Field *to,Field *from,bool save)
{
Expand Down

0 comments on commit 6c414fc

Please sign in to comment.