Skip to content

Commit 6c414fc

Browse files
sanja-byelkinsanja-byelkin
committed
MDEV-5542: GROUP_CONCAT truncate output to 65.536 chars when using DISTINCT or ORDER BY
port of mysql fix WL#6098
1 parent 66832b6 commit 6c414fc

13 files changed

+312
-101
lines changed

‎mysql-test/r/func_gconcat.result

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1119,3 +1119,84 @@ GROUP_CONCAT(t1a.a ORDER BY 1, t1a.a=0)
11191119
1,1
11201120
2,2
11211121
DROP TABLE t1;
1122+
#
1123+
# WL#6098 Eliminate GROUP_CONCAT intermediate result limitation.
1124+
# Bug#13387020 GROUP_CONCAT WITH ORDER BY RESULTS ARE TRUNCATED.
1125+
#
1126+
SET group_concat_max_len= 9999999;
1127+
CREATE TABLE t1 (f1 LONGTEXT , f2 INTEGER);
1128+
INSERT INTO t1 VALUES (REPEAT('a', 500000), 0), (REPEAT('b', 500000), 1), (REPEAT('c', 500000), 2);
1129+
SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1;
1130+
LENGTH(GROUP_CONCAT(f1 ORDER BY f2))
1131+
1500002
1132+
SELECT LENGTH(GROUP_CONCAT(DISTINCT f1 ORDER BY f1 DESC)) FROM t1;
1133+
LENGTH(GROUP_CONCAT(DISTINCT f1 ORDER BY f1 DESC))
1134+
1500002
1135+
SELECT SUBSTRING(GROUP_CONCAT(DISTINCT f1 ORDER BY f1 DESC), 1, 5) FROM t1;
1136+
SUBSTRING(GROUP_CONCAT(DISTINCT f1 ORDER BY f1 DESC), 1, 5)
1137+
ccccc
1138+
SELECT LENGTH(GROUP_CONCAT(DISTINCT f1)) FROM t1;
1139+
LENGTH(GROUP_CONCAT(DISTINCT f1))
1140+
1500002
1141+
SELECT LENGTH(GROUP_CONCAT(UPPER(f1) ORDER BY f2)) FROM t1;
1142+
LENGTH(GROUP_CONCAT(UPPER(f1) ORDER BY f2))
1143+
1500002
1144+
SELECT LENGTH(GROUP_CONCAT(DISTINCT UPPER(f1) ORDER BY f1)) FROM t1;
1145+
LENGTH(GROUP_CONCAT(DISTINCT UPPER(f1) ORDER BY f1))
1146+
1500002
1147+
SELECT SUBSTRING(GROUP_CONCAT(DISTINCT UPPER(f1) ORDER BY f1), 1, 5) FROM t1;
1148+
SUBSTRING(GROUP_CONCAT(DISTINCT UPPER(f1) ORDER BY f1), 1, 5)
1149+
AAAAA
1150+
SELECT LENGTH(GROUP_CONCAT(DISTINCT UPPER(f1))) FROM t1;
1151+
LENGTH(GROUP_CONCAT(DISTINCT UPPER(f1)))
1152+
1500002
1153+
CREATE TABLE t2 SELECT GROUP_CONCAT(f1 order by f2) FROM t1;
1154+
SHOW CREATE TABLE t2;
1155+
Table Create Table
1156+
t2 CREATE TABLE `t2` (
1157+
`GROUP_CONCAT(f1 order by f2)` mediumtext
1158+
) ENGINE=MyISAM DEFAULT CHARSET=latin1
1159+
DROP TABLE t2;
1160+
CREATE TABLE t2 SELECT GROUP_CONCAT(UPPER(f1) ORDER BY f2) FROM t1;
1161+
SHOW CREATE TABLE t2;
1162+
Table Create Table
1163+
t2 CREATE TABLE `t2` (
1164+
`GROUP_CONCAT(UPPER(f1) ORDER BY f2)` mediumtext
1165+
) ENGINE=MyISAM DEFAULT CHARSET=latin1
1166+
DROP TABLE t2;
1167+
SET group_concat_max_len= DEFAULT;
1168+
SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1;
1169+
LENGTH(GROUP_CONCAT(f1 ORDER BY f2))
1170+
1024
1171+
Warnings:
1172+
Warning 1260 Row 2 was cut by GROUP_CONCAT()
1173+
SET group_concat_max_len= 499999;
1174+
SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1 WHERE f2 = 0;
1175+
LENGTH(GROUP_CONCAT(f1 ORDER BY f2))
1176+
499999
1177+
Warnings:
1178+
Warning 1260 Row 1 was cut by GROUP_CONCAT()
1179+
SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1 GROUP BY f2;
1180+
LENGTH(GROUP_CONCAT(f1 ORDER BY f2))
1181+
499999
1182+
499999
1183+
499999
1184+
Warnings:
1185+
Warning 1260 Row 1 was cut by GROUP_CONCAT()
1186+
Warning 1260 Row 2 was cut by GROUP_CONCAT()
1187+
Warning 1260 Row 3 was cut by GROUP_CONCAT()
1188+
INSERT INTO t1 VALUES (REPEAT('a', 499999), 3), (REPEAT('b', 500000), 4);
1189+
SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1 GROUP BY f2;
1190+
LENGTH(GROUP_CONCAT(f1 ORDER BY f2))
1191+
499999
1192+
499999
1193+
499999
1194+
499999
1195+
499999
1196+
Warnings:
1197+
Warning 1260 Row 1 was cut by GROUP_CONCAT()
1198+
Warning 1260 Row 2 was cut by GROUP_CONCAT()
1199+
Warning 1260 Row 3 was cut by GROUP_CONCAT()
1200+
Warning 1260 Row 5 was cut by GROUP_CONCAT()
1201+
DROP TABLE t1;
1202+
SET group_concat_max_len= DEFAULT;

‎mysql-test/r/subselect_mat.result

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -762,6 +762,10 @@ select left(a1,7), left(a2,7)
762762
from t1_512
763763
where a1 in (select group_concat(b1) from t2_512 group by b2);
764764
left(a1,7) left(a2,7)
765+
Warnings:
766+
Warning 1260 Row 1 was cut by GROUP_CONCAT()
767+
Warning 1260 Row 2 was cut by GROUP_CONCAT()
768+
Warning 1260 Row 3 was cut by GROUP_CONCAT()
765769
set @@group_concat_max_len = 256;
766770
explain extended select left(a1,7), left(a2,7)
767771
from t1_512
@@ -775,6 +779,10 @@ select left(a1,7), left(a2,7)
775779
from t1_512
776780
where a1 in (select group_concat(b1) from t2_512 group by b2);
777781
left(a1,7) left(a2,7)
782+
Warnings:
783+
Warning 1260 Row 1 was cut by GROUP_CONCAT()
784+
Warning 1260 Row 2 was cut by GROUP_CONCAT()
785+
Warning 1260 Row 3 was cut by GROUP_CONCAT()
778786
drop table t1_512, t2_512, t3_512;
779787
set @blob_len = 1024;
780788
set @suffix_len = @blob_len - @prefix_len;
@@ -855,6 +863,10 @@ select left(a1,7), left(a2,7)
855863
from t1_1024
856864
where a1 in (select group_concat(b1) from t2_1024 group by b2);
857865
left(a1,7) left(a2,7)
866+
Warnings:
867+
Warning 1260 Row 1 was cut by GROUP_CONCAT()
868+
Warning 1260 Row 2 was cut by GROUP_CONCAT()
869+
Warning 1260 Row 3 was cut by GROUP_CONCAT()
858870
set @@group_concat_max_len = 256;
859871
explain extended select left(a1,7), left(a2,7)
860872
from t1_1024
@@ -868,6 +880,10 @@ select left(a1,7), left(a2,7)
868880
from t1_1024
869881
where a1 in (select group_concat(b1) from t2_1024 group by b2);
870882
left(a1,7) left(a2,7)
883+
Warnings:
884+
Warning 1260 Row 1 was cut by GROUP_CONCAT()
885+
Warning 1260 Row 2 was cut by GROUP_CONCAT()
886+
Warning 1260 Row 3 was cut by GROUP_CONCAT()
871887
drop table t1_1024, t2_1024, t3_1024;
872888
set @blob_len = 1025;
873889
set @suffix_len = @blob_len - @prefix_len;
@@ -948,6 +964,10 @@ select left(a1,7), left(a2,7)
948964
from t1_1025
949965
where a1 in (select group_concat(b1) from t2_1025 group by b2);
950966
left(a1,7) left(a2,7)
967+
Warnings:
968+
Warning 1260 Row 1 was cut by GROUP_CONCAT()
969+
Warning 1260 Row 2 was cut by GROUP_CONCAT()
970+
Warning 1260 Row 3 was cut by GROUP_CONCAT()
951971
set @@group_concat_max_len = 256;
952972
explain extended select left(a1,7), left(a2,7)
953973
from t1_1025
@@ -961,6 +981,10 @@ select left(a1,7), left(a2,7)
961981
from t1_1025
962982
where a1 in (select group_concat(b1) from t2_1025 group by b2);
963983
left(a1,7) left(a2,7)
984+
Warnings:
985+
Warning 1260 Row 1 was cut by GROUP_CONCAT()
986+
Warning 1260 Row 2 was cut by GROUP_CONCAT()
987+
Warning 1260 Row 3 was cut by GROUP_CONCAT()
964988
drop table t1_1025, t2_1025, t3_1025;
965989
create table t1bit (a1 bit(3), a2 bit(3));
966990
create table t2bit (b1 bit(3), b2 bit(3));

‎mysql-test/r/subselect_sj_mat.result

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -787,6 +787,10 @@ select left(a1,7), left(a2,7)
787787
from t1_512
788788
where a1 in (select group_concat(b1) from t2_512 group by b2);
789789
left(a1,7) left(a2,7)
790+
Warnings:
791+
Warning 1260 Row 1 was cut by GROUP_CONCAT()
792+
Warning 1260 Row 2 was cut by GROUP_CONCAT()
793+
Warning 1260 Row 3 was cut by GROUP_CONCAT()
790794
set @@group_concat_max_len = 256;
791795
explain extended select left(a1,7), left(a2,7)
792796
from t1_512
@@ -801,6 +805,10 @@ select left(a1,7), left(a2,7)
801805
from t1_512
802806
where a1 in (select group_concat(b1) from t2_512 group by b2);
803807
left(a1,7) left(a2,7)
808+
Warnings:
809+
Warning 1260 Row 1 was cut by GROUP_CONCAT()
810+
Warning 1260 Row 2 was cut by GROUP_CONCAT()
811+
Warning 1260 Row 3 was cut by GROUP_CONCAT()
804812
drop table t1_512, t2_512, t3_512;
805813
set @blob_len = 1024;
806814
set @suffix_len = @blob_len - @prefix_len;
@@ -882,6 +890,10 @@ select left(a1,7), left(a2,7)
882890
from t1_1024
883891
where a1 in (select group_concat(b1) from t2_1024 group by b2);
884892
left(a1,7) left(a2,7)
893+
Warnings:
894+
Warning 1260 Row 1 was cut by GROUP_CONCAT()
895+
Warning 1260 Row 2 was cut by GROUP_CONCAT()
896+
Warning 1260 Row 3 was cut by GROUP_CONCAT()
885897
set @@group_concat_max_len = 256;
886898
explain extended select left(a1,7), left(a2,7)
887899
from t1_1024
@@ -896,6 +908,10 @@ select left(a1,7), left(a2,7)
896908
from t1_1024
897909
where a1 in (select group_concat(b1) from t2_1024 group by b2);
898910
left(a1,7) left(a2,7)
911+
Warnings:
912+
Warning 1260 Row 1 was cut by GROUP_CONCAT()
913+
Warning 1260 Row 2 was cut by GROUP_CONCAT()
914+
Warning 1260 Row 3 was cut by GROUP_CONCAT()
899915
drop table t1_1024, t2_1024, t3_1024;
900916
set @blob_len = 1025;
901917
set @suffix_len = @blob_len - @prefix_len;
@@ -977,6 +993,10 @@ select left(a1,7), left(a2,7)
977993
from t1_1025
978994
where a1 in (select group_concat(b1) from t2_1025 group by b2);
979995
left(a1,7) left(a2,7)
996+
Warnings:
997+
Warning 1260 Row 1 was cut by GROUP_CONCAT()
998+
Warning 1260 Row 2 was cut by GROUP_CONCAT()
999+
Warning 1260 Row 3 was cut by GROUP_CONCAT()
9801000
set @@group_concat_max_len = 256;
9811001
explain extended select left(a1,7), left(a2,7)
9821002
from t1_1025
@@ -991,6 +1011,10 @@ select left(a1,7), left(a2,7)
9911011
from t1_1025
9921012
where a1 in (select group_concat(b1) from t2_1025 group by b2);
9931013
left(a1,7) left(a2,7)
1014+
Warnings:
1015+
Warning 1260 Row 1 was cut by GROUP_CONCAT()
1016+
Warning 1260 Row 2 was cut by GROUP_CONCAT()
1017+
Warning 1260 Row 3 was cut by GROUP_CONCAT()
9941018
drop table t1_1025, t2_1025, t3_1025;
9951019
create table t1bit (a1 bit(3), a2 bit(3));
9961020
create table t2bit (b1 bit(3), b2 bit(3));

‎mysql-test/t/func_gconcat.test

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -832,3 +832,44 @@ PREPARE stmt FROM "SELECT GROUP_CONCAT(t1a.a ORDER BY 1, t1a.a=0) FROM t1 AS t1a
832832
EXECUTE stmt;
833833
EXECUTE stmt;
834834
DROP TABLE t1;
835+
836+
--echo #
837+
--echo # WL#6098 Eliminate GROUP_CONCAT intermediate result limitation.
838+
--echo # Bug#13387020 GROUP_CONCAT WITH ORDER BY RESULTS ARE TRUNCATED.
839+
--echo #
840+
841+
842+
SET group_concat_max_len= 9999999;
843+
CREATE TABLE t1 (f1 LONGTEXT , f2 INTEGER);
844+
INSERT INTO t1 VALUES (REPEAT('a', 500000), 0), (REPEAT('b', 500000), 1), (REPEAT('c', 500000), 2);
845+
846+
SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1;
847+
SELECT LENGTH(GROUP_CONCAT(DISTINCT f1 ORDER BY f1 DESC)) FROM t1;
848+
SELECT SUBSTRING(GROUP_CONCAT(DISTINCT f1 ORDER BY f1 DESC), 1, 5) FROM t1;
849+
SELECT LENGTH(GROUP_CONCAT(DISTINCT f1)) FROM t1;
850+
851+
SELECT LENGTH(GROUP_CONCAT(UPPER(f1) ORDER BY f2)) FROM t1;
852+
SELECT LENGTH(GROUP_CONCAT(DISTINCT UPPER(f1) ORDER BY f1)) FROM t1;
853+
SELECT SUBSTRING(GROUP_CONCAT(DISTINCT UPPER(f1) ORDER BY f1), 1, 5) FROM t1;
854+
SELECT LENGTH(GROUP_CONCAT(DISTINCT UPPER(f1))) FROM t1;
855+
856+
CREATE TABLE t2 SELECT GROUP_CONCAT(f1 order by f2) FROM t1;
857+
SHOW CREATE TABLE t2;
858+
DROP TABLE t2;
859+
860+
CREATE TABLE t2 SELECT GROUP_CONCAT(UPPER(f1) ORDER BY f2) FROM t1;
861+
SHOW CREATE TABLE t2;
862+
DROP TABLE t2;
863+
864+
SET group_concat_max_len= DEFAULT;
865+
SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1;
866+
867+
SET group_concat_max_len= 499999;
868+
SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1 WHERE f2 = 0;
869+
SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1 GROUP BY f2;
870+
871+
INSERT INTO t1 VALUES (REPEAT('a', 499999), 3), (REPEAT('b', 500000), 4);
872+
SELECT LENGTH(GROUP_CONCAT(f1 ORDER BY f2)) FROM t1 GROUP BY f2;
873+
874+
DROP TABLE t1;
875+
SET group_concat_max_len= DEFAULT;

‎sql/field.cc

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7845,7 +7845,7 @@ int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs)
78457845
ASSERT_COLUMN_MARKED_FOR_WRITE_OR_COMPUTED;
78467846
uint copy_length, new_length;
78477847
String_copier copier;
7848-
const char *tmp;
7848+
char *tmp;
78497849
char buff[STRING_BUFFER_USUAL_SIZE];
78507850
String tmpstr(buff,sizeof(buff), &my_charset_bin);
78517851

@@ -7855,6 +7855,29 @@ int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs)
78557855
return 0;
78567856
}
78577857

7858+
if (table->blob_storage) // GROUP_CONCAT with ORDER BY | DISTINCT
7859+
{
7860+
DBUG_ASSERT(!f_is_hex_escape(flags));
7861+
DBUG_ASSERT(field_charset == cs);
7862+
DBUG_ASSERT(length <= max_data_length());
7863+
7864+
new_length= length;
7865+
copy_length= table->in_use->variables.group_concat_max_len;
7866+
if (new_length > copy_length)
7867+
{
7868+
int well_formed_error;
7869+
new_length= cs->cset->well_formed_len(cs, from, from + copy_length,
7870+
new_length, &well_formed_error);
7871+
table->blob_storage->set_truncated_value(true);
7872+
}
7873+
if (!(tmp= table->blob_storage->store(from, new_length)))
7874+
goto oom_error;
7875+
7876+
Field_blob::store_length(new_length);
7877+
bmove(ptr + packlength, (uchar*) &tmp, sizeof(char*));
7878+
return 0;
7879+
}
7880+
78587881
/*
78597882
If the 'from' address is in the range of the temporary 'value'-
78607883
object we need to copy the content to a different location or it will be
@@ -7881,23 +7904,21 @@ int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs)
78817904
new_length= MY_MIN(max_data_length(), field_charset->mbmaxlen * length);
78827905
if (value.alloc(new_length))
78837906
goto oom_error;
7884-
7907+
tmp= const_cast<char*>(value.ptr());
78857908

78867909
if (f_is_hex_escape(flags))
78877910
{
78887911
copy_length= my_copy_with_hex_escaping(field_charset,
7889-
(char*) value.ptr(), new_length,
7890-
from, length);
7912+
tmp, new_length,
7913+
from, length);
78917914
Field_blob::store_length(copy_length);
7892-
tmp= value.ptr();
78937915
bmove(ptr + packlength, (uchar*) &tmp, sizeof(char*));
78947916
return 0;
78957917
}
78967918
copy_length= copier.well_formed_copy(field_charset,
78977919
(char*) value.ptr(), new_length,
78987920
cs, from, length);
78997921
Field_blob::store_length(copy_length);
7900-
tmp= value.ptr();
79017922
bmove(ptr+packlength,(uchar*) &tmp,sizeof(char*));
79027923

79037924
return check_conversion_status(&copier, from + length, cs, true);

‎sql/field_conv.cc

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -629,9 +629,6 @@ void Copy_field::set(uchar *to,Field *from)
629629
Field_blob::store. Is this in order to trigger the call to
630630
well_formed_copy_nchars, by changing the pointer copy->tmp.ptr()?
631631
That call will take place anyway in all known cases.
632-
633-
- The above causes a truncation to MAX_FIELD_WIDTH. Is this the intended
634-
effect? Truncation is handled by well_formed_copy_nchars anyway.
635632
*/
636633
void Copy_field::set(Field *to,Field *from,bool save)
637634
{

0 commit comments

Comments
 (0)