Merge 10.11 into 11.0

CMakeLists.txt

@@ -484,7 +484,7 @@ ADD_SUBDIRECTORY(support-files)
 ADD_SUBDIRECTORY(extra/aws_sdk)
 
 IF(NOT CMAKE_CROSSCOMPILING OR DEFINED CMAKE_CROSSCOMPILING_EMULATOR)
-  SET(EXPORTED comp_err comp_sql factorial)
+  SET(EXPORTED comp_err comp_sql factorial uca-dump)
   IF(NOT WITHOUT_SERVER)
     SET(EXPORTED ${EXPORTED} gen_lex_hash gen_lex_token)
   ENDIF()

include/m_ctype.h

@@ -80,10 +80,26 @@ typedef const struct my_charset_handler_st MY_CHARSET_HANDLER;
 typedef const struct my_collation_handler_st MY_COLLATION_HANDLER;
 
 typedef const struct unicase_info_st MY_UNICASE_INFO;
+typedef const struct casefold_info_st MY_CASEFOLD_INFO;
 typedef const struct uni_ctype_st MY_UNI_CTYPE;
 typedef const struct my_uni_idx_st MY_UNI_IDX;
 typedef uint16 decimal_digits_t;
 
+
+typedef struct casefold_info_char_t
+{
+  uint32 toupper;
+  uint32 tolower;
+} MY_CASEFOLD_CHARACTER;
+
+
+struct casefold_info_st
+{
+  my_wc_t maxchar;
+  MY_CASEFOLD_CHARACTER **page;
+};
+
+
 typedef struct unicase_info_char_st
 {
   uint32 toupper;
@@ -720,6 +736,9 @@ struct my_charset_handler_st
   */
   my_charset_conv_wc_mb native_to_mb;
   my_charset_conv_wc_mb wc_to_printable;
+
+  uint (*caseup_multiply)(CHARSET_INFO *cs);
+  uint (*casedn_multiply)(CHARSET_INFO *cs);
 };
 
 extern MY_CHARSET_HANDLER my_charset_8bit_handler;
@@ -752,12 +771,11 @@ struct charset_info_st
   MY_UCA_INFO *uca;
   const uint16 *tab_to_uni;
   MY_UNI_IDX  *tab_from_uni;
+  MY_CASEFOLD_INFO *casefold;
   MY_UNICASE_INFO *caseinfo;
   const uchar  *state_map;
   const uchar  *ident_map;
   uint      strxfrm_multiply;
-  uchar     caseup_multiply;
-  uchar     casedn_multiply;
   uint      mbminlen;
   uint      mbmaxlen;
   /*
@@ -827,6 +845,16 @@ struct charset_info_st
     return (cset->casedn)(this, src, srclen, dst, dstlen);
   }
 
+  uint caseup_multiply() const
+  {
+    return (cset->caseup_multiply)(this);
+  }
+
+  uint casedn_multiply() const
+  {
+    return (cset->casedn_multiply)(this);
+  }
+
   size_t long10_to_str(char *dst, size_t dstlen,
                        int radix, long int val) const
   {

mysql-test/include/ctype_casefolding.inc

@@ -13,6 +13,14 @@ INSERT INTO case_folding (code) VALUES
 (0x26B),
 (0x271),
 (0x27D);
+
+INSERT INTO case_folding (code) VALUES
+(0x0049) /* LATIN CAPITAL LETTER I */,
+(0x0069) /* LATIN SMALL LETTER I */,
+(0x0130) /* LATIN CAPITAL LETTER I WITH DOT ABOVE */,
+(0x0131) /* LATIN SMALL LETTER DOTLESS I */
+;
+
 UPDATE case_folding SET c=CHAR(code USING ucs2);
 SELECT HEX(code), HEX(LOWER(c)), HEX(UPPER(c)), c FROM case_folding;
 DROP TABLE case_folding;

mysql-test/include/ctype_unicode_casefold_bmp.inc

@@ -0,0 +1,23 @@
+--source include/have_utf32.inc
+--source include/have_ucs2.inc
+--source include/have_sequence.inc
+
+EXECUTE IMMEDIATE SFORMAT('
+CREATE VIEW v_bmp AS
+SELECT
+  seq AS codepoint,
+  LPAD(HEX(seq),4,''0'') AS codepoint_hex4,
+  CONVERT(CHAR(seq USING utf32) USING {}) COLLATE {} AS c
+FROM
+  seq_0_to_65535', @@character_set_connection, @@collation_connection);
+
+SELECT COLLATION(c) FROM v_bmp LIMIT 1;
+
+SELECT
+  codepoint_hex4,
+  HEX(CAST(LOWER(c) AS CHAR CHARACTER SET ucs2)),
+  HEX(CAST(UPPER(c) AS CHAR CHARACTER SET ucs2))
+FROM v_bmp
+WHERE BINARY(c)<>BINARY(LOWER(c)) OR BINARY(c)<>BINARY(UPPER(c));
+
+DROP VIEW v_bmp;

mysql-test/include/ctype_unicode_casefold_supplementary.inc

@@ -0,0 +1,22 @@
+--source include/have_utf32.inc
+--source include/have_sequence.inc
+
+EXECUTE IMMEDIATE SFORMAT('
+CREATE VIEW v_supplementary AS
+SELECT
+  seq AS codepoint,
+  LPAD(HEX(seq),8,''0'') AS codepoint_hex8,
+  CONVERT(CHAR(seq USING utf32) USING {}) COLLATE {} AS c
+FROM
+  seq_65536_to_1114111', @@character_set_connection, @@collation_connection);
+
+SELECT COLLATION(c) FROM v_supplementary LIMIT 1;
+
+SELECT
+  codepoint_hex8,
+  HEX(CAST(LOWER(c) AS CHAR CHARACTER SET utf32)),
+  HEX(CAST(UPPER(c) AS CHAR CHARACTER SET utf32))
+FROM v_supplementary
+WHERE BINARY(c)<>BINARY(LOWER(c)) OR BINARY(c)<>BINARY(UPPER(c));
+
+DROP VIEW v_supplementary;

mysql-test/include/ctype_unicode_ws_bmp.inc

@@ -0,0 +1,26 @@
+--source include/have_utf32.inc
+--source include/have_sequence.inc
+
+EXECUTE IMMEDIATE SFORMAT('
+CREATE VIEW v_bmp AS
+SELECT
+  seq AS codepoint,
+  LPAD(HEX(seq),4,''0'') AS codepoint_hex4,
+  CONVERT(CHAR(seq USING utf32) USING {}) COLLATE {} AS c
+FROM
+  seq_0_to_65535', @@character_set_connection, @@collation_connection);
+
+SELECT COLLATION(c) FROM v_bmp LIMIT 1;
+
+SELECT HEX(codepoint) FROM v_bmp WHERE HEX(WEIGHT_STRING(c))='FFFD';
+
+SELECT
+  SUM(codepoint_hex4=HEX(WEIGHT_STRING(c))) AS count_bmp_weight_is_codepoint,
+  SUM(codepoint_hex4<>HEX(WEIGHT_STRING(c))) AS count_bmp_weight_is_not_codepoint
+FROM v_bmp;
+
+SELECT codepoint_hex4,HEX(WEIGHT_STRING(c))
+FROM v_bmp
+WHERE codepoint_hex4<>HEX(WEIGHT_STRING(c));
+
+DROP VIEW v_bmp;

mysql-test/include/ctype_unicode_ws_supplementary.inc

@@ -0,0 +1,19 @@
+--source include/have_utf32.inc
+--source include/have_sequence.inc
+
+EXECUTE IMMEDIATE SFORMAT('
+CREATE VIEW v_supplementary AS
+SELECT
+  seq AS codepoint,
+  CONVERT(CHAR(seq USING utf32) USING {}) COLLATE {} AS c
+FROM
+  seq_65536_to_1114111', @@character_set_connection, @@collation_connection);
+
+SELECT COLLATION(c) FROM v_supplementary LIMIT 1;
+
+SELECT
+  SUM(HEX(WEIGHT_STRING(c))<>'FFFD'),
+  SUM(HEX(WEIGHT_STRING(c))='FFFD')
+FROM v_supplementary;
+
+DROP VIEW v_supplementary;

mysql-test/main/ctype_ldml.result

@@ -3068,6 +3068,12 @@ INSERT INTO case_folding (code) VALUES
 (0x26B),
 (0x271),
 (0x27D);
+INSERT INTO case_folding (code) VALUES
+(0x0049) /* LATIN CAPITAL LETTER I */,
+(0x0069) /* LATIN SMALL LETTER I */,
+(0x0130) /* LATIN CAPITAL LETTER I WITH DOT ABOVE */,
+(0x0131) /* LATIN SMALL LETTER DOTLESS I */
+;
 UPDATE case_folding SET c=CHAR(code USING ucs2);
 SELECT HEX(code), HEX(LOWER(c)), HEX(UPPER(c)), c FROM case_folding;
 HEX(code)	HEX(LOWER(c))	HEX(UPPER(c))	c
@@ -3081,6 +3087,10 @@ HEX(code)	HEX(LOWER(c))	HEX(UPPER(c))	c
 26B	C9AB	E2B1A2	ɫ
 271	C9B1	E2B1AE	ɱ
 27D	C9BD	E2B1A4	ɽ
+49	69	49	I
+69	69	49	i
+130	69	C4B0	İ
+131	C4B1	49	ı
 DROP TABLE case_folding;
 #
 # End of 10.3 tests
@@ -3230,5 +3240,51 @@ is_520	0
 is_1400	1
 DROP TABLE t1;
 #
+# MDEV-30661 UPPER() returns an empty string for U+0251 in uca1400 collations for utf8
+#
+SET NAMES utf8mb4 COLLATE utf8mb4_uca1400_test01_as_ci;
+CREATE OR REPLACE TABLE case_folding AS SELECT 0 AS code, SPACE(32) AS c LIMIT 0;
+SHOW CREATE TABLE case_folding;
+Table	Create Table
+case_folding	CREATE TABLE `case_folding` (
+  `code` int(1) NOT NULL,
+  `c` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_uca1400_test01_as_ci DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
+INSERT INTO case_folding (code) VALUES
+(0x23A),
+(0x23E),
+(0x23F),
+(0x240),
+(0x250),
+(0x251),
+(0x252),
+(0x26B),
+(0x271),
+(0x27D);
+INSERT INTO case_folding (code) VALUES
+(0x0049) /* LATIN CAPITAL LETTER I */,
+(0x0069) /* LATIN SMALL LETTER I */,
+(0x0130) /* LATIN CAPITAL LETTER I WITH DOT ABOVE */,
+(0x0131) /* LATIN SMALL LETTER DOTLESS I */
+;
+UPDATE case_folding SET c=CHAR(code USING ucs2);
+SELECT HEX(code), HEX(LOWER(c)), HEX(UPPER(c)), c FROM case_folding;
+HEX(code)	HEX(LOWER(c))	HEX(UPPER(c))	c
+23A	E2B1A5	C8BA	Ⱥ
+23E	E2B1A6	C8BE	Ⱦ
+23F	C8BF	E2B1BE	ȿ
+240	C980	E2B1BF	ɀ
+250	C990	E2B1AF	ɐ
+251	C991	E2B1AD	ɑ
+252	C992	E2B1B0	ɒ
+26B	C9AB	E2B1A2	ɫ
+271	C9B1	E2B1AE	ɱ
+27D	C9BD	E2B1A4	ɽ
+49	69	49	I
+69	69	49	i
+130	69	C4B0	İ
+131	C4B1	49	ı
+DROP TABLE case_folding;
+#
 # End of 10.10 tests
 #

mysql-test/main/ctype_ldml.test

@@ -697,6 +697,13 @@ ORDER BY a, HEX(a);
 --horizontal_results
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-30661 UPPER() returns an empty string for U+0251 in uca1400 collations for utf8
+--echo #
+
+SET NAMES utf8mb4 COLLATE utf8mb4_uca1400_test01_as_ci;
+--source include/ctype_casefolding.inc
+
 --echo #
 --echo # End of 10.10 tests
 --echo #