From 1db4fc543bba0cadbdcecdd82a6584875559a667 Mon Sep 17 00:00:00 2001 From: anson1014 <56494179+anson1014@users.noreply.github.com> Date: Tue, 30 Aug 2022 04:21:40 -0400 Subject: [PATCH] Ensure that source files contain only valid UTF8 encodings (#2188) Modern software (including text editors, static analysis software, and web-based code review interfaces) often requires source code files to be interpretable via a consistent character encoding, with UTF-8 or ASCII (a strict subset of UTF-8) as the default. Several of the MariaDB source files contain bytes that are not valid in either the UTF-8 or ASCII encodings, but instead represent strings encoded in the ISO-8859-1/Latin-1 or ISO-8859-2/Latin-2 encodings. These inconsistent encodings may prevent software from correctly presenting or processing such files. Converting all source files to valid UTF8 characters will ensure correct handling. Comments written in Czech were replaced with lightly-corrected translations from Google Translate. Additionally, comments describing the proper handling of special characters were changed so that the comments are now purely UTF8. All new code of the whole pull request, including one or several files that are either new files or modified ones, are contributed under the BSD-new license. I am contributing on behalf of my employer Amazon Web Services, Inc. Co-authored-by: Andrew Hutchings --- mysys/my_win_popen.cc | 2 +- storage/connect/domdoc.cpp | 1 - strings/ctype-czech.c | 75 +++++++++++++------------------------- strings/ctype-latin1.c | 16 ++++---- 4 files changed, 34 insertions(+), 60 deletions(-) diff --git a/mysys/my_win_popen.cc b/mysys/my_win_popen.cc index f41f54100f1bf..cceb77e901966 100644 --- a/mysys/my_win_popen.cc +++ b/mysys/my_win_popen.cc @@ -92,7 +92,7 @@ extern "C" FILE *my_win_popen(const char *cmd, const char *mode) goto error; break; default: - /* Unknown mode, éxpected "r", "rt", "w", "wt" */ + /* Unknown mode, expected "r", "rt", "w", "wt" */ abort(); } if (!SetHandleInformation(parent_pipe_end, HANDLE_FLAG_INHERIT, 0)) diff --git a/storage/connect/domdoc.cpp b/storage/connect/domdoc.cpp index 7d5b87a264018..268ad771ef949 100644 --- a/storage/connect/domdoc.cpp +++ b/storage/connect/domdoc.cpp @@ -642,7 +642,6 @@ bool DOMNODELIST::DropItem(PGLOBAL g, int n) if (Listp == NULL || Listp->length < n) return true; -//Listp->item[n] = NULL; La propriété n'a pas de méthode 'set' return false; } // end of DeleteItem diff --git a/strings/ctype-czech.c b/strings/ctype-czech.c index 33d43d4dd4e54..cb5205f315e8e 100644 --- a/strings/ctype-czech.c +++ b/strings/ctype-czech.c @@ -23,13 +23,13 @@ solution was needed than the one-to-one conversion table. To note a few, here is an example of a Czech sorting sequence: - co < hlaska < hláska < hlava < chlapec < krtek + co < hlaska < hlĂĄska < hlava < chlapec < krtek It because some of the rules are: double char 'ch' is sorted - between 'h' and 'i'. Accented character 'á' (a with acute) is + between 'h' and 'i'. Accented character 'ĂĄ' (a with acute) is sorted after 'a' and before 'b', but only if the word is otherwise the same. However, because 's' is sorted before 'v' - in hlava, the accentness of 'á' is overridden. There are many + in hlava, the accentness of 'ĂĄ' is overridden. There are many more rules. This file defines functions my_strxfrm and my_strcoll for @@ -42,8 +42,9 @@ passes, that's why we need four times more space for expanded string. - This file also contains the ISO-Latin-2 definitions of - characters. + The non-ASCII literal strings in this file are encoded + in the iso-8859-2 / latin-2 character set + (https://en.wikipedia.org/wiki/ISO/IEC_8859-2) Author: (c) 1997--1998 Jan Pazdziora, adelton@fi.muni.cz Jan Pazdziora has a shared copyright for this code @@ -111,7 +112,7 @@ static const struct wordvalue doubles[] = { }; /* - Unformal description of the algorithm: + Informal description of the algorithm: We walk the string left to right. @@ -126,7 +127,7 @@ static const struct wordvalue doubles[] = { End of pass is marked with value 1 on the output. - For each character, we read it's value from the table. + For each character, we read its value from the table. If the value is ignore (0), we go straight to the next character. @@ -138,31 +139,6 @@ static const struct wordvalue doubles[] = { exists behind it, find its value. We append 0 to the end. ---- - Neformální popis algoritmu: - - Procházíme řetězec zleva doprava. - - Konec řetězce je předán buď jako parametr, nebo je to *p == 0. - Toto je ošetřeno makrem IS_END. - - Pokud jsme došli na konec řetězce při průchodu 0, nejdeme na - začátek, ale na uloženou pozici, protože první a druhý průchod - běží současně. - - Konec vstupu (průchodu) označíme na výstupu hodnotou 1. - - Pro každý znak řetězce načteme hodnotu z třídící tabulky. - - Jde-li o hodnotu ignorovat (0), skočíme ihned na další znak.. - - Jde-li o hodnotu konec slova (2) a je to průchod 0 nebo 1, - přeskočíme všechny další 0 -- 2 a prohodíme průchody. - - Jde-li o kompozitní znak (255), otestujeme, zda následuje - správný do dvojice, dohledáme správnou hodnotu. - - Na konci připojíme znak 0 */ #define ADD_TO_RESULT(dest, len, totlen, value) \ @@ -335,24 +311,23 @@ my_strnxfrm_czech(CHARSET_INFO *cs __attribute__((unused)), /* - Neformální popis algoritmu: - - procházíme řetězec zleva doprava - konec řetězce poznáme podle *p == 0 - pokud jsme došli na konec řetězce při průchodu 0, nejdeme na - začátek, ale na uloženou pozici, protože první a druhý - průchod běží současně - konec vstupu (průchodu) označíme na výstupu hodnotou 1 - - načteme hodnotu z třídící tabulky - jde-li o hodnotu ignorovat (0), skočíme na další průchod - jde-li o hodnotu konec slova (2) a je to průchod 0 nebo 1, - přeskočíme všechny další 0 -- 2 a prohodíme - průchody - jde-li o kompozitní znak (255), otestujeme, zda následuje - správný do dvojice, dohledáme správnou hodnotu - - na konci připojíme znak 0 + Informal description of the algorithm: + + we pass the chain from left to right + we know the end of the string by *p == 0 + if we reached the end of the string on transition 0, then we don't go to + start, but to the saved position, because the first and second + the passage runs concurrently + we mark the end of the input (transition) with the value 1 on the output + + then we load the value from the sorting table + if the value is ignore (0), we jump to the next pass + if the value is the end of the word (2) and it is a 0 or 1 transition, + we skip all the other 0 -- 2 and switch transitions + if it is a composite character (255), we test whether it follows + correct to the pair, we find the correct value + + then we add the character 0 at the end */ diff --git a/strings/ctype-latin1.c b/strings/ctype-latin1.c index bcf1cc6c9f150..b6ffd0906c8ef 100644 --- a/strings/ctype-latin1.c +++ b/strings/ctype-latin1.c @@ -499,19 +499,19 @@ struct charset_info_st my_charset_latin1_nopad= * * The modern sort order is used, where: * - * 'ä' -> "ae" - * 'ö' -> "oe" - * 'ü' -> "ue" - * 'ß' -> "ss" + * 'ä' -> "ae" + * 'Ăś' -> "oe" + * 'Ăź' -> "ue" + * 'ß' -> "ss" */ /* * This is a simple latin1 mapping table, which maps all accented * characters to their non-accented equivalents. Note: in this - * table, 'ä' is mapped to 'A', '˙' is mapped to 'Y', etc. - all + * table, 'ä' is mapped to 'A', 'Ăż' is mapped to 'Y', etc. - all * accented characters except the following are treated the same way. - * Ü, ü, Ö, ö, Ä, ä + * Ü, Ăź, Ö, Ăś, Ä, ä */ static const uchar sort_order_latin1_de[] = { @@ -577,7 +577,7 @@ static const uchar combo2map[]={ my_strnxfrm_latin_de() on both strings and compared the result strings. This means that: - Ä must also matches ÁE and Ač, because my_strxn_frm_latin_de() will convert + Ä must also matches ÁE and Aè, because my_strxn_frm_latin_de() will convert both to AE. The other option would be to not do any accent removal in @@ -703,7 +703,7 @@ void my_hash_sort_latin1_de(CHARSET_INFO *cs __attribute__((unused)), /* Remove end space. We have to do this to be able to compare - 'AE' and 'Ä' as identical + 'AE' and 'Ä' as identical */ end= skip_trailing_space(key, len);