WL#14467: Isolate character set and collation library

The patch is intended to make libstrings, the character set
and collation library, an independent self-contained library
suitable for use in separate projects where the whole mysys
is not needed or is not welcome.

API changes
===========

- include/m_ctype.h

  + m_ctype.h is a C++ source file for a long time, so,
    C-specific code has been removed/converted.

  + Error message data has been extracted from
    MY_CHARSET_LOADER into MY_CHARSET_ERRMSG

  + MY_CHARSET_LOADER has been converted into a C++
    abstract interface structure.

- include/my_sys.h

  + all_charsets[] has made an array of constant pointers
    (was not constant).

  + The variadic type my_error_reporter of the
    my_charset_error_reporter callback has been replaced
    with my_error_vreporter (the latest expects va_list
    instead of ellipsis).

  + my_charset_loader_init_mysys() has been removed.

  + my_collation_get_by_name() and my_charset_get_by_name()
    have been changed to return error messages via
    MY_CHARSET_ERRMSG instead of MY_CHARSET_LOADER.

Behavior changes
================

- On-demand initialization of lexer tables (former lex_init()
  and hint_lex_init_maps()).

Locks
=====

- Global mutex THR_LOCK_charset has been replaced with a local std::mutex.

Use cases of new API in server sources
======================================

- Lexer: new API call instead of get_charset_by_csname().

Added to public API
===================

- include/mysql/strings/collations.h: new public API of libstring

- include/mysql/attribute.h: public version of MY_ATTRIBUTE

- include/mysql/declspec.h: public version of MYSQL_PLUGIN_IMPORT

Moved to public API
===================

- include/mysql/strings/m_ctype.h

- include/mysql/my_loglevel.h

New important files
===================

- strings/collations.cc: hides interactions with implementation
  internals (i.e. with strings/collations_internal.{h,cc})

Change-Id: I95ab120ab32f8f1225cda9b5837c8b53c9b9c4a4

Author: Tor Didriksen

CMakeLists.txt

@@ -1576,6 +1576,17 @@ ADD_DEFINITIONS(-D__STDC_FORMAT_MACROS)    # Enable C99 printf format macros
 ADD_DEFINITIONS(-D_USE_MATH_DEFINES)       # Get access to M_PI, M_E, etc. in math.h
 ADD_DEFINITIONS(-DLZ4_DISABLE_DEPRECATE_WARNINGS) # C++14 deprecation warnings in LZ4.
 
+# Force the same macro definition MYSQL_PROJECT for all source files being
+# compiled as a part of the MySQL server, client tools, libraries etc.
+# The MYSQL_PROJECT macro helps to distinguish when e.g. some C++ header file
+# is included by the MySQL project source file or by the external project
+# source file.
+# For example, MYSQL_PROJECT helps to detect when include/mysql/strings/api.h
+# is used to a) build a static library, or b) build a Windows DLL, or
+# c) reference the server strings API from plugins (DLLs) loaded by the
+# MySQL server, or d) reference the libstrings DLL from external executables.
+ADD_DEFINITIONS(-DMYSQL_PROJECT)
+
 OPTION(ENABLE_EXPERIMENT_SYSVARS "Expose ussually hidden system variables to allow experiments" OFF)
 IF(ENABLE_EXPERIMENT_SYSVARS)
   ADD_DEFINITIONS(-DENABLE_EXPERIMENT_SYSVARS)
@@ -2222,6 +2233,13 @@ IF(NOT WITHOUT_SERVER AND WITH_UNIT_TESTS)
       # If some symbols are still missing, they will be picked up from
       # dependent libraries, since we LINK_PUBLIC.
       # To see what symbols we need to import, remove LINK_PUBLIC above.
+      #
+      # The strings library uses visibility=hidden for all symbols,
+      # except those explicitly tagged with MYSQL_STRINGS_EXPORT.
+      # If we get ODR violations for executables using server_unittest_library,
+      # it means the symbol has been found in strings and
+      # server_unittest_library, which means the unit test is using
+      # some non-exported symbol from strings.
       EXPORTS
       builtin_perfschema_plugin            # Pulls in the whole server.
       mysql_service_mysql_rwlock_v1        # Pulls in minchassis

client/CMakeLists.txt

@@ -176,7 +176,7 @@ MERGE_CONVENIENCE_LIBRARIES(json_binlog_static
 MYSQL_ADD_EXECUTABLE(json_client_library_main
   json_client_library_main.cc
   INCLUDE_DIRECTORIES ../sql
-  LINK_LIBRARIES json_client_library  strings
+  LINK_LIBRARIES json_client_library  strings mysys
   SKIP_INSTALL
   )
 

client/base/abstract_connection_program.cc

@@ -24,6 +24,8 @@
 
 #include "client/base/abstract_connection_program.h"
 
+struct CHARSET_INFO;
+
 using namespace Mysql::Tools::Base;
 
 Abstract_connection_program::Abstract_connection_program()
@@ -41,4 +43,4 @@ CHARSET_INFO *Abstract_connection_program::get_current_charset() const {
 
 void Abstract_connection_program::set_current_charset(CHARSET_INFO *charset) {
   m_connection_options.set_current_charset(charset);
-}
+}

client/base/abstract_connection_program.h

@@ -34,6 +34,8 @@
 #include "client/base/mysql_connection_options.h"
 #include "client/client_priv.h"
 
+struct CHARSET_INFO;
+
 namespace Mysql {
 namespace Tools {
 namespace Base {

client/base/message_data.cc

@@ -25,8 +25,9 @@
 #include "client/base/message_data.h"
 #include "errmsg.h"
 
-#include "m_ctype.h"
+#include "mysql/strings/m_ctype.h"
 #include "sql_string.h"
+#include "template_utils.h"
 
 using namespace Mysql::Tools::Base;
 

client/base/mysql_connection_options.cc

@@ -34,9 +34,11 @@
 #include "client/base/abstract_options_provider.h"
 #include "client/base/abstract_program.h"
 #include "compression.h"
-#include "m_ctype.h"
+#include "m_string.h"
 #include "multi_factor_passwordopt-vars.h"
+#include "mysql/strings/m_ctype.h"
 #include "mysys_err.h"
+#include "nulls.h"
 #include "typelib.h"
 
 using Mysql::Tools::Base::Abstract_program;

client/base/mysql_connection_options.h

@@ -35,6 +35,8 @@
 #include "my_compiler.h"
 #include "my_inttypes.h"
 
+struct CHARSET_INFO;
+
 namespace Mysql {
 namespace Tools {
 namespace Base {

client/base/mysql_query_runner.cc

@@ -29,7 +29,7 @@
 #include <atomic>
 #include <functional>
 
-#include "m_ctype.h"
+#include "mysql/strings/m_ctype.h"
 #include "sql_string.h"
 #include "template_utils.h"
 

client/check/mysqlcheck.cc

@@ -31,13 +31,14 @@
 #include "caching_sha2_passwordopt-vars.h"
 #include "client/client_priv.h"
 #include "compression.h"
-#include "m_ctype.h"
 #include "my_alloc.h"
 #include "my_dbug.h"
 #include "my_default.h"
 #include "my_inttypes.h"
 #include "my_macros.h"
 #include "mysql/service_mysql_alloc.h"
+#include "mysql/strings/m_ctype.h"
+#include "nulls.h"
 #include "print_version.h"
 #include "sslopt-vars.h"
 #include "typelib.h"

client/check/mysqlcheck_core.cc

@@ -30,9 +30,10 @@
 
 #include "client/check/mysqlcheck.h"
 #include "client/client_priv.h"
-#include "m_ctype.h"
+#include "m_string.h"
 #include "my_default.h"
 #include "my_inttypes.h"
+#include "mysql/strings/m_ctype.h"
 
 using namespace Mysql::Tools::Check;
 
@@ -354,15 +355,18 @@ static void print_result() {
   mysql_free_result(res);
 }
 
-void Mysql::Tools::Check::mysql_check(
-    MYSQL *connection, int what_to_do, bool opt_alldbs,
-    bool opt_check_only_changed, bool opt_extended, bool opt_databases,
-    bool opt_fast, bool opt_medium_check, bool opt_quick, bool opt_all_in_1,
-    bool opt_silent, bool opt_auto_repair, bool ignore_errors, bool opt_frm,
-    bool opt_fix_table_names, bool opt_fix_db_names, bool opt_upgrade,
-    bool opt_write_binlog, uint verbose, std::string opt_skip_database,
-    std::vector<std::string> arguments,
-    void (*dberror)(MYSQL *mysql, const std::string &when)) {
+namespace Mysql::Tools::Check {
+
+void mysql_check(MYSQL *connection, int what_to_do, bool opt_alldbs,
+                 bool opt_check_only_changed, bool opt_extended,
+                 bool opt_databases, bool opt_fast, bool opt_medium_check,
+                 bool opt_quick, bool opt_all_in_1, bool opt_silent,
+                 bool opt_auto_repair, bool ignore_errors, bool opt_frm,
+                 bool opt_fix_table_names, bool opt_fix_db_names,
+                 bool opt_upgrade, bool opt_write_binlog, uint verbose,
+                 std::string opt_skip_database,
+                 std::vector<std::string> arguments,
+                 void (*dberror)(MYSQL *mysql, const std::string &when)) {
   ::sock = connection;
   ::what_to_do = what_to_do;
   ::opt_alldbs = opt_alldbs;
@@ -419,6 +423,8 @@ void Mysql::Tools::Check::mysql_check(
   }
 }
 
+}  // namespace Mysql::Tools::Check
+
 Program::Program()
     : m_what_to_do(0),
       m_auto_repair(false),

client/client_priv.h

@@ -30,7 +30,6 @@
 #include <mysql.h>
 
 #include "errmsg.h"
-#include "m_string.h"
 #include "my_getopt.h"
 #include "my_sys.h"
 

client/dump/abstract_mysql_chain_element_extension.cc

@@ -30,7 +30,7 @@
 #include <optional>
 #include <sstream>
 
-#include "m_ctype.h"
+#include "mysql/strings/m_ctype.h"
 
 using namespace Mysql::Tools::Dump;
 

client/dump/abstract_mysql_chain_element_extension.h

@@ -40,6 +40,8 @@
 #define MYSQL_UNIVERSAL_CLIENT_CHARSET "utf8mb4"
 #define MAX_NAME_LEN (64 * 3)
 
+struct CHARSET_INFO;
+
 namespace Mysql {
 namespace Tools {
 namespace Dump {

client/dump/mysqldump_tool_chain_maker.cc

@@ -36,7 +36,7 @@
 #include "client/dump/sql_formatter.h"
 #include "client/dump/standard_writer.h"
 #include "client/dump/view.h"
-#include "m_ctype.h"
+#include "mysql/strings/m_ctype.h"
 
 using namespace Mysql::Tools::Dump;
 using std::placeholders::_1;

client/dump/mysqldump_tool_chain_maker_options.cc

@@ -28,7 +28,7 @@
 #include <sstream>
 #include <string>
 
-#include "m_ctype.h"
+#include "mysql/strings/m_ctype.h"
 
 using namespace Mysql::Tools::Dump;
 using std::placeholders::_1;

client/dump/sql_formatter.cc

@@ -38,7 +38,8 @@
 #include "client/dump/privilege.h"
 #include "client/dump/stored_procedure.h"
 #include "client/dump/view.h"
-#include "m_ctype.h"
+#include "m_string.h"
+#include "mysql/strings/m_ctype.h"
 
 using namespace Mysql::Tools::Dump;
 

client/dump/sql_formatter_options.h

@@ -29,6 +29,7 @@
 
 #include "client/base/abstract_options_provider.h"
 #include "client/dump/mysql_chain_element_options.h"
+#include "nulls.h"
 #include "template_utils.h"
 #include "typelib.h"
 

client/migrate_keyring/options.cc

@@ -22,11 +22,15 @@
    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
 */
 
+#include <sys/types.h>
+
+#include <cstddef>
+#include <cstdio>
 #include <cstring>
 #include <iostream>
 #include <string>
+#include <utility>
 
-#include <m_ctype.h>                  /* Character set */
 #include <my_alloc.h>                 /* MEM_ROOT */
 #include <my_default.h>               /* print_defaults */
 #include <my_getopt.h>                /* Options handling */
@@ -38,6 +42,10 @@
 #include <print_version.h>            /* print_version */
 #include <typelib.h>                  /* find_type_or_exit */
 #include <welcome_copyright_notice.h> /* ORACLE_WELCOME_COPYRIGHT_NOTICE */
+#include "m_string.h"
+#include "mysql/strings/m_ctype.h" /* Character set */
+#include "nulls.h"
+#include "template_utils.h"
 
 #include "options.h"
 #include "utilities.h"

client/multi_factor_passwordopt-vars.cc

@@ -28,6 +28,7 @@
 #include "my_getopt.h"
 #include "mysql.h"
 #include "mysql/service_mysql_alloc.h"  // my_free, my_strdup
+#include "nulls.h"
 
 char *opt_password[MAX_AUTH_FACTORS] = {nullptr};
 bool tty_password[MAX_AUTH_FACTORS] = {false};

client/mysql.cc

@@ -43,15 +43,24 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
 #include "client/pattern_matcher.h"
 #include "compression.h"
 #include "lex_string.h"
-#include "m_ctype.h"
+#include "m_string.h"
 #include "my_compiler.h"
 #include "my_dbug.h"
 #include "my_default.h"
 #include "my_dir.h"
 #include "my_inttypes.h"
 #include "my_io.h"
-#include "my_loglevel.h"
 #include "my_macros.h"
+#include "mysql/my_loglevel.h"
+#include "mysql/strings/int2str.h"
+#include "mysql/strings/m_ctype.h"
+#include "nulls.h"
+#include "str2int.h"
+#include "strcont.h"
+#include "string_with_len.h"
+#include "strmake.h"
+#include "strxmov.h"
+#include "strxnmov.h"
 #include "typelib.h"
 #include "user_registration.h"
 #include "violite.h"

client/mysql_config_editor.cc

@@ -50,6 +50,7 @@
 #include "my_rnd.h"
 #include "mysql/service_mysql_alloc.h"
 #include "mysys/my_default_priv.h"
+#include "nulls.h"
 #include "print_version.h"
 #include "welcome_copyright_notice.h"
 

client/mysql_secure_installation.cc

@@ -27,8 +27,9 @@
 #include <sys/types.h>
 
 #include "client/client_priv.h"
+#include "m_string.h"
 #ifdef _WIN32
-#include "m_ctype.h"
+#include "mysql/strings/m_ctype.h"
 #endif
 #include "my_alloc.h"
 #include "my_compiler.h"
@@ -39,6 +40,7 @@
 #include "my_shm_defaults.h"
 #include "mysql/service_mysql_alloc.h"
 #include "mysqld_error.h"
+#include "nulls.h"
 #include "print_version.h"
 #include "typelib.h"
 #include "welcome_copyright_notice.h"  // ORACLE_WELCOME_COPYRIGHT_NOTICE

client/mysql_ssl_rsa_setup.cc

@@ -40,7 +40,7 @@
 #include "client/logger.h"
 #include "client/path.h"
 #ifdef _WIN32
-#include "m_ctype.h"
+#include "mysql/strings/m_ctype.h"
 #endif
 #include "my_alloc.h"
 #include "my_compiler.h"

client/mysqladmin.cc

@@ -36,7 +36,7 @@
 
 #include "client/client_priv.h"
 #include "compression.h"
-#include "m_ctype.h"
+#include "m_string.h"
 #include "my_alloc.h"
 #include "my_compiler.h"
 #include "my_dbug.h"
@@ -46,8 +46,13 @@
 #include "my_macros.h"
 #include "my_thread.h" /* because of signal()	*/
 #include "mysql/service_mysql_alloc.h"
+#include "mysql/strings/int2str.h"
+#include "mysql/strings/m_ctype.h"
+#include "nulls.h"
 #include "print_version.h"
 #include "sql_common.h"
+#include "str2int.h"
+#include "strxmov.h"
 #include "typelib.h"
 #include "welcome_copyright_notice.h" /* ORACLE_WELCOME_COPYRIGHT_NOTICE */
 

client/mysqlbinlog.cc

@@ -52,13 +52,17 @@
 #include "libbinlogevents/include/compression/factory.h"
 #include "libbinlogevents/include/compression/iterator.h"
 #include "libbinlogevents/include/trx_boundary_parser.h"
+#include "m_string.h"
 #include "my_byteorder.h"
 #include "my_dbug.h"
 #include "my_default.h"
 #include "my_dir.h"
 #include "my_io.h"
 #include "my_macros.h"
 #include "my_time.h"
+#include "mysql/strings/int2str.h"
+#include "mysql/strings/m_ctype.h"
+#include "nulls.h"
 #include "prealloced_array.h"
 #include "print_version.h"
 #include "sql/binlog_reader.h"

client/mysqldump.cc

@@ -38,7 +38,6 @@
 
 #include "client/client_priv.h"
 #include "compression.h"
-#include "m_ctype.h"
 #include "m_string.h"
 #include "map_helpers.h"
 #include "my_compiler.h"
@@ -53,11 +52,16 @@
 #include "my_user.h"
 #include "mysql.h"
 #include "mysql/service_mysql_alloc.h"
+#include "mysql/strings/m_ctype.h"
 #include "mysql_version.h"
 #include "mysqld_error.h"
+#include "nulls.h"
 #include "prealloced_array.h"
 #include "print_version.h"
 #include "scope_guard.h"
+#include "string_with_len.h"
+#include "strmake.h"
+#include "strxmov.h"
 #include "template_utils.h"
 #include "typelib.h"
 #include "welcome_copyright_notice.h" /* ORACLE_WELCOME_COPYRIGHT_NOTICE */