Merge remote-tracking branch 'origin/bb-10.2-ext' into 10.3

.gitignore

@@ -97,6 +97,9 @@ pcre/pcre_chartables.c
 pcre/pcregrep
 pcre/pcretest
 pcre/test*grep
+plugin/aws_key_management/aws-sdk-cpp
+plugin/aws_key_management/aws_sdk_cpp
+plugin/aws_key_management/aws_sdk_cpp-prefix
 scripts/comp_sql
 scripts/make_binary_distribution
 scripts/msql2mysql
@@ -179,6 +182,9 @@ storage/myisam/myisamlog
 storage/myisam/myisampack
 storage/myisam/rt_test
 storage/myisam/sp_test
+storage/rocksdb/ldb
+storage/rocksdb/mysql_ldb
+storage/rocksdb/sst_dump
 storage/tokudb/PerconaFT/buildheader/db.h
 storage/tokudb/PerconaFT/buildheader/make_tdb
 storage/tokudb/PerconaFT/buildheader/runcat.sh

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "libmariadb"]
 	path = libmariadb
 	url = https://github.com/MariaDB/mariadb-connector-c
+[submodule "storage/rocksdb/rocksdb"]
+	path = storage/rocksdb/rocksdb
+	url = https://github.com/facebook/rocksdb.git

CMakeLists.txt

@@ -364,6 +364,10 @@ IF(WITH_UNIT_TESTS)
 ENDIF()
 
 SET (MYSQLD_STATIC_PLUGIN_LIBS "" CACHE INTERNAL "")
+
+# mariadb_connector_c fetches submodules which is useful for plugins
+INCLUDE(mariadb_connector_c) # this does ADD_SUBDIRECTORY(libmariadb)
+
 # Add storage engines and plugins.
 CONFIGURE_PLUGINS()
 
@@ -373,7 +377,6 @@ ADD_SUBDIRECTORY(strings)
 ADD_SUBDIRECTORY(vio)
 ADD_SUBDIRECTORY(mysys)
 ADD_SUBDIRECTORY(mysys_ssl)
-INCLUDE(mariadb_connector_c) # this does ADD_SUBDIRECTORY(libmariadb)
 ADD_SUBDIRECTORY(client)
 ADD_SUBDIRECTORY(extra)
 ADD_SUBDIRECTORY(libservices)

cmake/FindLZ4.cmake

@@ -0,0 +1,9 @@
+find_path(LZ4_INCLUDE_DIR NAMES lz4.h)
+find_library(LZ4_LIBRARY NAMES lz4)
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(
+    LZ4 DEFAULT_MSG
+    LZ4_LIBRARY LZ4_INCLUDE_DIR)
+
+mark_as_advanced(LZ4_INCLUDE_DIR LZ4_LIBRARY)

cmake/FindZSTD.cmake

@@ -0,0 +1,18 @@
+find_path(
+    ZSTD_INCLUDE_DIR
+    NAMES "zstd.h"
+)
+
+find_library(
+    ZSTD_LIBRARY
+    NAMES zstd
+)
+
+set(ZSTD_LIBRARIES ${ZSTD_LIBRARY})
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(
+    ZSTD DEFAULT_MSG ZSTD_INCLUDE_DIR ZSTD_LIBRARIES)
+
+mark_as_advanced(ZSTD_INCLUDE_DIR ZSTD_LIBRARIES ZSTD_FOUND)
+

cmake/os/Windows.cmake

@@ -63,6 +63,26 @@ IF(MINGW AND CMAKE_SIZEOF_VOID_P EQUAL 4)
 ENDIF()
 
 IF(MSVC)
+  SET(MSVC_CRT_TYPE /MT CACHE STRING
+    "Runtime library - specify runtime library for linking (/MT,/MTd,/MD,/MDd)"
+  )
+  SET(VALID_CRT_TYPES /MTd /MDd /MD  /MT)
+  IF (NOT ";${VALID_CRT_TYPES};" MATCHES ";${MSVC_CRT_TYPE};")
+    MESSAGE(FATAL_ERROR "Invalid value ${MSVC_CRT_TYPE} for MSVC_CRT_TYPE, choose one of /MT,/MTd,/MD,/MDd ")
+  ENDIF()
+
+  IF(MSVC_CRT_TYPE MATCHES "/MD")
+   # Dynamic runtime (DLLs), need to install CRT libraries.
+   SET(CMAKE_INSTALL_MFC_LIBRARIES TRUE)# upgrade wizard
+   SET(CMAKE_INSTALL_SYSTEM_RUNTIME_COMPONENT VCCRT)
+   SET(CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_NO_WARNINGS TRUE)
+   SET(CMAKE_INSTALL_UCRT_LIBRARIES TRUE)
+   IF(MSVC_CRT_TYPE STREQUAL "/MDd")
+     SET (CMAKE_INSTALL_DEBUG_LIBRARIES_ONLY TRUE)
+   ENDIF()
+   INCLUDE(InstallRequiredSystemLibraries)
+  ENDIF()
+
   # Enable debug info also in Release build,
   # and create PDB to be able to analyze crashes.
   FOREACH(type EXE SHARED MODULE)
@@ -77,6 +97,10 @@ IF(MSVC)
   #     information for use with the debugger. The symbolic debugging
   #     information includes the names and types of variables, as well as
   #     functions and line numbers. No .pdb file is produced by the compiler.
+  #
+  # - Remove preprocessor flag _DEBUG that older cmakes use with Config=Debug,
+  #   it is as defined by Debug runtimes itself (/MTd /MDd)
+
   FOREACH(lang C CXX)
     SET(CMAKE_${lang}_FLAGS_RELEASE "${CMAKE_${lang}_FLAGS_RELEASE} /Z7")
   ENDFOREACH()
@@ -85,7 +109,8 @@ IF(MSVC)
    CMAKE_C_FLAGS_DEBUG      CMAKE_C_FLAGS_DEBUG_INIT 
    CMAKE_CXX_FLAGS_RELEASE  CMAKE_CXX_FLAGS_RELWITHDEBINFO
    CMAKE_CXX_FLAGS_DEBUG    CMAKE_CXX_FLAGS_DEBUG_INIT)
-   STRING(REPLACE "/MD"  "/MT" "${flag}" "${${flag}}")
+   STRING(REGEX REPLACE "/M[TD][d]?"  "${MSVC_CRT_TYPE}" "${flag}"  "${${flag}}" )
+   STRING(REGEX REPLACE "/D[ ]?_DEBUG"  "" "${flag}" "${${flag}}")
    STRING(REPLACE "/Zi"  "/Z7" "${flag}" "${${flag}}")
   ENDFOREACH()
 
@@ -117,13 +142,6 @@ IF(MSVC)
   SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4800 /wd4805 /wd4996")
   SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4800 /wd4805 /wd4996 /wd4291 /wd4577 /we4099")
 
-  IF(CMAKE_SIZEOF_VOID_P MATCHES 8)
-    # _WIN64 is defined by the compiler itself. 
-    # Yet, we define it here again   to work around a bug with  Intellisense 
-    # described here: http://tinyurl.com/2cb428. 
-    # Syntax highlighting is important for proper debugger functionality.
-    ADD_DEFINITIONS("-D_WIN64")
-  ENDIF()
 ENDIF()
 
 # Always link with socket library

debian/autobake-deb.sh

@@ -72,6 +72,16 @@ then
   sed '/mariadb-service-convert/d' -i debian/mariadb-server-10.2.install
 fi
 
+# Convert gcc version to numberical value. Format is Mmmpp where M is Major
+# version, mm is minor version and p is patch.
+GCCVERSION=$(gcc -dumpversion | sed -e 's/\.\([0-9][0-9]\)/\1/g' -e 's/\.\([0-9]\)/0\1/g' -e 's/^[0-9]\{3,4\}$/&00/')
+# Don't build rocksdb package if gcc version is less than 4.8 or we are running on
+# x86 32 bit.
+if [[ $GCCVERSION -lt 40800 ]] || [[ $(arch) =~ i[346]86 ]]
+then
+  sed '/Package: mariadb-plugin-rocksdb/,+7d' -i debian/control
+fi
+
 # Adjust changelog, add new version
 echo "Incrementing changelog and starting build scripts"
 

debian/control

@@ -22,6 +22,7 @@ Build-Depends: bison,
                libpcre3-dev (>= 2:8.35-3.2~),
                libreadline-gplv2-dev,
                libssl-dev,
+               libsnappy-dev,
                libsystemd-dev,
                libxml2-dev,
                lsb-release,
@@ -452,6 +453,14 @@ Description: Connect storage engine for MariaDB
  other interesting features.
  This package contains the Connect plugin for MariaDB.
 
+Package: mariadb-plugin-rocksdb
+Architecture: any
+Depends: mariadb-server-10.2, ${misc:Depends}, ${shlibs:Depends}
+Description: RocksDB storage engine for MariaDB
+ The RocksDB storage engine is a high performance storage engine, aimed
+ at maximising storage efficiency while maintaining InnoDB-like performance.
+ This package contains the RocksDB plugin for MariaDB.
+
 Package: mariadb-plugin-oqgraph
 Architecture: any
 Depends: libjudydebian1, mariadb-server-10.2, ${misc:Depends}, ${shlibs:Depends}

debian/mariadb-plugin-rocksdb.install

@@ -0,0 +1,4 @@
+etc/mysql/conf.d/rocksdb.cnf etc/mysql/mariadb.conf.d
+usr/lib/mysql/plugin/ha_rocksdb.so
+usr/bin/mysql_ldb
+usr/bin/sst_dump

include/my_bit.h

@@ -25,7 +25,6 @@
 
 C_MODE_START
 
-extern const char _my_bits_nbits[256];
 extern const uchar _my_bits_reverse_table[256];
 
 /*
@@ -40,37 +39,32 @@ static inline uint my_bit_log2(ulong value)
   return bit;
 }
 
-static inline uint my_count_bits(ulonglong v)
+
+/*
+Count bits in 32bit integer
+
+  Algorithm by Sean Anderson, according to:
+  http://graphics.stanford.edu/~seander/bithacks.html
+  under "Counting bits set, in parallel"
+
+ (Orignal code public domain).
+*/
+static inline uint my_count_bits_uint32(uint32 v)
 {
-#if SIZEOF_LONG_LONG > 4
-  /* The following code is a bit faster on 16 bit machines than if we would
-     only shift v */
-  ulong v2=(ulong) (v >> 32);
-  return (uint) (uchar) (_my_bits_nbits[(uchar)  v] +
-                         _my_bits_nbits[(uchar) (v >> 8)] +
-                         _my_bits_nbits[(uchar) (v >> 16)] +
-                         _my_bits_nbits[(uchar) (v >> 24)] +
-                         _my_bits_nbits[(uchar) (v2)] +
-                         _my_bits_nbits[(uchar) (v2 >> 8)] +
-                         _my_bits_nbits[(uchar) (v2 >> 16)] +
-                         _my_bits_nbits[(uchar) (v2 >> 24)]);
-#else
-  return (uint) (uchar) (_my_bits_nbits[(uchar)  v] +
-                         _my_bits_nbits[(uchar) (v >> 8)] +
-                         _my_bits_nbits[(uchar) (v >> 16)] +
-                         _my_bits_nbits[(uchar) (v >> 24)]);
-#endif
+  v = v - ((v >> 1) & 0x55555555);
+  v = (v & 0x33333333) + ((v >> 2) & 0x33333333);
+  return (((v + (v >> 4)) & 0xF0F0F0F) * 0x1010101) >> 24;
 }
 
-static inline uint my_count_bits_uint32(uint32 v)
+
+static inline uint my_count_bits(ulonglong x)
 {
-  return (uint) (uchar) (_my_bits_nbits[(uchar)  v] +
-                         _my_bits_nbits[(uchar) (v >> 8)] +
-                         _my_bits_nbits[(uchar) (v >> 16)] +
-                         _my_bits_nbits[(uchar) (v >> 24)]);
+  return my_count_bits_uint32((uint32)x) + my_count_bits_uint32((uint32)(x >> 32));
 }
 
 
+
+
 /*
   Next highest power of two
 

include/mysql/plugin.h

@@ -393,6 +393,23 @@ DECLARE_MYSQL_SYSVAR_SIMPLE(name, unsigned long long) = { \
   PLUGIN_VAR_LONGLONG | PLUGIN_VAR_UNSIGNED | ((opt) & PLUGIN_VAR_MASK), \
   #name, comment, check, update, &varname, def, min, max, blk }
 
+#define MYSQL_SYSVAR_UINT64_T(name, varname, opt, comment, check, update, def, min, max, blk) \
+DECLARE_MYSQL_SYSVAR_SIMPLE(name, uint64_t) = { \
+  PLUGIN_VAR_LONGLONG | PLUGIN_VAR_UNSIGNED | ((opt) & PLUGIN_VAR_MASK), \
+  #name, comment, check, update, &varname, def, min, max, blk }
+
+#ifdef _WIN64
+#define MYSQL_SYSVAR_SIZE_T(name, varname, opt, comment, check, update, def, min, max, blk) \
+DECLARE_MYSQL_SYSVAR_SIMPLE(name, size_t) = { \
+  PLUGIN_VAR_LONGLONG | PLUGIN_VAR_UNSIGNED | ((opt) & PLUGIN_VAR_MASK), \
+  #name, comment, check, update, &varname, def, min, max, blk }
+#else
+#define MYSQL_SYSVAR_SIZE_T(name, varname, opt, comment, check, update, def, min, max, blk) \
+DECLARE_MYSQL_SYSVAR_SIMPLE(name, size_t) = { \
+  PLUGIN_VAR_LONG | PLUGIN_VAR_UNSIGNED | ((opt) & PLUGIN_VAR_MASK), \
+  #name, comment, check, update, &varname, def, min, max, blk }
+#endif
+
 #define MYSQL_SYSVAR_ENUM(name, varname, opt, comment, check, update, def, typelib) \
 DECLARE_MYSQL_SYSVAR_TYPELIB(name, unsigned long) = { \
   PLUGIN_VAR_ENUM | ((opt) & PLUGIN_VAR_MASK), \

mysql-test/collections/buildbot_suites.bat

@@ -0,0 +1,5 @@
+perl mysql-test-run.pl --verbose-restart --force --testcase-timeout=45 --suite-timeout=600 --max-test-fail=500 --retry=3  --parallel=4 --suite=^
+main,^
+innodb,^
+plugins,^
+rocksdb

mysql-test/extra/rpl_tests/rpl_checksum.inc

@@ -305,7 +305,6 @@ if(!$log_error_)
     let $log_error_ = $MYSQLTEST_VARDIR/log/mysqld.2.err;
 }
 --let SEARCH_FILE= $log_error_
---let SEARCH_RANGE=-50000
 --let SEARCH_PATTERN= Slave SQL: The incident LOST_EVENTS occurred on the master\. Message: error writing to the binary log, Internal MariaDB error code: 1590
 --source include/search_pattern_in_file.inc