MDEV-12933 sort out the compression library chaos

bzip2/lz4/lzma/lzo/snappy compression is now provided via *services*

they're almost like normal services, but in include/providers/
and they're supposed to provide exactly the same interface
as original compression libraries (but not everything,
only enough of if for the code to compile).

the services are implemented via dummy functions that return
corresponding error values (LZMA_PROG_ERROR, LZO_E_INTERNAL_ERROR, etc).

the actual compression libraries are linked into corresponding
provider plugins. Providers are daemon plugins that when loaded
replace service pointers to point to actual compression functions.

That is, run-time dependency on compression libraries is now on plugins,
and the server doesn't need any compression libraries to run, but
will automatically support the compression when a plugin is loaded.

InnoDB and Mroonga use compression plugins now. RocksDB doesn't,
because it comes with standalone utility binaries that cannot
load plugins.

Author: KartikSoneji

CMakeLists.txt

@@ -375,7 +375,8 @@ ADD_DEFINITIONS(-DHAVE_CONFIG_H)
 IF(_FILE_OFFSET_BITS)
   ADD_DEFINITIONS(-D_FILE_OFFSET_BITS=${_FILE_OFFSET_BITS})
 ENDIF()
-INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/include)
+INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/include
+                    ${CMAKE_SOURCE_DIR}/include/providers)
 
 # Add bundled or system zlib.
 MYSQL_CHECK_ZLIB_WITH_COMPRESS()

cmake/build_configurations/mysql_release.cmake

@@ -108,14 +108,10 @@ ELSEIF(RPM)
   # not yet, SLES 12.3 doesn't provide pcre2
   #SET(WITH_PCRE system CACHE STRING "")
   IF(RPM MATCHES "fedora|centos|rhel")
-    SET(WITH_INNODB_BZIP2 OFF CACHE STRING "")
-    SET(WITH_INNODB_LZO OFF CACHE STRING "")
     SET(WITH_ROCKSDB_BZip2 OFF CACHE STRING "")
   ENDIF()
   IF(RPM MATCHES "opensuse|sles|centos|rhel")
-    SET(WITH_INNODB_LZ4 OFF CACHE STRING "")
     SET(WITH_ROCKSDB_LZ4 OFF CACHE STRING "")
-    SET(GRN_WITH_LZ4 no CACHE STRING "")
   ENDIF()
 ELSEIF(DEB)
   SET(WITH_SSL system CACHE STRING "")
@@ -125,9 +121,6 @@ ELSEIF(DEB)
   SET(PLUGIN_AUTH_SOCKET YES CACHE STRING "")
   SET(WITH_EMBEDDED_SERVER ON CACHE BOOL "")
   SET(WITH_PCRE system CACHE STRING "")
-  SET(WITH_INNODB_BZIP2 OFF CACHE STRING "")
-  SET(WITH_INNODB_LZMA OFF CACHE STRING "")
-  SET(WITH_INNODB_LZO OFF CACHE STRING "")
   SET(WITH_ROCKSDB_BZip2 OFF CACHE STRING "")
 ELSE()
   SET(WITH_SSL bundled CACHE STRING "")
@@ -137,12 +130,8 @@ ELSE()
   SET(PLUGIN_AUTH_SOCKET STATIC CACHE STRING "")
   SET(WITH_STRIPPED_CLIENT ON CACHE BOOL "Strip all client binaries")
   SET(WITH_PCRE bundled CACHE STRING "")
-  SET(WITH_INNODB_BZIP2 OFF CACHE STRING "")
-  SET(WITH_INNODB_LZ4 OFF CACHE STRING "")
-  SET(WITH_INNODB_LZO OFF CACHE STRING "")
   SET(WITH_ROCKSDB_BZip2 OFF CACHE STRING "")
   SET(WITH_ROCKSDB_LZ4 OFF CACHE STRING "")
-  SET(GRN_WITH_LZ4 no CACHE STRING "")
 ENDIF()
 
 IF(NOT COMPILATION_COMMENT)

debian/control

@@ -18,6 +18,7 @@ Build-Depends: bison,
                libboost-regex-dev [amd64],
                libboost-system-dev [amd64],
                libboost-thread-dev [amd64],
+               libbz2-dev,
                libcrack2-dev (>= 2.9.0),
                libcurl4-openssl-dev | libcurl4-dev,
                libedit-dev,
@@ -26,6 +27,8 @@ Build-Depends: bison,
                libjudy-dev,
                libkrb5-dev,
                liblz4-dev,
+               liblzma-dev,
+               liblzo2-dev,
                libncurses5-dev (>= 5.0-6~),
                libncurses5-dev:native (>= 5.0-6~),
                libnuma-dev [linux-any],
@@ -935,6 +938,41 @@ Description: CrackLib Password Validation Plugin for MariaDB
  .
  Install and configure this to enforce stronger passwords for MariaDB users.
 
+Package: mariadb-plugin-provider-bzip2
+Architecture: any
+Depends: mariadb-server-10.7,
+         ${misc:Depends},
+         ${shlibs:Depends}
+Description: BZip2 compression support in the server and storage engines
+
+Package: mariadb-plugin-provider-lz4
+Architecture: any
+Depends: mariadb-server-10.7,
+         ${misc:Depends},
+         ${shlibs:Depends}
+Description: LZ4 compression support in the server and storage engines
+
+Package: mariadb-plugin-provider-lzma
+Architecture: any
+Depends: mariadb-server-10.7,
+         ${misc:Depends},
+         ${shlibs:Depends}
+Description: LZMA compression support in the server and storage engines
+
+Package: mariadb-plugin-provider-lzo
+Architecture: any
+Depends: mariadb-server-10.7,
+         ${misc:Depends},
+         ${shlibs:Depends}
+Description: LZO compression support in the server and storage engines
+
+Package: mariadb-plugin-provider-snappy
+Architecture: any
+Depends: mariadb-server-10.7,
+         ${misc:Depends},
+         ${shlibs:Depends}
+Description: Snappy compression support in the server and storage engines
+
 Package: mariadb-test
 Architecture: any
 Depends: mariadb-client-10.7 (= ${binary:Version}),

debian/mariadb-plugin-provider-bzip2.install

@@ -0,0 +1,2 @@
+etc/mysql/mariadb.conf.d/provider_bzip2.cnf
+usr/lib/mysql/plugin/provider_bzip2.so

debian/mariadb-plugin-provider-lz4.install

@@ -0,0 +1,2 @@
+etc/mysql/mariadb.conf.d/provider_lz4.cnf
+usr/lib/mysql/plugin/provider_lz4.so

debian/mariadb-plugin-provider-lzma.install

@@ -0,0 +1,2 @@
+etc/mysql/mariadb.conf.d/provider_lzma.cnf
+usr/lib/mysql/plugin/provider_lzma.so

debian/mariadb-plugin-provider-lzo.install

@@ -0,0 +1,2 @@
+etc/mysql/mariadb.conf.d/provider_lzo.cnf
+usr/lib/mysql/plugin/provider_lzo.so

debian/mariadb-plugin-provider-snappy.install

@@ -0,0 +1,2 @@
+etc/mysql/mariadb.conf.d/provider_snappy.cnf
+usr/lib/mysql/plugin/provider_snappy.so

extra/mariabackup/xtrabackup.cc

@@ -3309,7 +3309,8 @@ static void xb_load_single_table_tablespace(const char *dirname,
 			0, false, false);
 		node->deferred= defer;
 		mysql_mutex_lock(&fil_system.mutex);
-		space->read_page0();
+		if (!space->read_page0())
+                  err= DB_CANNOT_OPEN_FILE;
 		mysql_mutex_unlock(&fil_system.mutex);
 
 		if (srv_operation == SRV_OPERATION_RESTORE_DELTA

include/providers/bzlib.h

@@ -0,0 +1,122 @@
+/**
+  @file bzlib.h
+  This service provides dynamic access to BZip2.
+*/
+
+#ifndef BZIP2_INCLUDED
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef MYSQL_ABI_CHECK
+#include <stdbool.h>
+#endif
+
+#ifndef BZ_RUN
+#define BZ_RUN          0
+#define BZ_FINISH       2
+
+#define BZ_OK           0
+#define BZ_RUN_OK       1
+#define BZ_FINISH_OK    3
+#define BZ_STREAM_END   4
+
+typedef struct
+{
+  char *next_in;
+  unsigned int avail_in;
+  unsigned int total_in_lo32;
+  unsigned int total_in_hi32;
+
+  char *next_out;
+  unsigned int avail_out;
+  unsigned int total_out_lo32;
+  unsigned int total_out_hi32;
+
+  void *state;
+
+  void *(*bzalloc)(void *, int, int);
+  void (*bzfree)(void *, void *);
+  void *opaque;
+} bz_stream;
+
+#define BZ2_bzBuffToBuffCompress(...)   provider_service_bzip2->BZ2_bzBuffToBuffCompress_ptr   (__VA_ARGS__)
+#define BZ2_bzBuffToBuffDecompress(...) provider_service_bzip2->BZ2_bzBuffToBuffDecompress_ptr (__VA_ARGS__)
+#define BZ2_bzCompress(...)             provider_service_bzip2->BZ2_bzCompress_ptr             (__VA_ARGS__)
+#define BZ2_bzCompressEnd(...)          provider_service_bzip2->BZ2_bzCompressEnd_ptr          (__VA_ARGS__)
+#define BZ2_bzCompressInit(...)         provider_service_bzip2->BZ2_bzCompressInit_ptr         (__VA_ARGS__)
+#define BZ2_bzDecompress(...)           provider_service_bzip2->BZ2_bzDecompress_ptr           (__VA_ARGS__)
+#define BZ2_bzDecompressEnd(...)        provider_service_bzip2->BZ2_bzDecompressEnd_ptr        (__VA_ARGS__)
+#define BZ2_bzDecompressInit(...)       provider_service_bzip2->BZ2_bzDecompressInit_ptr       (__VA_ARGS__)
+#endif
+
+#define DEFINE_BZ2_bzBuffToBuffCompress(NAME) NAME( \
+    char *dest,                                     \
+    unsigned int *destLen,                          \
+    char *source,                                   \
+    unsigned int sourceLen,                         \
+    int blockSize100k,                              \
+    int verbosity,                                  \
+    int workFactor                                  \
+)
+
+#define DEFINE_BZ2_bzBuffToBuffDecompress(NAME) NAME( \
+    char *dest,                                       \
+    unsigned int *destLen,                            \
+    char *source,                                     \
+    unsigned int sourceLen,                           \
+    int small,                                        \
+    int verbosity                                     \
+)
+
+#define DEFINE_BZ2_bzCompress(NAME) NAME(   \
+    bz_stream *strm,                        \
+    int action                              \
+)
+
+#define DEFINE_BZ2_bzCompressEnd(NAME) NAME(    \
+    bz_stream *strm                             \
+)
+
+#define DEFINE_BZ2_bzCompressInit(NAME) NAME(   \
+    bz_stream *strm,                            \
+    int blockSize100k,                          \
+    int verbosity,                              \
+    int workFactor                              \
+)
+
+#define DEFINE_BZ2_bzDecompress(NAME) NAME( \
+    bz_stream *strm                         \
+)
+
+#define DEFINE_BZ2_bzDecompressEnd(NAME) NAME(  \
+    bz_stream *strm                             \
+)
+
+#define DEFINE_BZ2_bzDecompressInit(NAME) NAME( \
+    bz_stream *strm,                            \
+    int verbosity,                              \
+    int small                                   \
+)
+
+struct provider_service_bzip2_st{
+  int DEFINE_BZ2_bzBuffToBuffCompress((*BZ2_bzBuffToBuffCompress_ptr));
+  int DEFINE_BZ2_bzBuffToBuffDecompress((*BZ2_bzBuffToBuffDecompress_ptr));
+  int DEFINE_BZ2_bzCompress((*BZ2_bzCompress_ptr));
+  int DEFINE_BZ2_bzCompressEnd((*BZ2_bzCompressEnd_ptr));
+  int DEFINE_BZ2_bzCompressInit((*BZ2_bzCompressInit_ptr));
+  int DEFINE_BZ2_bzDecompress((*BZ2_bzDecompress_ptr));
+  int DEFINE_BZ2_bzDecompressEnd((*BZ2_bzDecompressEnd_ptr));
+  int DEFINE_BZ2_bzDecompressInit((*BZ2_bzDecompressInit_ptr));
+
+  bool is_loaded;
+};
+
+extern struct provider_service_bzip2_st *provider_service_bzip2;
+
+#ifdef __cplusplus
+}
+#endif
+
+#define BZIP2_INCLUDED
+#endif