WL#13759 MySQL 8.0: Support linking with 3rd party OpenLDAP and Cyrus SASL

This worklog affects standalone Linux builds only, i.e. where we build
MySQL with a custom version of OpenSSL 1.1.1 rather than the
system version.

In the past we have been using "system" versions of LDAP and SASL
(which then use system SSL), while still building the rest of MySQL
with the custom SSL. This does not work: all executables and libraries
must use the same version of OpenSSL.

Pushbuild will now build LDAP/SASL/SSL as one "package"
and invoke cmake with
-DWITH_LDAP=</path/to/ldap> -DWITH_SASL=</path/to/sasl> -DWITH_SSL=<path/to/ssl>

These builds depend on each other, so that the SASL build will use the
SSL build, the LDAP build will use the SASL and SSL build. The result
is a set of headers and shared libraries which consistently use
OpenSSL 1.1.1

The following libraries are copied into <build_dir>/library_output_directory
during build:
  libcrypto.so liblber.so libldap_r.so libsasl2.so libssl.so
The SASL SCRAM plugin (libscram.so) is copied to
  <build_dir>/library_output_directory/sasl2

During 'make install' they are all copied into <install dir>/lib/private/
and <install dir>/lib/private/sasl2.
Libraries are patched with patchelf(1), to set RPATH to $ORIGIN.

For standalone linux builds, the cmake options for LDAP/SASL/SSL must
either all be "system", or they must all be custom paths (in which
case the patchelf(1) utility is required on the build host).

The patch also changes the default value for the cmake option
WITH_AUTHENTICATION_LDAP to ON for commercial builds.

We also enable some tests which were previously skipped because of
bad feature tests.

Bugfix for windows: copy SCRAM plugins to runtime_output_directory.
This fixes four broken tests for mtr --suite=auth_ldap

Bugfix for windows: move code for finding and setting SASL plugin
path to plugin initialization, where it is thread safe.

Move calls to sasl_client_init() and sasl_done()/sasl_client_done()
to plugin initializer/deinitializer respectively.

Add LeakSanitizer and Valgrind suppressions for a known OpenLDAP bug.

The patch also implements -DWITH_KERBEROS=system for more Unices:
debian/ubuntu and Solaris

-DWITH_KERBEROS=<path/to/kerberos> is *not* implemented, we postpone
that to a followup patch.

Change-Id: Ibdfcd703243d34a33b35cc7df77c153edf5acbd0

Author: Tor Didriksen

CMakeLists.txt

@@ -516,6 +516,7 @@ INCLUDE(icu)
 INCLUDE(libevent)
 INCLUDE(ssl)
 INCLUDE(sasl)
+INCLUDE(ldap)
 INCLUDE(kerberos)
 INCLUDE(rpc)
 INCLUDE(readline)
@@ -558,8 +559,16 @@ IF(NOT WITHOUT_SERVER)
   OPTION(WITH_UNIT_TESTS "Compile MySQL with unit tests" ON)
   OPTION(WITH_ROUTER "Build MySQL Router" ON)
 ENDIF()
+
+IF(EXISTS ${CMAKE_SOURCE_DIR}/internal/CMakeLists.txt)
+  SET(WITH_AUTHENTICATION_LDAP_DEFAULT ON)
+ELSE()
+  SET(WITH_AUTHENTICATION_LDAP_DEFAULT OFF)
+ENDIF()
+
 OPTION(WITH_AUTHENTICATION_LDAP
-  "Report error if the LDAP authentication plugin cannot be built." OFF)
+  "Report error if the LDAP authentication plugin cannot be built."
+  ${WITH_AUTHENTICATION_LDAP_DEFAULT})
 
 OPTION(WITH_LOCK_ORDER
   "Build the lock order mutex instrumentation code." OFF)
@@ -1214,10 +1223,82 @@ IF(BUILD_BUNDLED_ZSTD)
   ADD_SUBDIRECTORY(extra/zstd)
 ENDIF()
 
+IF(LINUX_STANDALONE)
+  FIND_PROGRAM(PATCHELF_EXECUTABLE patchelf)
+  ADD_CUSTOM_TARGET(copy_linux_custom_dlls)
+  SET(KNOWN_CUSTOM_LIBRARIES "" CACHE INTERNAL "" FORCE)
+ENDIF()
+
 # Add system openssl.
 MYSQL_CHECK_SSL()
 MYSQL_CHECK_SSL_DLLS()
 
+# Add Kerberos library (custom DLLS not yet supported)
+MYSQL_CHECK_KERBEROS()
+
+# Add SASL library
+MYSQL_CHECK_SASL()
+MYSQL_CHECK_SASL_DLLS()
+
+# Add LDAP library
+MYSQL_CHECK_LDAP()
+MYSQL_CHECK_LDAP_DLLS()
+
+# Custom versions of LDAP / SASL / SSL.
+IF(LINUX_STANDALONE AND KNOWN_CUSTOM_LIBRARIES)
+  SET(LINUX_INSTALL_RPATH_ORIGIN 1)
+  # Debug versions of plugins may be installed to <root>/lib/plugin/debug
+  FOREACH(LINK_FLAG
+      CMAKE_MODULE_LINKER_FLAGS_DEBUG
+      CMAKE_SHARED_LINKER_FLAGS_DEBUG
+      )
+    STRING_APPEND(${LINK_FLAG} " -Wl,-rpath,'\$ORIGIN/../../private'")
+    MESSAGE(STATUS "${LINK_FLAG} ${${LINK_FLAG}}")
+  ENDFOREACH()
+  IF(NOT PATCHELF_EXECUTABLE)
+    MESSAGE(FATAL_ERROR "Please install the patchelf(1) utility.")
+  ENDIF()
+  VERIFY_CUSTOM_LIBRARY_DEPENDENCIES()
+ENDIF()
+
+IF(WITH_AUTHENTICATION_LDAP)
+  IF(LINUX_STANDALONE AND KNOWN_CUSTOM_LIBRARIES)
+    # LDAP / SASL / SSL must all be "system" or "custom", not a mix.
+    IF(WITH_LDAP STREQUAL "system" OR
+        WITH_SASL STREQUAL "system" OR
+        WITH_SSL STREQUAL "system")
+      MESSAGE(WARNING "-DWITH_AUTHENTICATION_LDAP=ON")
+      MESSAGE(FATAL_ERROR "Inconsistent options for LDAP/SASL/SSL")
+    ENDIF()
+  ELSEIF(WIN32)
+    # system LDAP, but SASL needs to be explicitly added
+    IF(NOT SASL_LIBRARY_DLL OR NOT SASL_SCRAM_PLUGIN)
+      MESSAGE(WARNING "-DWITH_AUTHENTICATION_LDAP=ON")
+      MESSAGE(FATAL_ERROR "Missing SASL and/or SCRAM libraries.")
+    ENDIF()
+  ELSEIF(APPLE)
+    # system LDAP and SASL, but there is no scram plugin.
+  ELSEIF(SOLARIS)
+    # 11.3 has system LDAP and SASL, but there is no scram plugin.
+  ELSE()
+    # We must have "system" LDAP and SASL (SSL is always required)
+    SET(LDAP_WARN_GIVEN)
+    SET(SASL_WARN_GIVEN)
+    WARN_MISSING_SYSTEM_LDAP(LDAP_WARN_GIVEN)
+    WARN_MISSING_SYSTEM_SASL(SASL_WARN_GIVEN)
+    IF(LDAP_WARN_GIVEN OR SASL_WARN_GIVEN)
+      # SUSE linux: may or may not have SCRAM, do not break the build.
+      IF(LINUX_SUSE)
+        MESSAGE(WARNING
+          "-DWITH_AUTHENTICATION_LDAP=ON, but missing system libraries")
+      ELSE()
+        MESSAGE(FATAL_ERROR
+          "-DWITH_AUTHENTICATION_LDAP=ON, but missing system libraries")
+      ENDIF()
+    ENDIF()
+  ENDIF()
+ENDIF()
+
 # Add system/bundled editline.
 MYSQL_CHECK_EDITLINE()
 # Add libevent
@@ -1226,10 +1307,6 @@ MYSQL_CHECK_LIBEVENT()
 MYSQL_CHECK_LZ4()
 # Add icu library
 MYSQL_CHECK_ICU()
-# Add SASL library
-MYSQL_CHECK_SASL()
-# Add Kerberos library
-MYSQL_CHECK_KERBEROS()
 # Add protoc and libprotobuf
 IF(NOT WITHOUT_SERVER)
   MYSQL_CHECK_PROTOBUF()
@@ -1546,7 +1623,12 @@ IF(UNIX)
     COMMAND ${CMAKE_COMMAND} -E make_directory plugin_output_directory
     COMMAND ${CMAKE_COMMAND} -E make_directory runtime_output_directory
     WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
-  )
+    )
+  IF(SASL_CUSTOM_LIBRARY)
+    EXECUTE_PROCESS(
+      COMMAND ${CMAKE_COMMAND} -E make_directory library_output_directory/sasl2
+    )
+  ENDIF()
 ENDIF()
 
 # Pre-create directories for library_output_directory and

cmake/copy_custom_library.cmake

@@ -0,0 +1,60 @@
+# Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License, version 2.0,
+# as published by the Free Software Foundation.
+#
+# This program is also distributed with certain software (including
+# but not limited to OpenSSL) that is licensed under separate terms,
+# as designated in a particular file or component or in included license
+# documentation.  The authors of MySQL hereby grant you an additional
+# permission to link the program and your derivative works with the
+# separately licensed software that they have included with MySQL.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License, version 2.0, for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
+
+IF(EXISTS "./${library_version}")
+  RETURN()
+ENDIF()
+
+EXECUTE_PROCESS(
+  COMMAND ${CMAKE_COMMAND} -E copy
+  "${library_directory}/${library_version}" "./${library_version}"
+  )
+
+IF(NOT "${library_version}" STREQUAL "${library_name}")
+  EXECUTE_PROCESS(
+    COMMAND ${CMAKE_COMMAND} -E create_symlink
+    "${library_version}" "${library_name}"
+    )
+ENDIF()
+
+IF(NOT "${library_version}" STREQUAL "${library_soname}")
+  EXECUTE_PROCESS(
+    COMMAND ${CMAKE_COMMAND} -E create_symlink
+    "${library_version}" "${library_soname}"
+    )
+ENDIF()
+
+# Some of the pre-built libraries come without execute bit set.
+EXECUTE_PROCESS(
+  COMMAND chmod +x "./${library_version}")
+
+# Patch RPATH so that we find NEEDED libraries at load time.
+IF(subdir)
+  EXECUTE_PROCESS(
+    COMMAND ${PATCHELF_EXECUTABLE} --set-rpath "$ORIGIN/.."
+    "./${library_version}"
+    )
+ELSE()
+  EXECUTE_PROCESS(
+    COMMAND ${PATCHELF_EXECUTABLE} --set-rpath "$ORIGIN" "./${library_version}"
+    )
+ENDIF()

cmake/fileutils.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License, version 2.0,
@@ -57,7 +57,7 @@ IF(WIN32)
       STRING(REPLACE "\n" ";" DUMPBIN_OUTPUT_LIST "${DUMPBIN_OUTPUT}")
       SET(DEPENDENCIES)
       FOREACH(LINE ${DUMPBIN_OUTPUT_LIST})
-        STRING(REGEX MATCH "^[\r\n\t ]*([A-Za-z0-9_]*\.dll)" XXX ${LINE})
+        STRING(REGEX MATCH "^[\r\n\t ]*([A-Za-z0-9_]*\.dll)" UNUSED ${LINE})
         IF(CMAKE_MATCH_1)
           LIST(APPEND DEPENDENCIES ${CMAKE_MATCH_1})
         ENDIF()
@@ -66,3 +66,60 @@ IF(WIN32)
     ENDIF()
   ENDFUNCTION()
 ENDIF()
+
+IF(LINUX)
+  FUNCTION(FIND_OBJECT_DEPENDENCIES FILE_NAME RETURN_VALUE)
+    SET(${RETURN_VALUE} PARENT_SCOPE)
+    EXECUTE_PROCESS(COMMAND
+      objdump -p "${FILE_NAME}"
+      OUTPUT_VARIABLE OBJDUMP_OUTPUT
+      RESULT_VARIABLE OBJDUMP_RESULT
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+      )
+    STRING(REPLACE "\n" ";" OBJDUMP_OUTPUT_LIST "${OBJDUMP_OUTPUT}")
+    SET(DEPENDENCIES)
+    FOREACH(LINE ${OBJDUMP_OUTPUT_LIST})
+      STRING(REGEX MATCH
+        "^[ ]+NEEDED[ ]+([-_A-Za-z0-9\\.]+)" UNUSED ${LINE})
+      IF(CMAKE_MATCH_1)
+        LIST(APPEND DEPENDENCIES ${CMAKE_MATCH_1})
+      ENDIF()
+    ENDFOREACH()
+    SET(${RETURN_VALUE} ${DEPENDENCIES} PARENT_SCOPE)
+  ENDFUNCTION()
+
+  FUNCTION(FIND_SONAME FILE_NAME RETURN_VALUE)
+    SET(${RETURN_VALUE} PARENT_SCOPE)
+    EXECUTE_PROCESS(COMMAND
+      objdump -p "${FILE_NAME}"
+      OUTPUT_VARIABLE OBJDUMP_OUTPUT
+      RESULT_VARIABLE OBJDUMP_RESULT
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+      )
+    STRING(REPLACE "\n" ";" OBJDUMP_OUTPUT_LIST "${OBJDUMP_OUTPUT}")
+    FOREACH(LINE ${OBJDUMP_OUTPUT_LIST})
+      STRING(REGEX MATCH
+        "^[ ]+SONAME[ ]+([-_A-Za-z0-9\\.]+)" UNUSED ${LINE})
+      IF(CMAKE_MATCH_1)
+        SET(${RETURN_VALUE} ${CMAKE_MATCH_1} PARENT_SCOPE)
+      ENDIF()
+    ENDFOREACH()
+  ENDFUNCTION()
+
+  FUNCTION(VERIFY_CUSTOM_LIBRARY_DEPENDENCIES)
+    FOREACH(lib ${KNOWN_CUSTOM_LIBRARIES})
+      FOREACH(lib_needs ${NEEDED_${lib}})
+        GET_FILENAME_COMPONENT(library_name_we "${lib_needs}" NAME_WE)
+        SET(SONAME ${SONAME_${library_name_we}})
+        IF(SONAME)
+          MESSAGE(STATUS
+            "${lib} needs ${lib_needs} from ${library_name_we}")
+          IF(NOT "${lib_needs}" STREQUAL "${SONAME}")
+            MESSAGE(WARNING "${library_name_we} provides ${SONAME}")
+          ENDIF()
+        ENDIF()
+      ENDFOREACH()
+    ENDFOREACH()
+  ENDFUNCTION()
+
+ENDIF()

cmake/install_layout.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2010, 2019, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2010, 2020, Oracle and/or its affiliates. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License, version 2.0,
@@ -110,6 +110,10 @@ IF(UNIX)
   MARK_AS_ADVANCED(SYSCONFDIR)
 ENDIF()
 
+IF(LINUX AND INSTALL_LAYOUT MATCHES "STANDALONE")
+  SET(LINUX_STANDALONE 1)
+ENDIF()
+
 IF(WIN32)
   SET(VALID_INSTALL_LAYOUTS "TARGZ" "STANDALONE")
   LIST(FIND VALID_INSTALL_LAYOUTS "${INSTALL_LAYOUT}" ind)

cmake/install_macros.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2009, 2019, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2009, 2020, Oracle and/or its affiliates. All rights reserved.
 # 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License, version 2.0,
@@ -385,3 +385,103 @@ FUNCTION(SET_PATH_TO_SSL target target_out_dir)
     ENDIF()
   ENDIF()
 ENDFUNCTION()
+
+
+# For standalone Linux build and -DWITH_LDAP -DWITH_SASL -DWITH_SSL
+# set to custom path.
+#
+# Move the custom shared library and symlinks to library_output_directory.
+# The subdir argument is typically empty, but set to "sasl2" for SASL plugins,
+# in which case we move library_full_filename and its symlinks to
+# library_output_directory/sasl2.
+#
+# We ensure that the copied custom libraries have the execute bit set.
+# We also update the RUNPATH of libraries to be '$ORIGIN' to ensure that
+# libraries get correct load-time dependencies. This is done using the
+# linux tool patchelf(1)
+#
+# Set ${OUTPUT_LIBRARY_NAME} to the new location.
+# Set ${OUTPUT_TARGET_NAME} to the name of a target which will do the copying.
+# Add an INSTALL(FILES ....) rule to install library and symlinks into
+#   ${INSTALL_PRIV_LIBDIR} or ${INSTALL_PRIV_LIBDIR}/sasl2
+FUNCTION(COPY_CUSTOM_SHARED_LIBRARY library_full_filename subdir
+    OUTPUT_LIBRARY_NAME
+    OUTPUT_TARGET_NAME
+    )
+  IF(NOT LINUX_STANDALONE)
+    RETURN()
+  ENDIF()
+  GET_FILENAME_COMPONENT(LIBRARY_EXT "${library_full_filename}" EXT)
+  IF(NOT LIBRARY_EXT STREQUAL ".so")
+    RETURN()
+  ENDIF()
+  EXECUTE_PROCESS(
+    COMMAND readlink "${library_full_filename}" OUTPUT_VARIABLE library_version
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+  GET_FILENAME_COMPONENT(library_directory "${library_full_filename}" DIRECTORY)
+  GET_FILENAME_COMPONENT(library_name "${library_full_filename}" NAME)
+  GET_FILENAME_COMPONENT(library_name_we "${library_full_filename}" NAME_WE)
+
+  FIND_SONAME(${library_full_filename} library_soname)
+  FIND_OBJECT_DEPENDENCIES(${library_full_filename} library_dependencies)
+
+  MESSAGE(STATUS "CUSTOM library ${library_full_filename}")
+  MESSAGE(STATUS "CUSTOM version ${library_version}")
+  MESSAGE(STATUS "CUSTOM directory ${library_directory}")
+  MESSAGE(STATUS "CUSTOM name ${library_name}")
+  MESSAGE(STATUS "CUSTOM name_we ${library_name_we}")
+  MESSAGE(STATUS "CUSTOM soname ${library_soname}")
+
+  SET(COPIED_LIBRARY_NAME
+    "${CMAKE_BINARY_DIR}/library_output_directory/${subdir}/${library_name}")
+  SET(COPY_TARGET_NAME "copy_${library_name_we}_dll")
+
+  # Keep track of libraries and dependencies.
+  SET(SONAME_${library_name_we} "${library_soname}"
+    CACHE INTERNAL "SONAME for ${library_name_we}" FORCE)
+  SET(NEEDED_${library_name_we} "${library_dependencies}"
+    CACHE INTERNAL "" FORCE)
+  SET(KNOWN_CUSTOM_LIBRARIES
+    ${KNOWN_CUSTOM_LIBRARIES} ${library_name_we} CACHE INTERNAL "" FORCE)
+
+  # Do copying and patching in a sub-process, so that we can skip it if
+  # already done. The BYPRODUCTS arguments is needed by Ninja, and is
+  # ignored on non-Ninja generators except to mark byproducts GENERATED.
+  ADD_CUSTOM_TARGET(${COPY_TARGET_NAME} ALL
+    COMMAND ${CMAKE_COMMAND}
+    -Dlibrary_directory="${library_directory}"
+    -Dlibrary_name="${library_name}"
+    -Dlibrary_soname="${library_soname}"
+    -Dlibrary_version="${library_version}"
+    -Dsubdir="${subdir}"
+    -DPATCHELF_EXECUTABLE="${PATCHELF_EXECUTABLE}"
+    -P ${CMAKE_SOURCE_DIR}/cmake/copy_custom_library.cmake
+
+    BYPRODUCTS
+    "${CMAKE_BINARY_DIR}/library_output_directory/${subdir}/${library_name}"
+
+    WORKING_DIRECTORY
+    "${CMAKE_BINARY_DIR}/library_output_directory/${subdir}"
+    )
+
+  # Link with the copied library, rather than the original one.
+  SET(${OUTPUT_LIBRARY_NAME} "${COPIED_LIBRARY_NAME}" PARENT_SCOPE)
+  SET(${OUTPUT_TARGET_NAME} "${COPY_TARGET_NAME}" PARENT_SCOPE)
+
+  ADD_DEPENDENCIES(copy_linux_custom_dlls ${COPY_TARGET_NAME})
+
+  MESSAGE(STATUS "INSTALL ${library_name} to ${INSTALL_PRIV_LIBDIR}")
+
+  # Cannot use INSTALL_PRIVATE_LIBRARY because these are not targets.
+  INSTALL(FILES
+    ${CMAKE_BINARY_DIR}/library_output_directory/${subdir}/${library_name}
+    ${CMAKE_BINARY_DIR}/library_output_directory/${subdir}/${library_soname}
+    ${CMAKE_BINARY_DIR}/library_output_directory/${subdir}/${library_version}
+    DESTINATION "${INSTALL_PRIV_LIBDIR}/${subdir}" COMPONENT SharedLibraries
+    PERMISSIONS
+    OWNER_READ OWNER_WRITE OWNER_EXECUTE
+    GROUP_READ GROUP_EXECUTE
+    WORLD_READ WORLD_EXECUTE
+    )
+
+ENDFUNCTION()