MDEV-30831 Cannot compile AWS KMS Plugin

Fix AWS SDK build, it has changed substantionally since the plugin was
introduced. There is now a bunch of intermediate C libraries, aws-cpp-crt
and others, and for static linking, the link dependency must be declared.

Also support AWS C++ SDK in vcpkg package manager.

Author: vaintroub

cmake/aws_sdk.cmake

@@ -4,19 +4,27 @@ MACRO (SKIP_AWS_SDK MSG)
   RETURN()
 ENDMACRO()
 
-FUNCTION (CHECK_AWS_SDK RETVAL REASON)
+FUNCTION (CHECK_AWS_SDK COMPONENTS RETVAL REASON)
   # AWS_SDK_EXTERNAL_PROJECT must be ON
   IF(NOT AWS_SDK_EXTERNAL_PROJECT)
-    SKIP_AWS_SDK("AWS_SDK_EXTERNAL_PROJECT is not ON")
+   FOREACH(comp ${COMPONENTS})
+    FIND_PACKAGE(aws-cpp-sdk-${comp} CONFIG QUIET)
+    IF (NOT aws-cpp-sdk-${comp}_FOUND)
+      SKIP_AWS_SDK("AWS_SDK_EXTERNAL_PROJECT is not ON and aws-cpp-sdk-${comp} not found")
+    ENDIF()
+   ENDFOREACH()
+   SET(${RETVAL} ON PARENT_SCOPE)
   ENDIF()
   IF(NOT NOT_FOR_DISTRIBUTION)
     SKIP_AWS_SDK("AWS SDK has Apache 2.0 License which is not compatible with GPLv2. Set -DNOT_FOR_DISTRIBUTION=ON if you need it")
   ENDIF()
+  IF(CMAKE_VERSION VERSION_LESS "3.15")
+    SKIP_AWS_SDK("CMake too old")
+  ENDIF()
   # Check compiler support
   IF(CMAKE_CXX_COMPILER_ID MATCHES "GNU")
-    EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION)
-    IF (GCC_VERSION VERSION_LESS 4.8)
-      SKIP_AWS_SDK("GCC VERSION too old (${GCC_VERSION}, required is 4.8 or later")
+    IF (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.9)
+      SKIP_AWS_SDK("GCC VERSION too old (${GCC_VERSION}, required is 4.9 or later")
     ENDIF()
   ELSEIF (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
     IF ((CMAKE_CXX_COMPILER_VERSION AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS  3.3) OR
@@ -36,35 +44,27 @@ FUNCTION (CHECK_AWS_SDK RETVAL REASON)
     SKIP_AWS_SDK("OS unsupported by AWS SDK")
   ENDIF()
 
-  # Build from source, using ExternalProject_Add
-  # AWS C++ SDK requires cmake 2.8.12
-  IF(CMAKE_VERSION VERSION_LESS "2.8.12")
-    SKIP_AWS_SDK("CMake is too old")
-  ENDIF()
-
   IF(UNIX)
-    # Check librairies required for building SDK
-    FIND_PACKAGE(CURL)
-    SET_PACKAGE_PROPERTIES(CURL PROPERTIES TYPE REQUIRED)
-    IF(NOT CURL_FOUND)
-      SKIP_AWS_SDK("AWS C++ SDK requires libcurl development package")
+    IF("${WITH_ZLIB}" STREQUAL "bundled")
+      # Breaks FIND_PACKAGE(ZLIB)
+      SKIP_AWS_SDK("Incompatible with WITH_ZLIB=bundled")
     ENDIF()
-    FIND_PATH(UUID_INCLUDE_DIR uuid/uuid.h)
-    IF(NOT UUID_INCLUDE_DIR)
-      SKIP_AWS_SDK("AWS C++ SDK requires uuid development package")
-    ENDIF()
-    IF(NOT APPLE)
-      FIND_LIBRARY(UUID_LIBRARIES uuid)
-      SET_PACKAGE_PROPERTIES(UUID_LIBRARIES PROPERTIES TYPE REQUIRED)
-      IF(NOT UUID_LIBRARIES)
-        SKIP_AWS_SDK("AWS C++ SDK requires uuid development package")
+    # Check libraries required for building SDK
+    FOREACH(pkg CURL ZLIB OpenSSL)
+      FIND_PACKAGE(${pkg})
+      IF(NOT ${pkg}_FOUND)
+        SKIP_AWS_SDK("AWS C++ SDK requires ${pkg}")
       ENDIF()
-      FIND_PACKAGE(OpenSSL)
-      SET_PACKAGE_PROPERTIES(OpenSSL PROPERTIES TYPE REQUIRED)
-      IF(NOT OPENSSL_FOUND)
-        SKIP_AWS_SDK("AWS C++ SDK requires openssl development package")
+      SET_PACKAGE_PROPERTIES(${pkg} PROPERTIES TYPE REQUIRED)
+    ENDFOREACH()
+    # Also check for required libraries explicitely - they might be
+    # missing, even if check above succeeds, e.g when using own copy
+    # of zlib
+    FOREACH(lib OpenSSL::Crypto ZLIB::ZLIB CURL::libcurl)
+      IF(NOT TARGET ${lib})
+        SKIP_AWS_SDK("AWS C++ SDK requires ${lib}")
       ENDIF()
-    ENDIF()
+    ENDFOREACH()
   ENDIF()
   SET(${RETVAL} ON PARENT_SCOPE)
 ENDFUNCTION()
@@ -85,14 +85,4 @@ FUNCTION(USE_AWS_SDK_LIBS)
     SET_PROPERTY(GLOBAL PROPERTY AWS_SDK_LIBS ${comp} APPEND)
     TARGET_LINK_LIBRARIES(${target} aws-cpp-sdk-${comp})
   ENDFOREACH()
-  TARGET_LINK_LIBRARIES(${target} aws-cpp-sdk-core)
-  TARGET_INCLUDE_DIRECTORIES(${target} PRIVATE ${PROJECT_BINARY_DIR}/extra/aws_sdk/aws_sdk_cpp/include)
-  # Link OS libraries that AWS SDK depends on
-  IF(WIN32)
-    TARGET_LINK_LIBRARIES(${target} bcrypt winhttp wininet userenv version)
-  ELSE()
-    FIND_PACKAGE(CURL REQUIRED)
-    FIND_PACKAGE(OpenSSL REQUIRED)
-    TARGET_LINK_LIBRARIES(${target} ${OPENSSL_LIBRARIES} ${CURL_LIBRARIES} ${UUID_LIBRARIES})
-  ENDIF()
 ENDFUNCTION()

extra/aws_sdk/CMakeLists.txt

@@ -8,67 +8,132 @@ IF(SDK_LIBS_COUNT EQUAL 0)
   RETURN()
 ENDIF()
 
-CHECK_AWS_SDK(RETVAL REASON)
+CHECK_AWS_SDK("${SDK_LIBS}" RETVAL REASON)
 IF(NOT RETVAL)
   MESSAGE(FATAL_ERROR 
    "AWS C++ will not be built (${REASON}), but dependency on following components is found ${SDK_LIBS}.
    Use CHECK_AWS_SDK() function before trying to build with SDK components")
 ENDIF()
 
+IF(NOT AWS_SDK_EXTERNAL_PROJECT)
+  # System aws sdk libraries found
+  RETURN()
+ENDIF()
 
-SET(byproducts)
+SET(all_libs
+  aws-crt-cpp
+  aws-c-event-stream
+  aws-c-common
+  aws-c-http
+  aws-c-mqtt
+  aws-c-cal
+  aws-c-auth
+  aws-c-io
+  aws-checksums
+  aws-c-s3
+  aws-c-sdkutils
+  aws-c-compression
+  aws-cpp-sdk-core
+)
+IF(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+  LIST(APPEND all_libs s2n)
+ENDIF()
 
-FOREACH(lib ${SDK_LIBS} core)
-  SET(lib aws-cpp-sdk-${lib})
-  ADD_LIBRARY(${lib} STATIC IMPORTED GLOBAL)
+FOREACH(lib ${SDK_LIBS})
+  LIST(APPEND all_libs aws-cpp-sdk-${lib})
+ENDFOREACH()
+LIST(REMOVE_DUPLICATES all_libs)
+
+SET(byproducts)
+MAKE_DIRECTORY(${PROJECT_BINARY_DIR}/extra/aws_sdk/aws_sdk_cpp/include)
+FOREACH(lib ${all_libs})
+  ADD_LIBRARY(${lib} UNKNOWN IMPORTED GLOBAL)
   ADD_DEPENDENCIES(${lib} aws_sdk_cpp)
 
   SET (loc "${CMAKE_CURRENT_BINARY_DIR}/aws_sdk_cpp/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${lib}${CMAKE_STATIC_LIBRARY_SUFFIX}")
   IF(CMAKE_VERSION VERSION_GREATER "3.1")
     SET(byproducts ${byproducts} BUILD_BYPRODUCTS ${loc})
   ENDIF()
   SET_TARGET_PROPERTIES(${lib} PROPERTIES IMPORTED_LOCATION ${loc})
+  TARGET_INCLUDE_DIRECTORIES(${lib} INTERFACE ${PROJECT_BINARY_DIR}/extra/aws_sdk/aws_sdk_cpp/include)
 ENDFOREACH()
 
-# To be compatible with older cmake, we use older version of the SDK
-IF(CMAKE_VERSION LESS "3.0")
-  SET(GIT_TAG "1.0.8")
-ELSE()
-  SET(GIT_TAG "1.8.29")
-ENDIF()
+# Define link dependencies  between AWS C libs
+TARGET_LINK_LIBRARIES(aws-cpp-sdk-core INTERFACE aws-crt-cpp aws-c-event-stream aws-c-common)
+TARGET_LINK_LIBRARIES(aws-crt-cpp INTERFACE aws-c-http aws-c-mqtt aws-c-cal aws-c-auth aws-c-common aws-c-io
+  aws-checksums aws-c-event-stream aws-c-s3 aws-c-sdkutils)
+TARGET_LINK_LIBRARIES(aws-c-event-stream INTERFACE  aws-c-io aws-c-common aws-checksums)
+TARGET_LINK_LIBRARIES(aws-c-http INTERFACE aws-c-io aws-c-compression aws-c-cal aws-c-common)
+TARGET_LINK_LIBRARIES(aws-c-mqtt INTERFACE aws-c-http aws-c-io aws-c-common aws-c-cal aws-c-common)
+TARGET_LINK_LIBRARIES(aws-c-auth INTERFACE aws-c-sdkutils  aws-c-cal  aws-c-http  aws-c-io  aws-c-common)
+TARGET_LINK_LIBRARIES(aws-c-io INTERFACE aws-c-common  aws-c-cal)
+TARGET_LINK_LIBRARIES(aws-checksums INTERFACE aws-c-common)
+TARGET_LINK_LIBRARIES(aws-c-compression INTERFACE aws-c-common)
+TARGET_LINK_LIBRARIES(aws-c-s3 INTERFACE aws-c-auth  aws-checksums aws-c-http aws-c-io aws-c-cal aws-c-common aws-c-sdkutils  aws-c-common)
+TARGET_LINK_LIBRARIES(aws-c-sdkutils INTERFACE aws-c-common)
 
-IF(MSVC_CRT_TYPE MATCHES "/MD")
-  SET(FORCE_SHARED_CRT ON)
+IF(WIN32)
+  TARGET_LINK_LIBRARIES(aws-c-io INTERFACE bcrypt crypt32 secur32 ncrypt ws2_32)
+  TARGET_LINK_LIBRARIES(aws-c-common INTERFACE  shlwapi)
+  TARGET_LINK_LIBRARIES(aws-cpp-sdk-core INTERFACE winhttp wininet version userenv)
 ELSE()
-  SET(FORCE_SHARED_CRT OFF)
+  TARGET_LINK_LIBRARIES(aws-c-cal INTERFACE OpenSSL::Crypto)
+  TARGET_LINK_LIBRARIES(aws-cpp-sdk-core INTERFACE ZLIB::ZLIB CURL::libcurl)
+
+  # Dependencies below are from CMakeLists.txt for aws-c-common
+  SET(THREADS_PREFER_PTHREAD_FLAG ON)
+  FIND_PACKAGE(Threads REQUIRED)
+  TARGET_LINK_LIBRARIES(aws-c-common INTERFACE ${CMAKE_DL_LIBS} Threads::Threads)
+  IF(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    TARGET_LINK_LIBRARIES(aws-c-common INTERFACE s2n m rt)
+  ELSEIF(APPLE)
+    TARGET_LINK_LIBRARIES(aws-c-common INTERFACE "-framework CoreFoundation")
+    TARGET_LINK_LIBRARIES(aws-c-io INTERFACE "-framework Security" "-framework Network")
+  ENDIF()
 ENDIF()
 
+FOREACH(lib ${SDK_LIBS})
+  TARGET_LINK_LIBRARIES(aws-cpp-sdk-${lib} INTERFACE aws-cpp-sdk-core)
+ENDFOREACH()
+
+SET(GIT_TAG "1.11.582")
+
 LIST(REMOVE_DUPLICATES SDK_LIBS)
 STRING( REPLACE ";" "!" SDK_LIBS_STR "${SDK_LIBS}")
-#MESSAGE("SDK_LIBS_STR=${SDK_LIBS_STR}")
+
+IF(MSVC)
+ SET(BUILD_AND_INSTALL_COMMANDS
+   BUILD_COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR> --config RelWithDebInfo --verbose --parallel
+   INSTALL_COMMAND ${CMAKE_COMMAND} --install <BINARY_DIR> --config RelWithDebInfo)
+ SET(BUILD_TYPE -DCMAKE_BUILD_TYPE=RelWithDebInfo)
+ELSE()
+ SET(BUILD_AND_INSTALL_COMMANDS)
+ SET(BUILD_TYPE)
+ENDIF()
 
 ExternalProject_Add(
   aws_sdk_cpp
   GIT_REPOSITORY "https://github.com/awslabs/aws-sdk-cpp.git"
   GIT_TAG ${GIT_TAG}
+  GIT_SHALLOW TRUE
   UPDATE_COMMAND ""
   SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/aws-sdk-cpp"
   LIST_SEPARATOR !
   ${byproducts}
   CMAKE_ARGS
+    ${BUILD_TYPE}
+    -DCMAKE_BUILD_TYPE=RelWithDebInfo
+    -DCMAKE_POSITION_INDEPENDENT_CODE=ON
     -DBUILD_ONLY=${SDK_LIBS_STR}
     -DBUILD_SHARED_LIBS=OFF
-    -DFORCE_SHARED_CRT=${FORCE_SHARED_CRT}
+    -DFORCE_SHARED_CRT=ON
     -DENABLE_TESTING=OFF
-    "-DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG} ${PIC_FLAG}"
-    "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${PIC_FLAG}"
-    "-DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE} ${PIC_FLAG}"
-    "-DCMAKE_CXX_FLAGS_MINSIZEREL=${CMAKE_CXX_FLAGS_MINSIZEREL} ${PIC_FLAG}"
-    "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
-    "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
+    -DAWS_WARNINGS_ARE_ERRORS=OFF
     ${EXTRA_SDK_CMAKE_FLAGS}
     -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR}/aws_sdk_cpp
     -DCMAKE_INSTALL_LIBDIR=lib
-    TEST_COMMAND ""
+  TEST_COMMAND ""
+  ${BUILD_AND_INSTALL_COMMANDS}
 )
+
 SET_TARGET_PROPERTIES(aws_sdk_cpp PROPERTIES EXCLUDE_FROM_ALL TRUE)

plugin/aws_key_management/CMakeLists.txt

@@ -1,7 +1,10 @@
 INCLUDE(aws_sdk)
-CHECK_AWS_SDK(HAVE_AWS_SDK REASON)
+CHECK_AWS_SDK("kms;core" HAVE_AWS_SDK REASON)
 IF(NOT HAVE_AWS_SDK)
   MESSAGE_ONCE(AWS_KEY_MANAGEMENT_NO_AWS_SDK "Can't build aws_key_management - AWS SDK not available (${REASON})")
+  IF(PLUGIN_AWS_KEY_MANAGEMENT MATCHES "^(STATIC|DYNAMIC)$")
+    MESSAGE(FATAL_ERROR "Can't build reqired plugin aws_key_management: ${REASON}")
+  ENDIF()
   ADD_FEATURE_INFO(AWS_KEY_MANAGEMENT "OFF" "AWS Encryption Key Management Plugin")
   RETURN()
 ENDIF()
@@ -11,7 +14,7 @@ MYSQL_ADD_PLUGIN(aws_key_management
   COMPONENT aws-key-management)
 
 IF(TARGET aws_key_management)
- USE_AWS_SDK_LIBS(aws_key_management kms)
+  USE_AWS_SDK_LIBS(aws_key_management kms core)
+  ADD_FEATURE_INFO(AWS_KEY_MANAGEMENT "ON" "AWS Encryption Key Management Plugin")
 ENDIF()
 
-ADD_FEATURE_INFO(AWS_KEY_MANAGEMENT "ON" "AWS Encryption Key Management Plugin")

plugin/aws_key_management/aws_key_management_plugin.cc

@@ -540,7 +540,7 @@ static int generate_and_save_datakey(uint keyid, uint version)
     return(-1);
   }
   unsigned int len= (unsigned int)byteBuffer.GetLength();
-  if (write(fd, byteBuffer.GetUnderlyingData(), len) != len)
+  if ((unsigned int)write(fd, byteBuffer.GetUnderlyingData(), len) != len)
   {
     my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: can't write to %s", ME_ERROR_LOG_ONLY, filename);
     close(fd);