MINIFICPP-1022 - Refactored third party build system

NOTICE

@@ -27,8 +27,8 @@ The derived and original works are listed below:
 This includes derived works from the cURL (MIT/X-style licensed) project (https://github.com/curl/curl):
 Copyright (c) 1996 - 2019, Daniel Stenberg, <daniel@haxx.se>, and many contributors, see the THANKS file.
 The derived work is adapted from
-  CMake/FindLibSSH2.cmake
-and can be found in cmake/libssh2/sys/FindLibSSH2.cmake
+  CMake/FindLibSSH2.cmake and can be found in cmake/libssh2/sys/FindLibSSH2.cmake
+  lib/base64.c and can be found in thirdparty/base64/base64.h
 
 This includes derived works from the CMake (BSD 3-Clause licensed) project (https://github.com/Kitware/CMake):
 Copyright 2000-2019 Kitware, Inc. and Contributors

README.md

@@ -530,6 +530,10 @@ On Windows it is suggested that MSI be used for installation.
 
 Please see [Extensions.md](Extensions.md) on how to build and run conditionally built dependencies and extensions.
 
+### Third parties
+
+Please see [ThirdParties.md](ThirdParties.md) on how MiNiFi builds and uses third party libraries and how you can add new ones.
+
 ## Operations
 See our [operations documentation for additional inforomation on how to manage instances](OPS.md)
 

cmake/BuildTests.cmake

@@ -37,9 +37,6 @@ endif()
 
 function(appendIncludes testName)
     target_include_directories(${testName} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/thirdparty/catch")
-    target_include_directories(${testName} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/thirdparty/cron")
-    target_include_directories(${testName} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/thirdparty/spdlog-20170710/include")
-    target_include_directories(${testName} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/thirdparty/yaml-cpp-yaml-cpp-0.5.3/include")
     target_include_directories(${testName} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/include")
     target_include_directories(${testName} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include/")
     target_include_directories(${testName} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include/c2/protocols")
@@ -74,17 +71,10 @@ function(createTests testName)
     if (Boost_FOUND)
         target_include_directories(${testName} BEFORE PRIVATE "${Boost_INCLUDE_DIRS}")
     endif()
-    target_link_libraries(${testName} ${CMAKE_DL_LIBS} ${SPD_LIB} ${TEST_BASE_LIB})
-    target_link_libraries(${testName} ${CMAKE_THREAD_LIBS_INIT}  core-minifi yaml-cpp)
+    target_link_libraries(${testName} ${CMAKE_DL_LIBS} ${TEST_BASE_LIB})
+    target_link_libraries(${testName} ${CMAKE_THREAD_LIBS_INIT} core-minifi yaml-cpp spdlog)
     if (NOT excludeBase)
-      if (APPLE)
-  		target_link_libraries (${testName} -Wl,-all_load minifi)
-	  elseif(WIN32)
-	    target_link_libraries (${testName} minifi)
-	    set_target_properties(${testName} PROPERTIES LINK_FLAGS "${LINK_FLAGS} /WHOLEARCHIVE:minifi")
-	  else ()
-  		target_link_libraries (${testName} -Wl,--whole-archive minifi -Wl,--no-whole-archive)
-	  endif ()
+      target_wholearchive_library(${testName} minifi)
 	endif()
 	add_dependencies(${testName} minifiexe nanofi)
     if (Boost_FOUND)
@@ -95,16 +85,11 @@ endfunction()
 
 enable_testing(test)
 
-SET(SPD_LIB spd_lib)
-add_library(${SPD_LIB} STATIC ${SPD_SOURCES})
-
 SET(TEST_BASE_LIB test_base)
 add_library(${TEST_BASE_LIB} STATIC "${TEST_DIR}/TestBase.cpp" "${TEST_DIR}/RandomServerSocket.cpp")
+target_link_libraries(${TEST_BASE_LIB} core-minifi)
 target_include_directories(${TEST_BASE_LIB} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/thirdparty/catch")
 target_include_directories(${TEST_BASE_LIB} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include/")
-target_include_directories(${TEST_BASE_LIB} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/thirdparty/cron")
-target_include_directories(${TEST_BASE_LIB} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/thirdparty/spdlog-20170710/include")
-target_include_directories(${TEST_BASE_LIB} BEFORE PRIVATE "${CIVET_THIRDPARTY_ROOT}/include")
 if(WIN32)
    	target_include_directories(${TEST_BASE_LIB} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/opsys/win")
 else()
@@ -143,21 +128,10 @@ FOREACH(testfile ${NANOFI_UNIT_TESTS})
     target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test")
     appendIncludes("${testfilename}")
     target_link_libraries(${testfilename} ${CMAKE_THREAD_LIBS_INIT} ${CATCH_MAIN_LIB} ${TEST_BASE_LIB}  nanofi)
-
-    if (APPLE)
-    	# minifi-standard-processors
-	target_link_libraries (${testfilename} -Wl,-all_load minifi-standard-processors nanofi)
-    elseif(NOT WIN32)
-	target_link_libraries (${testfilename} -Wl,--whole-archive minifi-standard-processors -Wl,--no-whole-archive)
-    else()
-	set(WIN32_ARCHIVES "${WIN32_ARCHIVES} /WHOLEARCHIVE:minifi-standard-processors")
-	target_link_libraries (${testfilename} minifi-standard-processors)
-    endif ()
-
-    if(WIN32)
-	set_target_properties(${testfilename} PROPERTIES LINK_FLAGS "${WIN32_ARCHIVES}")
-    endif()
-
+
+    target_wholearchive_library(${testfilename} minifi-standard-processors)
+#    target_wholearchive_library(${testfilename} nanofi)
+
     MATH(EXPR UNIT_TEST_COUNT "${UNIT_TEST_COUNT}+1")
     add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR})
 ENDFOREACH()

cmake/BundledAwsSdkCpp.cmake

@@ -0,0 +1,178 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+function(use_bundled_libaws SOURCE_DIR BINARY_DIR)
+    # Define byproducts
+    if (WIN32)
+        set(SUFFIX "lib")
+    else()
+        set(SUFFIX "a")
+    endif()
+    set(BYPRODUCTS
+            "lib/libaws-cpp-sdk-core.${SUFFIX}"
+            "lib/libaws-cpp-sdk-s3.${SUFFIX}")
+
+    FOREACH(BYPRODUCT ${BYPRODUCTS})
+        LIST(APPEND AWSSDK_LIBRARIES_LIST "${BINARY_DIR}/thirdparty/libaws-install/${BYPRODUCT}")
+    ENDFOREACH(BYPRODUCT)
+
+    # Set build options
+    set(AWS_C_COMMON_CMAKE_ARGS ${PASSTHROUGH_CMAKE_ARGS}
+            -DCMAKE_PREFIX_PATH=${BINARY_DIR}/thirdparty/libaws-install
+            -DCMAKE_INSTALL_PREFIX=${BINARY_DIR}/thirdparty/libaws-install
+            -DENABLE_TESTING=OFF
+            -DBUILD_SHARED_LIBS=OFF)
+
+    append_third_party_passthrough_args(AWS_C_COMMON_CMAKE_ARGS "${AWS_C_COMMON_CMAKE_ARGS}")
+
+    set(AWS_CHECKSUM_CMAKE_ARGS ${PASSTHROUGH_CMAKE_ARGS}
+            -DCMAKE_PREFIX_PATH=${BINARY_DIR}/thirdparty/libaws-install
+            -DCMAKE_INSTALL_PREFIX=${BINARY_DIR}/thirdparty/libaws-install
+            -DBUILD_SHARED_LIBS=OFF)
+
+    append_third_party_passthrough_args(AWS_CHECKSUM_CMAKE_ARGS "${AWS_CHECKSUM_CMAKE_ARGS}")
+
+    set(AWS_C_EVENT_STREAM_CMAKE_ARGS ${PASSTHROUGH_CMAKE_ARGS}
+            -DCMAKE_PREFIX_PATH=${BINARY_DIR}/thirdparty/libaws-install
+            -DCMAKE_INSTALL_PREFIX=${BINARY_DIR}/thirdparty/libaws-install
+            -DBUILD_SHARED_LIBS=OFF)
+
+    append_third_party_passthrough_args(AWS_C_EVENT_STREAM_CMAKE_ARGS "${AWS_C_EVENT_STREAM_CMAKE_ARGS}")
+
+    set(AWS_SDK_CPP_CMAKE_ARGS ${PASSTHROUGH_CMAKE_ARGS}
+            -DCMAKE_PREFIX_PATH=${BINARY_DIR}/thirdparty/libaws-install
+            -DCMAKE_INSTALL_PREFIX=${BINARY_DIR}/thirdparty/libaws-install
+            -DBUILD_ONLY=s3
+            -DENABLE_TESTING=OFF
+            -DBUILD_SHARED_LIBS=OFF
+            -DENABLE_UNITY_BUILD=ON
+            -DBUILD_DEPS=OFF)
+
+    append_third_party_passthrough_args(AWS_SDK_CPP_CMAKE_ARGS "${AWS_SDK_CPP_CMAKE_ARGS}")
+
+    # Build project
+    ExternalProject_Add(
+            aws-c-common-external
+            GIT_REPOSITORY "https://github.com/awslabs/aws-c-common.git"
+            GIT_TAG "ac02e1728d740bb9106b6ea727cd3378f8ea438a"
+            SOURCE_DIR "${BINARY_DIR}/thirdparty/aws-c-common-src"
+            INSTALL_DIR "${BINARY_DIR}/thirdparty/libaws-install"
+            LIST_SEPARATOR % # This is needed for passing semicolon-separated lists
+            CMAKE_ARGS ${AWS_C_COMMON_CMAKE_ARGS}
+            BUILD_BYPRODUCTS "${BINARY_DIR}/thirdparty/libaws-install/lib/libaws-c-common.${SUFFIX}"
+            EXCLUDE_FROM_ALL TRUE
+    )
+    ExternalProject_Add(
+            aws-checksum-external
+            GIT_REPOSITORY "https://github.com/awslabs/aws-checksums.git"
+            GIT_TAG "41dc36d14b0898bd34e3f91c808fcb00f5e21875"
+            SOURCE_DIR "${BINARY_DIR}/thirdparty/aws-checksums-src"
+            INSTALL_DIR "${BINARY_DIR}/thirdparty/libaws-install"
+            LIST_SEPARATOR % # This is needed for passing semicolon-separated lists
+            CMAKE_ARGS ${AWS_CHECKSUM_CMAKE_ARGS}
+            BUILD_BYPRODUCTS "${BINARY_DIR}/thirdparty/libaws-install/lib/libaws-checksums.${SUFFIX}"
+            EXCLUDE_FROM_ALL TRUE
+    )
+    ExternalProject_Add(
+            aws-c-event-stream-external
+            GIT_REPOSITORY "https://github.com/awslabs/aws-c-event-stream.git"
+            GIT_TAG "97ab2e57e83ad114679dbee0dcfb5048640debe7"
+            SOURCE_DIR "${BINARY_DIR}/thirdparty/aws-c-event-stream-src"
+            INSTALL_DIR "${BINARY_DIR}/thirdparty/libaws-install"
+            LIST_SEPARATOR % # This is needed for passing semicolon-separated lists
+            CMAKE_ARGS ${AWS_C_EVENT_STREAM_CMAKE_ARGS}
+            BUILD_BYPRODUCTS "${BINARY_DIR}/thirdparty/libaws-install/lib/libaws-c-event-stream.${SUFFIX}"
+            EXCLUDE_FROM_ALL TRUE
+    )
+    ExternalProject_Add(
+            aws-sdk-cpp-external
+            GIT_REPOSITORY "https://github.com/aws/aws-sdk-cpp.git"
+            GIT_TAG "1.7.109"
+            SOURCE_DIR "${BINARY_DIR}/thirdparty/aws-sdk-cpp-src"
+            INSTALL_DIR "${BINARY_DIR}/thirdparty/libaws-install"
+            LIST_SEPARATOR % # This is needed for passing semicolon-separated lists
+            CMAKE_ARGS ${AWS_SDK_CPP_CMAKE_ARGS}
+            BUILD_BYPRODUCTS "${AWSSDK_LIBRARIES_LIST}"
+            EXCLUDE_FROM_ALL TRUE
+    )
+
+    # Set dependencies
+    add_dependencies(aws-c-common-external CURL::libcurl OpenSSL::Crypto OpenSSL::SSL ZLIB::ZLIB)
+    add_dependencies(aws-checksum-external CURL::libcurl OpenSSL::Crypto OpenSSL::SSL ZLIB::ZLIB)
+    add_dependencies(aws-c-event-stream-external CURL::libcurl OpenSSL::Crypto OpenSSL::SSL ZLIB::ZLIB)
+    add_dependencies(aws-c-event-stream-external aws-c-common-external aws-checksum-external)
+    add_dependencies(aws-sdk-cpp-external CURL::libcurl OpenSSL::Crypto OpenSSL::SSL ZLIB::ZLIB)
+    add_dependencies(aws-sdk-cpp-external aws-c-event-stream-external aws-c-common-external aws-checksum-external)
+
+    # Set variables
+    set(LIBAWS_FOUND "YES" CACHE STRING "" FORCE)
+    set(LIBAWS_INCLUDE_DIR "${BINARY_DIR}/thirdparty/libaws-install/include" CACHE STRING "" FORCE)
+    set(LIBAWS_LIBRARIES
+            ${AWSSDK_LIBRARIES_LIST}
+            "${BINARY_DIR}/thirdparty/libaws-install/lib/libaws-c-event-stream.${SUFFIX}"
+            "${BINARY_DIR}/thirdparty/libaws-install/lib/libaws-c-common.${SUFFIX}"
+            "${BINARY_DIR}/thirdparty/libaws-install/lib/libaws-checksums.${SUFFIX}"
+            CACHE STRING "" FORCE)
+
+    # Create imported targets
+    file(MAKE_DIRECTORY ${LIBAWS_INCLUDE_DIR})
+
+    add_library(AWS::aws-c-common STATIC IMPORTED)
+    set_target_properties(AWS::aws-c-common PROPERTIES IMPORTED_LOCATION "${BINARY_DIR}/thirdparty/libaws-install/lib/libaws-c-common.${SUFFIX}")
+    add_dependencies(AWS::aws-c-common aws-c-common-external)
+    set_property(TARGET AWS::aws-c-common APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${LIBAWS_INCLUDE_DIR})
+    set_property(TARGET AWS::aws-c-common APPEND PROPERTY INTERFACE_LINK_LIBRARIES CURL::libcurl OpenSSL::Crypto OpenSSL::SSL ZLIB::ZLIB)
+    if (APPLE)
+        set_property(TARGET AWS::aws-c-common APPEND PROPERTY INTERFACE_LINK_LIBRARIES "-framework CoreFoundation")
+    endif()
+
+    add_library(AWS::aws-checksums STATIC IMPORTED)
+    set_target_properties(AWS::aws-checksums PROPERTIES IMPORTED_LOCATION "${BINARY_DIR}/thirdparty/libaws-install/lib/libaws-checksums.${SUFFIX}")
+    add_dependencies(AWS::aws-checksums aws-checksums-external)
+    set_property(TARGET AWS::aws-checksums APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${LIBAWS_INCLUDE_DIR})
+    set_property(TARGET AWS::aws-checksums APPEND PROPERTY INTERFACE_LINK_LIBRARIES CURL::libcurl OpenSSL::Crypto OpenSSL::SSL ZLIB::ZLIB)
+    if (APPLE)
+        set_property(TARGET AWS::aws-checksums APPEND PROPERTY INTERFACE_LINK_LIBRARIES "-framework CoreFoundation")
+    endif()
+
+    add_library(AWS::aws-c-event-stream STATIC IMPORTED)
+    set_target_properties(AWS::aws-c-event-stream PROPERTIES IMPORTED_LOCATION "${BINARY_DIR}/thirdparty/libaws-install/lib/libaws-c-event-stream.${SUFFIX}")
+    add_dependencies(AWS::aws-c-event-stream aws-c-event-stream-external)
+    set_property(TARGET AWS::aws-c-event-stream APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${LIBAWS_INCLUDE_DIR})
+    set_property(TARGET AWS::aws-c-event-stream APPEND PROPERTY INTERFACE_LINK_LIBRARIES AWS::aws-c-common AWS::aws-checksums CURL::libcurl OpenSSL::Crypto OpenSSL::SSL ZLIB::ZLIB)
+    if (APPLE)
+        set_property(TARGET AWS::aws-c-event-stream APPEND PROPERTY INTERFACE_LINK_LIBRARIES "-framework CoreFoundation")
+    endif()
+
+    add_library(AWS::aws-cpp-sdk-core STATIC IMPORTED)
+    set_target_properties(AWS::aws-cpp-sdk-core PROPERTIES IMPORTED_LOCATION "${BINARY_DIR}/thirdparty/libaws-install/lib/libaws-cpp-sdk-core.${SUFFIX}")
+    add_dependencies(AWS::aws-cpp-sdk-core aws-sdk-cpp-external)
+    set_property(TARGET AWS::aws-cpp-sdk-core APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${LIBAWS_INCLUDE_DIR})
+    set_property(TARGET AWS::aws-cpp-sdk-core APPEND PROPERTY INTERFACE_LINK_LIBRARIES AWS::aws-c-event-stream AWS::aws-c-common AWS::aws-checksums CURL::libcurl OpenSSL::Crypto OpenSSL::SSL ZLIB::ZLIB)
+    if (APPLE)
+        set_property(TARGET AWS::aws-cpp-sdk-core APPEND PROPERTY INTERFACE_LINK_LIBRARIES "-framework CoreFoundation")
+    endif()
+
+    add_library(AWS::aws-cpp-sdk-s3 STATIC IMPORTED)
+    set_target_properties(AWS::aws-cpp-sdk-s3 PROPERTIES IMPORTED_LOCATION "${BINARY_DIR}/thirdparty/libaws-install/lib/libaws-cpp-sdk-s3.${SUFFIX}")
+    add_dependencies(AWS::aws-cpp-sdk-s3 aws-sdk-cpp-external)
+    set_property(TARGET AWS::aws-cpp-sdk-s3 APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${LIBAWS_INCLUDE_DIR})
+    set_property(TARGET AWS::aws-cpp-sdk-s3 APPEND PROPERTY INTERFACE_LINK_LIBRARIES CURL::libcurl OpenSSL::Crypto OpenSSL::SSL ZLIB::ZLIB)
+    if (APPLE)
+        set_property(TARGET AWS::aws-cpp-sdk-s3 APPEND PROPERTY INTERFACE_LINK_LIBRARIES "-framework CoreFoundation")
+    endif()
+endfunction(use_bundled_libaws)

cmake/BundledBustache.cmake

@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+function(use_bundled_bustache SOURCE_DIR BINARY_DIR)
+    # Find Boost
+    find_package(Boost COMPONENTS system filesystem iostreams REQUIRED)
+
+    # Define byproducts
+    if (WIN32)
+        set(BYPRODUCT "lib/bustache.lib")
+    else()
+        set(BYPRODUCT "lib/libbustache.a")
+    endif()
+
+    # Set build options
+    set(BUSTACHE_BYPRODUCT_DIR "${CMAKE_CURRENT_BINARY_DIR}/thirdparty/bustache-install")
+
+    set(BUSTACHE_CMAKE_ARGS ${PASSTHROUGH_CMAKE_ARGS}
+            "-DCMAKE_INSTALL_PREFIX=${BUSTACHE_BYPRODUCT_DIR}"
+            "-DBUSTACHE_ENABLE_TESTING=OFF")
+
+    # Build project
+    ExternalProject_Add(
+            bustache-external
+            GIT "https://github.com/jamboree/bustache.git"
+            GIT_TAG "42dee8ef9bbcae7e9a33500a116cfd9c314662d6"
+            SOURCE_DIR "${BINARY_DIR}/thirdparty/bustache-src"
+            CMAKE_ARGS ${BUSTACHE_CMAKE_ARGS}
+            BUILD_BYPRODUCTS "${BUSTACHE_BYPRODUCT_DIR}/${BYPRODUCT}"
+            EXCLUDE_FROM_ALL TRUE
+    )
+
+    # Set variables
+    set(BUSTACHE_FOUND "YES" CACHE STRING "" FORCE)
+    set(BUSTACHE_INCLUDE_DIR "${BUSTACHE_BYPRODUCT_DIR}/include" CACHE STRING "" FORCE)
+    set(BUSTACHE_LIBRARY "${BUSTACHE_BYPRODUCT_DIR}/${BYPRODUCT}" CACHE STRING "" FORCE)
+
+    # Create imported targets
+    add_library(BUSTACHE::libbustache STATIC IMPORTED)
+    set_target_properties(BUSTACHE::libbustache PROPERTIES IMPORTED_LOCATION "${BUSTACHE_LIBRARY}")
+    add_dependencies(BUSTACHE::libbustache bustache-external)
+    file(MAKE_DIRECTORY ${BUSTACHE_INCLUDE_DIR})
+    set_property(TARGET BUSTACHE::libbustache APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${BUSTACHE_INCLUDE_DIR} ${Boost_INCLUDE_DIRS})
+    set_property(TARGET BUSTACHE::libbustache APPEND PROPERTY INTERFACE_LINK_LIBRARIES ${Boost_LIBRARIES})
+endfunction(use_bundled_bustache)