MINIFICPP-2346: Build MiNiFi Core, MainExe, Standard-Processors with Conan2

CMakeLists.txt

@@ -49,6 +49,10 @@ include(WholeArchive)
 
 include(MiNiFiOptions)
 
+message("Checking MiNiFi Options Updated by conanfile.py")
+message("USE_CONAN_PACKAGER = ${USE_CONAN_PACKAGER}")
+message("USE_CMAKE_FETCH_CONTENT = ${USE_CMAKE_FETCH_CONTENT}")
+
 include(DockerConfig)
 if (DOCKER_BUILD_ONLY)
     return()
@@ -221,6 +225,7 @@ else()
     message(VERBOSE "No custom malloc implementation")
 endif()
 
+# rocksdb needed for minifi C++ GTESTs
 if (ENABLE_BZIP2 AND (ENABLE_LIBARCHIVE OR (ENABLE_ROCKSDB AND NOT WIN32)))
     include(BundledBZip2)
     use_bundled_bzip2(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR})
@@ -230,6 +235,16 @@ endif()
 if(NOT WIN32)
     if (ENABLE_JNI)
         list(APPEND MINIFI_CPP_COMPILE_DEFINITIONS ENABLE_JNI)
+        # NOTE (JG): Mainly leverage conan to install the prebuilt Java OpenJDK and create custom Maven conan package for
+        # minifi exposing Maven Executable environment variable, so we can leverage conan to build Apache NiFi JNI Assembly
+        # and Framework without having to rely on user having Java & Maven installed on their system. Kept backward
+        # compatibility using find_package for Java and Maven
+        include(JavaOpenJDK)
+        include(JavaMaven)
+        use_javamaven(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR})
+
+        include(CheckJavaMaven)
+        check_javamaven_version()
     endif()
 endif()
 
@@ -252,17 +267,18 @@ if (MINIFI_OPENSSL)
     list(APPEND MINIFI_CPP_COMPILE_DEFINITIONS OPENSSL_SUPPORT)
 endif()
 
-# libsodium
+# libsodium (libminifi C++ core)
 include(BundledLibSodium)
 use_bundled_libsodium("${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}")
 
 list(APPEND MINIFI_CPP_COMPILE_DEFINITIONS SODIUM_STATIC=1)
 
-# zlib
+# zlib (libminifi C++ core)
 include(BundledZLIB)
 use_bundled_zlib(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/zlib/dummy")
 
+# TODO (JG): Add uthash.cmake with condition to use conan package
 # uthash
 add_library(ut INTERFACE)
 target_include_directories(ut SYSTEM INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/ut")
@@ -275,49 +291,37 @@ if(ENABLE_CURL)
     list(APPEND MINIFI_CPP_COMPILE_DEFINITIONS ENABLE_CURL)
 endif()
 
-# spdlog
+# spdlog (libminifi C++ core)
 include(Spdlog)
 
-# yaml-cpp
+# yaml-cpp (libminifi C++ core)
 include(BundledYamlCpp)
 use_bundled_yamlcpp(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
 
-# concurrentqueue
+# TODO (JG): Add concurrentqueue.cmake with condition to use conan package
+# concurrentqueue (libminifi C++ core)
 add_library(concurrentqueue INTERFACE)
 target_include_directories(concurrentqueue SYSTEM INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/concurrentqueue")
 
-# RapidJSON
+# TODO (JG): Add RapidJSON.cmake with condition to use conan package
+# RapidJSON (libminifi C++ core)
 add_library(RapidJSON INTERFACE)
 target_include_directories(RapidJSON SYSTEM INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/rapidjson-48fbd8cd202ca54031fe799db2ad44ffa8e77c13/include")
 target_compile_definitions(RapidJSON INTERFACE RAPIDJSON_HAS_STDSTRING)
 
 include(Coroutines)
 enable_coroutines()
 
-# gsl-lite
+# gsl-lite (libminifi C++ core)
 include(GslLite)
 
-# Add necessary definitions based on the value of STRICT_GSL_CHECKS, see gsl-lite README for more details
-list(APPEND GslDefinitions gsl_CONFIG_DEFAULTS_VERSION=1)
-list(APPEND GslDefinitionsNonStrict gsl_CONFIG_CONTRACT_VIOLATION_THROWS gsl_CONFIG_NARROW_THROWS_ON_TRUNCATION=1)
-if (STRICT_GSL_CHECKS STREQUAL "AUDIT")
-    list(APPEND GslDefinitions gsl_CONFIG_CONTRACT_CHECKING_AUDIT)
-endif()
-if (NOT STRICT_GSL_CHECKS)  # OFF (or any other falsey string) matches, AUDIT/ON/DEBUG_ONLY don't match
-    list(APPEND GslDefinitions ${GslDefinitionsNonStrict})
-endif()
-if (STRICT_GSL_CHECKS STREQUAL "DEBUG_ONLY")
-    list(APPEND GslDefinitions $<$<NOT:$<CONFIG:Debug>>:${GslDefinitionsNonStrict}>)
-endif()
-target_compile_definitions(gsl-lite INTERFACE ${GslDefinitions})
-
-# date
+# date (libminifi C++ core)
 include(Date)
 
-# expected-lite
+# expected-lite (libminifi C++ core)
 include(ExpectedLite)
 
-# magic_enum
+# magic_enum (libminifi C++ core)
 include(MagicEnum)
 
 # Setup warning flags
@@ -356,12 +360,15 @@ include(Extensions)
 add_subdirectory(libminifi)
 
 if ((ENABLE_OPENWSMAN AND ENABLE_CIVET AND ENABLE_CURL) OR ENABLE_ALL OR ENABLE_AZURE)
+    message("Enabled OpenWSMAN, Civet, and Curl for MiNiFi C++ Core")
+    message("Including BundledLibXml2")
     include(BundledLibXml2)
     use_bundled_libxml2(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
     list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/libxml2/dummy")
 endif()
 
 if (ENABLE_ALL OR ENABLE_PROMETHEUS OR ENABLE_CIVET)
+    message("Including CivetWeb")
     include(CivetWeb)
     list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/civetweb/dummy")
 endif()
@@ -629,28 +636,131 @@ if(NOT WIN32)
     cpack_add_component(conf DISPLAY_NAME "Default configuration files" REQUIRED)
 endif()
 
+# Speed up MiNIFi C++ builds by just having standard processors being included by default
+message("Adding minifi-standard-processors to EXTENSIONS_ENABLED_BY_DEFAULT list")
 cpack_add_component_group(extensions DISPLAY_NAME "Extensions" EXPANDED)
-set(EXTENSIONS_ENABLED_BY_DEFAULT (
-    minifi-aws
-    minifi-azure
-    minifi-civet-extensions
-    minifi-elasticsearch
-    minifi-expression-language-extensions
-    minifi-gcp
-    minifi-grafana-loki
-    minifi-http-curl
-    minifi-archive-extensions
-    minifi-mqtt-extensions
-    minifi-rdkafka-extensions
-    minifi-pdh
-    minifi-prometheus
-    minifi-rocksdb-repos
-    minifi-smb
-    minifi-splunk
-    minifi-sql
+set(EXTENSIONS_ENABLED_BY_DEFAULT
     minifi-standard-processors
-    minifi-wel
-))
+)
+
+# Only append the remaining extensions that are enabled, so they dont take up time in the build
+message("Checking for enabled extensions to append to EXTENSIONS_ENABLED_BY_DEFAULT list")
+if(ENABLE_AWS)
+    message("AWS Enabled appending to EXTENSIONS_ENABLED_BY_DEFAULT")
+    list(APPEND EXTENSIONS_ENABLED_BY_DEFAULT minifi-aws)
+endif()
+
+if(ENABLE_AZURE)
+    message("Azure Enabled appending to EXTENSIONS_ENABLED_BY_DEFAULT")
+    list(APPEND EXTENSIONS_ENABLED_BY_DEFAULT minifi-azure)
+endif()
+
+if(ENABLE_CIVET)
+    message("Civet Enabled appending to EXTENSIONS_ENABLED_BY_DEFAULT")
+    list(APPEND EXTENSIONS_ENABLED_BY_DEFAULT minifi-civet-extensions)
+endif()
+
+if(ENABLE_ELASTICSEARCH)
+    message("Elasticsearch Enabled appending to EXTENSIONS_ENABLED_BY_DEFAULT")
+    list(APPEND EXTENSIONS_ENABLED_BY_DEFAULT minifi-elasticsearch)
+endif()
+
+if(ENABLE_EXPRESSION_LANGUAGE)
+    message("Expression Language Enabled appending to EXTENSIONS_ENABLED_BY_DEFAULT")
+    list(APPEND EXTENSIONS_ENABLED_BY_DEFAULT minifi-expression-language-extensions)
+endif()
+
+if(ENABLE_GCP)
+    message("GCP Enabled appending to EXTENSIONS_ENABLED_BY_DEFAULT")
+    list(APPEND EXTENSIONS_ENABLED_BY_DEFAULT minifi-gcp)
+endif()
+
+if(ENABLE_GRAFANA_LOKI)
+    message("Grafana Loki Enabled appending to EXTENSIONS_ENABLED_BY_DEFAULT")
+    list(APPEND EXTENSIONS_ENABLED_BY_DEFAULT minifi-grafana-loki)
+endif()
+
+if(ENABLE_HTTP_CURL)
+    message("HTTP CURL Enabled appending to EXTENSIONS_ENABLED_BY_DEFAULT")
+    list(APPEND EXTENSIONS_ENABLED_BY_DEFAULT minifi-http-curl)
+endif()
+
+if(ENABLE_LIBARCHIVE)
+    message("Libarchive Enabled appending to EXTENSIONS_ENABLED_BY_DEFAULT")
+    list(APPEND EXTENSIONS_ENABLED_BY_DEFAULT minifi-archive-extensions)
+endif()
+
+if(ENABLE_MQTT)
+    message("MQTT Enabled appending to EXTENSIONS_ENABLED_BY_DEFAULT")
+    list(APPEND EXTENSIONS_ENABLED_BY_DEFAULT minifi-mqtt-extensions)
+endif()
+
+if(ENABLE_LIBRDKAFKA)
+    message("RDKafka Enabled appending to EXTENSIONS_ENABLED_BY_DEFAULT")
+    list(APPEND EXTENSIONS_ENABLED_BY_DEFAULT minifi-rdkafka-extensions)
+endif()
+
+if(ENABLE_PDH)
+    message("PDH Enabled appending to EXTENSIONS_ENABLED_BY_DEFAULT")
+    list(APPEND EXTENSIONS_ENABLED_BY_DEFAULT minifi-pdh)
+endif()
+
+if(ENABLE_PROMETHEUS)
+    message("Prometheus Enabled appending to EXTENSIONS_ENABLED_BY_DEFAULT")
+    list(APPEND EXTENSIONS_ENABLED_BY_DEFAULT minifi-prometheus)
+endif()
+
+if(ENABLE_ROCKSDB)
+    message("RocksDB Enabled appending to EXTENSIONS_ENABLED_BY_DEFAULT")
+    list(APPEND EXTENSIONS_ENABLED_BY_DEFAULT minifi-rocksdb-repos)
+endif()
+
+if(ENABLE_SMB)
+    message("SMB Enabled appending to EXTENSIONS_ENABLED_BY_DEFAULT")
+    list(APPEND EXTENSIONS_ENABLED_BY_DEFAULT minifi-smb)
+endif()
+
+if(ENABLE_SPLUNK)
+    message("Splunk Enabled appending to EXTENSIONS_ENABLED_BY_DEFAULT")
+    list(APPEND EXTENSIONS_ENABLED_BY_DEFAULT minifi-splunk)
+endif()
+
+if(ENABLE_SQL)
+    message("SQL Enabled appending to EXTENSIONS_ENABLED_BY_DEFAULT")
+    list(APPEND EXTENSIONS_ENABLED_BY_DEFAULT minifi-sql)
+endif()
+
+if(ENABLE_WEL)
+    message("WEL Enabled appending to EXTENSIONS_ENABLED_BY_DEFAULT")
+    list(APPEND EXTENSIONS_ENABLED_BY_DEFAULT minifi-wel)
+endif()
+
+
+# TODO (JG): Planning to replace below with the above conditional approach, so we save time on building
+# MiNiFi C++ Core library and Main binary executable and whats needed for building minifi-standard-processors
+# like external libraries Openwsman, Openssl, Civet, Curl and their appropriate extensions.
+# cpack_add_component_group(extensions DISPLAY_NAME "Extensions" EXPANDED)
+# set(EXTENSIONS_ENABLED_BY_DEFAULT (
+#     # minifi-aws
+#     # minifi-azure
+#     # minifi-civet-extensions
+#     # minifi-elasticsearch
+#     # minifi-expression-language-extensions
+#     # minifi-gcp
+#     # minifi-grafana-loki
+#     # minifi-http-curl
+#     # minifi-archive-extensions
+#     # minifi-mqtt-extensions
+#     # minifi-rdkafka-extensions
+#     # minifi-pdh
+#     # minifi-prometheus
+#     # minifi-rocksdb-repos
+#     # minifi-smb
+#     # minifi-splunk
+#     # minifi-sql
+#     minifi-standard-processors
+#     # minifi-wel
+# ))
 foreach(extension ${selected_extensions})
     get_component_name(${extension} component-name)
     list(APPEND CPACK_COMPONENTS_ALL ${component-name})

cmake/Abseil.cmake

@@ -16,20 +16,32 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-include(FetchContent)
-set(ABSL_PROPAGATE_CXX_STD ON CACHE INTERNAL absl-propagate-cxx)
-set(ABSL_ENABLE_INSTALL ON CACHE INTERNAL "")
-set(BUILD_TESTING OFF CACHE STRING "" FORCE)
+if(USE_CONAN_PACKAGER)
+    message("Using Conan Packager to manage installing prebuilt Abseil external lib")
+    include(${CMAKE_BINARY_DIR}/absl-config.cmake)
+    set(ABSL_INCLUDE_DIRS "${absl_INCLUDE_DIRS}" CACHE STRING "" FORCE)
 
-set(PATCH_FILE "${CMAKE_SOURCE_DIR}/thirdparty/abseil/rename-crc32.patch")
-set(PC ${Bash_EXECUTABLE}  -c "set -x &&\
-        (\\\"${Patch_EXECUTABLE}\\\" -p1 -R -s -f --dry-run -i \\\"${PATCH_FILE}\\\" || \\\"${Patch_EXECUTABLE}\\\" -p1 -N -i \\\"${PATCH_FILE}\\\")")
+elseif(USE_CMAKE_FETCH_CONTENT)
+    message("Using CMAKE's FetchContent to manage source building Abseil external lib")
+    include(FetchContent)
+    set(ABSL_PROPAGATE_CXX_STD ON CACHE INTERNAL absl-propagate-cxx)
+    set(ABSL_ENABLE_INSTALL ON CACHE INTERNAL "")
+    set(BUILD_TESTING OFF CACHE STRING "" FORCE)
 
-FetchContent_Declare(
-        absl
-        URL      https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.0.tar.gz
-        URL_HASH SHA256=59d2976af9d6ecf001a81a35749a6e551a335b949d34918cfade07737b9d93c5
-        PATCH_COMMAND "${PC}"
-        OVERRIDE_FIND_PACKAGE
-)
-FetchContent_MakeAvailable(absl)
+    set(PATCH_FILE "${CMAKE_SOURCE_DIR}/thirdparty/abseil/rename-crc32.patch")
+    set(PC ${Bash_EXECUTABLE}  -c "set -x &&\
+            (\\\"${Patch_EXECUTABLE}\\\" -p1 -R -s -f --dry-run -i \\\"${PATCH_FILE}\\\" || \\\"${Patch_EXECUTABLE}\\\" -p1 -N -i \\\"${PATCH_FILE}\\\")")
+
+    FetchContent_Declare(
+            absl
+            URL      https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.0.tar.gz
+            URL_HASH SHA256=59d2976af9d6ecf001a81a35749a6e551a335b949d34918cfade07737b9d93c5
+            PATCH_COMMAND "${PC}"
+            OVERRIDE_FIND_PACKAGE
+    )
+    FetchContent_MakeAvailable(absl)
+
+    add_library(abseil::abseil INTERFACE IMPORTED)
+    set(ABSL_INCLUDE_DIRS "${absl_SOURCE_DIR}/absl" CACHE STRING "" FORCE)
+    target_include_directories(abseil::abseil SYSTEM INTERFACE ${ABSL_INCLUDE_DIRS})
+endif()

cmake/ArgParse.cmake

@@ -14,11 +14,17 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-include(FetchContent)
-FetchContent_Declare(
-    argparse
-    URL https://github.com/p-ranav/argparse/archive/refs/tags/v3.0.tar.gz
-    URL_HASH SHA256=ba7b465759bb01069d57302855eaf4d1f7d677f21ad7b0b00b92939645c30f47
-)
-FetchContent_MakeAvailable(argparse)
+if(USE_CONAN_PACKAGER)
+    message("Using Conan Packager to manage installing prebuilt ArgParse external lib")
+    include(${CMAKE_BINARY_DIR}/argparse-config.cmake)
+elseif(USE_CMAKE_FETCH_CONTENT)
+    message("Using CMAKE's FetchContent to manage source building ArgParse external lib")
+
+    include(FetchContent)
+    FetchContent_Declare(
+        argparse
+        URL https://github.com/p-ranav/argparse/archive/refs/tags/v3.0.tar.gz
+        URL_HASH SHA256=ba7b465759bb01069d57302855eaf4d1f7d677f21ad7b0b00b92939645c30f47
+    )
+    FetchContent_MakeAvailable(argparse)
+endif()

cmake/Asio.cmake

@@ -14,18 +14,24 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+# under the License.
+if(USE_CONAN_PACKAGER)
+    message("Using Conan Packager to manage installing prebuilt Asio external lib")
+    include(${CMAKE_BINARY_DIR}/asio-config.cmake)
+elseif(USE_CMAKE_FETCH_CONTENT)
+    include(FetchContent)
+    message("Using CMAKE's FetchContent to manage source building Asio external lib")
+    FetchContent_Declare(asio
+            URL https://github.com/chriskohlhoff/asio/archive/refs/tags/asio-1-28-1.tar.gz
+            URL_HASH SHA256=5ff6111ec8cbe73a168d997c547f562713aa7bd004c5c02326f0e9d579a5f2ce)
 
-include(FetchContent)
-
-FetchContent_Declare(asio
-        URL https://github.com/chriskohlhoff/asio/archive/refs/tags/asio-1-28-1.tar.gz
-        URL_HASH SHA256=5ff6111ec8cbe73a168d997c547f562713aa7bd004c5c02326f0e9d579a5f2ce)
-
-FetchContent_GetProperties(asio)
-if(NOT asio_POPULATED)
-    FetchContent_Populate(asio)
-    add_library(asio INTERFACE)
-    target_include_directories(asio SYSTEM INTERFACE ${asio_SOURCE_DIR}/asio/include)
-    find_package(Threads)
-    target_link_libraries(asio INTERFACE Threads::Threads)
+    FetchContent_GetProperties(asio)
+    if(NOT asio_POPULATED)
+        FetchContent_Populate(asio)
+        add_library(asio INTERFACE)
+        add_library(asio::asio ALIAS asio)
+        target_include_directories(asio SYSTEM INTERFACE ${asio_SOURCE_DIR}/asio/include)
+        find_package(Threads)
+        target_link_libraries(asio INTERFACE Threads::Threads)
+    endif()
 endif()