diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e14505d68..3c929c8296 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,6 @@ jobs: -DENABLE_CIVET=ON -DENABLE_COAP=OFF -DENABLE_CONTROLLER=ON - -DENABLE_CURL=ON -DENABLE_ELASTICSEARCH=ON -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_EXPRESSION_LANGUAGE=ON @@ -60,7 +59,6 @@ jobs: -DMINIFI_ADVANCED_ASAN_BUILD=OFF -DMINIFI_ADVANCED_CODE_COVERAGE=OFF -DMINIFI_FAIL_ON_WARNINGS=OFF - -DMINIFI_OPENSSL=ON -DMINIFI_USE_REAL_ODBC_TEST_DRIVER=ON -DPORTABLE=ON -DSKIP_TESTS=OFF @@ -145,7 +143,6 @@ jobs: -DENABLE_COAP=OFF -DENABLE_CONTROLLER=ON -DENABLE_COVERAGE= - -DENABLE_CURL=ON -DENABLE_ELASTICSEARCH=ON -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_EXPRESSION_LANGUAGE=ON @@ -180,7 +177,6 @@ jobs: -DFORCE_COLORED_OUTPUT=ON -DINSTALLER_MERGE_MODULES=OFF -DMINIFI_FAIL_ON_WARNINGS=OFF - -DMINIFI_OPENSSL=ON -DMSI_REDISTRIBUTE_UCRT_NONASL=OFF -DPORTABLE=ON -DSKIP_TESTS=OFF @@ -260,7 +256,6 @@ jobs: -DENABLE_CIVET=ON -DENABLE_COAP=OFF -DENABLE_CONTROLLER=ON - -DENABLE_CURL=ON -DENABLE_ELASTICSEARCH=OFF -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_EXPRESSION_LANGUAGE=ON @@ -292,7 +287,6 @@ jobs: -DENABLE_USB_CAMERA=OFF -DFORCE_COLORED_OUTPUT=ON -DMINIFI_FAIL_ON_WARNINGS=ON - -DMINIFI_OPENSSL=ON -DPORTABLE=ON -DSKIP_TESTS=OFF -DMINIFI_USE_REAL_ODBC_TEST_DRIVER=OFF @@ -367,7 +361,6 @@ jobs: -DENABLE_CIVET=ON -DENABLE_COAP=ON -DENABLE_CONTROLLER=ON - -DENABLE_CURL=ON -DENABLE_ELASTICSEARCH=ON -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_EXPRESSION_LANGUAGE=ON @@ -400,7 +393,6 @@ jobs: -DENABLE_USB_CAMERA=ON -DFORCE_COLORED_OUTPUT=ON -DMINIFI_FAIL_ON_WARNINGS=ON - -DMINIFI_OPENSSL=ON -DPORTABLE=ON -DSKIP_TESTS=OFF -DMINIFI_USE_REAL_ODBC_TEST_DRIVER=OFF diff --git a/CMakeLists.txt b/CMakeLists.txt index a88aabe9ee..f765438914 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -221,6 +221,12 @@ else() message(VERBOSE "No custom malloc implementation") endif() +# OpenSSL +include(GetOpenSSL) +get_openssl("${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") +list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/ssl") + + if (ENABLE_BZIP2 AND (ENABLE_LIBARCHIVE OR (ENABLE_ROCKSDB AND NOT WIN32))) include(BundledBZip2) use_bundled_bzip2(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) @@ -244,14 +250,6 @@ if(NOT WIN32) use_bundled_osspuuid(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) endif() -# OpenSSL -if (MINIFI_OPENSSL) - include(BundledOpenSSL) - use_openssl("${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") - list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/ssl") - list(APPEND MINIFI_CPP_COMPILE_DEFINITIONS OPENSSL_SUPPORT) -endif() - # libsodium include(BundledLibSodium) use_bundled_libsodium("${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") @@ -259,8 +257,8 @@ use_bundled_libsodium("${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR} list(APPEND MINIFI_CPP_COMPILE_DEFINITIONS SODIUM_STATIC=1) # zlib -include(BundledZLIB) -use_bundled_zlib(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) +include(GetZLIB) +get_zlib(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/zlib/dummy") # uthash @@ -268,15 +266,13 @@ add_library(ut INTERFACE) target_include_directories(ut SYSTEM INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/ut") # cURL -if(ENABLE_CURL) - include(BundledLibcURL) - use_bundled_curl(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) - list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/curl/dummy") - list(APPEND MINIFI_CPP_COMPILE_DEFINITIONS ENABLE_CURL) -endif() +include(GetLibCURL) +get_curl(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) +list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/curl/dummy") # spdlog -include(Spdlog) +include(GetSpdlog) +get_spdlog() # yaml-cpp include(BundledYamlCpp) @@ -355,14 +351,15 @@ include(Extensions) add_subdirectory(libminifi) -if ((ENABLE_OPENWSMAN AND ENABLE_CIVET AND ENABLE_CURL) OR ENABLE_ALL OR ENABLE_AZURE) - include(BundledLibXml2) - use_bundled_libxml2(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) +if ((ENABLE_OPENWSMAN AND ENABLE_CIVET) OR ENABLE_ALL OR ENABLE_AZURE) + include(GetLibXml2) + get_libxml2(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/libxml2/dummy") endif() if (ENABLE_ALL OR ENABLE_PROMETHEUS OR ENABLE_CIVET) - include(CivetWeb) + include(GetCivetWeb) + get_civetweb() list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/civetweb/dummy") endif() @@ -375,6 +372,7 @@ foreach(extension-dir ${extension-directories}) endforeach() ## NOW WE CAN ADD LIBRARIES AND EXTENSIONS TO MAIN + add_subdirectory(minifi_main) if (ENABLE_NANOFI) @@ -385,7 +383,7 @@ if (ENABLE_ENCRYPT_CONFIG) add_subdirectory(encrypt-config) endif() -if (ENABLE_CURL AND ENABLE_CONTROLLER) +if (ENABLE_CONTROLLER) add_subdirectory(controller) endif() @@ -579,13 +577,9 @@ if (NOT WIN32) endif() endif() -install(FILES LICENSE NOTICE - DESTINATION . - COMPONENT bin) +install(FILES LICENSE NOTICE DESTINATION . COMPONENT bin) file(GLOB markdown_docs "*.md") -install(FILES ${markdown_docs} - DESTINATION . - COMPONENT bin) +install(FILES ${markdown_docs} DESTINATION . COMPONENT bin) include(CPackComponent) @@ -603,9 +597,11 @@ set(CPACK_PACKAGE_VENDOR "Apache NiFi") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) set(CPACK_PACKAGE_VERSION_MINOR ${PROJECT_VERSION_MINOR}) set(CPACK_PACKAGE_VERSION_PATCH ${PROJECT_VERSION_PATCH}) -set(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/README.md") -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/LICENSE" "${CMAKE_CURRENT_BINARY_DIR}/LICENSE.txt" COPYONLY) -set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_BINARY_DIR}/LICENSE.txt") +if(NOT MINIFI_BUILD_CONAN_PACKAGE) + set(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/README.md") + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/LICENSE" "${CMAKE_CURRENT_BINARY_DIR}/LICENSE.txt" COPYONLY) + set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_BINARY_DIR}/LICENSE.txt") +endif() if(NOT WIN32) set(CPACK_PACKAGE_FILE_NAME "${ASSEMBLY_BASE_NAME}") set(CPACK_GENERATOR "TGZ") @@ -638,7 +634,6 @@ set(EXTENSIONS_ENABLED_BY_DEFAULT ( minifi-expression-language-extensions minifi-gcp minifi-grafana-loki - minifi-http-curl minifi-archive-extensions minifi-mqtt-extensions minifi-rdkafka-extensions @@ -679,24 +674,30 @@ include(CPack) if (NOT SKIP_TESTS) include(BuildTests) -endif() -## Add KeyValueStorageService tests -registerTest("${TEST_DIR}/keyvalue-tests") + add_subdirectory("${TEST_DIR}/libtest") -registerTest("${TEST_DIR}/flow-tests") + add_subdirectory("${TEST_DIR}/unit") -registerTest("${TEST_DIR}/schema-tests") + add_subdirectory("${TEST_DIR}/integration") -if (ENABLE_ROCKSDB AND ENABLE_LIBARCHIVE) - registerTest("${TEST_DIR}/persistence-tests") -endif() + ## Add KeyValueStorageService tests + add_subdirectory("${TEST_DIR}/keyvalue-tests") + + add_subdirectory("${TEST_DIR}/flow-tests") -registerTest("minifi_main/tests") + add_subdirectory("${TEST_DIR}/schema-tests") -registerTest("encrypt-config/tests") + if (ENABLE_ROCKSDB AND ENABLE_LIBARCHIVE) + add_subdirectory("${TEST_DIR}/persistence-tests") + endif() + + add_subdirectory("minifi_main/tests") -registerTest("controller/tests") + add_subdirectory("encrypt-config/tests") + + add_subdirectory("controller/tests") +endif() include(BuildDocs) @@ -746,10 +747,10 @@ if (MINIFI_ADVANCED_CODE_COVERAGE) set(GCOVR_ADDITIONAL_ARGS --gcov-ignore-parse-errors=negative_hits.warn --gcov-ignore-errors=no_working_dir_found) setup_target_for_coverage_gcovr_html( NAME coverage - EXCLUDE "build/*" "cmake/*" "minifi_main/*" "nanofi/*" "thirdparty/*" "libminifi/test/*" "extensions/http-curl/tests/*" "encrypt-config/tests/*" "extensions/aws/tests/*" + EXCLUDE "build/*" "cmake/*" "minifi_main/*" "nanofi/*" "thirdparty/*" "libminifi/test/*" "encrypt-config/tests/*" "extensions/aws/tests/*" "extensions/civetweb/tests/*" "extensions/coap/tests/*" "extensions/elasticsearch/tests/*" "extensions/expression-language/tests/*" "extensions/gcp/tests/*" "extensions/grafana-loki/tests/*" "extensions/kubernetes/tests/*" "extensions/librdkafka/tests/*" "extensions/lua/tests/*" "extensions/mqtt/tests/*" "extensions/opencv/tests/*" "extensions/procfs/tests/*" "extensions/prometheus/tests/*" "extensions/script/tests/*" "extensions/sftp/tests/*" "extensions/splunk/tests/*" "extensions/standard-processors/tests/*" "extensions/systemd/tests/*" "extensions/test-processors/*" - "libminifi/include/utils/IntegrationTestUtils.h" "libminifi/include/utils/TestUtils.h" "controller/MiNiFiController.cpp" + "controller/MiNiFiController.cpp" ) endif() diff --git a/Extensions.md b/Extensions.md index 0e9b94626f..07fb211408 100644 --- a/Extensions.md +++ b/Extensions.md @@ -17,7 +17,7 @@ To enable all extensions for your platform, you may use -DENABLE_ALL=TRUE OR select the option to "Enable all Extensions" in the bootstrap script. [ReadMe](https://github.com/apache/nifi-minifi-cpp/#bootstrapping) # Extension internals -Extensions are dynamic libraries loaded at runtime by the agent. An extension makes its +Extensions are dynamic libraries loaded at runtime by the agent. An extension makes its capabilities (classes) available to the system through registrars. Registration must happen in source files, not headers. ```C++ @@ -32,19 +32,17 @@ REGISTER_RESOURCE(HTTPClient, InternalResource); REGISTER_RESOURCE(RESTSender, DescriptionOnly); ``` -Some extensions (e.g. `http-curl`) require initialization before use. +Some extensions (e.g. `OpenCVExtension`) require initialization before use. You need to create `init` and `deinit` functions and register them using `REGISTER_EXTENSION`. ```C++ -static bool init(const core::extension::ExtensionConfig& /*config*/) override { - return curl_global_init(CURL_GLOBAL_DEFAULT) == CURLE_OK; +static bool init(const std::shared_ptr& /*config*/) { + return org::apache::nifi::minifi::utils::Environment::setEnvironmentVariable("OPENCV_FFMPEG_CAPTURE_OPTIONS", "rtsp_transport;udp", false /*overwrite*/); } -static void deinit() override { - curl_global_cleanup(); -} +static void deinit() {} -REGISTER_EXTENSION("HttpCurlExtension", init, deinit); +REGISTER_EXTENSION("OpenCVExtension", init, deinit); ``` If you don't use `REGISTER_EXTENSION`, the registered resources still become available, so make sure to register the extension if you need special initialization. diff --git a/README.md b/README.md index ee85355b53..9a536de20a 100644 --- a/README.md +++ b/README.md @@ -65,9 +65,9 @@ The following table lists the base set of processors. | Extension Set | Processors | |---------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Base** | [AppendHostInfo](PROCESSORS.md#appendhostinfo)
[AttributesToJSON](PROCESSORS.md#attributestojson)
[DefragmentText](PROCESSORS.md#defragmenttext)
[ExecuteProcess](PROCESSORS.md#executeprocess)
[ExtractText](PROCESSORS.md#extracttext)
[FetchFile](PROCESSORS.md#fetchfile)
[GenerateFlowFile](PROCESSORS.md#generateflowfile)
[GetFile](PROCESSORS.md#getfile)
[GetTCP](PROCESSORS.md#gettcp)
[HashContent](PROCESSORS.md#hashcontent)
[ListenSyslog](PROCESSORS.md#listensyslog)
[ListenTCP](PROCESSORS.md#listentcp)
[ListenUDP](PROCESSORS.md#listenudp)
[ListFile](PROCESSORS.md#listfile)
[LogAttribute](PROCESSORS.md#logattribute)
[PutFile](PROCESSORS.md#putfile)
[PutTCP](PROCESSORS.md#puttcp)
[PutUDP](PROCESSORS.md#putudp)
[ReplaceText](PROCESSORS.md#replacetext)
[RetryFlowFile](PROCESSORS.md#retryflowfile)
[RouteOnAttribute](PROCESSORS.md#routeonattribute)
[RouteText](PROCESSORS.md#routetext)
[SplitText](PROCESSORS.md#splittext)
[TailFile](PROCESSORS.md#tailfile)
[UpdateAttribute](PROCESSORS.md#updateattribute) | +| **Base** | [AppendHostInfo](PROCESSORS.md#appendhostinfo)
[AttributesToJSON](PROCESSORS.md#attributestojson)
[DefragmentText](PROCESSORS.md#defragmenttext)
[ExecuteProcess](PROCESSORS.md#executeprocess)
[ExtractText](PROCESSORS.md#extracttext)
[FetchFile](PROCESSORS.md#fetchfile)
[GenerateFlowFile](PROCESSORS.md#generateflowfile)
[GetFile](PROCESSORS.md#getfile)
[GetTCP](PROCESSORS.md#gettcp)
[HashContent](PROCESSORS.md#hashcontent)
[InvokeHTTP](PROCESSORS.md#invokehttp)
[ListenSyslog](PROCESSORS.md#listensyslog)
[ListenTCP](PROCESSORS.md#listentcp)
[ListenUDP](PROCESSORS.md#listenudp)
[ListFile](PROCESSORS.md#listfile)
[LogAttribute](PROCESSORS.md#logattribute)
[PutFile](PROCESSORS.md#putfile)
[PutTCP](PROCESSORS.md#puttcp)
[PutUDP](PROCESSORS.md#putudp)
[ReplaceText](PROCESSORS.md#replacetext)
[RetryFlowFile](PROCESSORS.md#retryflowfile)
[RouteOnAttribute](PROCESSORS.md#routeonattribute)
[RouteText](PROCESSORS.md#routetext)
[SplitText](PROCESSORS.md#splittext)
[TailFile](PROCESSORS.md#tailfile)
[UpdateAttribute](PROCESSORS.md#updateattribute) | -The next table outlines CMAKE flags that correspond with MiNiFi extensions. Extensions that are enabled by default ( such as CURL ), can be disabled with the respective CMAKE flag on the command line. +The next table outlines CMAKE flags that correspond with MiNiFi extensions. Extensions that are enabled by default ( such as RocksDB ), can be disabled with the respective CMAKE flag on the command line. Through JNI extensions you can run NiFi processors using NARs. The JNI extension set allows you to run these Java processors. MiNiFi C++ will favor C++ implementations over Java implements. In the case where a processor is implemented in either language, the one in C++ will be selected; however, will remain transparent to the consumer. @@ -78,7 +78,6 @@ Through JNI extensions you can run NiFi processors using NARs. The JNI extension | AWS | [AWSCredentialsService](CONTROLLERS.md#awscredentialsservice)
[PutS3Object](PROCESSORS.md#puts3object)
[DeleteS3Object](PROCESSORS.md#deletes3object)
[FetchS3Object](PROCESSORS.md#fetchs3object)
[ListS3](PROCESSORS.md#lists3) | -DENABLE_AWS=ON | | Azure | [AzureStorageCredentialsService](CONTROLLERS.md#azurestoragecredentialsservice)
[PutAzureBlobStorage](PROCESSORS.md#putazureblobstorage)
[DeleteAzureBlobStorage](PROCESSORS.md#deleteazureblobstorage)
[FetchAzureBlobStorage](PROCESSORS.md#fetchazureblobstorage)
[ListAzureBlobStorage](PROCESSORS.md#listazureblobstorage)
[PutAzureDataLakeStorage](PROCESSORS.md#putazuredatalakestorage)
[DeleteAzureDataLakeStorage](PROCESSORS.md#deleteazuredatalakestorage)
[FetchAzureDataLakeStorage](PROCESSORS.md#fetchazuredatalakestorage)
[ListAzureDataLakeStorage](PROCESSORS.md#listazuredatalakestorage) | -DENABLE_AZURE=ON | | CivetWeb | [ListenHTTP](PROCESSORS.md#listenhttp) | -DENABLE_CIVET=ON | -| CURL | [InvokeHTTP](PROCESSORS.md#invokehttp) | -DENABLE_CURL=ON | | Elasticsearch | [ElasticsearchCredentialsControllerService](CONTROLLERS.md#elasticsearchcredentialscontrollerservice)
[PostElasticsearch](PROCESSORS.md#postelasticsearch) | -DENABLE_ELASTICSEARCH=ON | | GPS (Linux and macOS) | [GetGPS](PROCESSORS.md#getgps) | -DENABLE_GPS=ON | | Google Cloud Platform | [DeleteGCSObject](PROCESSORS.md#deletegcsobject)
[FetchGCSObject](PROCESSORS.md#fetchgcsobject)
[GCPCredentialsControllerService](CONTROLLERS.md#gcpcredentialscontrollerservice)
[ListGCSBucket](PROCESSORS.md#listgcsbucket)
[PutGCSObject](PROCESSORS.md#putgcsobject) | -DENABLE_GCP=ON | @@ -144,18 +143,18 @@ versions of OpenSSL, cURL, or zlib are used: and rebuild. -#### Libraries / Development Headers -* libcurl-openssl (If not available or desired, NSS will be used) -* libuuid and uuid-dev -* openssl +#### System Libraries / Development Headers Required * Python 3 and development headers -- Required if Python support is enabled * libgps-dev -- Required if building libGPS support -* Zlib headers * perl -- Required for OpenSSL configuration * NASM -- Required for OpenSSL only on Windows **NOTE:** On Windows if Strawberry Perl is used the `${StrawberryPerlRoot}\c\bin` directory should not be part of the PATH environment variable as the patch executable in this directory interferes with git's patch executable. Alternatively [scoop](https://scoop.sh/) package manager can also be used to install Strawberry Perl using the command `scoop install perl` that does not pollute the PATH variable. Also on Windows CMake's CPack is used for MSI generation, building WIX files and calling WIX toolset tools to create an MSI. If Chocolatey package manager is used its CPack can conflict with CMake, so make sure that CMake's CPack is found in the %PATH% before that. +#### Bundled Thirdparty Dependencies + +The [NOTICE](NOTICE) file lists all the bundled thirdparty dependencies that are built and linked statically to MiNiFi or one of its extensions. The licenses of these projects can be found in the [LICENSE](LICENSE) file. + #### CentOS 7 Additional environmental preparations are required for CentOS 7 support. Before @@ -181,13 +180,7 @@ On all distributions please use -DUSE_SHARED_LIBS=OFF to statically link zlib, l ### To run -#### Libraries -* libuuid -* librocksdb (built and statically linked) -* libcurl-openssl (If not available or desired, NSS will be used) -* libssl and libcrypto from openssl (built and statically linked) -* libarchive (built and statically linked) -* librdkafka (built and statically linked) +#### System Libraries Required * Python 3 -- Required if Python support is enabled * libusb -- Optional, unless USB Camera support is enabled * libpng -- Optional, unless USB Camera support is enabled @@ -351,7 +344,6 @@ This will set up a virtual environment in the bootstrap folder, and guide you th Select MiNiFi C++ Features to toggle. **************************************** A. Persistent Repositories .....Enabled - B. libcurl features ............Enabled C. libarchive features .........Enabled D. Python Scripting support ....Enabled E. Expression Language support .Enabled diff --git a/Windows.md b/Windows.md index a526051bff..bea1523df3 100644 --- a/Windows.md +++ b/Windows.md @@ -135,7 +135,7 @@ A basic working CMake configuration can be inferred from the `win_build_vs.bat`. ``` mkdir build cd build -cmake -G "Visual Studio 17 2022" -A x64 -DMINIFI_INCLUDE_VC_REDIST_MERGE_MODULES=OFF -DTEST_CUSTOM_WEL_PROVIDER=OFF -DENABLE_SQL=OFF -DMINIFI_USE_REAL_ODBC_TEST_DRIVER=OFF -DCMAKE_BUILD_TYPE_INIT=Release -DCMAKE_BUILD_TYPE=Release -DWIN32=WIN32 -DENABLE_LIBRDKAFKA=OFF -DENABLE_JNI=OFF -DMINIFI_OPENSSL=ON -DENABLE_COAP=OFF -DENABLE_AWS=OFF -DENABLE_PDH= -DENABLE_AZURE=OFF -DENABLE_SFTP=OFF -DENABLE_SPLUNK= -DENABLE_GCP= -DENABLE_NANOFI=OFF -DENABLE_OPENCV=OFF -DENABLE_PROMETHEUS=OFF -DENABLE_ELASTICSEARCH= -DUSE_SHARED_LIBS=OFF -DENABLE_CONTROLLER=ON -DENABLE_BUSTACHE=OFF -DENABLE_COAP=OFF -DENABLE_ENCRYPT_CONFIG=OFF -DENABLE_GPS=OFF -DENABLE_LUA_SCRIPTING=OFF -DENABLE_MQTT=OFF -DENABLE_OPC=OFF -DENABLE_OPENWSMAN=OFF -DENABLE_OPS=OFF -DENABLE_PCAP=OFF -DENABLE_PYTHON_SCRIPTING= -DENABLE_SENSORS=OFF -DENABLE_USB_CAMERA=OFF -DBUILD_ROCKSDB=ON -DUSE_SYSTEM_UUID=OFF -DENABLE_LIBARCHIVE=ON -DENABLE_WEL=ON -DMINIFI_FAIL_ON_WARNINGS=OFF -DSKIP_TESTS=OFF .. +cmake -G "Visual Studio 17 2022" -A x64 -DMINIFI_INCLUDE_VC_REDIST_MERGE_MODULES=OFF -DTEST_CUSTOM_WEL_PROVIDER=OFF -DENABLE_SQL=OFF -DMINIFI_USE_REAL_ODBC_TEST_DRIVER=OFF -DCMAKE_BUILD_TYPE_INIT=Release -DCMAKE_BUILD_TYPE=Release -DWIN32=WIN32 -DENABLE_LIBRDKAFKA=OFF -DENABLE_JNI=OFF -DENABLE_COAP=OFF -DENABLE_AWS=OFF -DENABLE_PDH= -DENABLE_AZURE=OFF -DENABLE_SFTP=OFF -DENABLE_SPLUNK= -DENABLE_GCP= -DENABLE_NANOFI=OFF -DENABLE_OPENCV=OFF -DENABLE_PROMETHEUS=OFF -DENABLE_ELASTICSEARCH= -DUSE_SHARED_LIBS=OFF -DENABLE_CONTROLLER=ON -DENABLE_BUSTACHE=OFF -DENABLE_COAP=OFF -DENABLE_ENCRYPT_CONFIG=OFF -DENABLE_GPS=OFF -DENABLE_LUA_SCRIPTING=OFF -DENABLE_MQTT=OFF -DENABLE_OPC=OFF -DENABLE_OPENWSMAN=OFF -DENABLE_OPS=OFF -DENABLE_PCAP=OFF -DENABLE_PYTHON_SCRIPTING= -DENABLE_SENSORS=OFF -DENABLE_USB_CAMERA=OFF -DBUILD_ROCKSDB=ON -DUSE_SYSTEM_UUID=OFF -DENABLE_LIBARCHIVE=ON -DENABLE_WEL=ON -DMINIFI_FAIL_ON_WARNINGS=OFF -DSKIP_TESTS=OFF .. msbuild /m nifi-minifi-cpp.sln /property:Configuration=Release /property:Platform=x64 copy minifi_main\Release\minifi.exe minifi_main\ cpack diff --git a/aptitude.sh b/aptitude.sh index 0206fe36ac..b70111a933 100644 --- a/aptitude.sh +++ b/aptitude.sh @@ -49,11 +49,10 @@ bootstrap_compiler() { sudo apt-get -y install $compiler_pkgs } build_deps(){ - COMMAND="sudo apt-get -y install zlib1g-dev libssl-dev uuid uuid-dev" + COMMAND="sudo apt-get -y install zlib1g-dev libssl-dev uuid uuid-dev perl libbz2-dev libcurl4-openssl-dev" export DEBIAN_FRONTEND=noninteractive INSTALLED=() - INSTALLED+=("libbz2-dev") sudo apt-get -y update for option in "${OPTIONS[@]}" ; do option_value="${!option}" @@ -65,9 +64,7 @@ build_deps(){ VALUE=${cmake_opt#*:} if [ "$KEY" = "$option" ]; then FOUND_VALUE="$VALUE" - if [ "$FOUND_VALUE" = "libcurl" ]; then - INSTALLED+=("libcurl4-openssl-dev") - elif [ "$FOUND_VALUE" = "libpcap" ]; then + if [ "$FOUND_VALUE" = "libpcap" ]; then INSTALLED+=("libpcap-dev") elif [ "$FOUND_VALUE" = "openssl" ]; then INSTALLED+=("openssl") @@ -96,8 +93,6 @@ build_deps(){ INSTALLED+=("libgps-dev") elif [ "$FOUND_VALUE" = "libarchive" ]; then INSTALLED+=("liblzma-dev") - elif [ "$FOUND_VALUE" = "opensslbuild" ]; then - INSTALLED+=("perl") fi fi done diff --git a/arch.sh b/arch.sh index 4e94f5cada..413dc5ba2a 100644 --- a/arch.sh +++ b/arch.sh @@ -31,10 +31,9 @@ bootstrap_compiler(){ sudo pacman -S --noconfirm gcc } build_deps(){ - COMMAND="sudo pacman -S --noconfirm --needed cmake zlib openssl util-linux make patch which pkgconf" + COMMAND="sudo pacman -S --noconfirm --needed cmake zlib openssl util-linux make patch which pkgconf perl bzip2 curl" INSTALLED=() - INSTALLED+=("bzip2") for option in "${OPTIONS[@]}" ; do option_value="${!option}" if [ "$option_value" = "${TRUE}" ]; then @@ -45,9 +44,7 @@ build_deps(){ VALUE=${cmake_opt#*:} if [ "$KEY" = "$option" ]; then FOUND_VALUE="$VALUE" - if [ "$FOUND_VALUE" = "libcurl" ]; then - INSTALLED+=("curl") - elif [ "$FOUND_VALUE" = "libpcap" ]; then + if [ "$FOUND_VALUE" = "libpcap" ]; then INSTALLED+=("libpcap") elif [ "$FOUND_VALUE" = "openssl" ]; then INSTALLED+=("openssl") @@ -74,8 +71,6 @@ build_deps(){ INSTALLED+=("gpsd") elif [ "$FOUND_VALUE" = "libarchive" ]; then INSTALLED+=("libarchive") - elif [ "$FOUND_VALUE" = "opensslbuild" ]; then - INSTALLED+=("perl") fi fi done diff --git a/bootstrap.sh b/bootstrap.sh index 49f908ed25..a376be5271 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -259,8 +259,6 @@ add_cmake_option DEBUG_SYMBOLS ${FALSE} add_cmake_option BUILD_ROCKSDB ${TRUE} ## uses the source from the third party directory add_option ROCKSDB_ENABLED ${TRUE} "ENABLE_ROCKSDB" -## need libcurl installed -add_option HTTP_CURL_ENABLED ${TRUE} "ENABLE_CURL" # third party directory add_option LIBARCHIVE_ENABLED ${TRUE} "ENABLE_LIBARCHIVE" @@ -325,23 +323,17 @@ fi add_option NANOFI_ENABLED ${FALSE} "ENABLE_NANOFI" add_option SPLUNK_ENABLED ${TRUE} "ENABLE_SPLUNK" -set_dependency SPLUNK_ENABLED HTTP_CURL_ENABLED add_option GCP_ENABLED ${TRUE} "ENABLE_GCP" add_option ELASTIC_ENABLED ${TRUE} "ENABLE_ELASTICSEARCH" -set_dependency ELASTIC_ENABLED HTTP_CURL_ENABLED add_option GRAFANA_LOKI_ENABLED ${FALSE} "ENABLE_GRAFANA_LOKI" -set_dependency GRAFANA_LOKI_ENABLED HTTP_CURL_ENABLED add_option PROCFS_ENABLED ${TRUE} "ENABLE_PROCFS" add_option PROMETHEUS_ENABLED ${TRUE} "ENABLE_PROMETHEUS" -add_option OPENSSL_ENABLED ${TRUE} "MINIFI_OPENSSL" -add_dependency OPENSSL_ENABLED "opensslbuild" - USE_SHARED_LIBS=${TRUE} ASAN_ENABLED=${FALSE} MINIFI_FAIL_ON_WARNINGS=${FALSE} diff --git a/bootstrap/system_dependency.py b/bootstrap/system_dependency.py index f4c273f3b9..3ef31dc700 100644 --- a/bootstrap/system_dependency.py +++ b/bootstrap/system_dependency.py @@ -23,7 +23,7 @@ def _create_system_dependencies(minifi_options: MinifiOptions) -> Dict[str, Set[str]]: - system_dependencies = {'patch': {'patch'}, 'make': {'make'}} + system_dependencies = {'patch': {'patch'}, 'make': {'make'}, 'perl': {'perl'}} if minifi_options.is_enabled("ENABLE_EXPRESSION_LANGUAGE"): system_dependencies['bison'] = {'bison'} system_dependencies['flex'] = {'flex'} @@ -43,8 +43,6 @@ def _create_system_dependencies(minifi_options: MinifiOptions) -> Dict[str, Set[ system_dependencies['libtool'] = {'libtool'} if minifi_options.is_enabled("ENABLE_PYTHON_SCRIPTING"): system_dependencies['python'] = {'python'} - if minifi_options.is_enabled("MINIFI_OPENSSL"): - system_dependencies['perl'] = {'perl'} if minifi_options.is_enabled("ENABLE_JNI"): system_dependencies['jni'] = {'openjdk'} system_dependencies['maven'] = {'maven'} diff --git a/bstrp_functions.sh b/bstrp_functions.sh index 6a1857182d..03d4c8a1ed 100755 --- a/bstrp_functions.sh +++ b/bstrp_functions.sh @@ -360,7 +360,6 @@ show_supported_features() { echo " Select MiNiFi C++ Features to toggle." echo "****************************************" echo "A. Persistent Repositories .....$(print_feature_status ROCKSDB_ENABLED)" - echo "B. libcurl features ............$(print_feature_status HTTP_CURL_ENABLED)" echo "C. libarchive features .........$(print_feature_status LIBARCHIVE_ENABLED)" echo "D. Python Scripting support ....$(print_feature_status PYTHON_SCRIPTING_ENABLED)" echo "E. Expression Language support .$(print_feature_status EXPRESSION_LANGUAGE_ENABLED)" @@ -400,7 +399,6 @@ show_supported_features() { echo "5. Build Profile ...............$(print_multi_option_status BUILD_PROFILE)" echo "6. Create ASAN build ...........$(print_feature_status ASAN_ENABLED)" echo "7. Treat warnings as errors.....$(print_feature_status MINIFI_FAIL_ON_WARNINGS)" - echo "8. Enable OpenSSL...............$(print_feature_status OPENSSL_ENABLED)" echo "P. Continue with these options" if [ "$GUIDED_INSTALL" = "${TRUE}" ]; then echo "R. Return to Main Menu" @@ -418,7 +416,6 @@ read_feature_options(){ choice=$(echo "${choice}" | tr '[:upper:]' '[:lower:]') case $choice in a) ToggleFeature ROCKSDB_ENABLED ;; - b) ToggleFeature HTTP_CURL_ENABLED ;; c) ToggleFeature LIBARCHIVE_ENABLED ;; d) ToggleFeature PYTHON_SCRIPTING_ENABLED ;; e) ToggleFeature EXPRESSION_LANGUAGE_ENABLED ;; @@ -453,7 +450,6 @@ read_feature_options(){ 5) ToggleMultiOption BUILD_PROFILE;; 6) ToggleFeature ASAN_ENABLED;; 7) ToggleFeature MINIFI_FAIL_ON_WARNINGS;; - 8) ToggleFeature OPENSSL_ENABLED;; p) export FEATURES_SELECTED="true" ;; r) if [ "$GUIDED_INSTALL" = "${TRUE}" ]; then export MENU="main" diff --git a/centos.sh b/centos.sh index ca17f62ffd..8cecc5a9cf 100644 --- a/centos.sh +++ b/centos.sh @@ -70,7 +70,7 @@ bootstrap_compiler() { } build_deps() { - COMMAND="install_pkgs libuuid libuuid-devel libtool patch epel-release" + COMMAND="install_pkgs libuuid libuuid-devel libtool patch epel-release perl" INSTALLED=() for option in "${OPTIONS[@]}" ; do option_value="${!option}" @@ -107,8 +107,6 @@ build_deps() { INSTALLED+=("bzip2-devel") elif [ "$FOUND_VALUE" = "libssh2" ]; then INSTALLED+=("libssh2-devel") - elif [ "$FOUND_VALUE" = "opensslbuild" ]; then - INSTALLED+=("perl") fi fi done diff --git a/cmake/BuildTests.cmake b/cmake/BuildTests.cmake index d299d67fbb..c0b1cb14a9 100644 --- a/cmake/BuildTests.cmake +++ b/cmake/BuildTests.cmake @@ -15,7 +15,8 @@ # specific language governing permissions and limitations # under the License. -include(Catch2) +include(GetCatch2) +get_catch2() ### test functions MACRO(GETSOURCEFILES result curdir) @@ -29,8 +30,6 @@ MACRO(GETSOURCEFILES result curdir) SET(${result} ${dirlist}) ENDMACRO() -set(NANOFI_TEST_DIR "${CMAKE_SOURCE_DIR}/nanofi/tests/") - function(copyTestResources SOURCE_DIR DEST_DIR) file(GLOB_RECURSE RESOURCE_FILES "${SOURCE_DIR}/*") foreach(RESOURCE_FILE ${RESOURCE_FILES}) @@ -77,102 +76,30 @@ function(createTests testName) appendIncludes("${testName}") target_link_libraries(${testName} ${CMAKE_DL_LIBS}) - target_wholearchive_library(${testName} ${TEST_BASE_LIB}) - target_link_libraries(${testName} core-minifi yaml-cpp spdlog Threads::Threads) + target_wholearchive_library(${testName} libminifi-unittest) + target_link_libraries(${testName} core-minifi yaml-cpp spdlog::spdlog Threads::Threads Catch2::Catch2WithMain) + target_include_directories(${testName} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test/libtest/") target_compile_definitions(${testName} PRIVATE LOAD_EXTENSIONS) set_target_properties(${testName} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") endfunction() -enable_testing(test) +function(createIntegrationTests testName) + message(DEBUG "-- Adding integration test: ${testName}") + appendIncludes("${testName}") -SET(TEST_BASE_LIB test_base) -set(TEST_BASE_SOURCES "TestBase.cpp" "StatefulProcessor.cpp" "WriteToFlowFileTestProcessor.cpp" "ReadFromFlowFileTestProcessor.cpp" "DummyProcessor.cpp") -list(TRANSFORM TEST_BASE_SOURCES PREPEND "${TEST_DIR}/") -add_minifi_library(${TEST_BASE_LIB} STATIC ${TEST_BASE_SOURCES}) -target_link_libraries(${TEST_BASE_LIB} core-minifi) -target_include_directories(${TEST_BASE_LIB} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include/") -if(WIN32) - target_include_directories(${TEST_BASE_LIB} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/opsys/win") -else() - target_include_directories(${TEST_BASE_LIB} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/opsys/posix") -endif() + target_link_libraries(${testName} ${CMAKE_DL_LIBS}) + target_wholearchive_library(${testName} libminifi-integrationtest) + target_link_libraries(${testName} core-minifi yaml-cpp spdlog::spdlog Threads::Threads Catch2::Catch2WithMain) + target_include_directories(${testName} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test/libtest/") + target_compile_definitions(${testName} PRIVATE LOAD_EXTENSIONS) + set_target_properties(${testName} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") +endfunction() + +enable_testing() file(COPY ${TEST_DIR}/resources DESTINATION ${CMAKE_BINARY_DIR}) SET(TEST_RESOURCES ${CMAKE_BINARY_DIR}/resources) -GETSOURCEFILES(UNIT_TESTS "${TEST_DIR}/unit/") -GETSOURCEFILES(TLS_UNIT_TESTS "${TEST_DIR}/unit/tls/") -GETSOURCEFILES(NANOFI_UNIT_TESTS "${NANOFI_TEST_DIR}") -GETSOURCEFILES(INTEGRATION_TESTS "${TEST_DIR}/integration/") - -if (NOT WIN32) - list(REMOVE_ITEM UNIT_TESTS WindowsCertStoreLocationTests.cpp) -endif() - -SET(UNIT_TEST_COUNT 0) -FOREACH(testfile ${UNIT_TESTS}) - get_filename_component(testfilename "${testfile}" NAME_WE) - add_minifi_executable("${testfilename}" "${TEST_DIR}/unit/${testfile}") - target_compile_definitions("${testfilename}" PRIVATE TZ_DATA_DIR="${CMAKE_BINARY_DIR}/tzdata") - createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) - MATH(EXPR UNIT_TEST_COUNT "${UNIT_TEST_COUNT}+1") - add_test(NAME "${testfilename}" COMMAND "${testfilename}") -ENDFOREACH() -message("-- Finished building ${UNIT_TEST_COUNT} unit test file(s)...") - -if (MINIFI_OPENSSL) - SET(UNIT_TEST_COUNT 0) - FOREACH(testfile ${TLS_UNIT_TESTS}) - get_filename_component(testfilename "${testfile}" NAME_WE) - add_minifi_executable("${testfilename}" "${TEST_DIR}/unit/tls/${testfile}") - createTests("${testfilename}") - MATH(EXPR UNIT_TEST_COUNT "${UNIT_TEST_COUNT}+1") - add_test(NAME "${testfilename}" COMMAND "${testfilename}" "${TEST_RESOURCES}/") - ENDFOREACH() - message("-- Finished building ${UNIT_TEST_COUNT} TLS unit test file(s)...") -endif() - -if(NOT WIN32 AND ENABLE_NANOFI) - SET(UNIT_TEST_COUNT 0) - FOREACH(testfile ${NANOFI_UNIT_TESTS}) - get_filename_component(testfilename "${testfile}" NAME_WE) - add_minifi_executable("${testfilename}" "${NANOFI_TEST_DIR}/${testfile}") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/nanofi/include") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/processors/") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test") - appendIncludes("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain Threads::Threads) - - target_wholearchive_library(${testfilename} nanofi) - - createTests(${testfilename}) - - target_link_libraries(${testfilename} minifi-standard-processors) - - MATH(EXPR UNIT_TEST_COUNT "${UNIT_TEST_COUNT}+1") - add_test(NAME "${testfilename}" COMMAND "${testfilename}") - ENDFOREACH() - message("-- Finished building ${UNIT_TEST_COUNT} NanoFi unit test file(s)...") -endif() - -SET(INT_TEST_COUNT 0) -FOREACH(testfile ${INTEGRATION_TESTS}) - get_filename_component(testfilename "${testfile}" NAME_WE) - add_minifi_executable("${testfilename}" "${TEST_DIR}/integration/${testfile}") - createTests("${testfilename}") - MATH(EXPR INT_TEST_COUNT "${INT_TEST_COUNT}+1") -ENDFOREACH() - -target_link_libraries(OnScheduleErrorHandlingTests minifi-test-processors) -target_wholearchive_library(StateTransactionalityTests minifi-standard-processors) - -add_test(NAME OnScheduleErrorHandlingTests COMMAND OnScheduleErrorHandlingTests "${TEST_RESOURCES}/TestOnScheduleRetry.yml" "${TEST_RESOURCES}/") -add_test(NAME StateTransactionalityTests COMMAND StateTransactionalityTests "${TEST_RESOURCES}/TestStateTransactionality.yml") - -message("-- Finished building ${INT_TEST_COUNT} integration test file(s)...") - get_property(extensions GLOBAL PROPERTY EXTENSION-TESTS) foreach(EXTENSION ${extensions}) add_subdirectory(${EXTENSION}) diff --git a/cmake/BundledLibArchive.cmake b/cmake/BundledLibArchive.cmake index 6ab55de11c..12a4bd3df9 100644 --- a/cmake/BundledLibArchive.cmake +++ b/cmake/BundledLibArchive.cmake @@ -47,13 +47,8 @@ function(use_bundled_libarchive SOURCE_DIR BINARY_DIR) -DENABLE_ACL=ON -DENABLE_ICONV=OFF -DENABLE_TEST=OFF - -DENABLE_WERROR=OFF) - - if (MINIFI_OPENSSL) - list(APPEND LIBARCHIVE_CMAKE_ARGS -DENABLE_OPENSSL=ON) - else() - list(APPEND LIBARCHIVE_CMAKE_ARGS -DENABLE_OPENSSL=OFF) - endif() + -DENABLE_WERROR=OFF + -DENABLE_OPENSSL=ON) if (NOT ENABLE_LZMA) list(APPEND LIBARCHIVE_CMAKE_ARGS -DENABLE_LZMA=OFF) @@ -83,10 +78,7 @@ function(use_bundled_libarchive SOURCE_DIR BINARY_DIR) ) # Set dependencies - add_dependencies(libarchive-external ZLIB::ZLIB) - if (MINIFI_OPENSSL) - add_dependencies(libarchive-external OpenSSL::Crypto) - endif() + add_dependencies(libarchive-external ZLIB::ZLIB OpenSSL::Crypto) if (ENABLE_LZMA) add_dependencies(libarchive-external LibLZMA::LibLZMA) endif() @@ -104,10 +96,7 @@ function(use_bundled_libarchive SOURCE_DIR BINARY_DIR) add_library(LibArchive::LibArchive STATIC IMPORTED) set_target_properties(LibArchive::LibArchive PROPERTIES IMPORTED_LOCATION "${LIBARCHIVE_LIBRARY}") add_dependencies(LibArchive::LibArchive libarchive-external) - set_property(TARGET LibArchive::LibArchive APPEND PROPERTY INTERFACE_LINK_LIBRARIES ZLIB::ZLIB) - if (MINIFI_OPENSSL) - set_property(TARGET LibArchive::LibArchive APPEND PROPERTY INTERFACE_LINK_LIBRARIES OpenSSL::Crypto) - endif() + set_property(TARGET LibArchive::LibArchive APPEND PROPERTY INTERFACE_LINK_LIBRARIES ZLIB::ZLIB OpenSSL::Crypto) if (ENABLE_LZMA) set_property(TARGET LibArchive::LibArchive APPEND PROPERTY INTERFACE_LINK_LIBRARIES LibLZMA::LibLZMA) endif() diff --git a/cmake/BundledLibcURL.cmake b/cmake/BundledLibcURL.cmake index 23735457db..bba5f70ab0 100644 --- a/cmake/BundledLibcURL.cmake +++ b/cmake/BundledLibcURL.cmake @@ -50,12 +50,8 @@ function(use_bundled_curl SOURCE_DIR BINARY_DIR) -DCURL_USE_LIBSSH2=OFF -DUSE_LIBIDN2=OFF -DCURL_USE_LIBPSL=OFF + -DCURL_USE_OPENSSL=ON ) - if (NOT MINIFI_OPENSSL) - list(APPEND CURL_CMAKE_ARGS -DCURL_USE_OPENSSL=OFF) - else() - list(APPEND CURL_CMAKE_ARGS -DCURL_USE_OPENSSL=ON) - endif() append_third_party_passthrough_args(CURL_CMAKE_ARGS "${CURL_CMAKE_ARGS}") @@ -73,10 +69,7 @@ function(use_bundled_curl SOURCE_DIR BINARY_DIR) ) # Set dependencies - add_dependencies(curl-external ZLIB::ZLIB) - if (MINIFI_OPENSSL) - add_dependencies(curl-external OpenSSL::SSL OpenSSL::Crypto) - endif() + add_dependencies(curl-external ZLIB::ZLIB OpenSSL::SSL OpenSSL::Crypto) # Set variables set(CURL_FOUND "YES" CACHE STRING "" FORCE) @@ -96,13 +89,10 @@ function(use_bundled_curl SOURCE_DIR BINARY_DIR) set_target_properties(CURL::libcurl PROPERTIES IMPORTED_LOCATION "${CURL_LIBRARY}") add_dependencies(CURL::libcurl curl-external) target_include_directories(CURL::libcurl INTERFACE ${CURL_INCLUDE_DIRS}) - target_link_libraries(CURL::libcurl INTERFACE ZLIB::ZLIB Threads::Threads) + target_link_libraries(CURL::libcurl INTERFACE ZLIB::ZLIB Threads::Threads OpenSSL::SSL OpenSSL::Crypto) if (APPLE) target_link_libraries(CURL::libcurl INTERFACE "-framework CoreFoundation") target_link_libraries(CURL::libcurl INTERFACE "-framework SystemConfiguration") target_link_libraries(CURL::libcurl INTERFACE "-framework CoreServices") endif() - if (MINIFI_OPENSSL) - target_link_libraries(CURL::libcurl INTERFACE OpenSSL::SSL OpenSSL::Crypto) - endif() endfunction(use_bundled_curl SOURCE_DIR BINARY_DIR) diff --git a/cmake/BundledOpenSSL.cmake b/cmake/BundledOpenSSL.cmake index 7a429a9f54..aa26e1616b 100644 --- a/cmake/BundledOpenSSL.cmake +++ b/cmake/BundledOpenSSL.cmake @@ -134,4 +134,5 @@ function(use_openssl SOURCE_DIR BINARY_DIR) set_property(TARGET OpenSSL::Crypto APPEND PROPERTY INTERFACE_LINK_LIBRARIES crypt32.lib ) set_property(TARGET OpenSSL::SSL APPEND PROPERTY INTERFACE_LINK_LIBRARIES crypt32.lib) endif() + endfunction(use_openssl) diff --git a/cmake/Bustache.cmake b/cmake/Bustache.cmake index 1160287752..6c037af6c9 100644 --- a/cmake/Bustache.cmake +++ b/cmake/Bustache.cmake @@ -17,7 +17,9 @@ # under the License. # include(FetchContent) -include(fmt) +include(GetFmt) +get_fmt() + set(BUSTACHE_USE_FMT ON CACHE STRING "" FORCE) FetchContent_Declare(Bustache GIT_REPOSITORY https://github.com/jamboree/bustache.git diff --git a/cmake/Extensions.cmake b/cmake/Extensions.cmake index 41d0cc9a9c..c497ec525a 100644 --- a/cmake/Extensions.cmake +++ b/cmake/Extensions.cmake @@ -105,12 +105,6 @@ macro(register_extension_test extension-dir) endif() endmacro() -function(registerTest dirName) - if(NOT SKIP_TESTS) - add_subdirectory(${dirName}) - endif() -endfunction(registerTest) - macro(register_extension_linter target-name) get_property(extensions GLOBAL PROPERTY EXTENSION-LINTERS) set_property(GLOBAL APPEND PROPERTY EXTENSION-LINTERS "${target-name}") diff --git a/cmake/GetCatch2.cmake b/cmake/GetCatch2.cmake new file mode 100644 index 0000000000..e0745ad246 --- /dev/null +++ b/cmake/GetCatch2.cmake @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_catch2) + if(MINIFI_CATCH2_SOURCE STREQUAL "CONAN") + message("Using Conan Packager to manage installing prebuilt Catch2 external lib") + include(${CMAKE_BINARY_DIR}/Catch2Config.cmake) + message("Catch2_INCLUDE_DIRS = ${Catch2_INCLUDE_DIRS}") + set(CATCH2_INCLUDE_DIRS "${Catch2_INCLUDE_DIRS}" CACHE STRING "" FORCE) + elseif(MINIFI_CATCH2_SOURCE STREQUAL "BUILD") + message("Using CMAKE's ExternalProject_Add to manage source building Catch2 external lib") + include(Catch2) + endif() +endfunction(get_catch2) diff --git a/cmake/GetCivetWeb.cmake b/cmake/GetCivetWeb.cmake new file mode 100644 index 0000000000..419c5d834c --- /dev/null +++ b/cmake/GetCivetWeb.cmake @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_civetweb) + if(MINIFI_CIVETWEB_SOURCE STREQUAL "CONAN") + message("Using Conan Packager to manage installing prebuilt CivetWeb external lib") + include(${CMAKE_BINARY_DIR}/civetweb-config.cmake) + elseif(MINIFI_CIVETWEB_SOURCE STREQUAL "BUILD") + message("Using CMAKE's ExternalProject_Add to manage source building CivetWeb external lib") + include(CivetWeb) + endif() +endfunction(get_civetweb) diff --git a/cmake/GetFmt.cmake b/cmake/GetFmt.cmake new file mode 100644 index 0000000000..bb9f7da346 --- /dev/null +++ b/cmake/GetFmt.cmake @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_fmt) + if(MINIFI_FMT_SOURCE STREQUAL "CONAN") + message("Using Conan Packager to manage installing prebuilt Fmt external lib") + include(${CMAKE_BINARY_DIR}/fmt-config.cmake) + elseif(MINIFI_FMT_SOURCE STREQUAL "BUILD") + message("Using CMAKE's ExternalProject_Add to manage source building Fmt external lib") + include(fmt) + endif() +endfunction(get_fmt) diff --git a/cmake/GetLibCURL.cmake b/cmake/GetLibCURL.cmake new file mode 100644 index 0000000000..6fb39c00c1 --- /dev/null +++ b/cmake/GetLibCURL.cmake @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_curl SOURCE_DIR BINARY_DIR) + if(MINIFI_LIBCURL_SOURCE STREQUAL "CONAN") + message("Using Conan Packager to manage installing prebuilt libcurl external lib") + include(${CMAKE_BINARY_DIR}/CURLConfig.cmake) + elseif(MINIFI_LIBCURL_SOURCE STREQUAL "BUILD") + message("Using CMAKE's ExternalProject_Add to manage source building libcurl external lib") + include(BundledLibcURL) + use_bundled_curl(${SOURCE_DIR} ${BINARY_DIR}) + endif() +endfunction(get_curl SOURCE_DIR BINARY_DIR) diff --git a/cmake/GetLibXml2.cmake b/cmake/GetLibXml2.cmake new file mode 100644 index 0000000000..6381e6dccd --- /dev/null +++ b/cmake/GetLibXml2.cmake @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_libxml2 SOURCE_DIR BINARY_DIR) + if(MINIFI_LIBXML2_SOURCE STREQUAL "CONAN") + message("Using Conan Packager to manage installing prebuilt LibXml2 external lib") + include(${CMAKE_BINARY_DIR}/libxml2-config.cmake) + elseif(MINIFI_LIBXML2_SOURCE STREQUAL "BUILD") + message("Using CMAKE's ExternalProject_Add to manage source building LibXml2 external lib") + include(BundledLibXml2) + use_bundled_libxml2(${SOURCE_DIR} ${BINARY_DIR}) + endif() +endfunction(get_libxml2) diff --git a/cmake/GetOpenSSL.cmake b/cmake/GetOpenSSL.cmake new file mode 100644 index 0000000000..9c8b2542d2 --- /dev/null +++ b/cmake/GetOpenSSL.cmake @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_openssl SOURCE_DIR BINARY_DIR) + if(MINIFI_OPENSSL_SOURCE STREQUAL "CONAN") + message("Using Conan Packager to manage installing prebuilt OpenSSL external lib") + include(${CMAKE_BINARY_DIR}/OpenSSLConfig.cmake) + elseif(MINIFI_OPENSSL_SOURCE STREQUAL "BUILD") + message("Using CMAKE's ExternalProject_Add to manage source building OpenSSL external lib") + include(BundledOpenSSL) + use_openssl(${SOURCE_DIR} ${BINARY_DIR}) + endif() +endfunction(get_openssl) diff --git a/cmake/GetSpdlog.cmake b/cmake/GetSpdlog.cmake new file mode 100644 index 0000000000..a30d1215de --- /dev/null +++ b/cmake/GetSpdlog.cmake @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_spdlog) + include(GetFmt) + get_fmt() + + if(MINIFI_SPDLOG_SOURCE STREQUAL "CONAN") + message("Using Conan Packager to manage installing prebuilt Spdlog external lib") + include(${CMAKE_BINARY_DIR}/spdlog-config.cmake) + elseif(MINIFI_SPDLOG_SOURCE STREQUAL "BUILD") + message("Using CMAKE's ExternalProject_Add to manage source building Spdlog external lib") + include(Spdlog) + endif() +endfunction(get_spdlog) diff --git a/cmake/GetZLIB.cmake b/cmake/GetZLIB.cmake new file mode 100644 index 0000000000..aae8d59e80 --- /dev/null +++ b/cmake/GetZLIB.cmake @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function(get_zlib SOURCE_DIR BINARY_DIR) + if(MINIFI_ZLIB_SOURCE STREQUAL "CONAN") + message("Using Conan Packager to manage installing prebuilt zlib external lib") + include(${CMAKE_BINARY_DIR}/ZLIBConfig.cmake) + elseif(MINIFI_ZLIB_SOURCE STREQUAL "BUILD") + message("Using CMAKE's ExternalProject_Add to manage source building zlib external lib") + include(BundledZLIB) + use_bundled_zlib(${SOURCE_DIR} ${BINARY_DIR}) + endif() +endfunction(get_zlib) diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake index de2b264ed5..47f2047e3c 100644 --- a/cmake/MiNiFiOptions.cmake +++ b/cmake/MiNiFiOptions.cmake @@ -23,6 +23,13 @@ function(add_minifi_option OPTION_NAME OPTION_DESCRIPTION OPTION_VALUE) set(MINIFI_OPTIONS ${MINIFI_OPTIONS} PARENT_SCOPE) endfunction() +function(add_minifi_multi_option OPTION_NAME OPTION_DESCRIPTION OPTION_VALUES DEFAULT_VALUE) + set(${OPTION_NAME} ${DEFAULT_VALUE} CACHE STRING ${OPTION_DESCRIPTION}) + set_property(CACHE ${OPTION_NAME} PROPERTY STRINGS ${OPTION_VALUES}) + list(APPEND MINIFI_OPTIONS ${OPTION_NAME}) + set(MINIFI_OPTIONS ${MINIFI_OPTIONS} PARENT_SCOPE) +endfunction() + function(add_minifi_dependent_option OPTION_NAME OPTION_DESCRIPTION OPTION_VALUE DEPENDS FORCE) cmake_dependent_option(${OPTION_NAME} ${OPTION_DESCRIPTION} ${OPTION_VALUE} ${DEPENDS} ${FORCE}) list(APPEND MINIFI_OPTIONS ${OPTION_NAME}) @@ -45,7 +52,6 @@ add_minifi_option(PORTABLE "Instructs the compiler to remove architecture specif add_minifi_option(USE_SHARED_LIBS "Builds using shared libraries" ON) add_minifi_dependent_option(STATIC_BUILD "Attempts to statically link as many dependencies as possible." ON "NOT USE_SHARED_LIBS" OFF) add_minifi_option(LIBC_STATIC "Instructs the build system to statically link libstdc++ and glibc into minifiexe. Experiemental" OFF) -add_minifi_option(MINIFI_OPENSSL "Enables OpenSSL" ON) add_minifi_option(ENABLE_OPS "Enable Operations/zlib Tools" ON) add_minifi_option(ENABLE_JNI "Instructs the build system to enable the JNI extension" OFF) add_minifi_option(ENABLE_OPC "Instructs the build system to enable the OPC extension" ON) @@ -53,7 +59,6 @@ add_minifi_option(ENABLE_NANOFI "Instructs the build system to enable nanofi lib add_minifi_option(BUILD_SHARED_LIBS "Build yaml cpp shared lib" OFF) add_minifi_option(BUILD_ROCKSDB "Instructs the build system to use RocksDB from the third party directory" ON) -add_minifi_option(ENABLE_CURL "Enables libCurl Properties." ON) add_minifi_option(MINIFI_FAIL_ON_WARNINGS "Treat warnings as errors" OFF) add_minifi_option(MINIFI_USE_REAL_ODBC_TEST_DRIVER "Use SQLite ODBC driver in SQL extenstion unit tests instead of a mock database" OFF) @@ -69,15 +74,6 @@ add_minifi_option(AWS_ENABLE_UNITY_BUILD "If enabled, AWS SDK libraries will be add_minifi_dependent_option(MINIFI_ADVANCED_ASAN_BUILD "Uses AddressSanitizer to instrument the code" OFF "NOT WIN32" OFF) add_minifi_dependent_option(MINIFI_ADVANCED_CODE_COVERAGE "Use coverage build options and enable coverage build target" OFF "NOT WIN32" OFF) -# Option: STRICT_GSL_CHECKS -# AUDIT: Enable all checks, including gsl_ExpectsAudit() and gsl_EnsuresAudit() -# ON: Enable all checks, excluding gsl_ExpectsAudit() and gsl_EnsuresAudit() (GSL default) -# DEBUG_ONLY: Like ON in the Debug configuration, OFF in others (MiNiFi C++ default) -# OFF: Throw on contract checking and assertion failures instead of calling std::terminate() -set(STRICT_GSL_CHECKS "DEBUG_ONLY" CACHE STRING "Contract checking and assertion failures call terminate") -list(APPEND STRICT_GSL_CHECKS_Values AUDIT ON DEBUG_ONLY OFF) -set_property(CACHE STRICT_GSL_CHECKS PROPERTY STRINGS ${STRICT_GSL_CHECKS_Values}) - if (WIN32) add_minifi_option(MINIFI_INCLUDE_VC_REDIST_MERGE_MODULES "Include merge modules for Visual C++ Redistributable with the MSI generated by CPack. The resulting MSI is not distributable under Apache 2.0." OFF) add_minifi_option(MINIFI_INCLUDE_VC_REDIST_DLLS "Include Visual C++ Redistributable DLLs with the MSI generated by CPack. The resulting MSI is not distributable under Apache 2.0." OFF) @@ -123,7 +119,7 @@ add_minifi_option(ENABLE_TEST_PROCESSORS "Enables test processors" OFF) add_minifi_option(ENABLE_PROMETHEUS "Enables Prometheus support." ON) add_minifi_option(ENABLE_GRAFANA_LOKI "Enable Grafana Loki support" OFF) add_minifi_option(ENABLE_GRPC_FOR_LOKI "Enable gRPC for Grafana Loki extension" ON) -add_minifi_dependent_option(ENABLE_CONTROLLER "Enables the build of MiNiFi controller binary." ON "ENABLE_CURL" OFF) +add_minifi_option(ENABLE_CONTROLLER "Enables the build of MiNiFi controller binary." ON) set_minifi_cache_variable(CUSTOM_MALLOC OFF "Overwrite malloc implementation.") set_property(CACHE CUSTOM_MALLOC PROPERTY STRINGS "jemalloc" "mimalloc" "rpmalloc" OFF) @@ -132,6 +128,26 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_minifi_option(ENABLE_PROCFS "Enables the procfs extension." ON) endif() +## Each Option provides a list of possible values + +# Option: STRICT_GSL_CHECKS +# AUDIT: Enable all checks, including gsl_ExpectsAudit() and gsl_EnsuresAudit() +# ON: Enable all checks, excluding gsl_ExpectsAudit() and gsl_EnsuresAudit() (GSL default) +# DEBUG_ONLY: Like ON in the Debug configuration, OFF in others (MiNiFi C++ default) +# OFF: Throw on contract checking and assertion failures instead of calling std::terminate() +set(STRICT_GSL_CHECKS "DEBUG_ONLY" CACHE STRING "Contract checking and assertion failures call terminate") +list(APPEND STRICT_GSL_CHECKS_Values AUDIT ON DEBUG_ONLY OFF) +set_property(CACHE STRICT_GSL_CHECKS PROPERTY STRINGS ${STRICT_GSL_CHECKS_Values}) + +add_minifi_option(MINIFI_BUILD_CONAN_PACKAGE "Instructs MiNiFi to install external libs, build & create MiNiFi conan package" OFF) +add_minifi_multi_option(MINIFI_LIBCURL_SOURCE "Retrieves LibCURL from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_OPENSSL_SOURCE "Retrieves OpenSSL from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_ZLIB_SOURCE "Retrieves ZLib from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_CIVETWEB_SOURCE "Retrieves CivetWeb from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_LIBXML2_SOURCE "Retrieves LibXml2 from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_CATCH2_SOURCE "Retrieves Catch2 from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_FMT_SOURCE "Retrieves Fmt from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_SPDLOG_SOURCE "Retrieves Spdlog from provided source" "BUILD;SYSTEM;CONAN" "BUILD") # Docker options diff --git a/cmake/PahoMqttC.cmake b/cmake/PahoMqttC.cmake index bc744c5caf..5ae355fa8a 100644 --- a/cmake/PahoMqttC.cmake +++ b/cmake/PahoMqttC.cmake @@ -21,12 +21,7 @@ include(FetchContent) set(PAHO_BUILD_STATIC ON CACHE BOOL "" FORCE) set(PAHO_BUILD_SHARED OFF CACHE BOOL "" FORCE) set(PAHO_ENABLE_TESTING OFF CACHE BOOL "" FORCE) - -if (NOT MINIFI_OPENSSL) - set(PAHO_WITH_SSL OFF CACHE BOOL "" FORCE) -else() - set(PAHO_WITH_SSL ON CACHE BOOL "" FORCE) -endif() +set(PAHO_WITH_SSL ON CACHE BOOL "" FORCE) set(PATCH_FILE "${CMAKE_SOURCE_DIR}/thirdparty/paho-mqtt/cmake-openssl.patch") set(PC ${Bash_EXECUTABLE} -c "set -x &&\ @@ -42,9 +37,5 @@ FetchContent_Declare( FetchContent_MakeAvailable(paho.mqtt.c-external) # Set dependencies and target to link to -if (MINIFI_OPENSSL) - add_library(paho.mqtt.c ALIAS paho-mqtt3as-static) - add_dependencies(common_ssl_obj_static OpenSSL::SSL OpenSSL::Crypto) -else() - add_library(paho.mqtt.c ALIAS paho-mqtt3a-static) -endif() +add_library(paho.mqtt.c ALIAS paho-mqtt3as-static) +add_dependencies(common_ssl_obj_static OpenSSL::SSL OpenSSL::Crypto) diff --git a/cmake/Spdlog.cmake b/cmake/Spdlog.cmake index 07eb0f103a..3b07298297 100644 --- a/cmake/Spdlog.cmake +++ b/cmake/Spdlog.cmake @@ -17,7 +17,7 @@ # under the License. # include(FetchContent) -include(fmt) + set(SPDLOG_FMT_EXTERNAL ON CACHE STRING "" FORCE) FetchContent_Declare(Spdlog URL https://github.com/gabime/spdlog/archive/refs/tags/v1.12.0.tar.gz diff --git a/conanfile.py b/conanfile.py new file mode 100644 index 0000000000..9b3cac5493 --- /dev/null +++ b/conanfile.py @@ -0,0 +1,115 @@ +from conan import ConanFile +from conan.tools.env import VirtualRunEnv +from conan.tools.cmake import CMake, CMakeToolchain +from conan.tools.files import collect_libs, copy +import os + +required_conan_version = ">=2.1.0" + +shared_requires = ("openssl/3.2.1", "libcurl/8.6.0", "civetweb/1.16", "libxml2/2.12.6", "catch2/3.5.4", + "fmt/10.2.1", "spdlog/1.14.0") +shared_sources = ("CMakeLists.txt", "libminifi/*", "extensions/*", "minifi_main/*", "nanofi/*", + "bin/*", "bootstrap/*", "cmake/*", "conf/*", "controller/*", "encrypt-config/*", + "etc/*", "examples/*", "msi/*", "thirdparty/*", "docker/*", "LICENSE", "NOTICE", + "README.md", "C2.md", "CONFIGURE.md", "CONTRIBUTING.md", "CONTROLLERS.md", "EXPRESSIONS.md", + "Extensions.md", "JNI.md", "METRICS.md", "OPS.md", "PROCESSORS.md", "ThirdParties.md", + "Windows.md", "aptitude.sh", "arch.sh", "bootstrap.sh", "bstrp_functions.sh", "centos.sh", + "CPPLINT.cfg", "darwin.sh", "debian.sh", "deploy.sh", "fedora.sh", "generateVersion.sh", + "linux.sh", "rheldistro.sh", "run_clang_tidy.sh", "run_clang_tidy.sh", "run_flake8.sh", + "run_shellcheck.sh", "suse.sh", "versioninfo.rc.in") + +class MiNiFiCppMain(ConanFile): + name = "minifi-cpp" + version = "0.99.0" + license = "Apache-2.0" + requires = shared_requires + settings = "os", "compiler", "build_type", "arch" + generators = "CMakeDeps" + options = {"shared": [True, False], "fPIC": [True, False]} + + default_options = {"shared": False, "fPIC": True,} + + exports_sources = shared_sources + + def generate(self): + tc = CMakeToolchain(self) + + tc.variables["MINIFI_BUILD_CONAN_PACKAGE"] = "ON" + tc.variables["MINIFI_LIBCURL_SOURCE"] = "CONAN" + tc.variables["MINIFI_OPENSSL_SOURCE"] = "CONAN" + tc.variables["MINIFI_ZLIB_SOURCE"] = "CONAN" + tc.variables["MINIFI_CIVETWEB_SOURCE"] = "CONAN" + tc.variables["MINIFI_LIBXML2_SOURCE"] = "CONAN" + tc.variables["MINIFI_CATCH2_SOURCE"] = "CONAN" + tc.variables["MINIFI_FMT_SOURCE"] = "CONAN" + tc.variables["MINIFI_SPDLOG_SOURCE"] = "CONAN" + + tc.variables["SKIP_TESTS"] = "OFF" + tc.variables["ENABLE_OPENWSMAN"] = "ON" + tc.variables["ENABLE_CIVET"] = "ON" + tc.variables["ENABLE_EXPRESSION_LANGUAGE"] = "ON" + tc.variables["ENABLE_BZIP2"] = "ON" + tc.variables["ENABLE_ROCKSDB"] = "ON" + tc.variables["BUILD_ROCKSDB"] = "ON" + + tc.variables["ENABLE_LIBARCHIVE"] = "OFF" + tc.variables["ENABLE_AWS"] = "OFF" + + tc.variables["ENABLE_OPC"] = "OFF" + + if self.settings.os == "Windows": + tc.variables["ENABLE_WEL"] = "OFF" + tc.variables["ENABLE_PDH"] = "OFF" + tc.variables["ENABLE_SMB"] = "OFF" + elif self.settings.os == "Linux": + tc.variables["ENABLE_SYSTEMD"] = "OFF" + tc.variables["ENABLE_PROCFS"] = "OFF" + + tc.variables["ENABLE_LZMA"] = "OFF" + tc.variables["ENABLE_GPS"] = "OFF" + tc.variables["ENABLE_COAP"] = "OFF" + tc.variables["ENABLE_SQL"] = "OFF" + tc.variables["ENABLE_MQTT"] = "OFF" + tc.variables["ENABLE_PCAP"] = "OFF" + tc.variables["ENABLE_LIBRDKAFKA"] = "OFF" + tc.variables["ENABLE_LUA_SCRIPTING"] = "OFF" + tc.variables["ENABLE_PYTHON_SCRIPTING"] = "OFF" + tc.variables["ENABLE_SENSORS"] = "OFF" + tc.variables["ENABLE_USB_CAMERA"] = "OFF" + tc.variables["ENABLE_OPENCV"] = "OFF" + tc.variables["ENABLE_BUSTACHE"] = "OFF" + tc.variables["ENABLE_SFTP"] = "OFF" + tc.variables["ENABLE_AZURE"] = "OFF" + tc.variables["ENABLE_ENCRYPT_CONFIG"] = "OFF" + tc.variables["ENABLE_SPLUNK"] = "OFF" + tc.variables["ENABLE_ELASTICSEARCH"] = "OFF" + tc.variables["ENABLE_GCP"] = "OFF" + tc.variables["ENABLE_KUBERNETES"] = "OFF" + tc.variables["ENABLE_TEST_PROCESSORS"] = "OFF" + tc.variables["ENABLE_PROMETHEUS"] = "OFF" + tc.variables["ENABLE_GRAFANA_LOKI"] = "OFF" + tc.variables["ENABLE_GRPC_FOR_LOKI"] = "OFF" + tc.variables["ENABLE_CONTROLLER"] = "OFF" + + tc.generate() + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def package(self): + cmake = CMake(self) + include_dir = os.path.join(self.source_folder) + built_dir = os.path.join(self.source_folder, self.folders.build) + copy(self, pattern="*.h*", dst=os.path.join(self.package_folder, "include"), src=include_dir, keep_path=True) + copy(self, pattern="*.i*", dst=os.path.join(self.package_folder, "include"), src=include_dir, keep_path=True) + copy(self, pattern="*.a", dst=os.path.join(self.package_folder, "lib"), src=built_dir, keep_path=False) + copy(self, pattern="*.so*", dst=os.path.join(self.package_folder, "lib"), src=built_dir, keep_path=False) + cmake.install() + + def package_info(self): + self.cpp_info.libs = collect_libs(self, folder=os.path.join(self.package_folder, "lib")) + self.cpp_info.set_property("cmake_file_name", "minifi-cpp") + self.cpp_info.set_property("cmake_target_name", "minifi-cpp::minifi-cpp") + self.cpp_info.set_property("pkg_config_name", "minifi-cpp") diff --git a/controller/tests/CMakeLists.txt b/controller/tests/CMakeLists.txt index b00d3aa8bd..056969fa60 100644 --- a/controller/tests/CMakeLists.txt +++ b/controller/tests/CMakeLists.txt @@ -33,11 +33,10 @@ foreach(testfile ${CONTROLLER_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/controller") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test") createTests(${testfilename}) - target_link_libraries(${testfilename} Catch2WithMain ${LIBMINIFI} ${TEST_BASE_LIB}) + target_link_libraries(${testfilename} Catch2::Catch2WithMain ${LIBMINIFI} libminifi-unittest) add_test(NAME ${testfilename} COMMAND ${testfilename} WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") math(EXPR CONTROLLER_TEST_COUNT "${CONTROLLER_TEST_COUNT}+1") diff --git a/controller/tests/ControllerTests.cpp b/controller/tests/ControllerTests.cpp index d4f5166787..9fdca02682 100644 --- a/controller/tests/ControllerTests.cpp +++ b/controller/tests/ControllerTests.cpp @@ -23,12 +23,12 @@ #include #include "range/v3/algorithm/find.hpp" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/Processor.h" #include "Controller.h" #include "c2/ControllerSocketProtocol.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" #include "c2/ControllerSocketMetricsPublisher.h" #include "core/controller/ControllerServiceProvider.h" #include "controllers/SSLContextService.h" @@ -308,7 +308,7 @@ TEST_CASE_METHOD(ControllerTestFixture, "Test listComponents", "[controllerTests minifi::controller::startComponent(controller_socket_data_, "TestStateController"); - using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; + using org::apache::nifi::minifi::test::utils::verifyEventHappenedInPollTime; REQUIRE(verifyEventHappenedInPollTime(5s, [&] { return controller_->isRunning(); }, 20ms)); minifi::controller::stopComponent(controller_socket_data_, "TestStateController"); @@ -338,7 +338,7 @@ TEST_CASE_METHOD(ControllerTestFixture, "TestClear", "[controllerTests]") { minifi::controller::startComponent(controller_socket_data_, "TestStateController"); - using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; + using org::apache::nifi::minifi::test::utils::verifyEventHappenedInPollTime; REQUIRE(verifyEventHappenedInPollTime(5s, [&] { return controller_->isRunning(); }, 20ms)); for (auto i = 0; i < 3; ++i) { @@ -364,7 +364,7 @@ TEST_CASE_METHOD(ControllerTestFixture, "TestUpdate", "[controllerTests]") { initalizeControllerSocket(); minifi::controller::startComponent(controller_socket_data_, "TestStateController"); - using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; + using org::apache::nifi::minifi::test::utils::verifyEventHappenedInPollTime; REQUIRE(verifyEventHappenedInPollTime(5s, [&] { return controller_->isRunning(); }, 20ms)); std::stringstream ss; diff --git a/darwin.sh b/darwin.sh index 0c536cf978..b81ed614f3 100644 --- a/darwin.sh +++ b/darwin.sh @@ -66,9 +66,8 @@ bootstrap_compiler() { } build_deps(){ - COMMAND="brew install cmake" + COMMAND="brew install cmake perl bzip2" INSTALLED=() - INSTALLED+=("bzip2") for option in "${OPTIONS[@]}" ; do option_value="${!option}" if [ "$option_value" = "${TRUE}" ]; then @@ -105,8 +104,6 @@ build_deps(){ INSTALLED+=("bzip2") elif [ "$FOUND_VALUE" = "libssh2" ]; then INSTALLED+=("libssh2") - elif [ "$FOUND_VALUE" = "opensslbuild" ]; then - INSTALLED+=("perl") fi fi done diff --git a/debian.sh b/debian.sh index f532e8826b..648a1fcba5 100644 --- a/debian.sh +++ b/debian.sh @@ -43,10 +43,9 @@ build_deps(){ if [ "$RETVAL" -ne "0" ]; then sudo apt-get install -y libssl-dev > /dev/null fi - COMMAND="sudo apt-get -y install gcc g++ zlib1g-dev uuid uuid-dev" + COMMAND="sudo apt-get -y install gcc g++ zlib1g-dev uuid uuid-dev perl libbz2-dev" export DEBIAN_FRONTEND=noninteractive INSTALLED=() - INSTALLED+=("libbz2-dev") sudo apt-get -y update for option in "${OPTIONS[@]}" ; do option_value="${!option}" @@ -85,8 +84,6 @@ build_deps(){ INSTALLED+=("liblzma-dev") elif [ "$FOUND_VALUE" = "libssh2" ]; then INSTALLED+=("libssh2-1-dev") - elif [ "$FOUND_VALUE" = "opensslbuild" ]; then - INSTALLED+=("perl") fi fi done diff --git a/encrypt-config/tests/CMakeLists.txt b/encrypt-config/tests/CMakeLists.txt index a394a3c97d..91d8e39dd3 100644 --- a/encrypt-config/tests/CMakeLists.txt +++ b/encrypt-config/tests/CMakeLists.txt @@ -28,11 +28,10 @@ foreach(testfile ${ENCRYPT_CONFIG_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/encrypt-config") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test") createTests(${testfilename}) - target_link_libraries(${testfilename} Catch2WithMain ${LIBMINIFI} ${TEST_BASE_LIB}) + target_link_libraries(${testfilename} Catch2::Catch2WithMain ${LIBMINIFI} libminifi-unittest) add_test(NAME ${testfilename} COMMAND ${testfilename} WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") math(EXPR ENCRYPT_CONFIG_TEST_COUNT "${ENCRYPT_CONFIG_TEST_COUNT}+1") diff --git a/encrypt-config/tests/ConfigFileEncryptorTests.cpp b/encrypt-config/tests/ConfigFileEncryptorTests.cpp index 3793df3e14..4fb309c061 100644 --- a/encrypt-config/tests/ConfigFileEncryptorTests.cpp +++ b/encrypt-config/tests/ConfigFileEncryptorTests.cpp @@ -23,8 +23,8 @@ #include "properties/Configuration.h" #include "utils/RegexUtils.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" using org::apache::nifi::minifi::encrypt_config::ConfigFile; using org::apache::nifi::minifi::encrypt_config::encryptSensitivePropertiesInFile; diff --git a/encrypt-config/tests/ConfigFileTests.cpp b/encrypt-config/tests/ConfigFileTests.cpp index 5f97bff8b9..0c72f1a532 100644 --- a/encrypt-config/tests/ConfigFileTests.cpp +++ b/encrypt-config/tests/ConfigFileTests.cpp @@ -25,8 +25,8 @@ #include "utils/gsl.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/file/FileUtils.h" using org::apache::nifi::minifi::encrypt_config::ConfigFile; diff --git a/etc/conan/profiles/release-linux b/etc/conan/profiles/release-linux new file mode 100644 index 0000000000..26b4fa6a08 --- /dev/null +++ b/etc/conan/profiles/release-linux @@ -0,0 +1,12 @@ +[settings] +os=Linux +arch=x86_64 +build_type=Release +compiler=gcc +compiler.cppstd=gnu20 +compiler.libcxx=libstdc++11 +compiler.version=11 +[conf] +tools.system.package_manager:mode=install +tools.system.package_manager:sudo=True +[options] diff --git a/extensions/aws/tests/AWSCredentialsServiceTest.cpp b/extensions/aws/tests/AWSCredentialsServiceTest.cpp index 7c467648df..4eacaf3cfa 100644 --- a/extensions/aws/tests/AWSCredentialsServiceTest.cpp +++ b/extensions/aws/tests/AWSCredentialsServiceTest.cpp @@ -19,10 +19,10 @@ #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "controllerservices/AWSCredentialsService.h" -#include "Utils.h" +#include "unit/TestUtils.h" #include "core/controller/ControllerServiceNode.h" class AWSCredentialsServiceTestAccessor { diff --git a/extensions/aws/tests/CMakeLists.txt b/extensions/aws/tests/CMakeLists.txt index 6d99060abc..7bc116b616 100644 --- a/extensions/aws/tests/CMakeLists.txt +++ b/extensions/aws/tests/CMakeLists.txt @@ -28,9 +28,8 @@ FOREACH(testfile ${AWS_INTEGRATION_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/aws/s3") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/aws/processors") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/expression-language") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-aws) target_link_libraries(${testfilename} minifi-standard-processors) target_link_libraries(${testfilename} minifi-expression-language-extensions) diff --git a/extensions/aws/tests/DeleteS3ObjectTests.cpp b/extensions/aws/tests/DeleteS3ObjectTests.cpp index 7fc40ee0e0..f75f3a4aa2 100644 --- a/extensions/aws/tests/DeleteS3ObjectTests.cpp +++ b/extensions/aws/tests/DeleteS3ObjectTests.cpp @@ -19,12 +19,12 @@ #include "S3TestsFixture.h" #include "processors/DeleteS3Object.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" namespace { using DeleteS3ObjectTestsFixture = FlowProcessorS3TestsFixture; -using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; +using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; TEST_CASE_METHOD(DeleteS3ObjectTestsFixture, "Test AWS credential setting", "[awsCredentials]") { setBucket(); diff --git a/extensions/aws/tests/FetchS3ObjectTests.cpp b/extensions/aws/tests/FetchS3ObjectTests.cpp index 7d5a82fb3c..eeaaa1c4b3 100644 --- a/extensions/aws/tests/FetchS3ObjectTests.cpp +++ b/extensions/aws/tests/FetchS3ObjectTests.cpp @@ -20,12 +20,12 @@ #include "S3TestsFixture.h" #include "processors/FetchS3Object.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" #include "utils/file/FileUtils.h" namespace { -using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; +using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; using org::apache::nifi::minifi::utils::file::get_content; class FetchS3ObjectTestsFixture : public FlowProcessorS3TestsFixture { diff --git a/extensions/aws/tests/MultipartUploadStateStorageTest.cpp b/extensions/aws/tests/MultipartUploadStateStorageTest.cpp index a2c6181921..3f2d010493 100644 --- a/extensions/aws/tests/MultipartUploadStateStorageTest.cpp +++ b/extensions/aws/tests/MultipartUploadStateStorageTest.cpp @@ -17,8 +17,8 @@ */ #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "s3/MultipartUploadStateStorage.h" #include "utils/Environment.h" #include "MockS3RequestSender.h" diff --git a/extensions/aws/tests/PutS3ObjectTests.cpp b/extensions/aws/tests/PutS3ObjectTests.cpp index 571977a6e6..aaa207ae58 100644 --- a/extensions/aws/tests/PutS3ObjectTests.cpp +++ b/extensions/aws/tests/PutS3ObjectTests.cpp @@ -18,11 +18,11 @@ #include "S3TestsFixture.h" #include "processors/PutS3Object.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" namespace { -using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; +using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; class PutS3ObjectTestsFixture : public FlowProcessorS3TestsFixture { public: diff --git a/extensions/aws/tests/S3TestsFixture.h b/extensions/aws/tests/S3TestsFixture.h index 578eb737ec..7e8055c6cf 100644 --- a/extensions/aws/tests/S3TestsFixture.h +++ b/extensions/aws/tests/S3TestsFixture.h @@ -26,15 +26,15 @@ #include "controllerservices/AWSCredentialsService.h" #include "core/Processor.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "processors/GetFile.h" #include "processors/LogAttribute.h" #include "processors/S3Processor.h" #include "processors/UpdateAttribute.h" #include "utils/file/FileUtils.h" #include "MockS3RequestSender.h" -#include "utils/TestUtils.h" +#include "unit/TestUtils.h" #include "AWSCredentialsProvider.h" #include "s3/MultipartUploadStateStorage.h" #include "s3/S3Wrapper.h" diff --git a/extensions/azure/CMakeLists.txt b/extensions/azure/CMakeLists.txt index 209aa028a1..7bdad325df 100644 --- a/extensions/azure/CMakeLists.txt +++ b/extensions/azure/CMakeLists.txt @@ -38,7 +38,7 @@ target_include_directories(minifi-azure BEFORE PRIVATE storage) target_include_directories(minifi-azure BEFORE PRIVATE ${CMAKE_SOURCE_DIR}/extensions/azure) target_link_libraries(minifi-azure ${LIBMINIFI} Threads::Threads) -target_link_libraries(minifi-azure CURL::libcurl LibXml2::LibXml2) +target_link_libraries(minifi-azure LibXml2::LibXml2) target_link_libraries(minifi-azure AZURE::azure-storage-files-datalake AZURE::azure-storage-blobs AZURE::azure-storage-common AZURE::azure-core AZURE::azure-identity) if (WIN32) diff --git a/extensions/azure/tests/AzureBlobStorageTestsFixture.h b/extensions/azure/tests/AzureBlobStorageTestsFixture.h index 002bd3e657..b847b5d68e 100644 --- a/extensions/azure/tests/AzureBlobStorageTestsFixture.h +++ b/extensions/azure/tests/AzureBlobStorageTestsFixture.h @@ -24,8 +24,8 @@ #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/Processor.h" #include "processors/GetFile.h" #include "processors/PutFile.h" diff --git a/extensions/azure/tests/AzureDataLakeStorageTestsFixture.h b/extensions/azure/tests/AzureDataLakeStorageTestsFixture.h index b2319ce63e..7d121726d3 100644 --- a/extensions/azure/tests/AzureDataLakeStorageTestsFixture.h +++ b/extensions/azure/tests/AzureDataLakeStorageTestsFixture.h @@ -25,10 +25,9 @@ #include #include "MockDataLakeStorageClient.h" -#include "TestBase.h" -#include "Catch.h" -#include "utils/TestUtils.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/TestUtils.h" #include "core/Processor.h" #include "processors/GetFile.h" #include "processors/PutFile.h" @@ -64,7 +63,7 @@ class AzureDataLakeStorageTestsFixture { azure_data_lake_storage_ = std::shared_ptr( new AzureDataLakeStorageProcessor("AzureDataLakeStorageProcessor", utils::Identifier(), std::move(mock_data_lake_storage_client))); auto input_dir = test_controller_.createTempDirectory(); - utils::putFileToDir(input_dir, GETFILE_FILE_NAME, TEST_DATA); + minifi::test::utils::putFileToDir(input_dir, GETFILE_FILE_NAME, TEST_DATA); get_file_ = plan_->addProcessor("GetFile", "GetFile"); plan_->setProperty(get_file_, minifi::processors::GetFile::Directory, input_dir.string()); diff --git a/extensions/azure/tests/CMakeLists.txt b/extensions/azure/tests/CMakeLists.txt index 195cea6f8f..9bce4a99d4 100644 --- a/extensions/azure/tests/CMakeLists.txt +++ b/extensions/azure/tests/CMakeLists.txt @@ -27,10 +27,9 @@ FOREACH(testfile ${AZURE_INTEGRATION_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/azure") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/azure/storage") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/expression-language") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test") target_compile_features(${testfilename} PRIVATE cxx_std_14) createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-azure) target_link_libraries(${testfilename} minifi-standard-processors) target_link_libraries(${testfilename} minifi-expression-language-extensions) diff --git a/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp b/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp index ea35be5316..40bacdca53 100644 --- a/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp +++ b/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp @@ -70,7 +70,7 @@ TEST_CASE_METHOD(DeleteAzureDataLakeStorageTestsFixture, "Test Azure credentials TEST_CASE_METHOD(DeleteAzureDataLakeStorageTestsFixture, "Filesystem name is not set", "[azureDataLakeStorageParameters]") { plan_->setDynamicProperty(update_attribute_, "test.filesystemname", ""); test_controller_.runSession(plan_, true); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; auto failed_flowfiles = getFailedFlowFileContents(); CHECK(failed_flowfiles.size() == 1); CHECK(failed_flowfiles[0] == TEST_DATA); @@ -86,7 +86,7 @@ TEST_CASE_METHOD(DeleteAzureDataLakeStorageTestsFixture, "Connection String is e TEST_CASE_METHOD(DeleteAzureDataLakeStorageTestsFixture, "Delete file succeeds", "[azureDataLakeStorageDelete]") { test_controller_.runSession(plan_, true); REQUIRE(getFailedFlowFileContents().empty()); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; auto passed_params = mock_data_lake_storage_client_ptr_->getPassedDeleteParams(); CHECK(passed_params.credentials.buildConnectionString() == CONNECTION_STRING); CHECK(passed_params.file_system_name == FILESYSTEM_NAME); @@ -98,7 +98,7 @@ TEST_CASE_METHOD(DeleteAzureDataLakeStorageTestsFixture, "Delete file succeeds", TEST_CASE_METHOD(DeleteAzureDataLakeStorageTestsFixture, "Delete file fails", "[azureDataLakeStorageDelete]") { mock_data_lake_storage_client_ptr_->setDeleteFailure(true); test_controller_.runSession(plan_, true); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; auto passed_params = mock_data_lake_storage_client_ptr_->getPassedDeleteParams(); CHECK(passed_params.credentials.buildConnectionString() == CONNECTION_STRING); CHECK(passed_params.file_system_name == FILESYSTEM_NAME); @@ -114,7 +114,7 @@ TEST_CASE_METHOD(DeleteAzureDataLakeStorageTestsFixture, "Delete result is false mock_data_lake_storage_client_ptr_->setDeleteResult(false); test_controller_.runSession(plan_, true); auto passed_params = mock_data_lake_storage_client_ptr_->getPassedDeleteParams(); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; CHECK(passed_params.credentials.buildConnectionString() == CONNECTION_STRING); CHECK(passed_params.file_system_name == FILESYSTEM_NAME); CHECK(passed_params.directory_name == DIRECTORY_NAME); diff --git a/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp b/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp index 52ccf8cb7d..72df7ee207 100644 --- a/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp +++ b/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp @@ -70,7 +70,7 @@ TEST_CASE_METHOD(FetchAzureDataLakeStorageTestsFixture, "Test Azure credentials TEST_CASE_METHOD(FetchAzureDataLakeStorageTestsFixture, "Filesystem name is not set", "[azureDataLakeStorageParameters]") { plan_->setDynamicProperty(update_attribute_, "test.filesystemname", ""); test_controller_.runSession(plan_, true); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; CHECK(verifyLogLinePresenceInPollTime(1s, "Filesystem Name '' is invalid or empty!")); auto failed_flowfiles = getFailedFlowFileContents(); REQUIRE(failed_flowfiles.size() == 1); diff --git a/extensions/azure/tests/ListAzureBlobStorageTests.cpp b/extensions/azure/tests/ListAzureBlobStorageTests.cpp index f658e68bf3..31e7bf973a 100644 --- a/extensions/azure/tests/ListAzureBlobStorageTests.cpp +++ b/extensions/azure/tests/ListAzureBlobStorageTests.cpp @@ -16,10 +16,10 @@ * limitations under the License. */ -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "MockBlobStorage.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" #include "processors/LogAttribute.h" #include "processors/ListAzureBlobStorage.h" #include "controllerservices/AzureStorageCredentialsService.h" @@ -226,7 +226,7 @@ TEST_CASE_METHOD(ListAzureBlobStorageTestsFixture, "List all files every time", plan_->setProperty(list_azure_blob_storage_, minifi::azure::processors::ListAzureBlobStorage::Prefix, PREFIX); plan_->setProperty(list_azure_blob_storage_, minifi::azure::processors::ListAzureBlobStorage::ListingStrategy, magic_enum::enum_name(minifi::azure::EntityTracking::none)); test_controller_.runSession(plan_, true); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; auto run_assertions = [this]() { auto passed_params = mock_blob_storage_ptr_->getPassedListParams(); CHECK(passed_params.container_name == CONTAINER_NAME); @@ -261,7 +261,7 @@ TEST_CASE_METHOD(ListAzureBlobStorageTestsFixture, "Do not list same files the s plan_->setProperty(list_azure_blob_storage_, minifi::azure::processors::ListAzureBlobStorage::Prefix, PREFIX); plan_->setProperty(list_azure_blob_storage_, minifi::azure::processors::ListAzureBlobStorage::ListingStrategy, magic_enum::enum_name(minifi::azure::EntityTracking::timestamps)); test_controller_.runSession(plan_, true); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; auto passed_params = mock_blob_storage_ptr_->getPassedListParams(); CHECK(passed_params.container_name == CONTAINER_NAME); CHECK(passed_params.prefix == PREFIX); diff --git a/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp b/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp index 071a7a3778..fdf3071e10 100644 --- a/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp +++ b/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp @@ -16,10 +16,10 @@ * limitations under the License. */ -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "MockDataLakeStorageClient.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" #include "processors/LogAttribute.h" #include "processors/ListAzureDataLakeStorage.h" #include "controllerservices/AzureStorageCredentialsService.h" @@ -89,7 +89,7 @@ TEST_CASE_METHOD(ListAzureDataLakeStorageTestsFixture, "Azure storage credential TEST_CASE_METHOD(ListAzureDataLakeStorageTestsFixture, "Filesystem name is not set", "[azureDataLakeStorageParameters]") { plan_->setProperty(list_azure_data_lake_storage_, minifi::azure::processors::ListAzureDataLakeStorage::FilesystemName, ""); REQUIRE_THROWS_AS(test_controller_.runSession(plan_, true), minifi::Exception); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; REQUIRE(verifyLogLinePresenceInPollTime(1s, "Filesystem Name '' is invalid or empty!")); } @@ -102,7 +102,7 @@ TEST_CASE_METHOD(ListAzureDataLakeStorageTestsFixture, "List all files every tim plan_->setProperty(list_azure_data_lake_storage_, minifi::azure::processors::ListAzureDataLakeStorage::ListingStrategy, magic_enum::enum_name(minifi::azure::EntityTracking::none)); plan_->setProperty(list_azure_data_lake_storage_, minifi::azure::processors::ListAzureDataLakeStorage::RecurseSubdirectories, "false"); test_controller_.runSession(plan_, true); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; auto run_assertions = [this]() { auto passed_params = mock_data_lake_storage_client_ptr_->getPassedListParams(); CHECK(passed_params.credentials.buildConnectionString() == CONNECTION_STRING); @@ -131,7 +131,7 @@ TEST_CASE_METHOD(ListAzureDataLakeStorageTestsFixture, "Do not list same files t plan_->setProperty(list_azure_data_lake_storage_, minifi::azure::processors::ListAzureDataLakeStorage::ListingStrategy, magic_enum::enum_name(minifi::azure::EntityTracking::timestamps)); plan_->setProperty(list_azure_data_lake_storage_, minifi::azure::processors::ListAzureDataLakeStorage::RecurseSubdirectories, "false"); test_controller_.runSession(plan_, true); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; auto passed_params = mock_data_lake_storage_client_ptr_->getPassedListParams(); CHECK(passed_params.credentials.buildConnectionString() == CONNECTION_STRING); CHECK(passed_params.file_system_name == FILESYSTEM_NAME); @@ -158,7 +158,7 @@ TEST_CASE_METHOD(ListAzureDataLakeStorageTestsFixture, "Do not list same files t TEST_CASE_METHOD(ListAzureDataLakeStorageTestsFixture, "Do not list filtered files", "[listAzureDataLakeStorage]") { plan_->setProperty(list_azure_data_lake_storage_, minifi::azure::processors::ListAzureDataLakeStorage::FileFilter, "item1.*g"); test_controller_.runSession(plan_, true); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; auto passed_params = mock_data_lake_storage_client_ptr_->getPassedListParams(); CHECK(passed_params.credentials.buildConnectionString() == CONNECTION_STRING); CHECK(passed_params.file_system_name == FILESYSTEM_NAME); @@ -181,7 +181,7 @@ TEST_CASE_METHOD(ListAzureDataLakeStorageTestsFixture, "Do not list filtered fil TEST_CASE_METHOD(ListAzureDataLakeStorageTestsFixture, "Do not list filtered paths", "[listAzureDataLakeStorage]") { plan_->setProperty(list_azure_data_lake_storage_, minifi::azure::processors::ListAzureDataLakeStorage::PathFilter, "su.*"); test_controller_.runSession(plan_, true); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; auto passed_params = mock_data_lake_storage_client_ptr_->getPassedListParams(); CHECK(passed_params.credentials.buildConnectionString() == CONNECTION_STRING); CHECK(passed_params.file_system_name == FILESYSTEM_NAME); diff --git a/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp b/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp index 41f45726c8..9ea94222d4 100644 --- a/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp +++ b/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp @@ -70,7 +70,7 @@ TEST_CASE_METHOD(PutAzureDataLakeStorageTestsFixture, "Test Azure credentials wi TEST_CASE_METHOD(PutAzureDataLakeStorageTestsFixture, "Filesystem name is not set", "[azureDataLakeStorageParameters]") { plan_->setDynamicProperty(update_attribute_, "test.filesystemname", ""); test_controller_.runSession(plan_, true); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; CHECK(verifyLogLinePresenceInPollTime(1s, "Filesystem Name '' is invalid or empty!")); auto failed_flowfiles = getFailedFlowFileContents(); REQUIRE(failed_flowfiles.size() == 1); @@ -92,7 +92,7 @@ TEST_CASE_METHOD(PutAzureDataLakeStorageTestsFixture, "Upload to Azure Data Lake CHECK(passed_params.filename == GETFILE_FILE_NAME); CHECK_FALSE(passed_params.replace_file); REQUIRE(getFailedFlowFileContents().empty()); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; CHECK(verifyLogLinePresenceInPollTime(1s, "key:azure.directory value:" + DIRECTORY_NAME)); CHECK(verifyLogLinePresenceInPollTime(1s, "key:azure.filename value:" + GETFILE_FILE_NAME)); CHECK(verifyLogLinePresenceInPollTime(1s, "key:azure.filesystem value:" + FILESYSTEM_NAME)); @@ -130,7 +130,7 @@ TEST_CASE_METHOD(PutAzureDataLakeStorageTestsFixture, "Transfer to success on 'i mock_data_lake_storage_client_ptr_->setFileCreation(false); test_controller_.runSession(plan_, true); REQUIRE(getFailedFlowFileContents().empty()); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; CHECK(verifyLogLinePresenceInPollTime(1s, "key:filename value:" + GETFILE_FILE_NAME)); CHECK_FALSE(LogTestController::getInstance().contains("key:azure", 0s, 0ms)); } @@ -147,7 +147,7 @@ TEST_CASE_METHOD(PutAzureDataLakeStorageTestsFixture, "Replace old file on 'repl CHECK(passed_params.filename == GETFILE_FILE_NAME); CHECK(passed_params.replace_file); REQUIRE(getFailedFlowFileContents().empty()); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; CHECK(verifyLogLinePresenceInPollTime(1s, "key:azure.directory value:" + DIRECTORY_NAME)); CHECK(verifyLogLinePresenceInPollTime(1s, "key:azure.filename value:" + GETFILE_FILE_NAME)); CHECK(verifyLogLinePresenceInPollTime(1s, "key:azure.filesystem value:" + FILESYSTEM_NAME)); @@ -161,7 +161,7 @@ TEST_CASE_METHOD(PutAzureDataLakeStorageTestsFixture, "Upload to Azure Data Lake auto passed_params = mock_data_lake_storage_client_ptr_->getPassedPutParams(); CHECK(passed_params.directory_name.empty()); REQUIRE(getFailedFlowFileContents().empty()); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; CHECK(verifyLogLinePresenceInPollTime(1s, "key:azure.directory value:\n")); } diff --git a/extensions/bustache/tests/ApplyTemplateTests.cpp b/extensions/bustache/tests/ApplyTemplateTests.cpp index a0144f9452..3755014d78 100644 --- a/extensions/bustache/tests/ApplyTemplateTests.cpp +++ b/extensions/bustache/tests/ApplyTemplateTests.cpp @@ -21,8 +21,8 @@ #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/Core.h" #include "core/FlowFile.h" diff --git a/extensions/bustache/tests/CMakeLists.txt b/extensions/bustache/tests/CMakeLists.txt index 9b03bc98aa..5ec7664f38 100644 --- a/extensions/bustache/tests/CMakeLists.txt +++ b/extensions/bustache/tests/CMakeLists.txt @@ -24,14 +24,13 @@ FOREACH(testfile ${BUSTACHE_INTEGRATION_TESTS}) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/bustache") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test/") target_include_directories(${testfilename} SYSTEM PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/thirdparty/bustache/include") target_link_libraries(${testfilename} minifi-bustache-extensions) target_link_libraries(${testfilename} minifi-standard-processors) createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) MATH(EXPR BUSTACHE-EXTENSIONS_TEST_COUNT "${BUSTACHE-EXTENSIONS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) ENDFOREACH() diff --git a/extensions/civetweb/CMakeLists.txt b/extensions/civetweb/CMakeLists.txt index 0eab2b98cd..0ce633937f 100644 --- a/extensions/civetweb/CMakeLists.txt +++ b/extensions/civetweb/CMakeLists.txt @@ -32,7 +32,8 @@ target_include_directories(minifi-civet-extensions BEFORE PUBLIC ${CMAKE_SOURCE_DIR}/thirdparty/ ${civetweb_SOURCE_DIR}/include ./include) -target_link_libraries(minifi-civet-extensions ${LIBMINIFI} Threads::Threads civetweb::civetweb-cpp civetweb::c-library) +# target_link_libraries(minifi-civet-extensions ${LIBMINIFI} Threads::Threads civetweb::civetweb-cpp civetweb::c-library) +target_link_libraries(minifi-civet-extensions ${LIBMINIFI} Threads::Threads civetweb::civetweb-cpp) register_extension(minifi-civet-extensions CIVETWEB CIVETWEB "This enables ListenHTTP" "extensions/civetweb/tests") register_extension_linter(minifi-civet-extensions-linter) diff --git a/extensions/civetweb/tests/CMakeLists.txt b/extensions/civetweb/tests/CMakeLists.txt index 180617872a..23dceab36a 100644 --- a/extensions/civetweb/tests/CMakeLists.txt +++ b/extensions/civetweb/tests/CMakeLists.txt @@ -17,34 +17,28 @@ # under the License. # -if(ENABLE_CURL) - file(GLOB CIVETWEB_INTEGRATION_TESTS "*.cpp") - SET(CIVETWEB-EXTENSIONS_TEST_COUNT 0) - FOREACH(testfile ${CIVETWEB_INTEGRATION_TESTS}) - get_filename_component(testfilename "${testfile}" NAME_WE) - add_minifi_executable("${testfilename}" "${testfile}") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test/") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/civetweb") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/http-curl") - target_include_directories(${testfilename} SYSTEM PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/thirdparty/civetweb-1.10/include") +file(GLOB CIVETWEB_INTEGRATION_TESTS "*.cpp") +SET(CIVETWEB-EXTENSIONS_TEST_COUNT 0) +FOREACH(testfile ${CIVETWEB_INTEGRATION_TESTS}) + get_filename_component(testfilename "${testfile}" NAME_WE) + add_minifi_executable("${testfilename}" "${testfile}") + target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/civetweb") + target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") - target_link_libraries(${testfilename} minifi-civet-extensions) - target_link_libraries(${testfilename} minifi-http-curl) - target_link_libraries(${testfilename} minifi-standard-processors) + target_link_libraries(${testfilename} minifi-civet-extensions) + target_link_libraries(${testfilename} minifi-standard-processors) - createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) - MATH(EXPR CIVETWEB-EXTENSIONS_TEST_COUNT "${CIVETWEB-EXTENSIONS_TEST_COUNT}+1") - add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) - # Copy test resources - add_custom_command( - TARGET "${testfilename}" - POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_directory - "${CMAKE_SOURCE_DIR}/extensions/civetweb/tests/resources" - "$/resources" - ) - ENDFOREACH() - message("-- Finished building ${CIVETWEB-EXTENSIONS_TEST_COUNT} civetweb related test file(s)...") -endif() + createTests("${testfilename}") + target_link_libraries(${testfilename} Catch2::Catch2WithMain) + MATH(EXPR CIVETWEB-EXTENSIONS_TEST_COUNT "${CIVETWEB-EXTENSIONS_TEST_COUNT}+1") + add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) + # Copy test resources + add_custom_command( + TARGET "${testfilename}" + POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_directory + "${CMAKE_SOURCE_DIR}/extensions/civetweb/tests/resources" + "$/resources" + ) +ENDFOREACH() +message("-- Finished building ${CIVETWEB-EXTENSIONS_TEST_COUNT} civetweb related test file(s)...") diff --git a/extensions/civetweb/tests/ListenHTTPTests.cpp b/extensions/civetweb/tests/ListenHTTPTests.cpp index 84ef99d84b..f9883515da 100644 --- a/extensions/civetweb/tests/ListenHTTPTests.cpp +++ b/extensions/civetweb/tests/ListenHTTPTests.cpp @@ -22,26 +22,26 @@ #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/file/FileUtils.h" #include "processors/GetFile.h" #include "processors/UpdateAttribute.h" #include "processors/LogAttribute.h" #include "processors/ListenHTTP.h" -#include "client/HTTPClient.h" +#include "http/HTTPClient.h" #include "controllers/SSLContextService.h" #include "properties/Configure.h" #include "FlowController.h" #include "SchedulingAgent.h" #include "core/ProcessGroup.h" -#include "SingleProcessorTestController.h" +#include "unit/SingleProcessorTestController.h" namespace org::apache::nifi::minifi::test { using namespace std::literals::chrono_literals; -using HttpRequestMethod = org::apache::nifi::minifi::utils::HttpRequestMethod; +using HttpRequestMethod = org::apache::nifi::minifi::http::HttpRequestMethod; class ListenHTTPTestsFixture { public: @@ -63,7 +63,7 @@ class ListenHTTPTestsFixture { LogTestController::getInstance().setTrace(); LogTestController::getInstance().setTrace(); LogTestController::getInstance().setDebug(); - LogTestController::getInstance().setDebug(); + LogTestController::getInstance().setDebug(); LogTestController::getInstance().setDebug(); // Create temporary directories @@ -154,7 +154,7 @@ class ListenHTTPTestsFixture { return; } - client = std::make_unique(); + client = std::make_unique(); client->initialize(method, url, ssl_context_service); client->setVerbose(false); for (const auto &header : headers) { @@ -239,7 +239,7 @@ class ListenHTTPTestsFixture { std::string payload; std::string endpoint = "test"; std::string url; - std::unique_ptr client; + std::unique_ptr client; std::size_t batch_size_ = 0; std::size_t buffer_size_ = 0; }; @@ -429,7 +429,6 @@ TEST_CASE_METHOD(ListenHTTPTestsFixture, "HTTP Batch tests", "[batch]") { test_connect(requests, expected_processed_request_count); } -#ifdef OPENSSL_SUPPORT TEST_CASE_METHOD(ListenHTTPTestsFixture, "HTTPS without CA", "[basic][https]") { plan->setProperty(listen_http, minifi::processors::ListenHTTP::SSLCertificate, (minifi::utils::file::FileUtils::get_executable_dir() / "resources" / "server.pem").string()); @@ -640,7 +639,6 @@ TEST_CASE_METHOD(ListenHTTPTestsFixture, "HTTPS with client cert with non-matchi test_connect({HttpResponseExpectations{true, response_code}}, expected_commited_requests); } -#if CURL_AT_LEAST_VERSION(7, 54, 0) TEST_CASE_METHOD(ListenHTTPTestsFixture, "HTTPS minimum SSL version", "[https]") { auto executable_dir = minifi::utils::file::FileUtils::get_executable_dir(); plan->setProperty(listen_http, minifi::processors::ListenHTTP::SSLCertificate, (executable_dir / "resources" / "server.pem").string()); @@ -661,18 +659,16 @@ TEST_CASE_METHOD(ListenHTTPTestsFixture, "HTTPS minimum SSL version", "[https]") run_server(); - client = std::make_unique(); + client = std::make_unique(); client->setVerbose(false); client->initialize(method, url, ssl_context_service); if (method == HttpRequestMethod::POST) { client->setPostFields(payload); } - REQUIRE(client->setSpecificSSLVersion(minifi::utils::SSLVersion::TLSv1_1)); + REQUIRE(client->setSpecificSSLVersion(minifi::http::SSLVersion::TLSv1_1)); test_connect({HttpResponseExpectations{false, 0}}, 0); } -#endif -#endif TEST_CASE("ListenHTTP bored yield", "[listenhttp][bored][yield]") { using processors::ListenHTTP; diff --git a/extensions/coap/CMakeLists.txt b/extensions/coap/CMakeLists.txt index 2ef66e72ef..3a374335a0 100644 --- a/extensions/coap/CMakeLists.txt +++ b/extensions/coap/CMakeLists.txt @@ -36,7 +36,7 @@ add_minifi_library(minifi-coap SHARED ${SOURCES}) target_link_libraries(nanofi-coap-c COAP::libcoap Threads::Threads) target_link_libraries(minifi-coap ${LIBMINIFI}) -target_link_libraries(minifi-coap nanofi-coap-c Threads::Threads COAP::libcoap minifi-http-curl) +target_link_libraries(minifi-coap nanofi-coap-c Threads::Threads COAP::libcoap) register_extension(minifi-coap "COAP EXTENSIONS" COAP-EXTENSION "Enables LibCOAP Functionality." "extensions/coap/tests/") register_extension_linter(minifi-coap-extensions-linter) diff --git a/extensions/coap/protocols/CoapC2Protocol.h b/extensions/coap/protocols/CoapC2Protocol.h index 5e0c032770..145cc61398 100644 --- a/extensions/coap/protocols/CoapC2Protocol.h +++ b/extensions/coap/protocols/CoapC2Protocol.h @@ -35,7 +35,7 @@ #include "coap2/coap.h" #include "coap2/uri.h" #include "coap2/address.h" -#include "protocols/RESTSender.h" +#include "c2/protocols/RESTSender.h" namespace org::apache::nifi::minifi::coap::c2 { diff --git a/extensions/coap/tests/CMakeLists.txt b/extensions/coap/tests/CMakeLists.txt index de962ee564..3aaa6e492c 100644 --- a/extensions/coap/tests/CMakeLists.txt +++ b/extensions/coap/tests/CMakeLists.txt @@ -21,36 +21,33 @@ file(GLOB COAP_INTEGRATION_TESTS "*.cpp") SET(CURL_INT_TEST_COUNT 0) -if (ENABLE_CURL) - FOREACH(testfile ${COAP_INTEGRATION_TESTS}) - get_filename_component(testfilename "${testfile}" NAME_WE) - add_minifi_executable("${testfilename}" "${testfile}") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test/") - target_include_directories(${testfilename} BEFORE PRIVATE "../") - target_include_directories(${testfilename} BEFORE PRIVATE "../server") - target_include_directories(${testfilename} BEFORE PRIVATE "../nanofi") - target_include_directories(${testfilename} BEFORE PRIVATE "../protocols") - target_include_directories(${testfilename} BEFORE PRIVATE "../controllerservice") - target_include_directories(${testfilename} BEFORE PRIVATE "../../http-curl/") - target_include_directories(${testfilename} BEFORE PRIVATE "../../http-curl/client/") - target_include_directories(${testfilename} BEFORE PRIVATE "../../http-curl/processors/") - target_include_directories(${testfilename} BEFORE PRIVATE "../../http-curl/protocols/") - target_include_directories(${testfilename} BEFORE PRIVATE "../../http-curl/sitetosite/") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/") - target_include_directories(${testfilename} BEFORE PRIVATE ./include) - createTests("${testfilename}") - target_link_libraries(${testfilename} minifi-coap) - target_link_libraries(${testfilename} minifi-civet-extensions) - target_link_libraries(${testfilename} minifi-http-curl) - target_link_libraries(${testfilename} minifi-standard-processors) +FOREACH(testfile ${COAP_INTEGRATION_TESTS}) + get_filename_component(testfilename "${testfile}" NAME_WE) + add_minifi_executable("${testfilename}" "${testfile}") + target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") + target_include_directories(${testfilename} BEFORE PRIVATE "../") + target_include_directories(${testfilename} BEFORE PRIVATE "../server") + target_include_directories(${testfilename} BEFORE PRIVATE "../nanofi") + target_include_directories(${testfilename} BEFORE PRIVATE "../protocols") + target_include_directories(${testfilename} BEFORE PRIVATE "../controllerservice") + target_include_directories(${testfilename} BEFORE PRIVATE "../../http-curl/") + target_include_directories(${testfilename} BEFORE PRIVATE "../../http-curl/client/") + target_include_directories(${testfilename} BEFORE PRIVATE "../../http-curl/processors/") + target_include_directories(${testfilename} BEFORE PRIVATE "../../http-curl/protocols/") + target_include_directories(${testfilename} BEFORE PRIVATE "../../http-curl/sitetosite/") + target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/") + target_include_directories(${testfilename} BEFORE PRIVATE ./include) + target_compile_definitions(${testfilename} PRIVATE TEST_RESOURCES="${TEST_RESOURCES}") + createIntegrationTests("${testfilename}") + target_link_libraries(${testfilename} minifi-coap) + target_link_libraries(${testfilename} minifi-civet-extensions) + target_link_libraries(${testfilename} minifi-standard-processors) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) + add_test(NAME "${testfilename}" COMMAND "${testfilename}") - MATH(EXPR CURL_INT_TEST_COUNT "${CURL_INT_TEST_COUNT}+1") - ENDFOREACH() + MATH(EXPR CURL_INT_TEST_COUNT "${CURL_INT_TEST_COUNT}+1") +ENDFOREACH() + +message("-- Finished building ${CURL_INT_TEST_COUNT} CoAP integration test file(s)...") - message("-- Finished building ${CURL_INT_TEST_COUNT} CoAP integration test file(s)...") - add_test(NAME CoapC2VerifyHeartbeat COMMAND CoapC2VerifyHeartbeat "${TEST_RESOURCES}/CoapC2VerifyServe.yml" "${TEST_RESOURCES}/" "http://localhost:9888/geturl") -else() - message("-- CoAP tests rely on minifi-http-curl, so they will not be built...") -endif() diff --git a/extensions/coap/tests/CoapC2VerifyHeartbeat.cpp b/extensions/coap/tests/CoapC2VerifyHeartbeat.cpp index a1f8b947c7..e0febdcbd1 100644 --- a/extensions/coap/tests/CoapC2VerifyHeartbeat.cpp +++ b/extensions/coap/tests/CoapC2VerifyHeartbeat.cpp @@ -15,10 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - #include -#undef NDEBUG -#include #include #include #include @@ -29,9 +26,9 @@ #include #include #include -#include "BaseHTTPClient.h" +#include "http/BaseHTTPClient.h" #include "processors/InvokeHTTP.h" -#include "TestBase.h" +#include "unit/TestBase.h" #include "utils/StringUtils.h" #include "core/Core.h" #include "core/logging/Logger.h" @@ -50,7 +47,8 @@ #include "CoapC2Protocol.h" #include "CoapServer.h" #include "concurrentqueue.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" +#include "unit/Catch.h" class VerifyCoAPServer : public CoapIntegrationBase { public: @@ -60,7 +58,7 @@ class VerifyCoAPServer : public CoapIntegrationBase { } void testSetup() override { - LogTestController::getInstance().setDebug(); + LogTestController::getInstance().setDebug(); LogTestController::getInstance().setOff(); LogTestController::getInstance().setDebug(); LogTestController::getInstance().setDebug(); @@ -80,8 +78,8 @@ class VerifyCoAPServer : public CoapIntegrationBase { } void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; - assert(verifyLogLinePresenceInPollTime(std::chrono::seconds(3), + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; + REQUIRE(verifyLogLinePresenceInPollTime(std::chrono::seconds(3), "Received ack. version 3. number of operations 1", "Received ack. version 3. number of operations 0", "Received read error event from protocol", @@ -90,11 +88,11 @@ class VerifyCoAPServer : public CoapIntegrationBase { void queryRootProcessGroup(std::shared_ptr pg) override { auto proc = pg->findProcessorByName("invoke"); - assert(proc != nullptr); + REQUIRE(proc != nullptr); const auto* const inv = dynamic_cast(proc); - assert(inv != nullptr); + REQUIRE(inv != nullptr); std::string url; inv->getProperty(minifi::processors::InvokeHTTP::URL, url); @@ -116,14 +114,14 @@ class VerifyCoAPServer : public CoapIntegrationBase { { // valid response version 3, 0 ops - auto data = std::unique_ptr(new uint8_t[5] { 0x00, 0x03, 0x00, 0x01, 0x00 }); + auto data = std::unique_ptr(new uint8_t[5] { 0x00, 0x03, 0x00, 0x01, 0x00 }); // NOLINT(cppcoreguidelines-avoid-c-arrays) minifi::coap::CoapResponse response(205, std::move(data), 5); responses.enqueue(std::move(response)); } { // valid response - auto data = std::unique_ptr(new uint8_t[5] { 0x00, 0x03, 0x00, 0x00, 0x00 }); + auto data = std::unique_ptr(new uint8_t[5] { 0x00, 0x03, 0x00, 0x00, 0x00 }); // NOLINT(cppcoreguidelines-avoid-c-arrays) minifi::coap::CoapResponse response(205, std::move(data), 5); responses.enqueue(std::move(response)); } @@ -140,7 +138,7 @@ class VerifyCoAPServer : public CoapIntegrationBase { stream.write("id"); stream.write("operand"); - auto data = std::make_unique(stream.size()); + auto data = std::make_unique(stream.size()); // NOLINT(cppcoreguidelines-avoid-c-arrays) memcpy(data.get(), stream.getBuffer().data(), stream.getBuffer().size()); minifi::coap::CoapResponse response(205, std::move(data), stream.size()); responses.enqueue(std::move(response)); @@ -174,12 +172,9 @@ class VerifyCoAPServer : public CoapIntegrationBase { TestController testController; }; -int main(int argc, char **argv) { - const cmd_args args = parse_cmdline_args(argc, argv); - const bool isSecure = args.isUrlSecure(); - - VerifyCoAPServer harness(isSecure); - harness.setKeyDir(args.key_dir); - harness.run(args.test_file); - return 0; +TEST_CASE("CoapC2VerifyHeartbeat", "[c2test]") { + VerifyCoAPServer harness(false); + harness.setKeyDir(TEST_RESOURCES); + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "CoapC2VerifyServe.yml"; + harness.run(test_file_path); } diff --git a/extensions/coap/tests/CoapIntegrationBase.h b/extensions/coap/tests/CoapIntegrationBase.h index 18bc2ee490..6974143ca3 100644 --- a/extensions/coap/tests/CoapIntegrationBase.h +++ b/extensions/coap/tests/CoapIntegrationBase.h @@ -23,9 +23,10 @@ #include #include -#include "../tests/TestServer.h" +#include "integration/TestServer.h" #include "CivetServer.h" #include "integration/IntegrationBase.h" +#include "core/yaml/YamlConfiguration.h" using namespace std::literals::chrono_literals; @@ -38,7 +39,7 @@ int ssl_enable(void *, void *) { return 0; } -class CoapIntegrationBase : public IntegrationBase { +class CoapIntegrationBase : public minifi::test::IntegrationBase { public: explicit CoapIntegrationBase(std::chrono::seconds waitTime = 5s) : IntegrationBase(waitTime), @@ -110,7 +111,7 @@ void CoapIntegrationBase::setUrl(std::string url, CivetHandler *handler) { } if (scheme == "https" && !key_dir.empty()) { std::string cert; - cert = key_dir + "nifi-cert.pem"; + cert = key_dir.string() + "nifi-cert.pem"; callback.init_ssl = ssl_enable; port += "s"; callback.log_message = log_message; diff --git a/extensions/elasticsearch/CMakeLists.txt b/extensions/elasticsearch/CMakeLists.txt index 3c6961e055..7231c4afb7 100644 --- a/extensions/elasticsearch/CMakeLists.txt +++ b/extensions/elasticsearch/CMakeLists.txt @@ -27,9 +27,7 @@ file(GLOB SOURCES "*.cpp") add_minifi_library(minifi-elasticsearch SHARED ${SOURCES}) -target_include_directories(minifi-elasticsearch PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/http-curl") target_link_libraries(minifi-elasticsearch ${LIBMINIFI}) -target_link_libraries(minifi-elasticsearch minifi-http-curl) register_extension(minifi-elasticsearch "ELASTICSEARCH EXTENSIONS" ELASTICSEARCH-EXTENSIONS "This enables elasticsearch support" "extensions/elasticsearch/tests") diff --git a/extensions/elasticsearch/ElasticsearchCredentialsControllerService.cpp b/extensions/elasticsearch/ElasticsearchCredentialsControllerService.cpp index fcb042585b..62c2a96131 100644 --- a/extensions/elasticsearch/ElasticsearchCredentialsControllerService.cpp +++ b/extensions/elasticsearch/ElasticsearchCredentialsControllerService.cpp @@ -38,7 +38,7 @@ void ElasticsearchCredentialsControllerService::onEnable() { throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Either an API Key or Username and Password must be provided"); } -void ElasticsearchCredentialsControllerService::authenticateClient(curl::HTTPClient& client) { +void ElasticsearchCredentialsControllerService::authenticateClient(http::HTTPClient& client) { gsl_Expects(api_key_.has_value() != username_password_.has_value()); if (api_key_) { client.setRequestHeader("Authorization", "ApiKey " + *api_key_); diff --git a/extensions/elasticsearch/ElasticsearchCredentialsControllerService.h b/extensions/elasticsearch/ElasticsearchCredentialsControllerService.h index d3c9f69e47..1069aa8406 100644 --- a/extensions/elasticsearch/ElasticsearchCredentialsControllerService.h +++ b/extensions/elasticsearch/ElasticsearchCredentialsControllerService.h @@ -22,7 +22,7 @@ #include #include -#include "client/HTTPClient.h" +#include "http/HTTPClient.h" #include "core/controller/ControllerService.h" #include "core/PropertyDefinition.h" #include "core/PropertyDefinitionBuilder.h" @@ -73,7 +73,7 @@ class ElasticsearchCredentialsControllerService : public core::controller::Contr void onEnable() override; - void authenticateClient(curl::HTTPClient& client); + void authenticateClient(http::HTTPClient& client); private: std::optional> username_password_; diff --git a/extensions/elasticsearch/PostElasticsearch.cpp b/extensions/elasticsearch/PostElasticsearch.cpp index ffc3f4ac3f..29798060de 100644 --- a/extensions/elasticsearch/PostElasticsearch.cpp +++ b/extensions/elasticsearch/PostElasticsearch.cpp @@ -69,7 +69,7 @@ void PostElasticsearch::onSchedule(core::ProcessContext& context, core::ProcessS if (!credentials_service_) throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Missing Elasticsearch credentials service"); - client_.initialize(utils::HttpRequestMethod::POST, host_url_ + "/_bulk", getSSLContextService(context)); + client_.initialize(http::HttpRequestMethod::POST, host_url_ + "/_bulk", getSSLContextService(context)); client_.setContentType("application/json"); credentials_service_->authenticateClient(client_); } @@ -165,7 +165,7 @@ class ElasticPayload { std::optional payload_; }; -nonstd::expected submitRequest(curl::HTTPClient& client, const std::string& payload, const size_t expected_items) { +nonstd::expected submitRequest(http::HTTPClient& client, const std::string& payload, const size_t expected_items) { client.setPostFields(payload); if (!client.submit()) return nonstd::make_unexpected("Submit failed"); diff --git a/extensions/elasticsearch/PostElasticsearch.h b/extensions/elasticsearch/PostElasticsearch.h index 864558bda1..1c0b857205 100644 --- a/extensions/elasticsearch/PostElasticsearch.h +++ b/extensions/elasticsearch/PostElasticsearch.h @@ -30,7 +30,7 @@ #include "core/PropertyType.h" #include "core/RelationshipDefinition.h" #include "utils/Enum.h" -#include "client/HTTPClient.h" +#include "http/HTTPClient.h" namespace org::apache::nifi::minifi::extensions::elasticsearch { @@ -112,7 +112,7 @@ class PostElasticsearch : public core::Processor { uint64_t max_batch_size_ = 100; std::string host_url_; std::shared_ptr credentials_service_; - curl::HTTPClient client_; + http::HTTPClient client_; std::shared_ptr logger_ = core::logging::LoggerFactory::getLogger(uuid_); }; diff --git a/extensions/elasticsearch/tests/CMakeLists.txt b/extensions/elasticsearch/tests/CMakeLists.txt index 516b606e91..82a7ff6432 100644 --- a/extensions/elasticsearch/tests/CMakeLists.txt +++ b/extensions/elasticsearch/tests/CMakeLists.txt @@ -25,15 +25,12 @@ FOREACH(testfile ${ELASTICSEARCH_TESTS}) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/elasticsearch") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test/") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/http-curl/") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-elasticsearch) target_link_libraries(${testfilename} minifi-civet-extensions) - target_link_libraries(${testfilename} minifi-http-curl) target_link_libraries(${testfilename} minifi-standard-processors) MATH(EXPR ELASTICSEARCH_TEST_COUNT "${ELASTICSEARCH_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) diff --git a/extensions/elasticsearch/tests/MockElastic.h b/extensions/elasticsearch/tests/MockElastic.h index d755e5be30..2a1a6d1e5f 100644 --- a/extensions/elasticsearch/tests/MockElastic.h +++ b/extensions/elasticsearch/tests/MockElastic.h @@ -22,7 +22,7 @@ #include #include #include -#include "tests/CivetLibrary.h" +#include "integration/CivetLibrary.h" #include "core/logging/Logger.h" #include "core/logging/LoggerConfiguration.h" #include "rapidjson/document.h" diff --git a/extensions/elasticsearch/tests/PostElasticsearchTests.cpp b/extensions/elasticsearch/tests/PostElasticsearchTests.cpp index 3d6dc08ffb..f8e7aba548 100644 --- a/extensions/elasticsearch/tests/PostElasticsearchTests.cpp +++ b/extensions/elasticsearch/tests/PostElasticsearchTests.cpp @@ -18,8 +18,8 @@ #include "../PostElasticsearch.h" #include "../ElasticsearchCredentialsControllerService.h" #include "MockElastic.h" -#include "SingleProcessorTestController.h" -#include "Catch.h" +#include "unit/SingleProcessorTestController.h" +#include "unit/Catch.h" namespace org::apache::nifi::minifi::extensions::elasticsearch::test { diff --git a/extensions/expression-language/CMakeLists.txt b/extensions/expression-language/CMakeLists.txt index 8f89c9958f..19653fda4a 100644 --- a/extensions/expression-language/CMakeLists.txt +++ b/extensions/expression-language/CMakeLists.txt @@ -132,7 +132,7 @@ endif() add_minifi_library(minifi-expression-language-extensions SHARED ${SOURCES} ${BISON_el-parser_OUTPUTS} ${FLEX_el-scanner_OUTPUTS}) target_link_libraries(minifi-expression-language-extensions ${LIBMINIFI}) -target_link_libraries(minifi-expression-language-extensions RapidJSON CURL::libcurl) +target_link_libraries(minifi-expression-language-extensions RapidJSON) if (BREW_FLEX_INCLUDE) target_include_directories(minifi-expression-language-extensions PRIVATE ${BREW_FLEX_INCLUDE}) endif() diff --git a/extensions/expression-language/Expression.cpp b/extensions/expression-language/Expression.cpp index 8c651c31f8..e242b98e46 100644 --- a/extensions/expression-language/Expression.cpp +++ b/extensions/expression-language/Expression.cpp @@ -35,7 +35,6 @@ #include "expression/Expression.h" #include "utils/RegexUtils.h" -#ifdef ENABLE_CURL #ifdef WIN32 #pragma comment(lib, "wldap32.lib" ) #pragma comment(lib, "crypt32.lib" ) @@ -44,8 +43,6 @@ #define CURL_STATICLIB #endif #include -#endif - #ifdef WIN32 #ifndef WIN32_LEAN_AND_MEAN @@ -702,7 +699,6 @@ Value expr_unescapeCsv(const std::vector &args) { } Value expr_urlEncode(const std::vector &args) { -#ifdef ENABLE_CURL auto arg_0 = args[0].asString(); CURL *curl = curl_easy_init(); if (curl != nullptr) { @@ -719,13 +715,9 @@ Value expr_urlEncode(const std::vector &args) { } else { throw std::runtime_error("Failed to initialize cURL"); } -#else - throw std::runtime_error("Failed to initialize cURL"); -#endif } Value expr_urlDecode(const std::vector &args) { -#ifdef ENABLE_CURL auto arg_0 = args[0].asString(); CURL *curl = curl_easy_init(); if (curl != nullptr) { @@ -743,9 +735,6 @@ Value expr_urlDecode(const std::vector &args) { } else { throw std::runtime_error("Failed to initialize cURL"); } -#else - throw std::runtime_error("Failed to initialize cURL"); -#endif } Value expr_base64Encode(const std::vector &args) { diff --git a/extensions/expression-language/tests/CMakeLists.txt b/extensions/expression-language/tests/CMakeLists.txt index c427efa289..c5e08eca89 100644 --- a/extensions/expression-language/tests/CMakeLists.txt +++ b/extensions/expression-language/tests/CMakeLists.txt @@ -24,16 +24,12 @@ SET(EXTENSIONS_TEST_COUNT 0) FOREACH(testfile ${EXPRESSION_LANGUAGE_TESTS}) get_filename_component(testfilename "${testfile}" NAME_WE) add_minifi_executable(${testfilename} "${testfile}") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/processors") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/expression-language") createTests(${testfilename}) - target_link_libraries(${testfilename} Catch2WithMain) - if(ENABLE_CURL) - target_link_libraries(${testfilename} CURL::libcurl) - endif() + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-expression-language-extensions) target_link_libraries(${testfilename} minifi-standard-processors) target_compile_definitions("${testfilename}" PRIVATE TZ_DATA_DIR="${CMAKE_BINARY_DIR}/tzdata") @@ -52,21 +48,16 @@ SET(INT_EXTENSIONS_TEST_COUNT 0) FOREACH(testfile ${INT_EXPRESSION_LANGUAGE_TESTS}) get_filename_component(testfilename "${testfile}" NAME_WE) add_minifi_executable(${testfilename} "${testfile}") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/processors") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/expression-language") - createTests(${testfilename}) - if(ENABLE_CURL) - target_link_libraries(${testfilename} CURL::libcurl) - endif() + createIntegrationTests(${testfilename}) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-expression-language-extensions) target_link_libraries(${testfilename} minifi-standard-processors) - + target_compile_definitions(${testfilename} PRIVATE TEST_RESOURCES="${TEST_RESOURCES}") + add_test(NAME "${testfilename}" COMMAND "${testfilename}") MATH(EXPR EXTENSIONS_TEST_COUNT "${INT_EXTENSIONS_TEST_COUNT}+1") ENDFOREACH() - -add_test(NAME UpdateAttributeIntegrationTest COMMAND UpdateAttributeIntegrationTest "${TEST_RESOURCES}/TestUpdateAttribute.yml" "${TEST_RESOURCES}/") - message("-- Finished building ${EXTENSIONS_TEST_COUNT} expression language related test file(s)...") diff --git a/extensions/expression-language/tests/ExpressionLanguageTests.cpp b/extensions/expression-language/tests/ExpressionLanguageTests.cpp index 7882e523e4..319a7ea9de 100644 --- a/extensions/expression-language/tests/ExpressionLanguageTests.cpp +++ b/extensions/expression-language/tests/ExpressionLanguageTests.cpp @@ -20,7 +20,6 @@ #include #include -#ifdef ENABLE_CURL #ifdef WIN32 #ifdef _DEBUG #pragma comment(lib, "libcurl-d.lib") @@ -32,7 +31,6 @@ #pragma comment(lib, "crypt32.lib") #endif #include -#endif #include "impl/expression/Expression.h" #include #include @@ -41,12 +39,12 @@ #include "core/FlowFile.h" #include #include "utils/gsl.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "catch2/catch_approx.hpp" #include "unit/ProvenanceTestHelper.h" #include "date/tz.h" -#include "Utils.h" +#include "unit/TestUtils.h" namespace expression = org::apache::nifi::minifi::expression; @@ -1215,7 +1213,6 @@ TEST_CASE("Encode Decode CSV", "[expressionEncodeDecodeCSV]") { REQUIRE("Zero > One < \"two!\" & 'true'" == expr(expression::Parameters{ flow_file_a.get() }).asString()); } -#ifdef ENABLE_CURL TEST_CASE("Encode URL", "[expressionEncodeURL]") { auto expr = expression::compile("${message:urlEncode()}"); @@ -1239,31 +1236,6 @@ TEST_CASE("Encode Decode URL", "[expressionEncodeDecodeURL]") { flow_file_a->addAttribute("message", "some value with spaces"); REQUIRE("some value with spaces" == expr(expression::Parameters{ flow_file_a.get() }).asString()); } -#else -TEST_CASE("Encode URL", "[expressionEncodeURLExcept]") { - auto expr = expression::compile("${message:urlEncode()}"); - - auto flow_file_a = std::make_shared(); - flow_file_a->addAttribute("message", "some value with spaces"); - REQUIRE_THROWS(expr(expression::Parameters{flow_file_a}).asString()); -} - -TEST_CASE("Decode URL", "[expressionDecodeURLExcept]") { - auto expr = expression::compile("${message:urlDecode()}"); - - auto flow_file_a = std::make_shared(); - flow_file_a->addAttribute("message", "some%20value%20with%20spaces"); - REQUIRE_THROWS(expr(expression::Parameters{flow_file_a}).asString()); -} - -TEST_CASE("Encode Decode URL", "[expressionEncodeDecodeURLExcept]") { - auto expr = expression::compile("${message:urlEncode():urlDecode()}"); - - auto flow_file_a = std::make_shared(); - flow_file_a->addAttribute("message", "some value with spaces"); - REQUIRE_THROWS(expr(expression::Parameters{flow_file_a}).asString()); -} -#endif TEST_CASE("Parse Date", "[expressionParseDate]") { #ifdef WIN32 diff --git a/extensions/expression-language/tests/ProcessContextExprTests.cpp b/extensions/expression-language/tests/ProcessContextExprTests.cpp index 20e0a7d26c..bf84350803 100644 --- a/extensions/expression-language/tests/ProcessContextExprTests.cpp +++ b/extensions/expression-language/tests/ProcessContextExprTests.cpp @@ -25,8 +25,8 @@ #include "PropertyDefinition.h" #include "PropertyDefinitionBuilder.h" #include "RelationshipDefinition.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" namespace org::apache::nifi::minifi { diff --git a/extensions/expression-language/tests/RouteOnAttributeTests.cpp b/extensions/expression-language/tests/RouteOnAttributeTests.cpp index f171eef06a..b5dc9ec0e4 100644 --- a/extensions/expression-language/tests/RouteOnAttributeTests.cpp +++ b/extensions/expression-language/tests/RouteOnAttributeTests.cpp @@ -17,8 +17,8 @@ */ #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include #include "processors/LogAttribute.h" #include "processors/UpdateAttribute.h" diff --git a/extensions/expression-language/tests/integration/UpdateAttributeIntegrationTest.cpp b/extensions/expression-language/tests/integration/UpdateAttributeIntegrationTest.cpp index 774edb3a5e..ead1f8fbb7 100644 --- a/extensions/expression-language/tests/integration/UpdateAttributeIntegrationTest.cpp +++ b/extensions/expression-language/tests/integration/UpdateAttributeIntegrationTest.cpp @@ -15,18 +15,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - #include -#undef NDEBUG -#include #include #include #include #include "processors/LogAttribute.h" #include "integration/IntegrationBase.h" #include "ProcessContextExpr.h" -#include "TestBase.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestBase.h" +#include "unit/TestUtils.h" +#include "unit/Catch.h" + +namespace org::apache::nifi::minifi::test { class TestHarness : public IntegrationBase { public: @@ -38,11 +38,10 @@ class TestHarness : public IntegrationBase { } void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; - assert(verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), "key:route_check_attr value:good", "key:variable_attribute value:replacement_value")); - assert(false == verifyLogLinePresenceInPollTime(std::chrono::milliseconds(200), "ProcessSession rollback")); // No rollback happened + REQUIRE_FALSE(minifi::test::utils::verifyLogLinePresenceInPollTime(std::chrono::milliseconds(200), "ProcessSession rollback")); // No rollback happened } void queryRootProcessGroup(std::shared_ptr /*pg*/) override { @@ -51,16 +50,10 @@ class TestHarness : public IntegrationBase { } }; -int main(int argc, char **argv) { - std::string key_dir; - std::string test_file_location; - std::string url; - if (argc > 1) { - test_file_location = argv[1]; - } - +TEST_CASE("UpdateAttributeIntegrationTest", "[updateattribute]") { TestHarness harness; - harness.run(test_file_location); - - return 0; + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestUpdateAttribute.yml"; + harness.run(test_file_path); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/gcp/tests/CMakeLists.txt b/extensions/gcp/tests/CMakeLists.txt index 1526e88016..7d07f73897 100644 --- a/extensions/gcp/tests/CMakeLists.txt +++ b/extensions/gcp/tests/CMakeLists.txt @@ -23,7 +23,6 @@ FOREACH(testfile ${GCS_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/expression-language") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/gcp") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test/") target_include_directories(${testfilename} PRIVATE BEFORE ${googletest_INCLUDE_DIRS} ${googletest_SOURCE_DIR}/googletest/include ${googletest_SOURCE_DIR}/googlemock/include) createTests("${testfilename}") diff --git a/extensions/gcp/tests/DeleteGCSObjectTests.cpp b/extensions/gcp/tests/DeleteGCSObjectTests.cpp index 1f54fccf0a..5d277fe8af 100644 --- a/extensions/gcp/tests/DeleteGCSObjectTests.cpp +++ b/extensions/gcp/tests/DeleteGCSObjectTests.cpp @@ -18,7 +18,7 @@ #include "../controllerservices/GCPCredentialsControllerService.h" #include "GCPAttributes.h" #include "core/Resource.h" -#include "SingleProcessorTestController.h" +#include "unit/SingleProcessorTestController.h" #include "google/cloud/storage/testing/mock_client.h" #include "google/cloud/storage/internal/object_metadata_parser.h" #include "google/cloud/storage/testing/canonical_errors.h" diff --git a/extensions/gcp/tests/FetchGCSObjectTests.cpp b/extensions/gcp/tests/FetchGCSObjectTests.cpp index f1489c07ac..d19473d65f 100644 --- a/extensions/gcp/tests/FetchGCSObjectTests.cpp +++ b/extensions/gcp/tests/FetchGCSObjectTests.cpp @@ -18,7 +18,7 @@ #include "../controllerservices/GCPCredentialsControllerService.h" #include "GCPAttributes.h" #include "core/Resource.h" -#include "SingleProcessorTestController.h" +#include "unit/SingleProcessorTestController.h" #include "google/cloud/storage/testing/mock_client.h" #include "google/cloud/storage/internal/object_metadata_parser.h" #include "google/cloud/storage/testing/canonical_errors.h" diff --git a/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp b/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp index e0a22e1d26..dccddba08b 100644 --- a/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp +++ b/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp @@ -16,7 +16,7 @@ */ #define EXTENSION_LIST "minifi-gcp" // NOLINT(cppcoreguidelines-macro-usage) -#include "TestBase.h" +#include "unit/TestBase.h" #include "gtest/gtest.h" #include "../controllerservices/GCPCredentialsControllerService.h" #include "core/Resource.h" @@ -25,7 +25,7 @@ #include "rapidjson/document.h" #include "rapidjson/stream.h" #include "rapidjson/writer.h" -#include "DummyProcessor.h" +#include "unit/DummyProcessor.h" #include "utils/Environment.h" namespace minifi_gcp = org::apache::nifi::minifi::extensions::gcp; diff --git a/extensions/gcp/tests/ListGCSBucketTests.cpp b/extensions/gcp/tests/ListGCSBucketTests.cpp index fa5ac5fdcc..89ccdc718e 100644 --- a/extensions/gcp/tests/ListGCSBucketTests.cpp +++ b/extensions/gcp/tests/ListGCSBucketTests.cpp @@ -17,7 +17,7 @@ #include "../processors/ListGCSBucket.h" #include "../controllerservices/GCPCredentialsControllerService.h" #include "core/Resource.h" -#include "SingleProcessorTestController.h" +#include "unit/SingleProcessorTestController.h" #include "google/cloud/storage/testing/mock_client.h" #include "google/cloud/storage/internal/object_metadata_parser.h" #include "google/cloud/storage/testing/canonical_errors.h" diff --git a/extensions/gcp/tests/PutGCSObjectTests.cpp b/extensions/gcp/tests/PutGCSObjectTests.cpp index 266e102384..5907ecb49e 100644 --- a/extensions/gcp/tests/PutGCSObjectTests.cpp +++ b/extensions/gcp/tests/PutGCSObjectTests.cpp @@ -18,7 +18,7 @@ #include "../controllerservices/GCPCredentialsControllerService.h" #include "GCPAttributes.h" #include "core/Resource.h" -#include "SingleProcessorTestController.h" +#include "unit/SingleProcessorTestController.h" #include "ProcessContextExpr.h" #include "google/cloud/storage/testing/mock_client.h" #include "google/cloud/storage/internal/object_metadata_parser.h" diff --git a/extensions/gps/tests/CMakeLists.txt b/extensions/gps/tests/CMakeLists.txt index 02e831b210..694ce87b5f 100644 --- a/extensions/gps/tests/CMakeLists.txt +++ b/extensions/gps/tests/CMakeLists.txt @@ -24,11 +24,10 @@ FOREACH(testfile ${GPS_INTEGRATION_TESTS}) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/gps/") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test/") createTests("${testfilename}") target_link_libraries(${testfilename} minifi-gps) target_link_libraries(${testfilename} minifi-standard-processors) - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) MATH(EXPR GPS_TEST_COUNT "${GPS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) ENDFOREACH() diff --git a/extensions/gps/tests/GPSTests.cpp b/extensions/gps/tests/GPSTests.cpp index 40a322de65..0376cfbc7b 100644 --- a/extensions/gps/tests/GPSTests.cpp +++ b/extensions/gps/tests/GPSTests.cpp @@ -33,8 +33,8 @@ #include "FlowController.h" #include "GetGPS.h" #include "processors/GetFile.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "unit/ProvenanceTestHelper.h" TEST_CASE("GPSD Create", "[gpsdtest1]") { diff --git a/extensions/grafana-loki/CMakeLists.txt b/extensions/grafana-loki/CMakeLists.txt index 79e4af7738..d21d775386 100644 --- a/extensions/grafana-loki/CMakeLists.txt +++ b/extensions/grafana-loki/CMakeLists.txt @@ -52,9 +52,7 @@ else() endif() add_minifi_library(minifi-grafana-loki SHARED ${SOURCES}) -target_include_directories(minifi-grafana-loki PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/http-curl") -target_link_libraries(minifi-grafana-loki ${LIBMINIFI} minifi-http-curl) -add_dependencies(minifi-grafana-loki minifi-http-curl) +target_link_libraries(minifi-grafana-loki ${LIBMINIFI}) if (ENABLE_GRPC_FOR_LOKI) target_include_directories(minifi-grafana-loki SYSTEM PRIVATE BEFORE "${LOKI_PROTOBUF_GENERATED_DIR}" "${GRPC_INCLUDE_DIR}" "${PROTOBUF_INCLUDE_DIR}") diff --git a/extensions/grafana-loki/PushGrafanaLokiREST.cpp b/extensions/grafana-loki/PushGrafanaLokiREST.cpp index 8cb43ab978..9f5470577d 100644 --- a/extensions/grafana-loki/PushGrafanaLokiREST.cpp +++ b/extensions/grafana-loki/PushGrafanaLokiREST.cpp @@ -82,7 +82,7 @@ void PushGrafanaLokiREST::initializeHttpClient(core::ProcessContext& context) { url += "/loki/api/v1/push"; } logger_->log_debug("PushGrafanaLokiREST push url is set to: {}", url); - client_.initialize(utils::HttpRequestMethod::POST, url, getSSLContextService(context)); + client_.initialize(http::HttpRequestMethod::POST, url, getSSLContextService(context)); } void PushGrafanaLokiREST::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) { diff --git a/extensions/grafana-loki/PushGrafanaLokiREST.h b/extensions/grafana-loki/PushGrafanaLokiREST.h index 8120f58f37..eb8414a65a 100644 --- a/extensions/grafana-loki/PushGrafanaLokiREST.h +++ b/extensions/grafana-loki/PushGrafanaLokiREST.h @@ -18,7 +18,7 @@ #include "PushGrafanaLoki.h" -#include "client/HTTPClient.h" +#include "http/HTTPClient.h" namespace org::apache::nifi::minifi::extensions::grafana::loki { @@ -68,7 +68,7 @@ class PushGrafanaLokiREST : public PushGrafanaLoki { void addLogLineMetadata(rapidjson::Value& log_line, rapidjson::Document::AllocatorType& allocator, core::FlowFile& flow_file) const; std::map stream_label_attributes_; - curl::HTTPClient client_; + http::HTTPClient client_; }; } // namespace org::apache::nifi::minifi::extensions::grafana::loki diff --git a/extensions/grafana-loki/tests/CMakeLists.txt b/extensions/grafana-loki/tests/CMakeLists.txt index a98a0e4240..d32fea604f 100644 --- a/extensions/grafana-loki/tests/CMakeLists.txt +++ b/extensions/grafana-loki/tests/CMakeLists.txt @@ -31,15 +31,12 @@ FOREACH(testfile ${GRAFANA_LOKI_TESTS}) if (ENABLE_GRPC_FOR_LOKI) target_include_directories(${testfilename} SYSTEM PRIVATE BEFORE "${CMAKE_BINARY_DIR}/grafana-loki-protobuf-generated") endif() - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test/") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/http-curl/") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-grafana-loki) target_link_libraries(${testfilename} minifi-civet-extensions) - target_link_libraries(${testfilename} minifi-http-curl) target_link_libraries(${testfilename} minifi-standard-processors) MATH(EXPR GRAFANA_LOKI_TEST_COUNT "${GRAFANA_LOKI_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) diff --git a/extensions/grafana-loki/tests/MockGrafanaLokiREST.h b/extensions/grafana-loki/tests/MockGrafanaLokiREST.h index 7260cd6e9c..0a92d3e077 100644 --- a/extensions/grafana-loki/tests/MockGrafanaLokiREST.h +++ b/extensions/grafana-loki/tests/MockGrafanaLokiREST.h @@ -22,7 +22,7 @@ #include #include #include -#include "tests/CivetLibrary.h" +#include "integration/CivetLibrary.h" #include "core/logging/Logger.h" #include "core/logging/LoggerConfiguration.h" #include "rapidjson/document.h" diff --git a/extensions/grafana-loki/tests/PushGrafanaLokiGrpcTest.cpp b/extensions/grafana-loki/tests/PushGrafanaLokiGrpcTest.cpp index 557b6bed07..21b861e996 100644 --- a/extensions/grafana-loki/tests/PushGrafanaLokiGrpcTest.cpp +++ b/extensions/grafana-loki/tests/PushGrafanaLokiGrpcTest.cpp @@ -17,10 +17,10 @@ #include "../PushGrafanaLokiGrpc.h" #include "MockGrafanaLokiGrpc.h" -#include "SingleProcessorTestController.h" -#include "Catch.h" +#include "unit/SingleProcessorTestController.h" +#include "unit/Catch.h" #include "utils/StringUtils.h" -#include "utils/TestUtils.h" +#include "unit/TestUtils.h" namespace org::apache::nifi::minifi::extensions::grafana::loki::test { diff --git a/extensions/grafana-loki/tests/PushGrafanaLokiRESTTest.cpp b/extensions/grafana-loki/tests/PushGrafanaLokiRESTTest.cpp index 4299dd7d91..a40950ec02 100644 --- a/extensions/grafana-loki/tests/PushGrafanaLokiRESTTest.cpp +++ b/extensions/grafana-loki/tests/PushGrafanaLokiRESTTest.cpp @@ -17,10 +17,10 @@ #include "../PushGrafanaLokiREST.h" #include "MockGrafanaLokiREST.h" -#include "SingleProcessorTestController.h" -#include "Catch.h" +#include "unit/SingleProcessorTestController.h" +#include "unit/Catch.h" #include "utils/StringUtils.h" -#include "utils/TestUtils.h" +#include "unit/TestUtils.h" #ifdef WIN32 #pragma push_macro("GetObject") @@ -292,7 +292,7 @@ TEST_CASE_METHOD(PushGrafanaLokiRESTTestFixture, "Basic authentication is set in TEST_CASE_METHOD(PushGrafanaLokiRESTTestFixture, "Bearer token is set for authentication", "[PushGrafanaLokiREST]") { auto temp_dir = test_controller_.createTempDirectory(); - auto test_file_path = minifi::utils::putFileToDir(temp_dir, "test1.txt", "mytoken\n"); + auto test_file_path = minifi::test::utils::putFileToDir(temp_dir, "test1.txt", "mytoken\n"); setProperty(PushGrafanaLokiREST::LogLineBatchSize, "1"); setProperty(PushGrafanaLokiREST::BearerTokenFile, test_file_path.string()); auto results = test_controller_.trigger({minifi::test::InputFlowFileData{"log line 1", {}}}); diff --git a/extensions/http-curl/CMakeLists.txt b/extensions/http-curl/CMakeLists.txt deleted file mode 100644 index 47c182d504..0000000000 --- a/extensions/http-curl/CMakeLists.txt +++ /dev/null @@ -1,41 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -if (NOT ENABLE_CURL OR NOT ENABLE_CIVET) - return() -endif() - -message("minifi-http-curl will depend on curl-external") - -include(${CMAKE_SOURCE_DIR}/extensions/ExtensionHeader.txt) -include_directories(protocols client processors sitetosite) - -file(GLOB SOURCES "*.cpp" "protocols/*.cpp" "client/*.cpp" "processors/*.cpp" "sitetosite/*.cpp") - -add_minifi_library(minifi-http-curl SHARED ${SOURCES}) - -target_link_libraries(minifi-http-curl ${LIBMINIFI} Threads::Threads) -target_link_libraries(minifi-http-curl CURL::libcurl RapidJSON) - -if (APPLE) - target_link_libraries(minifi-http-curl "-framework CoreFoundation -framework SystemConfiguration") -endif() - -register_extension(minifi-http-curl "HTTP CURL" HTTP-CURL "This enables RESTProtocol, InvokeHTTP, and the HTTPClient for Site to Site" "extensions/http-curl/tests/") -register_extension_linter(minifi-http-curl-linter) diff --git a/extensions/http-curl/tests/CMakeLists.txt b/extensions/http-curl/tests/CMakeLists.txt deleted file mode 100644 index 250b82ca53..0000000000 --- a/extensions/http-curl/tests/CMakeLists.txt +++ /dev/null @@ -1,113 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -file(GLOB CURL_UNIT_TESTS "unit/*.cpp") -file(GLOB CURL_INTEGRATION_TESTS "*.cpp") - -SET(CURL_INT_TEST_COUNT 0) - -FOREACH(testfile ${CURL_UNIT_TESTS}) - get_filename_component(testfilename "${testfile}" NAME_WE) - add_minifi_executable("${testfilename}" "${testfile}") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test/") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/") - target_include_directories(${testfilename} BEFORE PRIVATE "../") - target_include_directories(${testfilename} BEFORE PRIVATE "../client/") - target_include_directories(${testfilename} BEFORE PRIVATE "../processors/") - target_include_directories(${testfilename} BEFORE PRIVATE "../protocols/") - target_include_directories(${testfilename} BEFORE PRIVATE "../sitetosite/") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/") - target_include_directories(${testfilename} BEFORE PRIVATE ./include) - createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) - target_link_libraries(${testfilename} minifi-http-curl) - target_link_libraries(${testfilename} minifi-civet-extensions) - target_link_libraries(${testfilename} minifi-standard-processors) - - MATH(EXPR CURL_INT_TEST_COUNT "${CURL_INT_TEST_COUNT}+1") - add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) -ENDFOREACH() - -FOREACH(testfile ${CURL_INTEGRATION_TESTS}) - get_filename_component(testfilename "${testfile}" NAME_WE) - add_minifi_executable("${testfilename}" "${testfile}") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test/") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/") - target_include_directories(${testfilename} BEFORE PRIVATE "../") - target_include_directories(${testfilename} BEFORE PRIVATE "../client/") - target_include_directories(${testfilename} BEFORE PRIVATE "../processors/") - target_include_directories(${testfilename} BEFORE PRIVATE "../protocols/") - target_include_directories(${testfilename} BEFORE PRIVATE "../sitetosite/") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/") - target_include_directories(${testfilename} BEFORE PRIVATE ./include) - createTests("${testfilename}") - target_link_libraries(${testfilename} minifi-http-curl) - target_link_libraries(${testfilename} minifi-civet-extensions) - target_link_libraries(${testfilename} minifi-standard-processors) - - MATH(EXPR CURL_INT_TEST_COUNT "${CURL_INT_TEST_COUNT}+1") -ENDFOREACH() - -message("-- Finished building ${CURL_INT_TEST_COUNT} libcURL integration test file(s)...") - -add_test(NAME VerifyInvokeHTTPGetTest COMMAND VerifyInvokeHTTPGetTest "${TEST_RESOURCES}/TestHTTPGet.yml") -add_test(NAME C2UpdateTest COMMAND C2UpdateTest "${TEST_RESOURCES}/TestHTTPGet.yml" "${TEST_RESOURCES}/") -add_test(NAME C2FetchFlowIfMissingTest COMMAND C2FetchFlowIfMissingTest "${TEST_RESOURCES}/TestEmpty.yml" "${TEST_RESOURCES}/") -add_test(NAME C2ConfigEncryptionYaml COMMAND C2ConfigEncryption "${TEST_RESOURCES}/decrypted.config.yml" "${TEST_RESOURCES}/") -add_test(NAME C2ConfigEncryptionJson COMMAND C2ConfigEncryption "${TEST_RESOURCES}/decrypted.config.json" "${TEST_RESOURCES}/") -add_test(NAME C2JstackTest COMMAND C2JstackTest "${TEST_RESOURCES}/TestHTTPGet.yml" "${TEST_RESOURCES}/") -add_test(NAME C2DescribeManifestTest COMMAND C2DescribeManifestTest "${TEST_RESOURCES}/TestHTTPGet.yml" "${TEST_RESOURCES}/") -add_test(NAME C2DescribeCoreComponentStateTest COMMAND C2DescribeCoreComponentStateTest "${TEST_RESOURCES}/TestC2DescribeCoreComponentState.yml" "${TEST_RESOURCES}/") -add_test(NAME C2FailedUpdateTest COMMAND C2FailedUpdateTest "${TEST_RESOURCES}/TestHTTPGet.yml" "${TEST_RESOURCES}/" "${TEST_RESOURCES}/TestBad.yml") -add_test(NAME C2NullConfiguration COMMAND C2NullConfiguration "${TEST_RESOURCES}/TestNull.yml" "${TEST_RESOURCES}/") -add_test(NAME C2RequestClassTest COMMAND C2RequestClassTest) -if (MINIFI_OPENSSL) - add_test(NAME VerifyInvokeHTTPGetTestSecure COMMAND VerifyInvokeHTTPGetTest "${TEST_RESOURCES}/TestHTTPGetSecure.yml" "${TEST_RESOURCES}/") - add_test(NAME C2VerifyHeartbeatAndStopSecure COMMAND C2VerifyHeartbeatAndStop "${TEST_RESOURCES}/C2VerifyHeartbeatAndStopSecure.yml" "${TEST_RESOURCES}/") - add_test(NAME C2VerifyLightweightHeartbeatAndStopSecure COMMAND C2VerifyLightweightHeartbeatAndStop "${TEST_RESOURCES}/C2VerifyHeartbeatAndStopSecure.yml" "${TEST_RESOURCES}/") - add_test(NAME VerifyInvokeHTTPPostTestSecure COMMAND VerifyInvokeHTTPPostTest "${TEST_RESOURCES}/TestInvokeHTTPPostSecure.yml" "${TEST_RESOURCES}/") -endif() -add_test(NAME HttpPostIntegrationTest COMMAND HttpPostIntegrationTest "${TEST_RESOURCES}/TestHTTPPost.yml" "${TEST_RESOURCES}/") -if (NOT APPLE) - add_test(NAME HttpPostIntegrationTestChunked COMMAND HttpPostIntegrationTest "${TEST_RESOURCES}/TestHTTPPostChunkedEncoding.yml" "${TEST_RESOURCES}/") -endif() -add_test(NAME C2VerifyServeResults COMMAND C2VerifyServeResults "${TEST_RESOURCES}/C2VerifyServeResults.yml" "${TEST_RESOURCES}/") -add_test(NAME C2VerifyHeartbeatAndStop COMMAND C2VerifyHeartbeatAndStop "${TEST_RESOURCES}/C2VerifyHeartbeatAndStop.yml") -add_test(NAME C2VerifyLightweightHeartbeatAndStop COMMAND C2VerifyLightweightHeartbeatAndStop "${TEST_RESOURCES}/C2VerifyHeartbeatAndStop.yml") -add_test(NAME C2VerifyResourceConsumptionInHeartbeat COMMAND C2VerifyResourceConsumptionInHeartbeat "${TEST_RESOURCES}/C2VerifyHeartbeatAndStop.yml") -add_test(NAME HTTPSiteToSiteTests COMMAND HTTPSiteToSiteTests "${TEST_RESOURCES}/TestHTTPSiteToSite.yml" "${TEST_RESOURCES}/" "http://localhost:8099/nifi-api") -add_test(NAME TimeoutHTTPSiteToSiteTests COMMAND TimeoutHTTPSiteToSiteTests "${TEST_RESOURCES}/TestTimeoutHTTPSiteToSite.yml" "${TEST_RESOURCES}/" "http://localhost:8098/nifi-api") -add_test(NAME SiteToSiteRestTest COMMAND SiteToSiteRestTest "${TEST_RESOURCES}/TestSite2SiteRest.yml" "${TEST_RESOURCES}/" "http://localhost:8077/nifi-api/site-to-site") -add_test(NAME ControllerServiceIntegrationTests COMMAND ControllerServiceIntegrationTests "${TEST_RESOURCES}/TestControllerServices.yml" "${TEST_RESOURCES}/") -add_test(NAME VerifyInvokeHTTPPostTest COMMAND VerifyInvokeHTTPPostTest "${TEST_RESOURCES}/TestInvokeHTTPPost.yml") -add_test(NAME AbsoluteTimeoutTest COMMAND AbsoluteTimeoutTest) -add_test(NAME C2PauseResumeTest COMMAND C2PauseResumeTest "${TEST_RESOURCES}/C2PauseResumeTest.yml") -add_test(NAME C2LogHeartbeatTest COMMAND C2LogHeartbeatTest) -if (ENABLE_LIBARCHIVE) - add_test(NAME C2DebugBundleTest COMMAND C2DebugBundleTest) -endif() -add_test(NAME C2PropertiesUpdateTests COMMAND C2PropertiesUpdateTests) -add_test(NAME C2ClearCoreComponentStateTest COMMAND C2ClearCoreComponentStateTest "${TEST_RESOURCES}/TestC2DescribeCoreComponentState.yml") -add_test(NAME C2MultipleCommandsTest COMMAND C2MultipleCommandsTest "${TEST_RESOURCES}/TestC2DescribeCoreComponentState.yml") -add_test(NAME C2UpdateAssetTest COMMAND C2UpdateAssetTest) -add_test(NAME C2CompressTest COMMAND C2CompressTest) -add_test(NAME C2MetricsTest COMMAND C2MetricsTest "${TEST_RESOURCES}/TestC2Metrics.yml") -add_test(NAME C2EmptyMetricTest COMMAND C2EmptyMetricTest "${TEST_RESOURCES}/TestEmpty.yml") -add_test(NAME C2SameProcessorMetrics COMMAND C2SameProcessorMetrics "${TEST_RESOURCES}/TestSameProcessorMetrics.yml") -add_test(NAME C2DescribeMetricsTest COMMAND C2DescribeMetricsTest "${TEST_RESOURCES}/TestSameProcessorMetrics.yml") diff --git a/extensions/http-curl/tests/HTTPHandlers.h b/extensions/http-curl/tests/HTTPHandlers.h deleted file mode 100644 index 57bf83ae17..0000000000 --- a/extensions/http-curl/tests/HTTPHandlers.h +++ /dev/null @@ -1,886 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "civetweb.h" -#include "CivetServer.h" -#include "concurrentqueue.h" -#include "CivetStream.h" -#include "io/CRCStream.h" -#include "rapidjson/document.h" -#include "rapidjson/error/en.h" -#include "HTTPUtils.h" -#include "ServerAwareHandler.h" -#include "utils/gsl.h" -#include "agent/build_description.h" -#include "c2/C2Payload.h" -#include "properties/Configuration.h" -#include "range/v3/algorithm/contains.hpp" -#include "range/v3/view/filter.hpp" -#include "range/v3/view/view.hpp" -#include "utils/net/DNS.h" - -static std::atomic transaction_id; -static std::atomic transaction_id_output; - -struct FlowObj { - FlowObj() = default; - - FlowObj(FlowObj &&other) noexcept - : total_size(other.total_size), - attributes(std::move(other.attributes)), - data(std::move(other.data)) - { } - - uint64_t total_size{0}; - std::map attributes; - std::vector data; -}; - -class SiteToSiteLocationResponder : public ServerAwareHandler { - public: - explicit SiteToSiteLocationResponder(bool isSecure) - : isSecure(isSecure) { - } - bool handleGet(CivetServer* /*server*/, struct mg_connection *conn) override { - std::string site2site_rest_resp = "{" - "\"revision\": {" - "\"clientId\": \"483d53eb-53ec-4e93-b4d4-1fc3d23dae6f\"" - "}," - "\"controller\": {" - "\"id\": \"fe4a3a42-53b6-4af1-a80d-6fdfe60de97f\"," - "\"name\": \"NiFi Flow\"," - "\"siteToSiteSecure\": "; - site2site_rest_resp += (isSecure ? "true" : "false"); - site2site_rest_resp += "}}"; - mg_printf(conn, "HTTP/1.1 200 OK\r\nContent-Type: " - "text/plain\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", - site2site_rest_resp.length()); - mg_printf(conn, "%s", site2site_rest_resp.c_str()); - return true; - } - - protected: - bool isSecure; -}; - -class PeerResponder : public ServerAwareHandler { - public: - explicit PeerResponder(std::string base_url) { - (void)base_url; // unused in release builds - std::string scheme; - assert(minifi::utils::parse_http_components(base_url, port, scheme, path)); - } - - bool handleGet(CivetServer* /*server*/, struct mg_connection *conn) override { -#ifdef WIN32 - std::string hostname = org::apache::nifi::minifi::utils::net::getMyHostName(); -#else - std::string hostname = "localhost"; -#endif - std::string site2site_rest_resp = "{\"peers\" : [{ \"hostname\": \"" + hostname + "\", \"port\": " + port + ", \"secure\": false, \"flowFileCount\" : 0 }] }"; - std::stringstream headers; - headers << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << site2site_rest_resp.length() << "\r\nConnection: close\r\n\r\n"; - mg_printf(conn, "%s", headers.str().c_str()); - mg_printf(conn, "%s", site2site_rest_resp.c_str()); - return true; - } - - protected: - std::string base_url; - std::string port; - std::string path; -}; - -class SiteToSiteBaseResponder : public ServerAwareHandler { - public: - explicit SiteToSiteBaseResponder(std::string base_url) - : base_url(std::move(base_url)) { - } - - bool handleGet(CivetServer* /*server*/, struct mg_connection *conn) override { - std::string site2site_rest_resp = - "{\"controller\":{\"id\":\"96dab149-0162-1000-7924-ed3122d6ea2b\",\"name\":\"NiFi Flow\",\"comments\":\"\",\"runningCount\":3,\"stoppedCount\":6,\"invalidCount\":1,\"disabledCount\":0,\"inputPortCount\":1,\"outputPortCount\":1,\"remoteSiteListeningPort\":10443,\"siteToSiteSecure\":false,\"instanceId\":\"13881505-0167-1000-be72-aa29341a3e9a\",\"inputPorts\":[{\"id\":\"471deef6-2a6e-4a7d-912a-81cc17e3a204\",\"name\":\"RPGIN\",\"comments\":\"\",\"state\":\"RUNNING\"}],\"outputPorts\":[{\"id\":\"9cf15a63-0166-1000-1b29-027406d96013\",\"name\":\"ddsga\",\"comments\":\"\",\"state\":\"STOPPED\"}]}}"; // NOLINT line length - std::stringstream headers; - headers << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << site2site_rest_resp.length() << "\r\nConnection: close\r\n\r\n"; - mg_printf(conn, "%s", headers.str().c_str()); - mg_printf(conn, "%s", site2site_rest_resp.c_str()); - return true; - } - - protected: - std::string base_url; -}; - -class TransactionResponder : public ServerAwareHandler { - public: - explicit TransactionResponder(std::string base_url, std::string port_id, bool input_port, bool wrong_uri = false, bool empty_transaction_uri = false) - : base_url(std::move(base_url)), - wrong_uri(wrong_uri), - empty_transaction_uri(empty_transaction_uri), - input_port(input_port), - port_id(std::move(port_id)), - flow_files_feed_(nullptr) { - if (input_port) { - transaction_id_str = "fe4a3a42-53b6-4af1-a80d-6fdfe60de96"; - transaction_id_str += std::to_string(transaction_id.load()); - transaction_id++; - } else { - transaction_id_str = "fe4a3a42-53b6-4af1-a80d-6fdfe60de95"; - transaction_id_str += std::to_string(transaction_id_output.load()); - transaction_id_output++; - } - } - - bool handlePost(CivetServer* /*server*/, struct mg_connection *conn) override { - std::string site2site_rest_resp; - std::stringstream headers; - headers << "HTTP/1.1 201 OK\r\nContent-Type: application/json\r\nContent-Length: " << site2site_rest_resp.length() << "\r\nX-Location-Uri-Intent: "; - if (wrong_uri) - headers << "ohstuff\r\n"; - else - headers << "transaction-url\r\n"; - - std::string port_type; - - if (input_port) - port_type = "input-ports"; - else - port_type = "output-ports"; - if (!empty_transaction_uri) - headers << "locAtion: " << base_url << "/site-to-site/" << port_type << "/" << port_id << "/transactions/" << transaction_id_str << "\r\n"; - headers << "Connection: close\r\n\r\n"; - mg_printf(conn, "%s", headers.str().c_str()); - mg_printf(conn, "%s", site2site_rest_resp.c_str()); - return true; - } - - void setFeed(moodycamel::ConcurrentQueue> *feed) { - flow_files_feed_ = feed; - } - - std::string getTransactionId() { - return transaction_id_str; - } - - protected: - std::string base_url; - std::string transaction_id_str; - bool wrong_uri; - bool empty_transaction_uri; - bool input_port; - std::string port_id; - moodycamel::ConcurrentQueue> *flow_files_feed_; -}; - -class FlowFileResponder : public ServerAwareHandler { - public: - explicit FlowFileResponder(bool input_port, bool wrong_uri = false, bool invalid_checksum = false) - : wrong_uri(wrong_uri), - input_port(input_port), - invalid_checksum(invalid_checksum), - flow_files_feed_(nullptr) { - } - - moodycamel::ConcurrentQueue> *getFlows() { - return &flow_files_; - } - - void setFeed(moodycamel::ConcurrentQueue> *feed) { - flow_files_feed_ = feed; - } - - bool handlePost(CivetServer* /*server*/, struct mg_connection *conn) override { - std::string site2site_rest_resp; - std::stringstream headers; - - if (!wrong_uri) { - minifi::io::CivetStream civet_stream(conn); - minifi::io::CRCStream < minifi::io::CivetStream > stream(gsl::make_not_null(&civet_stream)); - uint32_t num_attributes = 0; - uint64_t total_size = 0; - { - const auto read = stream.read(num_attributes); - if (!isServerRunning()) return false; - assert(read > 0); - total_size += read; - } - - const auto flow = std::make_shared(); - - for (uint32_t i = 0; i < num_attributes; i++) { - std::string name, value; - { - const auto read = stream.read(name, true); - if (!isServerRunning()) return false; - assert(read > 0); - total_size += read; - } - { - const auto read = stream.read(value, true); - if (!isServerRunning()) return false; - assert(read > 0); - total_size += read; - } - flow->attributes[name] = value; - } - uint64_t length{}; - { - const auto read = stream.read(length); - if (!isServerRunning()) return false; - assert(read > 0); - total_size += read; - } - - total_size += length; - flow->data.resize(gsl::narrow(length)); - flow->total_size = total_size; - - { - const auto read = stream.read(flow->data); - if (!isServerRunning()) return false; - (void)read; - assert(read == length); - } - - if (!invalid_checksum) { - site2site_rest_resp = std::to_string(stream.getCRC()); - flow_files_.enqueue(flow); - } else { - site2site_rest_resp = "Imawrongchecksumshortandstout"; - } - - headers << "HTTP/1.1 202 OK\r\nContent-Type: application/json\r\nContent-Length: " << site2site_rest_resp.length() << "\r\nConnection: close\r\n\r\n"; - } else { - headers << "HTTP/1.1 404\r\nConnection: close\r\n\r\n"; - } - - mg_printf(conn, "%s", headers.str().c_str()); - mg_printf(conn, "%s", site2site_rest_resp.c_str()); - return true; - } - - bool handleGet(CivetServer* /*server*/, struct mg_connection *conn) override { - if (flow_files_feed_->size_approx() > 0) { - std::shared_ptr flowobj; - std::vector> flows; - uint64_t total = 0; - - while (flow_files_feed_->try_dequeue(flowobj)) { - flows.push_back(flowobj); - total += flowobj->total_size; - } - mg_printf(conn, - "HTTP/1.1 200 OK\r\n" - "Content-Length: %" PRIu64 "\r\n" - "Content-Type: application/octet-stream\r\n" - "Connection: close\r\n\r\n", - total); - minifi::io::BufferStream serializer; - minifi::io::CRCStream stream(gsl::make_not_null(&serializer)); - for (const auto& flow : flows) { - uint32_t num_attributes = gsl::narrow(flow->attributes.size()); - stream.write(num_attributes); - for (const auto& entry : flow->attributes) { - stream.write(entry.first); - stream.write(entry.second); - } - uint64_t length = flow->data.size(); - stream.write(length); - stream.write(flow->data); - } - } else { - mg_printf(conn, "HTTP/1.1 200 OK\r\nConnection: " - "close\r\nContent-Length: 0\r\n"); - mg_printf(conn, "Content-Type: text/plain\r\n\r\n"); - } - return true; - } - - void setFlowUrl(std::string flowUrl) { - base_url = std::move(flowUrl); - } - - protected: - // base url - std::string base_url; - // set the wrong url - bool wrong_uri; - // we are running an input port - bool input_port; - // invalid checksum is returned. - bool invalid_checksum; - moodycamel::ConcurrentQueue> flow_files_; - moodycamel::ConcurrentQueue> *flow_files_feed_; -}; - -class DeleteTransactionResponder : public ServerAwareHandler { - public: - explicit DeleteTransactionResponder(std::string base_url, std::string response_code, int expected_resp_code) - : flow_files_feed_(nullptr), - base_url(std::move(base_url)), - response_code(std::move(response_code)) { - expected_resp_code_str = std::to_string(expected_resp_code); - } - - explicit DeleteTransactionResponder(std::string base_url, std::string response_code, moodycamel::ConcurrentQueue> *feed) - : flow_files_feed_(feed), - base_url(std::move(base_url)), - response_code(std::move(response_code)) { - } - - bool handleDelete(CivetServer* /*server*/, struct mg_connection *conn) override { - std::string site2site_rest_resp; - std::stringstream headers; - std::string resp; - CivetServer::getParam(conn, "responseCode", resp); - headers << "HTTP/1.1 " << response_code << "\r\nContent-Type: application/json\r\nContent-Length: " << site2site_rest_resp.length() << "\r\n"; - headers << "Connection: close\r\n\r\n"; - mg_printf(conn, "%s", headers.str().c_str()); - mg_printf(conn, "%s", site2site_rest_resp.c_str()); - return true; - } - - void setFeed(moodycamel::ConcurrentQueue> *feed) { - flow_files_feed_ = feed; - } - - protected: - moodycamel::ConcurrentQueue> *flow_files_feed_; - std::string base_url; - std::string expected_resp_code_str; - std::string response_code; -}; - -class HeartbeatHandler : public ServerAwareHandler { - public: - explicit HeartbeatHandler(std::shared_ptr configuration) : configuration_(std::move(configuration)) {} - - virtual void handleHeartbeat(const rapidjson::Document& root, struct mg_connection *) { - verifyJsonHasAgentManifest(root); - } - - virtual void handleAcknowledge(const rapidjson::Document&) { - } - - bool handlePost(CivetServer *, struct mg_connection *conn) override { - verify(conn); - return true; - } - - protected: - struct C2Operation { - std::string operation; - std::string operand; - std::string operation_id; - std::unordered_map args; - }; - - void sendHeartbeatResponse(const std::string& operation, const std::string& operand, const std::string& operation_id, struct mg_connection* conn, - const std::unordered_map& args = {}) { - sendHeartbeatResponse({{operation, operand, operation_id, args}}, conn); - } - - void sendHeartbeatResponse(const std::vector& operations, struct mg_connection * conn) { - std::string operation_jsons; - for (const auto& c2_operation : operations) { - std::string resp_args; - if (!c2_operation.args.empty()) { - resp_args = ", \"args\": {"; - auto it = c2_operation.args.begin(); - while (it != c2_operation.args.end()) { - resp_args += "\"" + it->first + "\": \"" + it->second + "\""; - ++it; - if (it != c2_operation.args.end()) { - resp_args += ", "; - } - } - resp_args += "}"; - } - - std::string operation_json = "{" - "\"operation\" : \"" + c2_operation.operation + "\"," - "\"operationid\" : \"" + c2_operation.operation_id + "\"," - "\"operand\": \"" + c2_operation.operand + "\"" + - resp_args + "}"; - - if (operation_jsons.empty()) { - operation_jsons += operation_json; - } else { - operation_jsons += ", " + operation_json; - } - } - - std::string heartbeat_response = "{\"operation\" : \"heartbeat\",\"requested_operations\": [ " + operation_jsons + " ]}"; - - mg_printf(conn, "HTTP/1.1 200 OK\r\nContent-Type: " - "text/plain\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", - heartbeat_response.length()); - mg_printf(conn, "%s", heartbeat_response.c_str()); - } - - void verifyJsonHasAgentManifest(const rapidjson::Document& root, const std::vector& verify_components = {}, const std::vector& disallowed_properties = {}) { - bool found = false; - assert(root.HasMember("agentInfo")); - assert(root["agentInfo"].HasMember("agentManifest")); - assert(root["agentInfo"]["agentManifest"].HasMember("bundles")); - assert(root["agentInfo"].HasMember("agentManifestHash")); - const std::string manifestHash = root["agentInfo"]["agentManifestHash"].GetString(); - assert(manifestHash.length() == 128); - - // throws if not a valid hexadecimal hash - const auto hashVec = utils::string::from_hex(manifestHash); - assert(hashVec.size() == 64); - - for (auto &bundle : root["agentInfo"]["agentManifest"]["bundles"].GetArray()) { - assert(bundle.HasMember("artifact")); - std::string str = bundle["artifact"].GetString(); - if (str == "minifi-standard-processors") { - std::vector classes; - for (auto &proc : bundle["componentManifest"]["processors"].GetArray()) { - classes.push_back(proc["type"].GetString()); - } - - auto group = minifi::BuildDescription{}.getClassDescriptions(str); - for (const auto& proc : group.processors_) { - assert(std::find(classes.begin(), classes.end(), proc.full_name_) != std::end(classes)); - (void)proc; - found = true; - } - } - } - assert(found); - (void)found; // unused in release builds - - verifySupportedOperations(root, verify_components, disallowed_properties); - } - - void verify(struct mg_connection *conn) { - auto post_data = readPayload(conn); - if (!isServerRunning()) { - return; - } - if (!IsNullOrEmpty(post_data)) { - rapidjson::Document root; - rapidjson::ParseResult result = root.Parse(post_data.data(), post_data.size()); - if (!result) { - throw std::runtime_error(fmt::format("JSON parse error: {0}\n JSON data: {1}", std::string(rapidjson::GetParseError_En(result.Code())), post_data)); - } - std::string operation = root["operation"].GetString(); - if (operation == "heartbeat") { - handleHeartbeat(root, conn); - } else if (operation == "acknowledge") { - handleAcknowledge(root); - } else { - throw std::runtime_error("operation not supported " + operation); - } - } - } - - private: - using Metadata = std::unordered_map>>; - - static std::set getOperandsOfProperties(const rapidjson::Value& operation_node) { - std::set operands; - assert(operation_node.HasMember("properties")); - const auto& properties_node = operation_node["properties"]; - for (auto it = properties_node.MemberBegin(); it != properties_node.MemberEnd(); ++it) { - operands.insert(it->name.GetString()); - } - return operands; - } - - static void verifyMetadata(const rapidjson::Value& operation_node, const std::unordered_map& operand_with_metadata) { - std::unordered_map operand_with_metadata_found; - const auto& properties_node = operation_node["properties"]; - for (auto prop_it = properties_node.MemberBegin(); prop_it != properties_node.MemberEnd(); ++prop_it) { - if (prop_it->value.ObjectEmpty()) { - continue; - } - Metadata metadata_item; - for (auto metadata_it = prop_it->value.MemberBegin(); metadata_it != prop_it->value.MemberEnd(); ++metadata_it) { - std::vector> values; - for (const auto& value : metadata_it->value.GetArray()) { - std::unordered_map value_item; - for (auto value_it = value.MemberBegin(); value_it != value.MemberEnd(); ++value_it) { - value_item.emplace(value_it->name.GetString(), value_it->value.GetString()); - } - values.push_back(value_item); - } - metadata_item.emplace(metadata_it->name.GetString(), values); - } - operand_with_metadata_found.emplace(prop_it->name.GetString(), metadata_item); - } - assert(operand_with_metadata_found == operand_with_metadata); - } - - template - void verifyOperands(const rapidjson::Value& operation_node, const std::unordered_map& operand_with_metadata = {}) { - auto operands = getOperandsOfProperties(operation_node); - assert(operands == std::set(magic_enum::enum_names().begin(), magic_enum::enum_names().end())); - verifyMetadata(operation_node, operand_with_metadata); - } - - void verifyProperties(const rapidjson::Value& operation_node, minifi::c2::Operation operation, - const std::vector& verify_components, const std::vector& disallowed_properties) { - switch (operation) { - case minifi::c2::Operation::describe: { - verifyOperands(operation_node); - break; - } - case minifi::c2::Operation::update: { - std::vector> config_properties; - const auto prop_reader = [this](const std::string& sensitive_props) { return configuration_->getString(sensitive_props); }; - const auto sensitive_props = minifi::Configuration::getSensitiveProperties(prop_reader); - - auto allowed_not_sensitive_configuration_properties = minifi::Configuration::CONFIGURATION_PROPERTIES | ranges::views::filter([&](const auto& configuration_property) { - const auto& configuration_property_name = configuration_property.first; - return !ranges::contains(sensitive_props, configuration_property_name) && !ranges::contains(disallowed_properties, configuration_property_name); - }); - for (const auto& [property_name, property_validator] : allowed_not_sensitive_configuration_properties) { - std::unordered_map config_property; - config_property.emplace("propertyName", property_name); - if (auto value = configuration_->getRawValue(std::string(property_name))) { - config_property.emplace("propertyValue", *value); - } - config_property.emplace("validator", property_validator->getValidatorName()); - config_properties.push_back(config_property); - } - Metadata metadata; - metadata.emplace("availableProperties", config_properties); - std::unordered_map operand_with_metadata; - operand_with_metadata.emplace("properties", metadata); - verifyOperands(operation_node, operand_with_metadata); - break; - } - case minifi::c2::Operation::transfer: { - verifyOperands(operation_node); - break; - } - case minifi::c2::Operation::clear: { - verifyOperands(operation_node); - break; - } - case minifi::c2::Operation::start: - case minifi::c2::Operation::stop: { - auto operands = getOperandsOfProperties(operation_node); - assert(operands.find("c2") != operands.end()); - // FlowController is also present, but this handler has no way of knowing its UUID to test it - for (const auto& component : verify_components) { - assert(operands.find(component) != operands.end()); - } - break; - } - default: - break; - } - } - - void verifySupportedOperations(const rapidjson::Document& root, const std::vector& verify_components, const std::vector& disallowed_properties) { - auto& agent_manifest = root["agentInfo"]["agentManifest"]; - assert(agent_manifest.HasMember("supportedOperations")); - - std::set operations; - for (const auto& operation_node : agent_manifest["supportedOperations"].GetArray()) { - assert(operation_node.HasMember("type")); - operations.insert(operation_node["type"].GetString()); - verifyProperties(operation_node, utils::enumCast(operation_node["type"].GetString(), true), verify_components, disallowed_properties); - } - - assert(operations == std::set(magic_enum::enum_names().begin(), magic_enum::enum_names().end())); - } - - std::shared_ptr configuration_; -}; - -class StoppingHeartbeatHandler : public HeartbeatHandler { - public: - explicit StoppingHeartbeatHandler(std::shared_ptr configuration) : HeartbeatHandler(std::move(configuration)) {} - - bool handlePost(CivetServer *, struct mg_connection *conn) override { - verify(conn); - sendStopOperation(conn); - return true; - } - - private: - static void sendStopOperation(struct mg_connection *conn) { - std::string resp = "{\"operation\" : \"heartbeat\", \"requested_operations\" : [{ \"operationid\" : 41, \"operation\" : \"stop\", \"operand\" : \"2438e3c8-015a-1000-79ca-83af40ec1991\" }, " - "{ \"operationid\" : 42, \"operation\" : \"stop\", \"operand\" : \"FlowController\" } ]}"; - mg_printf(conn, "HTTP/1.1 200 OK\r\nContent-Type: " - "text/plain\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", - resp.length()); - mg_printf(conn, "%s", resp.c_str()); - } -}; - -class C2FlowProvider : public ServerAwareHandler { - public: - explicit C2FlowProvider(std::string test_file_location) - : test_file_location_(std::move(test_file_location)) { - } - - bool handleGet(CivetServer* /*server*/, struct mg_connection *conn) override { - std::ifstream myfile(test_file_location_.c_str(), std::ios::in | std::ios::binary); - if (myfile.good()) { - std::string str((std::istreambuf_iterator(myfile)), (std::istreambuf_iterator())); - mg_printf(conn, "HTTP/1.1 200 OK\r\nContent-Type: " - "text/plain\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", - str.length()); - mg_printf(conn, "%s", str.c_str()); - } else { - mg_printf(conn, "HTTP/1.1 500 Internal Server Error\r\n"); - } - - return true; - } - - private: - const std::string test_file_location_; -}; - -class C2UpdateHandler : public C2FlowProvider { - public: - using C2FlowProvider::C2FlowProvider; - - bool handlePost(CivetServer* /*server*/, struct mg_connection *conn) override { - calls_++; - if (!response_.empty()) { - mg_printf(conn, "HTTP/1.1 200 OK\r\nContent-Type: " - "text/plain\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", - response_.length()); - mg_printf(conn, "%s", response_.c_str()); - response_.clear(); - } else { - mg_printf(conn, "HTTP/1.1 500 Internal Server Error\r\n"); - } - - return true; - } - - void setC2RestResponse(const std::string& url, const std::string& name, const std::optional& persist = {}) { - std::string content = "{\"location\": \"" + url + "\""; - if (persist) { - content += ", \"persist\": \"" + *persist + "\""; - } - content += "}"; - response_ = - "{\"operation\" : \"heartbeat\", " - "\"requested_operations\": [ {" - "\"operation\" : \"update\", " - "\"operationid\" : \"8675309\", " - "\"name\": \"" + name + "\", " - "\"content\": " + content + "}]}"; - } - - size_t getCallCount() const { - return calls_; - } - - protected: - std::atomic calls_{0}; - - private: - std::string response_; -}; - -class C2FailedUpdateHandler : public C2UpdateHandler { - public: - explicit C2FailedUpdateHandler(const std::string& test_file_location) : C2UpdateHandler(test_file_location) { - } - - bool handlePost(CivetServer *server, struct mg_connection *conn) override { - calls_++; - const auto data = readPayload(conn); - - if (data.find("operationState") != std::string::npos) { - assert(data.find("state\": \"NOT_APPLIED") != std::string::npos); - } - - return C2UpdateHandler::handlePost(server, conn); - } -}; - -class InvokeHTTPCouldNotConnectHandler : public ServerAwareHandler { -}; - -class InvokeHTTPResponseOKHandler : public ServerAwareHandler { - public: - bool handlePost(CivetServer *, struct mg_connection *conn) override { - mg_printf(conn, "HTTP/1.1 201 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); - return true; - } -}; - -class InvokeHTTPRedirectHandler : public ServerAwareHandler { - public: - bool handlePost(CivetServer *, struct mg_connection *conn) override { - mg_printf(conn, "HTTP/1.1 301 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\nLocation: /\r\n\r\n"); - return true; - } -}; - -class InvokeHTTPResponse404Handler : public ServerAwareHandler { - public: - bool handlePost(CivetServer *, struct mg_connection *conn) override { - mg_printf(conn, "HTTP/1.1 404 Not Found\r\nContent-Type: text/plain\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); - return true; - } -}; - -class InvokeHTTPResponse501Handler : public ServerAwareHandler { - public: - bool handlePost(CivetServer *, struct mg_connection *conn) override { - mg_printf(conn, "HTTP/1.1 501 Not Implemented\r\nContent-Type: text/plain\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); - return true; - } -}; - -class TimeoutingHTTPHandler : public ServerAwareHandler { - public: - explicit TimeoutingHTTPHandler(std::vector wait_times) - : wait_times_(wait_times) { - } - bool handlePost(CivetServer *, struct mg_connection *conn) override { - respond(conn); - return true; - } - bool handleGet(CivetServer *, struct mg_connection *conn) override { - respond(conn); - return true; - } - bool handleDelete(CivetServer *, struct mg_connection *conn) override { - respond(conn); - return true; - } - bool handlePut(CivetServer *, struct mg_connection *conn) override { - respond(conn); - return true; - } - - private: - void respond(struct mg_connection *conn) { - if (!wait_times_.empty() && wait_times_[0] > std::chrono::seconds(0)) { - sleep_for(wait_times_[0]); - } - int chunk_count = std::max(static_cast(wait_times_.size()) - 1, 0); - mg_printf(conn, "HTTP/1.1 201 OK\r\nContent-Type: text/plain\r\nContent-Length: %d\r\nConnection: close\r\n\r\n", chunk_count); - for (int chunkIdx = 0; chunkIdx < chunk_count; ++chunkIdx) { - mg_printf(conn, "a"); - if (wait_times_[chunkIdx + 1].count() > 0) { - sleep_for(wait_times_[chunkIdx + 1]); - } - } - } - std::vector wait_times_; -}; - -class HttpGetResponder : public ServerAwareHandler { - public: - bool handleGet(CivetServer* /*server*/, struct mg_connection *conn) override { - puts("handle get"); - static const std::string site2site_rest_resp = "hi this is a get test"; - mg_printf(conn, "HTTP/1.1 200 OK\r\nContent-Type: " - "text/plain\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", - site2site_rest_resp.length()); - mg_printf(conn, "%s", site2site_rest_resp.c_str()); - return true; - } -}; - -class RetryHttpGetResponder : public ServerAwareHandler { - public: - bool handleGet(CivetServer* /*server*/, struct mg_connection *conn) override { - puts("handle get with retry"); - mg_printf(conn, "HTTP/1.1 501 Not Implemented\r\nContent-Type: text/plain\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); - return true; - } -}; - -class C2AcknowledgeHandler : public ServerAwareHandler { - struct OpResult { - std::string state; - std::string details; - }; - - public: - bool handlePost(CivetServer* /*server*/, struct mg_connection* conn) override { - std::string req = readPayload(conn); - rapidjson::Document root; - root.Parse(req.data(), req.size()); - - std::string result_state; - std::string details; - - if (root.IsObject() && root.HasMember("operationState")) { - if (root["operationState"].IsObject()) { - if (root["operationState"].HasMember("state")) { - result_state = root["operationState"]["state"].GetString(); - std::lock_guard guard(apply_count_mtx_); - ++apply_count_[result_state]; - } - if (root["operationState"].HasMember("details")) { - details = root["operationState"]["details"].GetString(); - } - } - } - if (root.IsObject() && root.HasMember("operationId")) { - std::lock_guard guard(ack_operations_mtx_); - acknowledged_operations_.insert({root["operationId"].GetString(), OpResult{result_state, details}}); - } - - mg_printf(conn, "HTTP/1.1 200 OK\r\nContent-Type: " - "text/plain\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); - return true; - } - - bool isAcknowledged(const std::string& operation_id) const { - std::lock_guard guard(ack_operations_mtx_); - return acknowledged_operations_.count(operation_id) > 0; - } - - std::optional getState(const std::string& operation_id) const { - std::lock_guard guard(ack_operations_mtx_); - if (auto it = acknowledged_operations_.find(operation_id); it != acknowledged_operations_.end()) { - return it->second; - } - return std::nullopt; - } - - uint32_t getApplyCount(const std::string& result_state) const { - std::lock_guard guard(apply_count_mtx_); - return apply_count_.find(result_state) != apply_count_.end() ? apply_count_.at(result_state) : 0; - } - - private: - mutable std::mutex ack_operations_mtx_; - mutable std::mutex apply_count_mtx_; - std::unordered_map acknowledged_operations_; - std::unordered_map apply_count_; -}; diff --git a/extensions/http-curl/tests/HTTPIntegrationBase.h b/extensions/http-curl/tests/HTTPIntegrationBase.h deleted file mode 100644 index a4b54d55a4..0000000000 --- a/extensions/http-curl/tests/HTTPIntegrationBase.h +++ /dev/null @@ -1,262 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include -#include -#include - -#include "CivetServer.h" -#include "integration/IntegrationBase.h" -#include "c2/C2Agent.h" -#include "protocols/RESTSender.h" -#include "ServerAwareHandler.h" -#include "TestBase.h" -#include "utils/IntegrationTestUtils.h" -#include "TestServer.h" -#include "properties/Configuration.h" - -int log_message(const struct mg_connection* /*conn*/, const char *message) { - puts(message); - return 1; -} - -int ssl_enable(void* /*ssl_context*/, void* /*user_data*/) { - return 0; -} - -class HTTPIntegrationBase : public IntegrationBase { - public: - explicit HTTPIntegrationBase(std::chrono::milliseconds waitTime = std::chrono::milliseconds(DEFAULT_WAITTIME_MSECS)) - : IntegrationBase(waitTime), - server(nullptr) { - } - HTTPIntegrationBase(const HTTPIntegrationBase&) = delete; - HTTPIntegrationBase(HTTPIntegrationBase&&) = default; - HTTPIntegrationBase& operator=(const HTTPIntegrationBase&) = delete; - HTTPIntegrationBase& operator=(HTTPIntegrationBase&&) = default; - - virtual void setUrl(const std::string &url, ServerAwareHandler *handler); - - void setC2Url(const std::string& heartbeat_path, const std::string& acknowledge_path); - - void shutdownBeforeFlowController() override { - server.reset(); - } - - std::string getWebPort() { - std::string ret_val = port; - if (ret_val.back() == 's') { - ret_val = ret_val.substr(0, ret_val.size() - 1); - } - return ret_val; - } - - std::string getC2RestUrl() const { - std::string c2_rest_url; - configuration->get(org::apache::nifi::minifi::Configuration::nifi_c2_rest_url, c2_rest_url); - return c2_rest_url; - } - - protected: - std::unique_ptr server; -}; - -void HTTPIntegrationBase::setUrl(const std::string &url, ServerAwareHandler *handler) { - std::string url_port, url_scheme, url_path; - minifi::utils::parse_http_components(url, url_port, url_scheme, url_path); - if (server) { - if (url_port != "0" && url_port != port) { - throw std::logic_error("Inconsistent port requirements"); - } - if (url_scheme != scheme) { - throw std::logic_error("Inconsistent scheme requirements"); - } - server->addHandler(url_path, handler); - return; - } - // initialize server - scheme = url_scheme; - port = url_port; - CivetCallbacks callback{}; - if (scheme == "https" && !key_dir.empty()) { - std::string cert = key_dir + "nifi-cert.pem"; - callback.init_ssl = ssl_enable; - port += "s"; - callback.log_message = log_message; - server = std::make_unique(port, url_path, handler, &callback, cert, cert); - } else { - server = std::make_unique(port, url_path, handler); - } - bool secure{false}; - if (port == "0" || port == "0s") { - secure = (port == "0s"); - port = std::to_string(server->getListeningPorts()[0]); - if (secure) { - port += "s"; - } - } - std::string c2_url = std::string("http") + (secure ? "s" : "") + "://localhost:" + getWebPort() + url_path; - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_rest_url, c2_url); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_rest_url_ack, c2_url); -} - -void HTTPIntegrationBase::setC2Url(const std::string &heartbeat_path, const std::string &acknowledge_path) { - if (port.empty()) { - throw std::logic_error("Port is not yet initialized"); - } - bool secure = port.back() == 's'; - std::string base = std::string("http") + (secure ? "s" : "") + "://localhost:" + getWebPort(); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_rest_url, base + heartbeat_path); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_rest_url_ack, base + acknowledge_path); -} - -class VerifyC2Base : public HTTPIntegrationBase { - public: - using HTTPIntegrationBase::HTTPIntegrationBase; - void testSetup() override { - LogTestController::getInstance().setDebug(); - LogTestController::getInstance().setDebug(); - } - - void configureC2() override { - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_agent_protocol_class, "RESTSender"); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_enable, "true"); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_agent_class, "test"); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_agent_heartbeat_period, "1000"); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_root_classes, "DeviceInfoNode,AgentInformation,FlowInformation"); - } - - void cleanup() override { - LogTestController::getInstance().reset(); - HTTPIntegrationBase::cleanup(); - } -}; - -class VerifyC2Describe : public VerifyC2Base { - public: - explicit VerifyC2Describe(std::atomic& verified) - : verified_(verified) { - } - - void testSetup() override { - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setDebug(); - LogTestController::getInstance().setInfo(); - VerifyC2Base::testSetup(); - } - - void configureFullHeartbeat() override { - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_full_heartbeat, "false"); - } - - void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; - assert(verifyEventHappenedInPollTime(std::chrono::milliseconds(wait_time_), [&] { return verified_.load(); })); - } - - protected: - std::atomic& verified_; -}; - -class VerifyC2Update : public HTTPIntegrationBase { - public: - explicit VerifyC2Update(std::chrono::milliseconds waitTime) - : HTTPIntegrationBase(waitTime) { - } - - void testSetup() override { - LogTestController::getInstance().setInfo(); - LogTestController::getInstance().setDebug(); - LogTestController::getInstance().setDebug(); - LogTestController::getInstance().setDebug(); - } - - void configureC2() override { - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_agent_protocol_class, "RESTSender"); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_enable, "true"); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_agent_class, "test"); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_agent_heartbeat_period, "1000"); - } - - void cleanup() override { - LogTestController::getInstance().reset(); - HTTPIntegrationBase::cleanup(); - } - - void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; - assert(verifyLogLinePresenceInPollTime(std::chrono::seconds(10), "Starting to reload Flow Controller with flow control name MiNiFi Flow, version")); - } -}; - -class VerifyFlowFetched : public HTTPIntegrationBase { - public: - using HTTPIntegrationBase::HTTPIntegrationBase; - - void testSetup() override { - LogTestController::getInstance().setInfo(); - LogTestController::getInstance().setDebug(); - LogTestController::getInstance().setDebug(); - LogTestController::getInstance().setDebug(); - } - - void configureC2() override { - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_agent_protocol_class, "RESTSender"); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_enable, "true"); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_agent_class, "test"); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_agent_heartbeat_period, "1000"); - } - - void setFlowUrl(const std::string& url) { - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_flow_url, url); - } - - void cleanup() override { - LogTestController::getInstance().reset(); - HTTPIntegrationBase::cleanup(); - } - - void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; - assert(verifyLogLinePresenceInPollTime(std::chrono::seconds(10), "Successfully fetched valid flow configuration")); - } -}; - -class VerifyC2FailedUpdate : public VerifyC2Update { - public: - explicit VerifyC2FailedUpdate(std::chrono::milliseconds waitTime) - : VerifyC2Update(waitTime) { - } - - void testSetup() override { - LogTestController::getInstance().setInfo(); - LogTestController::getInstance().setDebug(); - utils::file::create_dir("content_repository"); - } - - void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; - assert(verifyLogLinePresenceInPollTime(std::chrono::seconds(10), "Invalid configuration payload", "update failed")); - } - - void cleanup() override { - utils::file::delete_dir("content_repository", true); - VerifyC2Update::cleanup(); - } -}; diff --git a/extensions/http-curl/tests/VerifyInvokeHTTPGetTest.cpp b/extensions/http-curl/tests/VerifyInvokeHTTPGetTest.cpp deleted file mode 100644 index 31cf122e79..0000000000 --- a/extensions/http-curl/tests/VerifyInvokeHTTPGetTest.cpp +++ /dev/null @@ -1,61 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "VerifyInvokeHTTP.h" - -#include "HTTPHandlers.h" -#include "utils/IntegrationTestUtils.h" - -class VerifyHTTPGet : public VerifyInvokeHTTP { - public: - void runAssertions() override { - assert(org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime( - std::chrono::seconds(10), - "key:invokehttp.status.code value:200", - "key:flow.id")); - } -}; - -class VerifyRetryHTTPGet : public VerifyInvokeHTTP { - public: - void runAssertions() override { - assert(org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime( - std::chrono::seconds(10), - "isSuccess: false, response code 501")); - assert(org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime( - std::chrono::seconds(10), - "from InvokeHTTP to relationship retry")); - } -}; - -int main(int argc, char **argv) { - const cmd_args args = parse_cmdline_args(argc, argv); - - { - HttpGetResponder http_handler; - VerifyHTTPGet harness; - harness.run(args.url, args.test_file, args.key_dir, &http_handler); - } - - { - RetryHttpGetResponder http_handler; - VerifyRetryHTTPGet harness; - harness.run(args.url, args.test_file, args.key_dir, &http_handler); - } - - return 0; -} diff --git a/extensions/http-curl/tests/VerifyInvokeHTTPPostTest.cpp b/extensions/http-curl/tests/VerifyInvokeHTTPPostTest.cpp deleted file mode 100644 index 8a72eab83f..0000000000 --- a/extensions/http-curl/tests/VerifyInvokeHTTPPostTest.cpp +++ /dev/null @@ -1,152 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include - -#include "VerifyInvokeHTTP.h" - -#include "HTTPHandlers.h" -#include "utils/IntegrationTestUtils.h" - -class VerifyInvokeHTTPOKResponse : public VerifyInvokeHTTP { - public: - void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; - assert(verifyLogLinePresenceInPollTime(std::chrono::seconds(6), - "key:invokehttp.status.code value:201", - "response code 201")); - } -}; - -class VerifyInvokeHTTPOK200Response : public VerifyInvokeHTTP { - public: - void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; - assert(verifyLogLinePresenceInPollTime(std::chrono::seconds(6), - "key:invokehttp.status.code value:200", - "response code 200")); - } -}; - -class VerifyInvokeHTTPRedirectResponse : public VerifyInvokeHTTP { - public: - void setupFlow(const std::optional& flow_yml_path) override { - VerifyInvokeHTTP::setupFlow(flow_yml_path); - setProperty(minifi::processors::InvokeHTTP::FollowRedirects, "false"); - } - - void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; - assert(verifyLogLinePresenceInPollTime(std::chrono::seconds(6), - "key:invokehttp.status.code value:301", - "response code 301")); - } -}; - -class VerifyCouldNotConnectInvokeHTTP : public VerifyInvokeHTTP { - public: - void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; - assert(verifyLogLinePresenceInPollTime(std::chrono::seconds(6), "key:invoke_http value:failure")); - } -}; - -class VerifyNoRetryInvokeHTTP : public VerifyInvokeHTTP { - public: - void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; - assert(verifyLogLinePresenceInPollTime(std::chrono::seconds(6), - "key:invokehttp.status.message value:HTTP/1.1 404 Not Found", - "isSuccess: false, response code 404")); - } -}; - -class VerifyRetryInvokeHTTP : public VerifyInvokeHTTP { - public: - void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; - assert(verifyLogLinePresenceInPollTime(std::chrono::seconds(6), - "key:invokehttp.status.message value:HTTP/1.1 501 Not Implemented", - "isSuccess: false, response code 501")); - } -}; - -class VerifyRWTimeoutInvokeHTTP : public VerifyInvokeHTTP { - public: - void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; - assert(verifyLogLinePresenceInPollTime(std::chrono::seconds(6), - "key:invoke_http value:failure", - "limit (1000ms) reached, terminating connection")); - } -}; - -int main(int argc, char ** argv) { - const cmd_args args = parse_cmdline_args(argc, argv); - - // Stop civet server to simulate - // unreachable remote end point - { - InvokeHTTPCouldNotConnectHandler handler; - VerifyCouldNotConnectInvokeHTTP harness; - harness.setKeyDir(args.key_dir); - harness.setUrl(args.url, &handler); - harness.setupFlow(args.test_file); - harness.shutdownBeforeFlowController(); - harness.startFlowController(); - harness.runAssertions(); - harness.stopFlowController(); - } - - { - InvokeHTTPResponseOKHandler handler; - VerifyInvokeHTTPOKResponse harness; - harness.run(args.url, args.test_file, args.key_dir, &handler); - } - - { - InvokeHTTPRedirectHandler handler; - VerifyInvokeHTTPOK200Response harness; - harness.run(args.url, args.test_file, args.key_dir, &handler); - } - - { - InvokeHTTPRedirectHandler handler; - VerifyInvokeHTTPRedirectResponse harness; - harness.run(args.url, args.test_file, args.key_dir, &handler); - } - - { - InvokeHTTPResponse404Handler handler; - VerifyNoRetryInvokeHTTP harness; - harness.run(args.url, args.test_file, args.key_dir, &handler); - } - - { - InvokeHTTPResponse501Handler handler; - VerifyRetryInvokeHTTP harness; - harness.run(args.url, args.test_file, args.key_dir, &handler); - } - - { - TimeoutingHTTPHandler handler({std::chrono::seconds(2)}); - VerifyRWTimeoutInvokeHTTP harness; - harness.run(args.url, args.test_file, args.key_dir, &handler); - } - - return 0; -} diff --git a/extensions/http-curl/tests/unit/ConnectionCountingServer.h b/extensions/http-curl/tests/unit/ConnectionCountingServer.h deleted file mode 100644 index 0d52d77686..0000000000 --- a/extensions/http-curl/tests/unit/ConnectionCountingServer.h +++ /dev/null @@ -1,151 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include -#include "CivetServer.h" - -namespace org::apache::nifi::minifi::extensions::curl::testing { - -namespace details { - -class NumberedMethodResponder : public CivetHandler { - public: - explicit NumberedMethodResponder(std::set>& connections) : connections_(connections) {} - - bool handleGet(CivetServer*, struct mg_connection* conn) override { - sendNumberedMessage("GET", conn); - return true; - } - - bool handlePost(CivetServer*, struct mg_connection* conn) override { - sendNumberedMessage("POST", conn); - return true; - } - - bool handlePut(CivetServer*, struct mg_connection* conn) override { - sendNumberedMessage("PUT", conn); - return true; - } - - bool handleHead(CivetServer*, struct mg_connection* conn) override { - sendNumberedMessage("HEAD", conn); - return true; - } - - private: - void sendNumberedMessage(std::string body, struct mg_connection* conn) { - saveConnectionId(conn); - body.append(std::to_string(response_id_)); - mg_printf(conn, "HTTP/1.1 200 OK\r\n"); - mg_printf(conn, "Content-length: %lu\r\n", body.length()); - mg_printf(conn, "Response-number: %" PRIu64 "\r\n", response_id_); - mg_printf(conn, "\r\n"); - mg_printf(conn, body.data(), body.length()); - ++response_id_; - } - - void saveConnectionId(struct mg_connection* conn) { - auto user_connection_data = reinterpret_cast*>(mg_get_user_connection_data(conn)); - assert(user_connection_data); - connections_.emplace(*user_connection_data); - } - - uint64_t response_id_ = 0; - std::set>& connections_; -}; - -class ReverseBodyPostHandler : public CivetHandler { - public: - explicit ReverseBodyPostHandler(std::set>& connections) : connections_(connections) {} - - bool handlePost(CivetServer* /*server*/, struct mg_connection* conn) override { - saveConnectionId(conn); - std::vector request_body; - request_body.reserve(2048); - size_t read_size = mg_read(conn, request_body.data(), 2048); - assert(read_size < 2048); - std::string response_body{request_body.begin(), request_body.begin() + read_size}; - std::reverse(std::begin(response_body), std::end(response_body)); - mg_printf(conn, "HTTP/1.1 200 OK\r\n"); - mg_printf(conn, "Content-length: %zu\r\n", read_size); - mg_printf(conn, "\r\n"); - mg_printf(conn, response_body.data(), read_size); - - return true; - } - - private: - void saveConnectionId(struct mg_connection* conn) { - auto user_connection_data = reinterpret_cast*>(mg_get_user_connection_data(conn)); - connections_.emplace(*user_connection_data); - } - - std::set>& connections_; -}; - -struct AddIdToUserConnectionData : public CivetCallbacks { - AddIdToUserConnectionData() { - init_connection = [](const struct mg_connection*, void** user_connection_data) -> int { - utils::SmallString<36>* id = new utils::SmallString<36>(utils::IdGenerator::getIdGenerator()->generate().to_string()); - *user_connection_data = reinterpret_cast(id); - return 0; - }; - - connection_close = [](const struct mg_connection* conn) -> void { - auto user_connection_data = reinterpret_cast*>(mg_get_user_connection_data(conn)); - delete user_connection_data; - }; - } -}; -} // namespace details - -class ConnectionCountingServer { - public: - ConnectionCountingServer() { - server_.addHandler("/method", numbered_method_responder_); - server_.addHandler("/reverse", reverse_body_post_handler_); - } - - size_t getConnectionCounter() { return connections_.size(); } - - std::string getPort() { - const auto& listening_ports = server_.getListeningPorts(); - assert(!listening_ports.empty()); - return std::to_string(listening_ports[0]); - } - - private: - static inline std::vector options = { - "enable_keep_alive", "yes", - "keep_alive_timeout_ms", "15000", - "num_threads", "1", - "listening_ports", "0"}; - - std::set> connections_; - details::AddIdToUserConnectionData add_id_to_user_connection_data_; - CivetServer server_{options, &add_id_to_user_connection_data_}; - details::ReverseBodyPostHandler reverse_body_post_handler_{connections_}; - details::NumberedMethodResponder numbered_method_responder_{connections_}; -}; - -} // namespace org::apache::nifi::minifi::extensions::curl::testing diff --git a/extensions/kubernetes/CMakeLists.txt b/extensions/kubernetes/CMakeLists.txt index 128e644d9c..4ac885c99f 100644 --- a/extensions/kubernetes/CMakeLists.txt +++ b/extensions/kubernetes/CMakeLists.txt @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -if (NOT (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND (ENABLE_ALL OR ENABLE_KUBERNETES)) OR NOT MINIFI_OPENSSL) +if (NOT (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND (ENABLE_ALL OR ENABLE_KUBERNETES))) return() endif() @@ -24,7 +24,7 @@ include(${CMAKE_SOURCE_DIR}/extensions/ExtensionHeader.txt) file(GLOB SOURCES "*.cpp" "controllerservice/*.cpp" "processors/*.cpp") add_minifi_library(minifi-kubernetes-extensions SHARED ${SOURCES}) -target_link_libraries(minifi-kubernetes-extensions ${LIBMINIFI} kubernetes CURL::libcurl) +target_link_libraries(minifi-kubernetes-extensions ${LIBMINIFI} kubernetes) register_extension(minifi-kubernetes-extensions "KUBERNETES EXTENSIONS" KUBERNETES-EXTENSIONS "This enables Kubernetes support" "extensions/kubernetes/tests") register_extension_linter(minifi-kubernetes-extensions-linter) diff --git a/extensions/kubernetes/tests/CMakeLists.txt b/extensions/kubernetes/tests/CMakeLists.txt index 09d982d581..0c2c0d295e 100644 --- a/extensions/kubernetes/tests/CMakeLists.txt +++ b/extensions/kubernetes/tests/CMakeLists.txt @@ -23,10 +23,9 @@ FOREACH(TEST_FILE ${KUBERNETES_UNIT_TESTS}) get_filename_component(TEST_TARGET "${TEST_FILE}" NAME_WE) add_minifi_executable("${TEST_TARGET}" "${TEST_FILE}") target_include_directories(${TEST_TARGET} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi") - target_include_directories(${TEST_TARGET} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test") target_include_directories(${TEST_TARGET} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/kubernetes") target_link_libraries(${TEST_TARGET} minifi-kubernetes-extensions) - target_link_libraries(${TEST_TARGET} Catch2WithMain) + target_link_libraries(${TEST_TARGET} Catch2::Catch2WithMain) createTests("${TEST_TARGET}") add_test(NAME ${TEST_TARGET} COMMAND "${TEST_TARGET}" WORKING_DIRECTORY "${TEST_DIR}") MATH(EXPR KUBERNETES_UNIT_TEST_COUNT "${KUBERNETES_UNIT_TEST_COUNT}+1") diff --git a/extensions/kubernetes/tests/KubernetesMetricsFilterTests.cpp b/extensions/kubernetes/tests/KubernetesMetricsFilterTests.cpp index 9e9fb71785..219208009a 100644 --- a/extensions/kubernetes/tests/KubernetesMetricsFilterTests.cpp +++ b/extensions/kubernetes/tests/KubernetesMetricsFilterTests.cpp @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "Catch.h" +#include "unit/Catch.h" #include "ContainerInfo.h" #include "MetricsFilter.h" diff --git a/extensions/libarchive/tests/ArchiveStreamTests.cpp b/extensions/libarchive/tests/ArchiveStreamTests.cpp index 072ed71835..6540eaa317 100644 --- a/extensions/libarchive/tests/ArchiveStreamTests.cpp +++ b/extensions/libarchive/tests/ArchiveStreamTests.cpp @@ -16,8 +16,8 @@ * limitations under the License. */ -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "WriteArchiveStream.h" #include "ReadArchiveStream.h" diff --git a/extensions/libarchive/tests/CMakeLists.txt b/extensions/libarchive/tests/CMakeLists.txt index 10bd7d5580..1358371492 100644 --- a/extensions/libarchive/tests/CMakeLists.txt +++ b/extensions/libarchive/tests/CMakeLists.txt @@ -25,11 +25,10 @@ FOREACH(testfile ${ARCHIVE_INTEGRATION_TESTS}) add_minifi_executable("${testfilename}" "${testfile}" "util/ArchiveTests.cpp") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/libarchive") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test/") target_link_libraries(${testfilename} minifi-archive-extensions) target_link_libraries(${testfilename} minifi-standard-processors) createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) ENDFOREACH() diff --git a/extensions/libarchive/tests/CompressContentTests.cpp b/extensions/libarchive/tests/CompressContentTests.cpp index 41e7815aef..7594151b45 100644 --- a/extensions/libarchive/tests/CompressContentTests.cpp +++ b/extensions/libarchive/tests/CompressContentTests.cpp @@ -27,8 +27,8 @@ #include #include "FlowController.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/Core.h" #include "../../include/core/FlowFile.h" #include "unit/ProvenanceTestHelper.h" @@ -43,7 +43,7 @@ #include "processors/GetFile.h" #include "processors/PutFile.h" #include "utils/file/FileUtils.h" -#include "Utils.h" +#include "unit/TestUtils.h" #include "utils/gsl.h" class ReadCallback { diff --git a/extensions/libarchive/tests/FocusArchiveTests.cpp b/extensions/libarchive/tests/FocusArchiveTests.cpp index 8b54fd8cb3..56c7c83dfe 100644 --- a/extensions/libarchive/tests/FocusArchiveTests.cpp +++ b/extensions/libarchive/tests/FocusArchiveTests.cpp @@ -32,8 +32,8 @@ #include "processors/LogAttribute.h" #include "processors/PutFile.h" #include "UnfocusArchiveEntry.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "repository/VolatileContentRepository.h" #include "unit/ProvenanceTestHelper.h" diff --git a/extensions/libarchive/tests/ManipulateArchiveTests.cpp b/extensions/libarchive/tests/ManipulateArchiveTests.cpp index bddea2128a..67ae36b183 100644 --- a/extensions/libarchive/tests/ManipulateArchiveTests.cpp +++ b/extensions/libarchive/tests/ManipulateArchiveTests.cpp @@ -21,8 +21,8 @@ #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "ArchiveTests.h" #include "core/PropertyDefinition.h" #include "processors/GetFile.h" diff --git a/extensions/libarchive/tests/MergeFileTests.cpp b/extensions/libarchive/tests/MergeFileTests.cpp index 8c3015712c..5a6cb5ca43 100644 --- a/extensions/libarchive/tests/MergeFileTests.cpp +++ b/extensions/libarchive/tests/MergeFileTests.cpp @@ -35,13 +35,13 @@ #include "../../include/core/FlowFile.h" #include "MergeContent.h" #include "processors/LogAttribute.h" -#include "TestBase.h" -#include "SingleProcessorTestController.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/SingleProcessorTestController.h" #include "unit/ProvenanceTestHelper.h" #include "serialization/FlowFileV3Serializer.h" #include "serialization/PayloadSerializer.h" -#include "Utils.h" +#include "unit/TestUtils.h" #include "utils/gsl.h" #include "utils/span.h" diff --git a/extensions/libarchive/tests/util/ArchiveTests.cpp b/extensions/libarchive/tests/util/ArchiveTests.cpp index 52eb8797c6..c5d09d83bd 100644 --- a/extensions/libarchive/tests/util/ArchiveTests.cpp +++ b/extensions/libarchive/tests/util/ArchiveTests.cpp @@ -27,8 +27,8 @@ #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/gsl.h" TAE_MAP_T build_test_archive_map(int NUM_FILES, const char* const* FILE_NAMES, const char* const* FILE_CONTENT) { diff --git a/extensions/librdkafka/tests/CMakeLists.txt b/extensions/librdkafka/tests/CMakeLists.txt index aaec9c4969..c8d0798951 100644 --- a/extensions/librdkafka/tests/CMakeLists.txt +++ b/extensions/librdkafka/tests/CMakeLists.txt @@ -24,9 +24,8 @@ FOREACH(testfile ${KAFKA_INTEGRATION_TESTS}) get_filename_component(testfilename "${testfile}" NAME_WE) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/librdkafka") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-rdkafka-extensions) target_link_libraries(${testfilename} minifi-standard-processors) MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1") diff --git a/extensions/librdkafka/tests/PublishKafkaTests.cpp b/extensions/librdkafka/tests/PublishKafkaTests.cpp index 78d65b01c5..8180edab6b 100644 --- a/extensions/librdkafka/tests/PublishKafkaTests.cpp +++ b/extensions/librdkafka/tests/PublishKafkaTests.cpp @@ -15,10 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "PublishKafka.h" -#include "SingleProcessorTestController.h" +#include "unit/SingleProcessorTestController.h" namespace org::apache::nifi::minifi::test { diff --git a/extensions/lua/tests/CMakeLists.txt b/extensions/lua/tests/CMakeLists.txt index a64bbcaef6..9922d94e35 100644 --- a/extensions/lua/tests/CMakeLists.txt +++ b/extensions/lua/tests/CMakeLists.txt @@ -25,10 +25,9 @@ FOREACH(testfile ${EXECUTESCRIPT_LUA_TESTS}) get_filename_component(testfilename "${testfile}" NAME_WE) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/lua") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test/") target_include_directories(${testfilename} SYSTEM PRIVATE BEFORE "${SOL2_INCLUDE_DIR}") target_link_libraries(${testfilename} minifi-lua-script-extension minifi-script-extension) - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) createTests("${testfilename}") MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/extensions/lua/tests/LuaScriptEngineTests.cpp b/extensions/lua/tests/LuaScriptEngineTests.cpp index 43a879cbe2..737d8d6b2e 100644 --- a/extensions/lua/tests/LuaScriptEngineTests.cpp +++ b/extensions/lua/tests/LuaScriptEngineTests.cpp @@ -15,9 +15,9 @@ * limitations under the License. */ -#include "TestBase.h" -#include "Catch.h" -#include "Utils.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/TestUtils.h" #include "LuaScriptEngine.h" namespace org::apache::nifi::minifi::extensions::lua::test { diff --git a/extensions/lua/tests/TestExecuteScriptProcessorWithLuaScript.cpp b/extensions/lua/tests/TestExecuteScriptProcessorWithLuaScript.cpp index 992a4bb5df..d680b64420 100644 --- a/extensions/lua/tests/TestExecuteScriptProcessorWithLuaScript.cpp +++ b/extensions/lua/tests/TestExecuteScriptProcessorWithLuaScript.cpp @@ -20,9 +20,9 @@ #include #include -#include "SingleProcessorTestController.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/SingleProcessorTestController.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "../../script/ExecuteScript.h" #include "utils/file/FileUtils.h" diff --git a/extensions/mqtt/tests/CMakeLists.txt b/extensions/mqtt/tests/CMakeLists.txt index d5b408a64a..913eb1ef77 100644 --- a/extensions/mqtt/tests/CMakeLists.txt +++ b/extensions/mqtt/tests/CMakeLists.txt @@ -25,7 +25,7 @@ FOREACH(testfile ${MQTT_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_CURRENT_SOURCE_DIR}/../../extensions/standard-processors") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_CURRENT_SOURCE_DIR}/../../../libminifi/test/") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-mqtt-extensions) target_link_libraries(${testfilename} minifi-standard-processors) add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/extensions/mqtt/tests/ConsumeMQTTTests.cpp b/extensions/mqtt/tests/ConsumeMQTTTests.cpp index c351338465..84729e40f4 100644 --- a/extensions/mqtt/tests/ConsumeMQTTTests.cpp +++ b/extensions/mqtt/tests/ConsumeMQTTTests.cpp @@ -16,9 +16,9 @@ * limitations under the License. */ -#include "Catch.h" +#include "unit/Catch.h" #include "catch2/matchers/catch_matchers_string.hpp" -#include "TestBase.h" +#include "unit/TestBase.h" #include "../processors/ConsumeMQTT.h" namespace { diff --git a/extensions/mqtt/tests/PublishMQTTTests.cpp b/extensions/mqtt/tests/PublishMQTTTests.cpp index 6002f34f2f..24301f4909 100644 --- a/extensions/mqtt/tests/PublishMQTTTests.cpp +++ b/extensions/mqtt/tests/PublishMQTTTests.cpp @@ -18,9 +18,9 @@ #include "range/v3/algorithm/find_if.hpp" -#include "Catch.h" +#include "unit/Catch.h" #include "catch2/matchers/catch_matchers_string.hpp" -#include "TestBase.h" +#include "unit/TestBase.h" #include "../processors/PublishMQTT.h" using namespace std::literals::chrono_literals; diff --git a/extensions/opc/CMakeLists.txt b/extensions/opc/CMakeLists.txt index 9f1267cf92..0d120de83d 100644 --- a/extensions/opc/CMakeLists.txt +++ b/extensions/opc/CMakeLists.txt @@ -38,7 +38,7 @@ file(GLOB SOURCES "src/*.cpp") add_minifi_library(minifi-opc-extensions SHARED ${SOURCES}) target_link_libraries(minifi-opc-extensions ${LIBMINIFI} Threads::Threads) -target_link_libraries(minifi-opc-extensions ${CMAKE_DL_LIBS} spdlog open62541::open62541) +target_link_libraries(minifi-opc-extensions ${CMAKE_DL_LIBS} spdlog::spdlog open62541::open62541) register_extension(minifi-opc-extensions "OPC EXTENSIONS" OPC-EXTENSIONS "This enables OPC-UA support") register_extension_linter(minifi-opc-extensions-linter) diff --git a/extensions/opencv/tests/CMakeLists.txt b/extensions/opencv/tests/CMakeLists.txt index d095c7e9a2..cc34d8550f 100644 --- a/extensions/opencv/tests/CMakeLists.txt +++ b/extensions/opencv/tests/CMakeLists.txt @@ -31,6 +31,6 @@ FOREACH(testfile ${OPENCV_TESTS}) createTests("${testfilename}") MATH(EXPR OPENCV_TEST_COUNT "${OPENCV_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) ENDFOREACH() message("-- Finished building ${OPENCV_TEST_COUNT} OpenCV related test file(s)...") diff --git a/extensions/opencv/tests/CaptureRTSPFrameTest.cpp b/extensions/opencv/tests/CaptureRTSPFrameTest.cpp index 2607ad8116..62890f28e7 100644 --- a/extensions/opencv/tests/CaptureRTSPFrameTest.cpp +++ b/extensions/opencv/tests/CaptureRTSPFrameTest.cpp @@ -28,15 +28,15 @@ #include "FlowFile.h" #include "core/Core.h" -#include "../../../libminifi/test/TestBase.h" -#include "../../../libminifi/test/Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "FlowController.h" #include "core/Processor.h" #include "core/ProcessorNode.h" #include "core/ProcessContext.h" #include "core/ProcessSession.h" #include "processors/LogAttribute.h" -#include "../../../libminifi/test/unit/ProvenanceTestHelper.h" +#include "unit/ProvenanceTestHelper.h" // TODO(_): valid capture test needs to be fixed TEST_CASE("CaptureRTSPFrame::ValidCapture", "[!mayfail]") { diff --git a/extensions/openwsman/CMakeLists.txt b/extensions/openwsman/CMakeLists.txt index f7152cad9b..cc1dd45adf 100644 --- a/extensions/openwsman/CMakeLists.txt +++ b/extensions/openwsman/CMakeLists.txt @@ -17,7 +17,7 @@ # under the License. # -if (WIN32 OR NOT ((ENABLE_ALL OR ENABLE_OPENWSMAN) AND ENABLE_CIVET AND ENABLE_CURL)) +if (WIN32 OR NOT ((ENABLE_ALL OR ENABLE_OPENWSMAN) AND ENABLE_CIVET)) return() endif() @@ -31,7 +31,8 @@ file(GLOB SOURCES "processors/*.cpp") add_minifi_library(minifi-openwsman SHARED ${SOURCES}) target_link_libraries(minifi-openwsman ${LIBMINIFI} Threads::Threads) -target_link_libraries(minifi-openwsman OpenWSMAN::libwsman civetweb::civetweb-cpp civetweb::c-library LibXml2::LibXml2) +# target_link_libraries(minifi-openwsman OpenWSMAN::libwsman civetweb::civetweb-cpp civetweb::c-library LibXml2::LibXml2) +target_link_libraries(minifi-openwsman OpenWSMAN::libwsman civetweb::civetweb-cpp LibXml2::LibXml2) register_extension(minifi-openwsman "OPENWSMAN EXTENSIONS" OPENWSMAN-EXTENSIONS "This enables Openwsman support") register_extension_linter(minifi-openwsman-linter) diff --git a/extensions/pcap/tests/CMakeLists.txt b/extensions/pcap/tests/CMakeLists.txt index ee23e165f7..a6c70f8275 100644 --- a/extensions/pcap/tests/CMakeLists.txt +++ b/extensions/pcap/tests/CMakeLists.txt @@ -20,23 +20,17 @@ file(GLOB PCAP_TESTS "*.cpp") SET(PCAP_INT_TEST_COUNT 0) - FOREACH(testfile ${PCAP_TESTS}) get_filename_component(testfilename "${testfile}" NAME_WE) - add_minifi_executable("${testfilename}" "${testfile}" ) - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test/") + add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/pcap/") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_BINARY_DIR}/extensions/pcap/pcap++/Dist/header/") - createTests("${testfilename}") + createIntegrationTests("${testfilename}") + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-pcap) target_link_libraries(${testfilename} minifi-standard-processors) + target_compile_definitions(${testfilename} PRIVATE TEST_RESOURCES="${TEST_RESOURCES}") MATH(EXPR PCAP_INT_TEST_COUNT "${PCAP_INT_TEST_COUNT}+1") + add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) ENDFOREACH() - message("-- Finished building ${PCAP_INT_TEST_COUNT} libPCAP test file(s)...") -if(APPLE) - add_test(NAME PcapTest COMMAND PcapTest "${TEST_RESOURCES}/TestPcap.yml" "${TEST_RESOURCES}/") -else() - add_test(NAME PcapTest COMMAND PcapTest "${TEST_RESOURCES}/TestPcapLinux.yml" "${TEST_RESOURCES}/") -endif() - diff --git a/extensions/pcap/tests/PcapTest.cpp b/extensions/pcap/tests/PcapTest.cpp index 9d79eee640..e73aead180 100644 --- a/extensions/pcap/tests/PcapTest.cpp +++ b/extensions/pcap/tests/PcapTest.cpp @@ -15,19 +15,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG -#include #include #include -#include "TestBase.h" +#include "unit/TestBase.h" #include "core/ProcessGroup.h" #include "FlowController.h" #include "core/ConfigurableComponent.h" #include "core/state/ProcessorController.h" #include "integration/IntegrationBase.h" #include "CapturePacket.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" +#include "unit/Catch.h" + +namespace org::apache::nifi::minifi::test { class PcapTestHarness : public IntegrationBase { public: @@ -49,8 +49,7 @@ class PcapTestHarness : public IntegrationBase { } void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; - assert(verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), // FIXME(fgerlits): These assertions don't work, but the test is still useful to check that the processor starts // "Starting capture", // "Stopping capture", @@ -75,14 +74,15 @@ class PcapTestHarness : public IntegrationBase { std::string dir = testController.createTempDirectory(); }; -int main(int argc, char **argv) { - std::string test_file_location; - if (argc > 1) { - test_file_location = argv[1]; - } - +TEST_CASE("PcapTest", "[pcap]") { PcapTestHarness harness; - harness.setKeyDir(""); - harness.run(test_file_location); - return 0; +#ifdef __APPLE__ + std::string test_file_name = "TestPcap.yml"; +#else + std::string test_file_name = "TestPcapLinux.yml"; +#endif + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / test_file_name; + harness.run(test_file_path); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/pdh/tests/CMakeLists.txt b/extensions/pdh/tests/CMakeLists.txt index d50549f337..5ba1b57585 100644 --- a/extensions/pdh/tests/CMakeLists.txt +++ b/extensions/pdh/tests/CMakeLists.txt @@ -25,9 +25,8 @@ FOREACH(testfile ${PDH_TESTS}) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/pdh") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test/") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-pdh) target_link_libraries(${testfilename} minifi-standard-processors) MATH(EXPR PDH_TEST_COUNT "${PDH_TEST_COUNT}+1") diff --git a/extensions/pdh/tests/PerformanceDataCounterTests.cpp b/extensions/pdh/tests/PerformanceDataCounterTests.cpp index 775c7cf3ff..969763e0d4 100644 --- a/extensions/pdh/tests/PerformanceDataCounterTests.cpp +++ b/extensions/pdh/tests/PerformanceDataCounterTests.cpp @@ -16,8 +16,8 @@ * limitations under the License. */ -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "PDHCounters.h" #include "MemoryConsumptionCounter.h" #include "utils/gsl.h" diff --git a/extensions/pdh/tests/PerformanceDataMonitorTests.cpp b/extensions/pdh/tests/PerformanceDataMonitorTests.cpp index 4d57969b96..2515909047 100644 --- a/extensions/pdh/tests/PerformanceDataMonitorTests.cpp +++ b/extensions/pdh/tests/PerformanceDataMonitorTests.cpp @@ -22,11 +22,11 @@ #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "processors/PutFile.h" #include "utils/file/FileUtils.h" -#include "utils/TestUtils.h" +#include "unit/TestUtils.h" #include "PerformanceDataMonitor.h" #include "rapidjson/filereadstream.h" diff --git a/extensions/procfs/tests/CMakeLists.txt b/extensions/procfs/tests/CMakeLists.txt index ec0a8e8e86..060f51dadd 100644 --- a/extensions/procfs/tests/CMakeLists.txt +++ b/extensions/procfs/tests/CMakeLists.txt @@ -25,10 +25,9 @@ FOREACH(testfile ${PROCFS_TESTS}) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/procfs") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test/") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-procfs) target_link_libraries(${testfilename} minifi-standard-processors) MATH(EXPR PROCFS_TEST_COUNT "${PROCFS_TEST_COUNT}+1") diff --git a/extensions/procfs/tests/CPUStatTests.cpp b/extensions/procfs/tests/CPUStatTests.cpp index 27507b25b9..be4d478426 100644 --- a/extensions/procfs/tests/CPUStatTests.cpp +++ b/extensions/procfs/tests/CPUStatTests.cpp @@ -15,7 +15,7 @@ * limitations under the License. */ -#include "Catch.h" +#include "unit/Catch.h" #include "catch2/catch_approx.hpp" #include "ProcFs.h" #include "MockProcFs.h" diff --git a/extensions/procfs/tests/DiskStatTests.cpp b/extensions/procfs/tests/DiskStatTests.cpp index 47a4468abe..309a75069c 100644 --- a/extensions/procfs/tests/DiskStatTests.cpp +++ b/extensions/procfs/tests/DiskStatTests.cpp @@ -15,7 +15,7 @@ * limitations under the License. */ -#include "Catch.h" +#include "unit/Catch.h" #include "ProcFs.h" #include "DiskStat.h" #include "MockProcFs.h" diff --git a/extensions/procfs/tests/MemInfoTests.cpp b/extensions/procfs/tests/MemInfoTests.cpp index fe3b9c1ce3..ca5ee64c3a 100644 --- a/extensions/procfs/tests/MemInfoTests.cpp +++ b/extensions/procfs/tests/MemInfoTests.cpp @@ -15,7 +15,7 @@ * limitations under the License. */ -#include "Catch.h" +#include "unit/Catch.h" #include "ProcFs.h" #include "Literals.h" #include "MockProcFs.h" diff --git a/extensions/procfs/tests/NetDevTests.cpp b/extensions/procfs/tests/NetDevTests.cpp index be21926557..85a11d62fb 100644 --- a/extensions/procfs/tests/NetDevTests.cpp +++ b/extensions/procfs/tests/NetDevTests.cpp @@ -15,7 +15,7 @@ * limitations under the License. */ -#include "Catch.h" +#include "unit/Catch.h" #include "ProcFs.h" #include "MockProcFs.h" diff --git a/extensions/procfs/tests/ProcFsMonitorTests.cpp b/extensions/procfs/tests/ProcFsMonitorTests.cpp index 7c4a744a5c..1e40eac4f8 100644 --- a/extensions/procfs/tests/ProcFsMonitorTests.cpp +++ b/extensions/procfs/tests/ProcFsMonitorTests.cpp @@ -16,8 +16,8 @@ * limitations under the License. */ -#include "SingleProcessorTestController.h" -#include "Catch.h" +#include "unit/SingleProcessorTestController.h" +#include "unit/Catch.h" #include "processors/ProcFsMonitor.h" namespace org::apache::nifi::minifi::extensions::procfs::tests { diff --git a/extensions/procfs/tests/ProcessStatTests.cpp b/extensions/procfs/tests/ProcessStatTests.cpp index 8d797e348c..6913250228 100644 --- a/extensions/procfs/tests/ProcessStatTests.cpp +++ b/extensions/procfs/tests/ProcessStatTests.cpp @@ -15,7 +15,7 @@ * limitations under the License. */ -#include "Catch.h" +#include "unit/Catch.h" #include "ProcFs.h" #include "MockProcFs.h" diff --git a/extensions/prometheus/tests/CMakeLists.txt b/extensions/prometheus/tests/CMakeLists.txt index 6f0e72c3e3..ebe0db595d 100644 --- a/extensions/prometheus/tests/CMakeLists.txt +++ b/extensions/prometheus/tests/CMakeLists.txt @@ -24,9 +24,8 @@ FOREACH(testfile ${PROMETHEUS_TESTS}) get_filename_component(testfilename "${testfile}" NAME_WE) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/prometheus") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test/") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-prometheus) MATH(EXPR PROMETHEUS_TEST_COUNT "${PROMETHEUS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}") diff --git a/extensions/prometheus/tests/PrometheusMetricsPublisherTest.cpp b/extensions/prometheus/tests/PrometheusMetricsPublisherTest.cpp index 365d892efe..377befea0e 100644 --- a/extensions/prometheus/tests/PrometheusMetricsPublisherTest.cpp +++ b/extensions/prometheus/tests/PrometheusMetricsPublisherTest.cpp @@ -18,7 +18,7 @@ #include #include -#include "Catch.h" +#include "unit/Catch.h" #include "PrometheusMetricsPublisher.h" #include "properties/Configure.h" #include "MetricsExposer.h" diff --git a/extensions/python/tests/CMakeLists.txt b/extensions/python/tests/CMakeLists.txt index 93ac1c8d24..e2f12ed72a 100644 --- a/extensions/python/tests/CMakeLists.txt +++ b/extensions/python/tests/CMakeLists.txt @@ -28,8 +28,7 @@ FOREACH(testfile ${EXECUTESCRIPT_PYTHON_TESTS}) get_filename_component(testfilename "${testfile}" NAME_WE) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/python") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test/") - target_link_libraries(${testfilename} minifi-python-script-extension minifi-script-extension Catch2WithMain Python::Python) + target_link_libraries(${testfilename} minifi-python-script-extension minifi-script-extension Catch2::Catch2WithMain Python::Python) createTests("${testfilename}") MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) @@ -44,9 +43,8 @@ FOREACH(testfile ${EXECUTEPYTHONPROCESSOR_UNIT_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/python") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test/") - target_link_libraries(${testfilename} minifi-python-script-extension minifi-script-extension minifi-standard-processors Catch2WithMain Python::Python) + target_link_libraries(${testfilename} minifi-python-script-extension minifi-script-extension minifi-standard-processors Catch2::Catch2WithMain Python::Python) createTests("${testfilename}") MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1") diff --git a/extensions/python/tests/ExecutePythonProcessorTests.cpp b/extensions/python/tests/ExecutePythonProcessorTests.cpp index 50c6b6d7eb..28736fbcd8 100644 --- a/extensions/python/tests/ExecutePythonProcessorTests.cpp +++ b/extensions/python/tests/ExecutePythonProcessorTests.cpp @@ -19,8 +19,8 @@ #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "processors/GetFile.h" #include "ExecutePythonProcessor.h" @@ -28,11 +28,11 @@ #include "processors/PutFile.h" #include "utils/file/FileUtils.h" #include "utils/file/PathUtils.h" -#include "utils/TestUtils.h" +#include "unit/TestUtils.h" namespace { -using org::apache::nifi::minifi::utils::putFileToDir; -using org::apache::nifi::minifi::utils::getFileContent; +using org::apache::nifi::minifi::test::utils::putFileToDir; +using org::apache::nifi::minifi::test::utils::getFileContent; using org::apache::nifi::minifi::utils::file::resolve; class ExecutePythonProcessorTestBase { diff --git a/extensions/python/tests/PythonManifestTests.cpp b/extensions/python/tests/PythonManifestTests.cpp index 7d0792a97f..61b76e43bf 100644 --- a/extensions/python/tests/PythonManifestTests.cpp +++ b/extensions/python/tests/PythonManifestTests.cpp @@ -17,13 +17,12 @@ #define CUSTOM_EXTENSION_INIT -#undef NDEBUG #include -#include "TestBase.h" -#include "Catch.h" -#include "flow-tests/TestControllerWithFlow.h" -#include "EmptyFlow.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/TestControllerWithFlow.h" +#include "unit/EmptyFlow.h" #include "c2/C2MetricsPublisher.h" #include "utils/gsl.h" @@ -60,7 +59,7 @@ TEST_CASE("Python processor's description is part of the manifest") { " proc.addProperty('Prop1', 'A great property', 'banana', True, False)\n"; controller.configuration_->set(minifi::Configuration::nifi_python_processor_dir, python_dir.string()); - controller.configuration_->set(minifi::Configuration::nifi_extension_path, "*minifi-python-script*,*minifi-http-curl*"); + controller.configuration_->set(minifi::Configuration::nifi_extension_path, "*minifi-python-script*"); core::extension::ExtensionManager::get().initialize(controller.configuration_); diff --git a/extensions/python/tests/PythonScriptEngineTests.cpp b/extensions/python/tests/PythonScriptEngineTests.cpp index f6d974c6df..eac90f98e5 100644 --- a/extensions/python/tests/PythonScriptEngineTests.cpp +++ b/extensions/python/tests/PythonScriptEngineTests.cpp @@ -16,9 +16,9 @@ */ #include -#include "TestBase.h" -#include "Catch.h" -#include "Utils.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/TestUtils.h" #include "PythonScriptEngine.h" namespace org::apache::nifi::minifi::extensions::python { diff --git a/extensions/python/tests/TestExecuteScriptProcessorWithPythonScript.cpp b/extensions/python/tests/TestExecuteScriptProcessorWithPythonScript.cpp index 86f2fbde00..e49af68be5 100644 --- a/extensions/python/tests/TestExecuteScriptProcessorWithPythonScript.cpp +++ b/extensions/python/tests/TestExecuteScriptProcessorWithPythonScript.cpp @@ -19,9 +19,9 @@ #include #include -#include "SingleProcessorTestController.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/SingleProcessorTestController.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "../../script/ExecuteScript.h" #include "utils/file/FileUtils.h" diff --git a/extensions/rocksdb-repos/tests/CMakeLists.txt b/extensions/rocksdb-repos/tests/CMakeLists.txt index 9f4754c120..18088e0c91 100644 --- a/extensions/rocksdb-repos/tests/CMakeLists.txt +++ b/extensions/rocksdb-repos/tests/CMakeLists.txt @@ -24,9 +24,8 @@ FOREACH(testfile ${ROCKSDB_UNIT_TESTS}) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/rocksdb-repos/") target_include_directories(${testfilename} SYSTEM BEFORE PRIVATE "${ROCKSDB_THIRDPARTY_ROOT}/include") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-rocksdb-repos) target_link_libraries(${testfilename} minifi-standard-processors) MATH(EXPR ROCKSDB_TEST_COUNT "${ROCKSDB_TEST_COUNT}+1") diff --git a/extensions/rocksdb-repos/tests/ContentSessionTests.cpp b/extensions/rocksdb-repos/tests/ContentSessionTests.cpp index b6985bade5..8d35589e1b 100644 --- a/extensions/rocksdb-repos/tests/ContentSessionTests.cpp +++ b/extensions/rocksdb-repos/tests/ContentSessionTests.cpp @@ -26,8 +26,8 @@ #include "DatabaseContentRepository.h" #include "BufferedContentSession.h" #include "FlowFileRecord.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/gsl.h" template diff --git a/extensions/rocksdb-repos/tests/DBContentRepositoryTests.cpp b/extensions/rocksdb-repos/tests/DBContentRepositoryTests.cpp index 5f9eb7b25b..2cd1898665 100644 --- a/extensions/rocksdb-repos/tests/DBContentRepositoryTests.cpp +++ b/extensions/rocksdb-repos/tests/DBContentRepositoryTests.cpp @@ -24,11 +24,11 @@ #include "FlowFileRecord.h" #include "properties/Configure.h" #include "provenance/Provenance.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "unit/ProvenanceTestHelper.h" #include "unit/ContentRepositoryDependentTests.h" -#include "IntegrationTestUtils.h" +#include "unit/TestUtils.h" class TestDatabaseContentRepository : public core::repository::DatabaseContentRepository { public: @@ -139,7 +139,7 @@ TEST_CASE("Delete Claim", "[TestDBCR2]") { // an immediate read will still be able to access the content REQUIRE_FALSE(minifi::io::isError(content_repo->read(*claim)->read(readstr))); - REQUIRE(minifi::utils::verifyEventHappenedInPollTime(1s, [&] { + REQUIRE(minifi::test::utils::verifyEventHappenedInPollTime(1s, [&] { return minifi::io::isError(content_repo->read(*claim)->read(readstr)); })); } @@ -318,7 +318,6 @@ TEST_CASE("ProcessSession::read reads the flowfile from offset to size (RocksDB) ContentRepositoryDependentTests::testReadOnSmallerClonedFlowFiles(std::make_shared()); } - TEST_CASE("ProcessSession::append should append to the flowfile and set its size correctly (RocksDB)" "[appendsetsize]") { ContentRepositoryDependentTests::testAppendToUnmanagedFlowFile(std::make_shared()); diff --git a/extensions/rocksdb-repos/tests/DBProvenanceRepositoryTests.cpp b/extensions/rocksdb-repos/tests/DBProvenanceRepositoryTests.cpp index ecd2d0266c..cab212cc3e 100644 --- a/extensions/rocksdb-repos/tests/DBProvenanceRepositoryTests.cpp +++ b/extensions/rocksdb-repos/tests/DBProvenanceRepositoryTests.cpp @@ -22,8 +22,8 @@ #include #include "ProvenanceRepository.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" static constexpr size_t TEST_PROVENANCE_STORAGE_SIZE = 100_KiB; static constexpr size_t TEST_MAX_PROVENANCE_STORAGE_SIZE = 100_MiB; diff --git a/extensions/rocksdb-repos/tests/EncryptionTests.cpp b/extensions/rocksdb-repos/tests/EncryptionTests.cpp index 5c5377f29c..41f285d5d4 100644 --- a/extensions/rocksdb-repos/tests/EncryptionTests.cpp +++ b/extensions/rocksdb-repos/tests/EncryptionTests.cpp @@ -16,10 +16,10 @@ * limitations under the License. */ -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "FlowFileRepository.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" #include "repository/VolatileContentRepository.h" #include "FlowFileRecord.h" #include "utils/span.h" @@ -93,7 +93,7 @@ TEST_CASE_METHOD(FFRepoFixture, "FlowFileRepository creates checkpoint and loads runWithNewRepository([&] (const std::shared_ptr& /*repo*/) { // wait for the flowfiles to be loaded - bool success = utils::verifyEventHappenedInPollTime(std::chrono::seconds{5}, [&] { + bool success = minifi::test::utils::verifyEventHappenedInPollTime(std::chrono::seconds{5}, [&] { return !container_->isEmpty(); }); REQUIRE(success); diff --git a/extensions/rocksdb-repos/tests/ProvenanceTests.cpp b/extensions/rocksdb-repos/tests/ProvenanceTests.cpp index 7afa9a5178..402397cb80 100644 --- a/extensions/rocksdb-repos/tests/ProvenanceTests.cpp +++ b/extensions/rocksdb-repos/tests/ProvenanceTests.cpp @@ -28,8 +28,8 @@ #include "FlowFileRecord.h" #include "provenance/Provenance.h" #include "unit/ProvenanceTestHelper.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" namespace provenance = minifi::provenance; using namespace std::literals::chrono_literals; diff --git a/extensions/rocksdb-repos/tests/RepoTests.cpp b/extensions/rocksdb-repos/tests/RepoTests.cpp index f31e29fcfa..70e31c86f8 100644 --- a/extensions/rocksdb-repos/tests/RepoTests.cpp +++ b/extensions/rocksdb-repos/tests/RepoTests.cpp @@ -31,10 +31,10 @@ #include "provenance/Provenance.h" #include "properties/Configure.h" #include "unit/ProvenanceTestHelper.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/gsl.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" #include "core/repository/VolatileFlowFileRepository.h" #include "core/repository/VolatileProvenanceRepository.h" #include "DatabaseContentRepository.h" @@ -350,13 +350,12 @@ TEST_CASE("Test FlowFile Restore", "[TestFFR6]") { // check if the @input Connection's FlowFile was restored // upon the FlowFileRepository's startup std::shared_ptr newFlow = nullptr; - using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; + using org::apache::nifi::minifi::test::utils::verifyEventHappenedInPollTime; const auto flowFileArrivedInOutput = [&newFlow, &expiredFiles, inputPtr] { newFlow = inputPtr->poll(expiredFiles); return newFlow != nullptr; }; - assert(verifyEventHappenedInPollTime(std::chrono::seconds(10), flowFileArrivedInOutput, std::chrono::milliseconds(50))); - (void)flowFileArrivedInOutput; // unused in release builds + REQUIRE(verifyEventHappenedInPollTime(std::chrono::seconds(10), flowFileArrivedInOutput, std::chrono::milliseconds(50))); REQUIRE(expiredFiles.empty()); LogTestController::getInstance().reset(); @@ -445,8 +444,8 @@ TEST_CASE("Flush deleted flowfiles before shutdown", "[TestFFR7]") { ff_repository->loadComponent(content_repo); ff_repository->start(); std::this_thread::sleep_for(std::chrono::milliseconds(100)); - using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; - assert(verifyEventHappenedInPollTime(std::chrono::seconds(1), [&connection]{ return connection->getQueueSize() == 50; }, std::chrono::milliseconds(50))); + using org::apache::nifi::minifi::test::utils::verifyEventHappenedInPollTime; + REQUIRE(verifyEventHappenedInPollTime(std::chrono::seconds(1), [&connection]{ return connection->getQueueSize() == 50; }, std::chrono::milliseconds(50))); } } @@ -587,7 +586,7 @@ TEST_CASE("Test getting flow file repository size properties", "[TestGettingRepo } auto original_size = repository->getRepositorySize(); - using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; + using org::apache::nifi::minifi::test::utils::verifyEventHappenedInPollTime; REQUIRE(verifyEventHappenedInPollTime(std::chrono::seconds(5), [&original_size, &repository] { auto old_size = original_size; original_size = repository->getRepositorySize(); @@ -700,7 +699,7 @@ TEST_CASE("Test getting content repository size properties", "[TestGettingReposi repository->flush(); repository->stop(); - using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; + using org::apache::nifi::minifi::test::utils::verifyEventHappenedInPollTime; REQUIRE(verifyEventHappenedInPollTime(std::chrono::seconds(5), [&original_content_repo_size, &content_repo] { auto new_content_repo_size = content_repo->getRepositorySize(); return new_content_repo_size > original_content_repo_size; @@ -814,14 +813,30 @@ TEST_CASE("FlowFileRepository can filter out too small contents") { config->set(minifi::Configure::nifi_flowfile_repository_directory_default, ff_dir.string()); config->set(minifi::Configure::nifi_dbcontent_repository_directory_default, content_dir.string()); - const auto [check_health, expected_flowfiles] = GENERATE( - std::make_tuple("false", size_t{2}), - std::make_tuple("true", size_t{1})); - config->set(minifi::Configure::nifi_flow_file_repository_check_health, check_health); - const auto content_repo = GENERATE( - static_cast>(std::make_shared()), - static_cast>(std::make_shared())); + size_t expected_flowfiles = std::numeric_limits::max(); + std::shared_ptr content_repo; + // Sections are used instead of GENERATE macro to have the content_repo destructed before testController and be able to delete the temp directories + SECTION("nifi.flowfile.repository.check.health set to false") { + config->set(minifi::Configure::nifi_flow_file_repository_check_health, "false"); + expected_flowfiles = 2; + SECTION("FileSystemRepository") { + content_repo = std::make_shared(); + } + SECTION("RocksDB") { + content_repo = std::make_shared(); + } + } + SECTION("nifi.flowfile.repository.check.health set to true") { + config->set(minifi::Configure::nifi_flow_file_repository_check_health, "true"); + expected_flowfiles = 1; + SECTION("FileSystemRepository") { + content_repo = std::make_shared(); + } + SECTION("RocksDB") { + content_repo = std::make_shared(); + } + } auto ff_repo = std::make_shared(); REQUIRE(ff_repo->initialize(config)); diff --git a/extensions/rocksdb-repos/tests/RocksDBStreamTests.cpp b/extensions/rocksdb-repos/tests/RocksDBStreamTests.cpp index e9433e576d..5773b697b5 100644 --- a/extensions/rocksdb-repos/tests/RocksDBStreamTests.cpp +++ b/extensions/rocksdb-repos/tests/RocksDBStreamTests.cpp @@ -16,8 +16,8 @@ * limitations under the License. */ -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "../RocksDbStream.h" #include "../DatabaseContentRepository.h" #include "../database/StringAppender.h" diff --git a/extensions/rocksdb-repos/tests/RocksDBTests.cpp b/extensions/rocksdb-repos/tests/RocksDBTests.cpp index 8a923a0514..7991e1eef7 100644 --- a/extensions/rocksdb-repos/tests/RocksDBTests.cpp +++ b/extensions/rocksdb-repos/tests/RocksDBTests.cpp @@ -17,18 +17,16 @@ #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "../database/RocksDatabase.h" #include "../database/RocksDbInstance.h" #include "../database/ColumnHandle.h" #include "../database/DbHandle.h" -#include "IntegrationTestUtils.h" +#include "unit/TestUtils.h" #include "database/StringAppender.h" #include "../encryption/RocksDbEncryptionProvider.h" -#undef NDEBUG - using core::repository::StringAppender; struct OpenDatabase { @@ -72,7 +70,7 @@ void new_db_opts(minifi::internal::Writable& db_opts) { TEST_CASE_METHOD(RocksDBTest, "Malformed database uri - Missing column name", "[rocksDBTest1]") { auto db = minifi::internal::RocksDatabase::create({}, {}, "minifidb://malformed", {}); REQUIRE(!db); - REQUIRE(utils::verifyLogLinePresenceInPollTime( + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime( std::chrono::seconds{1}, "Couldn't detect the column name in 'minifidb://malformed'")); } @@ -157,7 +155,7 @@ TEST_CASE_METHOD(RocksDBTest, "Logged if the options are incompatible with an ex }; auto default_db = minifi::internal::RocksDatabase::create(new_db_opts, cf_opts, "minifidb://" + db_dir + "/default", {}); REQUIRE(default_db->open()); - REQUIRE(utils::verifyLogLinePresenceInPollTime( + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime( std::chrono::seconds{1}, "Could not determine if we definitely need to reopen or we are definitely safe, requesting reopen")); } @@ -174,7 +172,7 @@ TEST_CASE_METHOD(RocksDBTest, "Error is logged if different DBOptions are used", REQUIRE(col_1->open()); auto col_2 = minifi::internal::RocksDatabase::create(db_opt_2, {}, "minifidb://" + db_dir + "/column_two", {}); REQUIRE_FALSE(col_2->open()); - REQUIRE(utils::verifyLogLinePresenceInPollTime( + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime( std::chrono::seconds{1}, "Conflicting database options requested for '" + db_dir + "'")); } @@ -251,14 +249,14 @@ TEST_CASE_METHOD(RocksDBTest, "Error is logged if different encryption keys are auto db_opt_2 = createEncrSetter(home_dir, "two", "encryption.key.two"); auto col_2 = minifi::internal::RocksDatabase::create(db_opt_2, {}, "minifidb://" + db_dir + "/column_two", {}); REQUIRE_FALSE(col_2->open()); - REQUIRE(utils::verifyLogLinePresenceInPollTime( + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime( std::chrono::seconds{1}, "Conflicting database options requested for '" + db_dir + "'")); } SECTION("Using no encryption key") { auto col_2 = minifi::internal::RocksDatabase::create(withDefaultEnv, {}, "minifidb://" + db_dir + "/column_two", {}); REQUIRE_FALSE(col_2->open()); - REQUIRE(utils::verifyLogLinePresenceInPollTime( + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime( std::chrono::seconds{1}, "Conflicting database options requested for '" + db_dir + "'")); } } diff --git a/extensions/rocksdb-repos/tests/RocksDBUtilsTests.cpp b/extensions/rocksdb-repos/tests/RocksDBUtilsTests.cpp index 9b7f662f5f..a13e2493f2 100644 --- a/extensions/rocksdb-repos/tests/RocksDBUtilsTests.cpp +++ b/extensions/rocksdb-repos/tests/RocksDBUtilsTests.cpp @@ -18,8 +18,8 @@ #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "../database/RocksDbUtils.h" #include "properties/Configure.h" diff --git a/extensions/rocksdb-repos/tests/SwapTests.cpp b/extensions/rocksdb-repos/tests/SwapTests.cpp index 0c61dded2d..62822e8d7b 100644 --- a/extensions/rocksdb-repos/tests/SwapTests.cpp +++ b/extensions/rocksdb-repos/tests/SwapTests.cpp @@ -15,14 +15,13 @@ * limitations under the License. */ -#include "Catch.h" +#include "unit/Catch.h" #include "core/RepositoryFactory.h" #include "core/repository/VolatileContentRepository.h" #include "FlowFileRepository.h" -#include "TestBase.h" -#include "Utils.h" +#include "unit/TestBase.h" #include "StreamPipe.h" -#include "IntegrationTestUtils.h" +#include "unit/TestUtils.h" #include "core/Processor.h" #include "core/ProcessSession.h" #include "core/PropertyDefinition.h" @@ -125,7 +124,7 @@ TEST_CASE("Connection will on-demand swap flow files") { std::set> expired; for (size_t i = 0; i < 200; ++i) { std::shared_ptr ff; - bool got_non_null_flow_file = minifi::utils::verifyEventHappenedInPollTime(std::chrono::seconds{5}, [&] { + bool got_non_null_flow_file = utils::verifyEventHappenedInPollTime(std::chrono::seconds{5}, [&] { ff = connection->poll(expired); return static_cast(ff); }); diff --git a/extensions/script/tests/CMakeLists.txt b/extensions/script/tests/CMakeLists.txt index 121428ec5e..5c88322264 100644 --- a/extensions/script/tests/CMakeLists.txt +++ b/extensions/script/tests/CMakeLists.txt @@ -24,10 +24,9 @@ FOREACH(testfile ${EXECUTE_SCRIPT_TESTS}) get_filename_component(testfilename "${testfile}" NAME_WE) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/script") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test/") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-script-extension) MATH(EXPR EXECUTE_SCRIPT_TEST_COUNT "${EXECUTE_SCRIPT_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) diff --git a/extensions/script/tests/ExecuteScriptTests.cpp b/extensions/script/tests/ExecuteScriptTests.cpp index 3e8ff2e605..40d7c31693 100644 --- a/extensions/script/tests/ExecuteScriptTests.cpp +++ b/extensions/script/tests/ExecuteScriptTests.cpp @@ -17,9 +17,9 @@ */ #define EXTENSION_LIST "*minifi-*,!*python*,!*lua*" // NOLINT(cppcoreguidelines-macro-usage) -#include "SingleProcessorTestController.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/SingleProcessorTestController.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "../ExecuteScript.h" diff --git a/extensions/sensors/tests/CMakeLists.txt b/extensions/sensors/tests/CMakeLists.txt index 3a5170de73..a8fef5a98b 100644 --- a/extensions/sensors/tests/CMakeLists.txt +++ b/extensions/sensors/tests/CMakeLists.txt @@ -24,17 +24,18 @@ SET(SENSORS_INT_TEST_COUNT 0) FOREACH(testfile ${SENSORS_TESTS}) get_filename_component(testfilename "${testfile}" NAME_WE) add_minifi_executable("${testfilename}" "${testfile}" ) - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test/") target_include_directories(${testfilename} SYSTEM PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/thirdparty/RTIMULib/RTIMULib/") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/sensors/") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_BINARY_DIR}/extensions/sensors/") - createTests("${testfilename}") + createIntegrationTests("${testfilename}") + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-sensors) target_link_libraries(${testfilename} minifi-standard-processors) + target_compile_definitions(${testfilename} PRIVATE TEST_RESOURCES="${TEST_RESOURCES}") + add_test(NAME "${testfilename}" COMMAND "${testfilename}") MATH(EXPR SENSORS_INT_TEST_COUNT "${SENSORS_INT_TEST_COUNT}+1") ENDFOREACH() message("-- Finished building ${SENSORS_INT_TEST_COUNT} sensor(s) test file(s)...") -add_test(NAME SensorTests COMMAND SensorTests "${TEST_RESOURCES}/TestEnvironmental.yml" "${TEST_RESOURCES}/") diff --git a/extensions/sensors/tests/SensorTests.cpp b/extensions/sensors/tests/SensorTests.cpp index d8a4678f22..958d0e46a7 100644 --- a/extensions/sensors/tests/SensorTests.cpp +++ b/extensions/sensors/tests/SensorTests.cpp @@ -15,10 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - #include -#undef NDEBUG -#include #include #include #include @@ -29,7 +26,7 @@ #include #include #include -#include "TestBase.h" +#include "unit/TestBase.h" #include "utils/StringUtils.h" #include "core/Core.h" #include "core/logging/Logger.h" @@ -40,11 +37,14 @@ #include "core/ConfigurableComponent.h" #include "integration/IntegrationBase.h" #include "GetEnvironmentalSensors.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" +#include "unit/Catch.h" + +namespace org::apache::nifi::minifi::test { -class PcapTestHarness : public IntegrationBase { +class SensorTestHarness : public IntegrationBase { public: - PcapTestHarness() { + SensorTestHarness() { dir = testController.createTempDirectory(); } @@ -63,16 +63,15 @@ class PcapTestHarness : public IntegrationBase { } void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; - assert(verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), "Initializing EnvironmentalSensors")); + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), "Initializing EnvironmentalSensors")); } void queryRootProcessGroup(std::shared_ptr pg) override { const auto proc = pg->findProcessorByName("pcap"); - assert(proc != nullptr); + REQUIRE(proc != nullptr); auto inv = dynamic_cast(proc); - assert(inv != nullptr); + REQUIRE(inv != nullptr); configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_enable, "false"); } @@ -82,21 +81,10 @@ class PcapTestHarness : public IntegrationBase { TestController testController; }; -int main(int argc, char **argv) { - std::string key_dir; - std::string test_file_location; - std::string url; - - if (argc > 1) { - test_file_location = argv[1]; - } - - - PcapTestHarness harness; - - harness.setKeyDir(key_dir); - - harness.run(test_file_location); - - return 0; +TEST_CASE("SensorTests", "[sensor]") { + SensorTestHarness harness; + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestEnvironmental.yml"; + harness.run(test_file_path); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/sftp/CMakeLists.txt b/extensions/sftp/CMakeLists.txt index 169c7f8ccc..2cea03f2ba 100644 --- a/extensions/sftp/CMakeLists.txt +++ b/extensions/sftp/CMakeLists.txt @@ -17,7 +17,7 @@ # under the License. # -if (NOT ((ENABLE_ALL OR ENABLE_SFTP) AND ENABLE_CURL)) +if (NOT (ENABLE_ALL OR ENABLE_SFTP)) return() endif() @@ -33,7 +33,7 @@ file(GLOB SOURCES "*.cpp" "client/*.cpp" "processors/*.cpp") add_minifi_library(minifi-sftp SHARED ${SOURCES}) target_link_libraries(minifi-sftp ${LIBMINIFI} Threads::Threads) -target_link_libraries(minifi-sftp CURL::libcurl libssh2 RapidJSON) +target_link_libraries(minifi-sftp libssh2 RapidJSON) register_extension(minifi-sftp "SFTP EXTENSIONS" SFTP "This enables SFTP support" "extensions/sftp/tests") register_extension_linter(minifi-sftp-linter) diff --git a/extensions/sftp/client/SFTPClient.cpp b/extensions/sftp/client/SFTPClient.cpp index 06dd1c8371..4706b33aac 100644 --- a/extensions/sftp/client/SFTPClient.cpp +++ b/extensions/sftp/client/SFTPClient.cpp @@ -222,7 +222,7 @@ void SFTPClient::setPublicKeyAuthenticationCredentials(const std::string& privat private_key_passphrase_ = private_key_passphrase; } -bool SFTPClient::setProxy(ProxyType type, const utils::HTTPProxy& proxy) { +bool SFTPClient::setProxy(ProxyType type, const http::HTTPProxy& proxy) { switch (type) { case ProxyType::Http: if (curl_easy_setopt(easy_, CURLOPT_PROXYTYPE, CURLPROXY_HTTP) != CURLE_OK) { diff --git a/extensions/sftp/client/SFTPClient.h b/extensions/sftp/client/SFTPClient.h index 4d7242b1cd..28d9024a89 100644 --- a/extensions/sftp/client/SFTPClient.h +++ b/extensions/sftp/client/SFTPClient.h @@ -29,7 +29,7 @@ #include "Exception.h" #include "utils/Enum.h" -#include "utils/BaseHTTPClient.h" +#include "http/BaseHTTPClient.h" #include "core/logging/Logger.h" namespace org::apache::nifi::minifi::utils { @@ -126,7 +126,7 @@ class SFTPClient { Socks }; - bool setProxy(ProxyType type, const utils::HTTPProxy& proxy); + bool setProxy(ProxyType type, const http::HTTPProxy& proxy); bool setConnectionTimeout(std::chrono::milliseconds timeout); diff --git a/extensions/sftp/processors/SFTPProcessorBase.cpp b/extensions/sftp/processors/SFTPProcessorBase.cpp index 753c0a0af1..574ea06852 100644 --- a/extensions/sftp/processors/SFTPProcessorBase.cpp +++ b/extensions/sftp/processors/SFTPProcessorBase.cpp @@ -300,7 +300,7 @@ std::unique_ptr SFTPProcessorBase::getOrCreateConnection( client->setPublicKeyAuthenticationCredentials(private_key_path, private_key_passphrase); } if (connection_cache_key.proxy_type != PROXY_TYPE_DIRECT) { - utils::HTTPProxy proxy; + http::HTTPProxy proxy; proxy.host = connection_cache_key.proxy_host; proxy.port = connection_cache_key.proxy_port; proxy.username = connection_cache_key.proxy_username; diff --git a/extensions/sftp/tests/CMakeLists.txt b/extensions/sftp/tests/CMakeLists.txt index 5dd84b3486..81729d84c5 100644 --- a/extensions/sftp/tests/CMakeLists.txt +++ b/extensions/sftp/tests/CMakeLists.txt @@ -32,7 +32,6 @@ if (NOT SKIP_TESTS AND Java_FOUND AND Maven_FOUND AND ENABLE_EXPRESSION_LANGUAGE target_include_directories(${testfilename} BEFORE PRIVATE ${LIBSSH2_INCLUDE_DIR}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/processors") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test/") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/nanofi/include") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/sftp") target_include_directories(${testfilename} BEFORE PRIVATE "../") @@ -42,7 +41,7 @@ if (NOT SKIP_TESTS AND Java_FOUND AND Maven_FOUND AND ENABLE_EXPRESSION_LANGUAGE target_include_directories(${testfilename} BEFORE PRIVATE ./tools) createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain core-minifi sftp-test-tools) + target_link_libraries(${testfilename} Catch2::Catch2WithMain core-minifi sftp-test-tools) target_link_libraries(${testfilename} minifi-sftp) target_link_libraries(${testfilename} minifi-expression-language-extensions) target_link_libraries(${testfilename} minifi-standard-processors) diff --git a/extensions/sftp/tests/FetchSFTPTests.cpp b/extensions/sftp/tests/FetchSFTPTests.cpp index 32202e22a6..2669a79320 100644 --- a/extensions/sftp/tests/FetchSFTPTests.cpp +++ b/extensions/sftp/tests/FetchSFTPTests.cpp @@ -24,8 +24,8 @@ #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/file/FileUtils.h" #include "core/ProcessGroup.h" #include "FlowController.h" diff --git a/extensions/sftp/tests/ListSFTPTests.cpp b/extensions/sftp/tests/ListSFTPTests.cpp index 6f038d777f..c6c13aeb2a 100644 --- a/extensions/sftp/tests/ListSFTPTests.cpp +++ b/extensions/sftp/tests/ListSFTPTests.cpp @@ -17,8 +17,6 @@ */ #include -#undef NDEBUG -#include #include #include #include @@ -37,8 +35,8 @@ #include #endif -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/file/FileUtils.h" #include "core/Core.h" #include "core/logging/Logger.h" @@ -50,7 +48,7 @@ #include "processors/GenerateFlowFile.h" #include "processors/LogAttribute.h" #include "tools/SFTPTestServer.h" -#include "utils/TestUtils.h" +#include "unit/TestUtils.h" using namespace std::literals::chrono_literals; diff --git a/extensions/sftp/tests/ListThenFetchSFTPTests.cpp b/extensions/sftp/tests/ListThenFetchSFTPTests.cpp index db8c30ece7..ca72cc90b1 100644 --- a/extensions/sftp/tests/ListThenFetchSFTPTests.cpp +++ b/extensions/sftp/tests/ListThenFetchSFTPTests.cpp @@ -17,8 +17,6 @@ */ #include -#undef NDEBUG -#include #include #include #include @@ -34,8 +32,8 @@ #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/file/FileUtils.h" #include "core/logging/Logger.h" #include "core/ProcessGroup.h" diff --git a/extensions/sftp/tests/PutSFTPTests.cpp b/extensions/sftp/tests/PutSFTPTests.cpp index 034145f5e2..0dec1faf29 100644 --- a/extensions/sftp/tests/PutSFTPTests.cpp +++ b/extensions/sftp/tests/PutSFTPTests.cpp @@ -17,7 +17,6 @@ */ #include -#undef NDEBUG #include #include #include @@ -37,8 +36,8 @@ #include #endif -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "Exception.h" #include "date/date.h" #include "utils/file/FileUtils.h" diff --git a/extensions/smb/tests/CMakeLists.txt b/extensions/smb/tests/CMakeLists.txt index 79ee9471b8..1a5ab5eec5 100644 --- a/extensions/smb/tests/CMakeLists.txt +++ b/extensions/smb/tests/CMakeLists.txt @@ -25,9 +25,8 @@ FOREACH(testfile ${SMB_TESTS}) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/smb") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test/") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-smb) target_link_libraries(${testfilename} minifi-standard-processors Netapi32) MATH(EXPR SMB_TEST_COUNT "${SMB_TEST_COUNT}+1") diff --git a/extensions/smb/tests/FetchSmbTests.cpp b/extensions/smb/tests/FetchSmbTests.cpp index 9f6c870f41..6afe9f7a62 100644 --- a/extensions/smb/tests/FetchSmbTests.cpp +++ b/extensions/smb/tests/FetchSmbTests.cpp @@ -16,12 +16,12 @@ * limitations under the License. */ -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "FetchSmb.h" #include "SmbConnectionControllerService.h" #include "utils/MockSmbConnectionControllerService.h" -#include "SingleProcessorTestController.h" +#include "unit/SingleProcessorTestController.h" #include "OsUtils.h" #include "core/Resource.h" diff --git a/extensions/smb/tests/ListAndFetchSmbTests.cpp b/extensions/smb/tests/ListAndFetchSmbTests.cpp index a0481c1d9b..d4779424eb 100644 --- a/extensions/smb/tests/ListAndFetchSmbTests.cpp +++ b/extensions/smb/tests/ListAndFetchSmbTests.cpp @@ -16,12 +16,12 @@ * limitations under the License. */ -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "ListSmb.h" #include "FetchSmb.h" #include "utils/MockSmbConnectionControllerService.h" -#include "ReadFromFlowFileTestProcessor.h" +#include "unit/ReadFromFlowFileTestProcessor.h" #include "core/Resource.h" namespace org::apache::nifi::minifi::extensions::smb::test { diff --git a/extensions/smb/tests/ListSmbTests.cpp b/extensions/smb/tests/ListSmbTests.cpp index d94f40bb0f..4b9d1d9bb1 100644 --- a/extensions/smb/tests/ListSmbTests.cpp +++ b/extensions/smb/tests/ListSmbTests.cpp @@ -16,12 +16,12 @@ * limitations under the License. */ -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "ListSmb.h" -#include "Utils.h" +#include "unit/TestUtils.h" #include "utils/MockSmbConnectionControllerService.h" -#include "SingleProcessorTestController.h" +#include "unit/SingleProcessorTestController.h" #include "range/v3/algorithm/count_if.hpp" #include "range/v3/algorithm/find_if.hpp" #include "core/Resource.h" diff --git a/extensions/smb/tests/PutSmbTests.cpp b/extensions/smb/tests/PutSmbTests.cpp index 14f5ed6490..3fdec6dec4 100644 --- a/extensions/smb/tests/PutSmbTests.cpp +++ b/extensions/smb/tests/PutSmbTests.cpp @@ -16,11 +16,11 @@ * limitations under the License. */ -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "PutSmb.h" #include "utils/MockSmbConnectionControllerService.h" -#include "SingleProcessorTestController.h" +#include "unit/SingleProcessorTestController.h" #include "range/v3/algorithm/count_if.hpp" #include "core/Resource.h" diff --git a/extensions/smb/tests/SmbConnectionControllerServiceTests.cpp b/extensions/smb/tests/SmbConnectionControllerServiceTests.cpp index 3ac0b4f63e..c9693dc460 100644 --- a/extensions/smb/tests/SmbConnectionControllerServiceTests.cpp +++ b/extensions/smb/tests/SmbConnectionControllerServiceTests.cpp @@ -16,8 +16,8 @@ * limitations under the License. */ -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "SmbConnectionControllerService.h" #include "utils/TempSmbShare.h" #include "utils/UnicodeConversion.h" diff --git a/extensions/smb/tests/utils/MockSmbConnectionControllerService.h b/extensions/smb/tests/utils/MockSmbConnectionControllerService.h index 6e85b006cf..22a53363b3 100644 --- a/extensions/smb/tests/utils/MockSmbConnectionControllerService.h +++ b/extensions/smb/tests/utils/MockSmbConnectionControllerService.h @@ -23,7 +23,7 @@ #include "utils/OsUtils.h" #include "utils/file/FileUtils.h" #include "ListSmb.h" -#include "Catch.h" +#include "unit/Catch.h" using namespace std::literals::chrono_literals; diff --git a/extensions/smb/tests/utils/TempSmbShare.h b/extensions/smb/tests/utils/TempSmbShare.h index 5024a0d59c..170261a6ef 100644 --- a/extensions/smb/tests/utils/TempSmbShare.h +++ b/extensions/smb/tests/utils/TempSmbShare.h @@ -24,7 +24,7 @@ #include "lm.h" #include "utils/OsUtils.h" #include "utils/expected.h" -#include "TestUtils.h" +#include "unit/TestUtils.h" #include "ListSmb.h" using namespace std::literals::chrono_literals; diff --git a/extensions/splunk/CMakeLists.txt b/extensions/splunk/CMakeLists.txt index e2b64382ba..ff18d2d9a5 100644 --- a/extensions/splunk/CMakeLists.txt +++ b/extensions/splunk/CMakeLists.txt @@ -27,9 +27,7 @@ file(GLOB SOURCES "*.cpp") add_minifi_library(minifi-splunk SHARED ${SOURCES}) -target_include_directories(minifi-splunk PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/http-curl") target_link_libraries(minifi-splunk ${LIBMINIFI}) -target_link_libraries(minifi-splunk minifi-http-curl) register_extension(minifi-splunk "SPLUNK EXTENSIONS" SPLUNK-EXTENSIONS "This enables Splunk support" "extensions/splunk/tests") diff --git a/extensions/splunk/PutSplunkHTTP.cpp b/extensions/splunk/PutSplunkHTTP.cpp index 68f0ae3f2e..330c7b0795 100644 --- a/extensions/splunk/PutSplunkHTTP.cpp +++ b/extensions/splunk/PutSplunkHTTP.cpp @@ -26,8 +26,8 @@ #include "core/ProcessSession.h" #include "core/Resource.h" #include "utils/StringUtils.h" -#include "client/HTTPClient.h" -#include "utils/BaseHTTPClient.h" +#include "http/HTTPClient.h" +#include "http/BaseHTTPClient.h" #include "utils/OptionalUtils.h" #include "utils/ByteArrayCallback.h" @@ -40,7 +40,7 @@ void PutSplunkHTTP::initialize() { setSupportedRelationships(Relationships); } -std::string PutSplunkHTTP::getEndpoint(curl::HTTPClient& client) { +std::string PutSplunkHTTP::getEndpoint(http::HTTPClient& client) { std::stringstream endpoint; endpoint << "/services/collector/raw"; std::vector parameters; @@ -67,7 +67,7 @@ std::optional getContentType(core::ProcessContext& context, const c return context.getProperty(PutSplunkHTTP::ContentType) | utils::orElse ([&flow_file] {return flow_file.getAttribute("mime.type");}); } -bool setAttributesFromClientResponse(core::FlowFile& flow_file, curl::HTTPClient& client) { +bool setAttributesFromClientResponse(core::FlowFile& flow_file, http::HTTPClient& client) { rapidjson::Document response_json; rapidjson::ParseResult parse_result = response_json.Parse(client.getResponseBody().data()); bool result = true; @@ -87,7 +87,7 @@ bool setAttributesFromClientResponse(core::FlowFile& flow_file, curl::HTTPClient return result; } -bool enrichFlowFileWithAttributes(core::FlowFile& flow_file, curl::HTTPClient& client) { +bool enrichFlowFileWithAttributes(core::FlowFile& flow_file, http::HTTPClient& client) { flow_file.setAttribute(SPLUNK_STATUS_CODE, std::to_string(client.getResponseCode())); flow_file.setAttribute(SPLUNK_RESPONSE_TIME, std::to_string(std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count())); @@ -96,9 +96,9 @@ bool enrichFlowFileWithAttributes(core::FlowFile& flow_file, curl::HTTPClient& c void setFlowFileAsPayload(core::ProcessSession& session, core::ProcessContext& context, - curl::HTTPClient& client, + http::HTTPClient& client, const gsl::not_null>& flow_file) { - auto payload = std::make_unique(); + auto payload = std::make_unique(); session.read(flow_file, std::ref(*payload)); payload->pos = 0; client.setRequestHeader("Content-Length", std::to_string(flow_file->getSize())); @@ -115,13 +115,13 @@ void setFlowFileAsPayload(core::ProcessSession& session, void PutSplunkHTTP::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) { SplunkHECProcessor::onSchedule(context, session_factory); ssl_context_service_ = getSSLContextService(context); - auto create_client = [this]() -> std::unique_ptr { - auto client = std::make_unique(); + auto create_client = [this]() -> std::unique_ptr { + auto client = std::make_unique(); initializeClient(*client, getNetworkLocation().append(getEndpoint(*client)), ssl_context_service_); return client; }; - client_queue_ = utils::ResourceQueue::create(create_client, getMaxConcurrentTasks(), std::nullopt, logger_); + client_queue_ = utils::ResourceQueue::create(create_client, getMaxConcurrentTasks(), std::nullopt, logger_); source_type_ = context.getProperty(PutSplunkHTTP::SourceType); source_ = context.getProperty(PutSplunkHTTP::Source); host_ = context.getProperty(PutSplunkHTTP::Host); diff --git a/extensions/splunk/PutSplunkHTTP.h b/extensions/splunk/PutSplunkHTTP.h index af422fefb6..06580f388e 100644 --- a/extensions/splunk/PutSplunkHTTP.h +++ b/extensions/splunk/PutSplunkHTTP.h @@ -22,7 +22,7 @@ #include #include "SplunkHECProcessor.h" -#include "client/HTTPClient.h" +#include "http/HTTPClient.h" #include "core/PropertyDefinition.h" #include "core/PropertyDefinitionBuilder.h" #include "core/RelationshipDefinition.h" @@ -107,7 +107,7 @@ class PutSplunkHTTP final : public SplunkHECProcessor { void onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) override; private: - std::string getEndpoint(curl::HTTPClient& client); + std::string getEndpoint(http::HTTPClient& client); std::shared_ptr ssl_context_service_; std::optional source_type_; @@ -115,7 +115,7 @@ class PutSplunkHTTP final : public SplunkHECProcessor { std::optional host_; std::optional index_; std::shared_ptr logger_{core::logging::LoggerFactory::getLogger(uuid_)}; - std::shared_ptr> client_queue_; + std::shared_ptr> client_queue_; }; } // namespace org::apache::nifi::minifi::extensions::splunk diff --git a/extensions/splunk/QuerySplunkIndexingStatus.cpp b/extensions/splunk/QuerySplunkIndexingStatus.cpp index b7fa1f9aab..c9cd890e96 100644 --- a/extensions/splunk/QuerySplunkIndexingStatus.cpp +++ b/extensions/splunk/QuerySplunkIndexingStatus.cpp @@ -27,7 +27,7 @@ #include "core/ProcessContext.h" #include "core/ProcessSession.h" #include "core/Resource.h" -#include "client/HTTPClient.h" +#include "http/HTTPClient.h" #include "rapidjson/document.h" #include "rapidjson/stringbuffer.h" @@ -86,7 +86,7 @@ std::string getAckIdsAsPayload(const std::unordered_map& undetermined_flow_files) { +void getIndexingStatusFromSplunk(http::HTTPClient& client, std::unordered_map& undetermined_flow_files) { rapidjson::Document response; if (!client.submit()) return; diff --git a/extensions/splunk/QuerySplunkIndexingStatus.h b/extensions/splunk/QuerySplunkIndexingStatus.h index 24021cb4c6..9a7bc2d8c3 100644 --- a/extensions/splunk/QuerySplunkIndexingStatus.h +++ b/extensions/splunk/QuerySplunkIndexingStatus.h @@ -28,7 +28,7 @@ #include "core/RelationshipDefinition.h" #include "utils/ArrayUtils.h" #include "utils/gsl.h" -#include "client/HTTPClient.h" +#include "http/HTTPClient.h" #include "rapidjson/stringbuffer.h" namespace org::apache::nifi::minifi::extensions::splunk { @@ -123,7 +123,7 @@ class QuerySplunkIndexingStatus final : public SplunkHECProcessor { protected: uint32_t batch_size_ = 1000; std::chrono::milliseconds max_age_ = std::chrono::hours(1); - curl::HTTPClient client_; + http::HTTPClient client_; }; } // namespace org::apache::nifi::minifi::extensions::splunk diff --git a/extensions/splunk/SplunkHECProcessor.cpp b/extensions/splunk/SplunkHECProcessor.cpp index 997d36e795..aa3d3e13b6 100644 --- a/extensions/splunk/SplunkHECProcessor.cpp +++ b/extensions/splunk/SplunkHECProcessor.cpp @@ -18,7 +18,7 @@ #include "SplunkHECProcessor.h" #include #include "core/ProcessContext.h" -#include "client/HTTPClient.h" +#include "http/HTTPClient.h" namespace org::apache::nifi::minifi::extensions::splunk { @@ -51,8 +51,8 @@ std::shared_ptr SplunkHECProcessor::getS return nullptr; } -void SplunkHECProcessor::initializeClient(curl::HTTPClient& client, const std::string &url, std::shared_ptr ssl_context_service) const { - client.initialize(utils::HttpRequestMethod::POST, url, std::move(ssl_context_service)); +void SplunkHECProcessor::initializeClient(http::HTTPClient& client, const std::string &url, std::shared_ptr ssl_context_service) const { + client.initialize(http::HttpRequestMethod::POST, url, std::move(ssl_context_service)); client.setRequestHeader("Authorization", token_); client.setRequestHeader("X-Splunk-Request-Channel", request_channel_); } diff --git a/extensions/splunk/SplunkHECProcessor.h b/extensions/splunk/SplunkHECProcessor.h index 7be366e5bf..398921c511 100644 --- a/extensions/splunk/SplunkHECProcessor.h +++ b/extensions/splunk/SplunkHECProcessor.h @@ -26,6 +26,7 @@ #include "core/PropertyDefinition.h" #include "core/PropertyDefinitionBuilder.h" #include "core/PropertyType.h" +#include "http/HTTPClient.h" namespace org::apache::nifi::minifi::extensions::curl { class HTTPClient; @@ -80,7 +81,7 @@ class SplunkHECProcessor : public core::Processor { protected: std::string getNetworkLocation() const; static std::shared_ptr getSSLContextService(core::ProcessContext& context); - void initializeClient(curl::HTTPClient& client, const std::string &url, std::shared_ptr ssl_context_service) const; + void initializeClient(http::HTTPClient& client, const std::string &url, std::shared_ptr ssl_context_service) const; std::string token_; std::string hostname_; diff --git a/extensions/splunk/tests/CMakeLists.txt b/extensions/splunk/tests/CMakeLists.txt index 945c18ea30..313844c52d 100644 --- a/extensions/splunk/tests/CMakeLists.txt +++ b/extensions/splunk/tests/CMakeLists.txt @@ -25,15 +25,12 @@ FOREACH(testfile ${SPLUNK_TESTS}) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/splunk") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test/") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/http-curl/") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-splunk) target_link_libraries(${testfilename} minifi-civet-extensions) - target_link_libraries(${testfilename} minifi-http-curl) target_link_libraries(${testfilename} minifi-standard-processors) MATH(EXPR SPLUNK_TEST_COUNT "${SPLUNK_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) diff --git a/extensions/splunk/tests/MockSplunkHEC.h b/extensions/splunk/tests/MockSplunkHEC.h index a83014ea82..53b548ba18 100644 --- a/extensions/splunk/tests/MockSplunkHEC.h +++ b/extensions/splunk/tests/MockSplunkHEC.h @@ -22,7 +22,7 @@ #include #include #include -#include "tests/CivetLibrary.h" +#include "integration/CivetLibrary.h" #include "core/logging/Logger.h" #include "core/logging/LoggerConfiguration.h" #include "rapidjson/document.h" diff --git a/extensions/splunk/tests/PutSplunkHTTPTests.cpp b/extensions/splunk/tests/PutSplunkHTTPTests.cpp index d402e0f07e..2b34cbd5f2 100644 --- a/extensions/splunk/tests/PutSplunkHTTPTests.cpp +++ b/extensions/splunk/tests/PutSplunkHTTPTests.cpp @@ -17,10 +17,10 @@ #include "PutSplunkHTTP.h" #include "SplunkAttributes.h" -#include "TestBase.h" -#include "Catch.h" -#include "ReadFromFlowFileTestProcessor.h" -#include "WriteToFlowFileTestProcessor.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/ReadFromFlowFileTestProcessor.h" +#include "unit/WriteToFlowFileTestProcessor.h" #include "core/Relationship.h" #include "processors/UpdateAttribute.h" #include "MockSplunkHEC.h" diff --git a/extensions/splunk/tests/QuerySplunkIndexingStatusTests.cpp b/extensions/splunk/tests/QuerySplunkIndexingStatusTests.cpp index 29716ad9e5..de4e7dde94 100644 --- a/extensions/splunk/tests/QuerySplunkIndexingStatusTests.cpp +++ b/extensions/splunk/tests/QuerySplunkIndexingStatusTests.cpp @@ -20,11 +20,11 @@ #include "QuerySplunkIndexingStatus.h" #include "MockSplunkHEC.h" #include "SplunkAttributes.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "processors/UpdateAttribute.h" -#include "ReadFromFlowFileTestProcessor.h" -#include "WriteToFlowFileTestProcessor.h" +#include "unit/ReadFromFlowFileTestProcessor.h" +#include "unit/WriteToFlowFileTestProcessor.h" using QuerySplunkIndexingStatus = org::apache::nifi::minifi::extensions::splunk::QuerySplunkIndexingStatus; using ReadFromFlowFileTestProcessor = org::apache::nifi::minifi::processors::ReadFromFlowFileTestProcessor; diff --git a/extensions/sql/tests/CMakeLists.txt b/extensions/sql/tests/CMakeLists.txt index 15b22d8566..2996cbad1e 100644 --- a/extensions/sql/tests/CMakeLists.txt +++ b/extensions/sql/tests/CMakeLists.txt @@ -32,10 +32,9 @@ foreach(testfile ${SQL_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/sql") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-sql-mocks) target_link_libraries(${testfilename} minifi-sql) target_link_libraries(${testfilename} minifi-standard-processors) diff --git a/extensions/sql/tests/ExecuteSQLTests.cpp b/extensions/sql/tests/ExecuteSQLTests.cpp index c8dec61d8b..5dd686b790 100644 --- a/extensions/sql/tests/ExecuteSQLTests.cpp +++ b/extensions/sql/tests/ExecuteSQLTests.cpp @@ -15,14 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG - #include #include "SQLTestController.h" #include "processors/ExecuteSQL.h" -#include "Utils.h" +#include "unit/TestUtils.h" #include "FlowFileMatcher.h" namespace org::apache::nifi::minifi::test { diff --git a/extensions/sql/tests/FlowFileMatcher.h b/extensions/sql/tests/FlowFileMatcher.h index 97f3657bd9..9733ae9891 100644 --- a/extensions/sql/tests/FlowFileMatcher.h +++ b/extensions/sql/tests/FlowFileMatcher.h @@ -25,8 +25,8 @@ #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/FlowFile.h" struct AttributeValue { diff --git a/extensions/sql/tests/PutSQLTests.cpp b/extensions/sql/tests/PutSQLTests.cpp index 581f28ffc9..cbeadf74e2 100644 --- a/extensions/sql/tests/PutSQLTests.cpp +++ b/extensions/sql/tests/PutSQLTests.cpp @@ -15,11 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG - -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "SQLTestController.h" #include "processors/PutSQL.h" diff --git a/extensions/sql/tests/QueryDatabaseTableTests.cpp b/extensions/sql/tests/QueryDatabaseTableTests.cpp index 7a903eaa65..7053cd26d2 100644 --- a/extensions/sql/tests/QueryDatabaseTableTests.cpp +++ b/extensions/sql/tests/QueryDatabaseTableTests.cpp @@ -15,13 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG - -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "SQLTestController.h" -#include "Utils.h" +#include "unit/TestUtils.h" #include "FlowFileMatcher.h" namespace org::apache::nifi::minifi::test { diff --git a/extensions/sql/tests/SQLColumnIdentifierTests.cpp b/extensions/sql/tests/SQLColumnIdentifierTests.cpp index e80a5544db..9a379b85ef 100644 --- a/extensions/sql/tests/SQLColumnIdentifierTests.cpp +++ b/extensions/sql/tests/SQLColumnIdentifierTests.cpp @@ -15,11 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG - -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "data/SQLColumnIdentifier.h" using org::apache::nifi::minifi::sql::SQLColumnIdentifier; diff --git a/extensions/sql/tests/SQLTestController.h b/extensions/sql/tests/SQLTestController.h index a590acfdb5..e8d86367d0 100644 --- a/extensions/sql/tests/SQLTestController.h +++ b/extensions/sql/tests/SQLTestController.h @@ -22,8 +22,8 @@ #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "processors/PutSQL.h" #include "processors/GenerateFlowFile.h" diff --git a/extensions/sql/tests/SQLTestPlan.h b/extensions/sql/tests/SQLTestPlan.h index 81854fbc9a..985a72923c 100644 --- a/extensions/sql/tests/SQLTestPlan.h +++ b/extensions/sql/tests/SQLTestPlan.h @@ -26,8 +26,8 @@ #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "processors/SQLProcessor.h" #ifdef MINIFI_USE_REAL_ODBC_TEST_DRIVER diff --git a/extensions/standard-processors/processors/GetTCP.cpp b/extensions/standard-processors/processors/GetTCP.cpp index e1a27e9a93..d41e34774e 100644 --- a/extensions/standard-processors/processors/GetTCP.cpp +++ b/extensions/standard-processors/processors/GetTCP.cpp @@ -205,7 +205,7 @@ asio::awaitable GetTCP::TcpClient::readLoop(auto& socket) { auto dynamic_buffer = max_message_size_ ? asio::dynamic_buffer(read_message, *max_message_size_) : asio::dynamic_buffer(read_message); auto [read_error, bytes_read] = co_await asio::async_read_until(socket, dynamic_buffer, delimiter_, utils::net::use_nothrow_awaitable); // NOLINT - if (*max_message_size_ && read_error == asio::error::not_found) { + if (max_message_size_ && *max_message_size_ && read_error == asio::error::not_found) { current_doesnt_end_with_delimiter = true; bytes_read = *max_message_size_; } else if (read_error) { diff --git a/extensions/standard-processors/processors/HashContent.cpp b/extensions/standard-processors/processors/HashContent.cpp index 7774608e6f..26499d166e 100644 --- a/extensions/standard-processors/processors/HashContent.cpp +++ b/extensions/standard-processors/processors/HashContent.cpp @@ -17,9 +17,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#ifdef OPENSSL_SUPPORT - #include #include #include @@ -86,5 +83,3 @@ void HashContent::onTrigger(core::ProcessContext&, core::ProcessSession& session REGISTER_RESOURCE(HashContent, Processor); } // namespace org::apache::nifi::minifi::processors - -#endif // OPENSSL_SUPPORT diff --git a/extensions/standard-processors/processors/HashContent.h b/extensions/standard-processors/processors/HashContent.h index dec63b8674..583c2f9230 100644 --- a/extensions/standard-processors/processors/HashContent.h +++ b/extensions/standard-processors/processors/HashContent.h @@ -19,8 +19,6 @@ */ #pragma once -#ifdef OPENSSL_SUPPORT - #include #include #include @@ -188,5 +186,3 @@ class HashContent : public core::Processor { }; } // namespace org::apache::nifi::minifi::processors - -#endif // OPENSSL_SUPPORT diff --git a/extensions/http-curl/processors/InvokeHTTP.cpp b/extensions/standard-processors/processors/InvokeHTTP.cpp similarity index 90% rename from extensions/http-curl/processors/InvokeHTTP.cpp rename to extensions/standard-processors/processors/InvokeHTTP.cpp index cced54c0ae..6049630ac8 100644 --- a/extensions/http-curl/processors/InvokeHTTP.cpp +++ b/extensions/standard-processors/processors/InvokeHTTP.cpp @@ -45,7 +45,7 @@ void InvokeHTTP::initialize() { } namespace { -void setupClientTimeouts(extensions::curl::HTTPClient& client, +void setupClientTimeouts(http::HTTPClient& client, std::optional connection_timeout, std::optional read_timeout) { if (connection_timeout) @@ -55,7 +55,7 @@ void setupClientTimeouts(extensions::curl::HTTPClient& client, client.setReadTimeout(*read_timeout); } -void setupClientTransferEncoding(extensions::curl::HTTPClient& client, bool use_chunked_encoding) { +void setupClientTransferEncoding(http::HTTPClient& client, bool use_chunked_encoding) { if (use_chunked_encoding) client.setRequestHeader("Transfer-Encoding", "chunked"); else @@ -67,7 +67,7 @@ void InvokeHTTP::setupMembersFromProperties(const core::ProcessContext& context) if (!context.getProperty(URL, url_)) throw Exception(PROCESS_SCHEDULE_EXCEPTION, "URL property missing or invalid"); - method_ = utils::parseEnumProperty(context, Method); + method_ = utils::parseEnumProperty(context, Method); context.getProperty(SendMessageBody, send_message_body_); @@ -121,8 +121,8 @@ void InvokeHTTP::setupMembersFromProperties(const core::ProcessContext& context) } } -std::unique_ptr InvokeHTTP::createHTTPClientFromMembers() const { - auto client = std::make_unique(); +std::unique_ptr InvokeHTTP::createHTTPClientFromMembers() const { + auto client = std::make_unique(); client->initialize(method_, url_, ssl_context_service_); setupClientTimeouts(*client, connect_timeout_, read_timeout_); client->setHTTPProxy(proxy_); @@ -141,15 +141,15 @@ std::unique_ptr InvokeHTTP::createHTTPClie void InvokeHTTP::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory&) { setupMembersFromProperties(context); - auto create_client = [this]() -> std::unique_ptr { + auto create_client = [this]() -> std::unique_ptr { return createHTTPClientFromMembers(); }; - client_queue_ = utils::ResourceQueue::create(create_client, getMaxConcurrentTasks(), std::nullopt, logger_); + client_queue_ = utils::ResourceQueue::create(create_client, getMaxConcurrentTasks(), std::nullopt, logger_); } bool InvokeHTTP::shouldEmitFlowFile() const { - return (utils::HttpRequestMethod::POST == method_ || utils::HttpRequestMethod::PUT == method_ || utils::HttpRequestMethod::PATCH == method_); + return (http::HttpRequestMethod::POST == method_ || http::HttpRequestMethod::PUT == method_ || http::HttpRequestMethod::PATCH == method_); } /** @@ -168,17 +168,17 @@ bool InvokeHTTP::appendHeaders(const core::FlowFile& flow_file, /*std::invocable | ranges::views::filter([this](const auto& key) { return utils::regexMatch(key, *attributes_to_send_); }, key_fn); switch (invalid_http_header_field_handling_strategy_) { case invoke_http::InvalidHTTPHeaderFieldHandlingOption::fail: - if (ranges::any_of(matching_attributes, std::not_fn(&extensions::curl::HTTPClient::isValidHttpHeaderField), key_fn)) return false; + if (ranges::any_of(matching_attributes, std::not_fn(&http::HTTPClient::isValidHttpHeaderField), key_fn)) return false; for (const auto& header: matching_attributes) append_header(header.first, header.second); return true; case invoke_http::InvalidHTTPHeaderFieldHandlingOption::drop: - for (const auto& header: matching_attributes | ranges::views::filter(&extensions::curl::HTTPClient::isValidHttpHeaderField, key_fn)) { + for (const auto& header: matching_attributes | ranges::views::filter(&http::HTTPClient::isValidHttpHeaderField, key_fn)) { append_header(header.first, header.second); } return true; case invoke_http::InvalidHTTPHeaderFieldHandlingOption::transform: for (const auto& header: matching_attributes) { - append_header(extensions::curl::HTTPClient::replaceInvalidCharactersInHttpHeaderFieldName(header.first), header.second); + append_header(http::HTTPClient::replaceInvalidCharactersInHttpHeaderFieldName(header.first), header.second); } return true; } @@ -209,7 +209,7 @@ void InvokeHTTP::onTrigger(core::ProcessContext& context, core::ProcessSession& } void InvokeHTTP::onTriggerWithClient(core::ProcessContext& context, core::ProcessSession& session, - const std::shared_ptr& flow_file, minifi::extensions::curl::HTTPClient& client) { + const std::shared_ptr& flow_file, minifi::http::HTTPClient& client) { logger_->log_debug("onTrigger InvokeHTTP with {} to {}", magic_enum::enum_name(method_), client.getURL()); const auto remove_callback_from_client_at_exit = gsl::finally([&client] { @@ -222,11 +222,11 @@ void InvokeHTTP::onTriggerWithClient(core::ProcessContext& context, core::Proces logger_->log_trace("InvokeHTTP -- reading flowfile"); const auto flow_file_reader_stream = session.getFlowFileContentStream(*flow_file); if (flow_file_reader_stream) { - std::unique_ptr callback_obj; + std::unique_ptr callback_obj; if (send_message_body_) { - callback_obj = std::make_unique(flow_file_reader_stream); + callback_obj = std::make_unique(flow_file_reader_stream); } else { - callback_obj = std::make_unique(); + callback_obj = std::make_unique(); } client.setUploadCallback(std::move(callback_obj)); logger_->log_trace("InvokeHTTP -- Setting callback, size is {}", flow_file->getSize()); diff --git a/extensions/http-curl/processors/InvokeHTTP.h b/extensions/standard-processors/processors/InvokeHTTP.h similarity index 96% rename from extensions/http-curl/processors/InvokeHTTP.h rename to extensions/standard-processors/processors/InvokeHTTP.h index 9da89866b4..d53f8081e3 100644 --- a/extensions/http-curl/processors/InvokeHTTP.h +++ b/extensions/standard-processors/processors/InvokeHTTP.h @@ -36,7 +36,7 @@ #include "core/logging/LoggerConfiguration.h" #include "utils/Id.h" #include "utils/ResourceQueue.h" -#include "../client/HTTPClient.h" +#include "../http/HTTPClient.h" #include "utils/Export.h" #include "utils/Enum.h" #include "utils/RegexUtils.h" @@ -67,10 +67,10 @@ class InvokeHTTP : public core::Processor { "The destination URL and HTTP Method are configurable. FlowFile attributes are converted to HTTP headers and the " "FlowFile contents are included as the body of the request (if the HTTP Method is PUT, POST or PATCH)."; - EXTENSIONAPI static constexpr auto Method = core::PropertyDefinitionBuilder()>::createProperty("HTTP Method") + EXTENSIONAPI static constexpr auto Method = core::PropertyDefinitionBuilder()>::createProperty("HTTP Method") .withDescription("HTTP request method. Methods other than POST, PUT and PATCH will be sent without a message body.") - .withAllowedValues(magic_enum::enum_names()) - .withDefaultValue(magic_enum::enum_name(utils::HttpRequestMethod::GET)) + .withAllowedValues(magic_enum::enum_names()) + .withDefaultValue(magic_enum::enum_name(http::HttpRequestMethod::GET)) .build(); EXTENSIONAPI static constexpr auto URL = core::PropertyDefinitionBuilder<>::createProperty("Remote URL") .withDescription("Remote URL which will be connected to, including scheme, host, port, path.") @@ -271,14 +271,14 @@ class InvokeHTTP : public core::Processor { core::ProcessContext& context, bool is_success, int64_t status_code); [[nodiscard]] bool shouldEmitFlowFile() const; void onTriggerWithClient(core::ProcessContext& context, core::ProcessSession& session, - const std::shared_ptr& flow_file, minifi::extensions::curl::HTTPClient& client); + const std::shared_ptr& flow_file, minifi::http::HTTPClient& client); [[nodiscard]] bool appendHeaders(const core::FlowFile& flow_file, /*std::invocable*/ auto append_header); void setupMembersFromProperties(const core::ProcessContext& context); - std::unique_ptr createHTTPClientFromMembers() const; + std::unique_ptr createHTTPClientFromMembers() const; - utils::HttpRequestMethod method_{}; + http::HttpRequestMethod method_{}; std::optional attributes_to_send_; std::optional put_response_body_in_attribute_; bool always_output_response_{false}; @@ -295,7 +295,7 @@ class InvokeHTTP : public core::Processor { std::chrono::milliseconds connect_timeout_{std::chrono::seconds(30)}; std::chrono::milliseconds read_timeout_{std::chrono::seconds(30)}; - utils::HTTPProxy proxy_{}; + http::HTTPProxy proxy_{}; bool follow_redirects_ = false; std::optional content_type_; @@ -303,7 +303,7 @@ class InvokeHTTP : public core::Processor { invoke_http::InvalidHTTPHeaderFieldHandlingOption invalid_http_header_field_handling_strategy_{}; std::shared_ptr logger_{core::logging::LoggerFactory::getLogger(uuid_)}; - std::shared_ptr> client_queue_; + std::shared_ptr> client_queue_; }; } // namespace org::apache::nifi::minifi::processors diff --git a/extensions/standard-processors/tests/CMakeLists.txt b/extensions/standard-processors/tests/CMakeLists.txt index 6defa1a982..7594c8c444 100644 --- a/extensions/standard-processors/tests/CMakeLists.txt +++ b/extensions/standard-processors/tests/CMakeLists.txt @@ -34,15 +34,14 @@ FOREACH(testfile ${PROCESSOR_UNIT_TESTS}) get_filename_component(testfilename "${testfile}" NAME_WE) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} BEFORE PRIVATE ${PROCESSOR_INCLUDE_DIRS}) - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test/") target_include_directories(${testfilename} BEFORE PRIVATE "../") target_include_directories(${testfilename} BEFORE PRIVATE "../processors") target_include_directories(${testfilename} BEFORE PRIVATE ./include) + target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-expression-language-extensions) target_link_libraries(${testfilename} minifi-standard-processors) - target_link_libraries(${testfilename} minifi-civet-extensions) target_compile_definitions("${testfilename}" PRIVATE JOLT_TESTS_DIR="${jolt_tests_SOURCE_DIR}/jolt-core/src/test/resources/json/shiftr") if (ENABLE_ROCKSDB) target_link_libraries(${testfilename} minifi-rocksdb-repos) @@ -60,27 +59,28 @@ FOREACH(testfile ${PROCESSOR_INTEGRATION_TESTS}) get_filename_component(testfilename "${testfile}" NAME_WE) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} BEFORE PRIVATE ${PROCESSOR_INCLUDE_DIRS}) - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test/") target_include_directories(${testfilename} BEFORE PRIVATE "../") target_include_directories(${testfilename} BEFORE PRIVATE "../processors") + target_include_directories(${testfilename} BEFORE PRIVATE "../http/") + target_include_directories(${testfilename} BEFORE PRIVATE "../protocols/") + target_include_directories(${testfilename} BEFORE PRIVATE "../sitetosite/") + target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/") target_include_directories(${testfilename} BEFORE PRIVATE ./include) - createTests("${testfilename}") + createIntegrationTests("${testfilename}") target_link_libraries(${testfilename}) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-standard-processors) - target_link_libraries(${testfilename} minifi-civet-extensions) target_compile_definitions("${testfilename}" PRIVATE TZ_DATA_DIR="${CMAKE_BINARY_DIR}/tzdata") + target_compile_definitions("${testfilename}" PRIVATE TEST_RESOURCES="${TEST_RESOURCES}") if (ENABLE_ROCKSDB) target_link_libraries(${testfilename} minifi-rocksdb-repos) endif() + add_test(NAME "${testfilename}" COMMAND "${testfilename}") MATH(EXPR INT_TEST_COUNT "${INT_TEST_COUNT}+1") ENDFOREACH() message("-- Finished building ${INT_TEST_COUNT} integration test file(s)...") -add_test(NAME TailFileTest COMMAND TailFileTest "${TEST_RESOURCES}/TestTailFile.yml" "${TEST_RESOURCES}/") -add_test(NAME TailFileCronTest COMMAND TailFileTest "${TEST_RESOURCES}/TestTailFileCron.yml" "${TEST_RESOURCES}/") -add_test(NAME ProcessGroupTest COMMAND ProcessGroupTest "${TEST_RESOURCES}/TestProcessGroup.yml") - FOREACH(resourcefile ${RESOURCE_APPS}) get_filename_component(resourcefilename "${resourcefile}" NAME_WE) add_minifi_executable("${resourcefilename}" "${resourcefile}") diff --git a/extensions/http-curl/tests/unit/InvokeHTTPTests.cpp b/extensions/standard-processors/tests/integration/InvokeHTTPTests.cpp similarity index 98% rename from extensions/http-curl/tests/unit/InvokeHTTPTests.cpp rename to extensions/standard-processors/tests/integration/InvokeHTTPTests.cpp index 3ec0067fb0..e543302fa0 100644 --- a/extensions/http-curl/tests/unit/InvokeHTTPTests.cpp +++ b/extensions/standard-processors/tests/integration/InvokeHTTPTests.cpp @@ -18,10 +18,10 @@ #include #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/Core.h" -#include "HTTPClient.h" +#include "http/HTTPClient.h" #include "InvokeHTTP.h" #include "processors/ListenHTTP.h" #include "core/FlowFile.h" @@ -31,8 +31,8 @@ #include "core/ProcessSession.h" #include "core/ProcessorNode.h" #include "processors/LogAttribute.h" -#include "SingleProcessorTestController.h" -#include "ConnectionCountingServer.h" +#include "unit/SingleProcessorTestController.h" +#include "integration/ConnectionCountingServer.h" namespace org::apache::nifi::minifi::test { @@ -449,7 +449,7 @@ TEST_CASE("HTTPTestsResponseBodyinAttribute", "[InvokeHTTP]") { auto invoke_http = std::make_shared("InvokeHTTP"); test::SingleProcessorTestController test_controller{invoke_http}; - minifi::extensions::curl::testing::ConnectionCountingServer connection_counting_server; + minifi::test::ConnectionCountingServer connection_counting_server; invoke_http->setProperty(InvokeHTTP::Method, "POST"); invoke_http->setProperty(InvokeHTTP::URL, "http://localhost:" + connection_counting_server.getPort() + "/reverse"); @@ -474,7 +474,7 @@ TEST_CASE("HTTPTestsResponseBody", "[InvokeHTTP]") { auto invoke_http = std::make_shared("InvokeHTTP"); test::SingleProcessorTestController test_controller{invoke_http}; - minifi::extensions::curl::testing::ConnectionCountingServer connection_counting_server; + minifi::test::ConnectionCountingServer connection_counting_server; invoke_http->setProperty(InvokeHTTP::Method, "POST"); invoke_http->setProperty(InvokeHTTP::URL, "http://localhost:" + connection_counting_server.getPort() + "/reverse"); @@ -497,7 +497,7 @@ TEST_CASE("Test Keepalive", "[InvokeHTTP]") { auto invoke_http = std::make_shared("InvokeHTTP"); test::SingleProcessorTestController test_controller{invoke_http}; - minifi::extensions::curl::testing::ConnectionCountingServer connection_counting_server; + minifi::test::ConnectionCountingServer connection_counting_server; invoke_http->setProperty(InvokeHTTP::Method, "GET"); invoke_http->setProperty(InvokeHTTP::URL, "http://localhost:" + connection_counting_server.getPort() + "/method"); diff --git a/extensions/standard-processors/tests/integration/ProcessGroupTest.cpp b/extensions/standard-processors/tests/integration/ProcessGroupTest.cpp index e6e71dc149..6eb6b2a5cc 100644 --- a/extensions/standard-processors/tests/integration/ProcessGroupTest.cpp +++ b/extensions/standard-processors/tests/integration/ProcessGroupTest.cpp @@ -15,22 +15,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG -#include #include #include "core/logging/Logger.h" #include "FlowController.h" -#include "TestBase.h" +#include "unit/TestBase.h" #include "processors/GenerateFlowFile.h" #include "processors/LogAttribute.h" #include "processors/UpdateAttribute.h" #include "integration/IntegrationBase.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" +#include "unit/Catch.h" using namespace std::literals::chrono_literals; +namespace org::apache::nifi::minifi::test { + class ProcessGroupTestHarness : public IntegrationBase { public: ProcessGroupTestHarness() : IntegrationBase(2s) { @@ -43,20 +43,16 @@ class ProcessGroupTestHarness : public IntegrationBase { } void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; - assert(verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; + REQUIRE(verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), "key:test_attribute value:success")); } }; -int main(int argc, char **argv) { - std::string test_file_location; - if (argc > 1) { - test_file_location = argv[1]; - } - +TEST_CASE("ProcessGroupTest", "[ProcessGroupTest]") { ProcessGroupTestHarness harness; - harness.run(test_file_location); - - return 0; + auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestProcessGroup.yml"; + harness.run(test_file_path); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/standard-processors/tests/integration/TailFileTest.cpp b/extensions/standard-processors/tests/integration/TailFileTest.cpp index 0486c575f9..6b4ec5a989 100644 --- a/extensions/standard-processors/tests/integration/TailFileTest.cpp +++ b/extensions/standard-processors/tests/integration/TailFileTest.cpp @@ -15,24 +15,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG -#include #include #include #include +#include #include "FlowController.h" -#include "TestBase.h" +#include "unit/TestBase.h" #include "processors/TailFile.h" #include "processors/LogAttribute.h" #include "state/ProcessorController.h" #include "integration/IntegrationBase.h" -#include "utils/IntegrationTestUtils.h" -#include "utils/TestUtils.h" +#include "unit/TestUtils.h" +#include "unit/Catch.h" +#include "utils/TimeUtil.h" using namespace std::literals::chrono_literals; +namespace org::apache::nifi::minifi::test { + class TailFileTestHarness : public IntegrationBase { public: TailFileTestHarness() { @@ -50,7 +51,7 @@ class TailFileTestHarness : public IntegrationBase { LogTestController::getInstance().setTrace(); LogTestController::getInstance().setTrace(); #ifdef WIN32 - utils::dateSetInstall(TZ_DATA_DIR); + minifi::utils::timeutils::dateSetInstall(TZ_DATA_DIR); #endif } @@ -61,8 +62,7 @@ class TailFileTestHarness : public IntegrationBase { } void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; - assert(verifyLogLinePresenceInPollTime(wait_time_, + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(wait_time_, "5 flowfiles were received from TailFile input", "Looking for delimiter 0xA", "li\\ne5")); @@ -84,15 +84,16 @@ class TailFileTestHarness : public IntegrationBase { TestController testController; }; -int main(int argc, char **argv) { - std::string test_file_location; - if (argc > 1) { - test_file_location = argv[1]; - } - +TEST_CASE("TailFile integration test", "[tailfile]") { TailFileTestHarness harness; - - harness.run(test_file_location); - - return 0; + std::filesystem::path test_file_path; + SECTION("Timer driven") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestTailFile.yml"; + } + SECTION("Cron driven") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestTailFileCron.yml"; + } + harness.run(test_file_path); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/VerifyInvokeHTTP.h b/extensions/standard-processors/tests/integration/VerifyInvokeHTTP.h similarity index 86% rename from extensions/http-curl/tests/VerifyInvokeHTTP.h rename to extensions/standard-processors/tests/integration/VerifyInvokeHTTP.h index 3f33ae2933..650ef2143e 100644 --- a/extensions/http-curl/tests/VerifyInvokeHTTP.h +++ b/extensions/standard-processors/tests/integration/VerifyInvokeHTTP.h @@ -18,22 +18,26 @@ #pragma once #include "PropertyDefinition.h" -#undef NDEBUG #include #include #include -#include "TestBase.h" -#include "HTTPClient.h" +#include "unit/TestBase.h" +#include "http/HTTPClient.h" #include "InvokeHTTP.h" #include "processors/LogAttribute.h" #include "controllers/SSLContextService.h" #include "core/state/ProcessorController.h" -#include "HTTPIntegrationBase.h" +#include "integration/HTTPIntegrationBase.h" +#include "unit/Catch.h" +#include "unit/ProvenanceTestHelper.h" +#include "core/yaml/YamlConfiguration.h" using namespace std::literals::chrono_literals; +namespace org::apache::nifi::minifi::test { + class VerifyInvokeHTTP : public HTTPIntegrationBase { public: VerifyInvokeHTTP() @@ -41,7 +45,7 @@ class VerifyInvokeHTTP : public HTTPIntegrationBase { } void testSetup() override { - LogTestController::getInstance().setDebug(); + LogTestController::getInstance().setDebug(); LogTestController::getInstance().setDebug(); LogTestController::getInstance().setTrace(); LogTestController::getInstance().setTrace(); @@ -75,7 +79,7 @@ class VerifyInvokeHTTP : public HTTPIntegrationBase { executed = true; }); - assert(executed); + REQUIRE(executed); } virtual void setupFlow(const std::optional& flow_yml_path) { @@ -90,14 +94,14 @@ class VerifyInvokeHTTP : public HTTPIntegrationBase { configuration->set(minifi::Configure::nifi_c2_agent_heartbeat_period, "200"); std::shared_ptr content_repo = std::make_shared(); content_repo->initialize(configuration); - auto encryption_key = utils::string::from_hex("e4bce4be67f417ed2530038626da57da7725ff8c0b519b692e4311e4d4fe8a28"); + auto encryption_key = minifi::utils::string::from_hex("e4bce4be67f417ed2530038626da57da7725ff8c0b519b692e4311e4d4fe8a28"); auto configuration_context = core::ConfigurationContext{ .flow_file_repo = test_repo, .content_repo = content_repo, .configuration = configuration, .path = flow_yml_path, - .filesystem = std::make_shared(), - .sensitive_properties_encryptor = utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{encryption_key}} + .filesystem = std::make_shared(), + .sensitive_properties_encryptor = minifi::utils::crypto::EncryptionProvider{minifi::utils::crypto::XSalsa20Cipher{encryption_key}} }; auto yaml_ptr = std::make_unique(configuration_context); flowController_ = std::make_unique(test_repo, test_flow_repo, configuration, std::move(yaml_ptr), content_repo); @@ -139,3 +143,5 @@ class VerifyInvokeHTTP : public HTTPIntegrationBase { private: std::optional path_; }; + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/standard-processors/tests/integration/VerifyInvokeHTTPGetTest.cpp b/extensions/standard-processors/tests/integration/VerifyInvokeHTTPGetTest.cpp new file mode 100644 index 0000000000..1fb361aa1a --- /dev/null +++ b/extensions/standard-processors/tests/integration/VerifyInvokeHTTPGetTest.cpp @@ -0,0 +1,78 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "VerifyInvokeHTTP.h" + +#include "integration/HTTPHandlers.h" +#include "unit/TestUtils.h" +#include "unit/Catch.h" + +namespace org::apache::nifi::minifi::test { + +class VerifyHTTPGet : public VerifyInvokeHTTP { + public: + void runAssertions() override { + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime( + std::chrono::seconds(10), + "key:invokehttp.status.code value:200", + "key:flow.id")); + } +}; + +class VerifyRetryHTTPGet : public VerifyInvokeHTTP { + public: + void runAssertions() override { + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime( + std::chrono::seconds(10), + "isSuccess: false, response code 501")); + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime( + std::chrono::seconds(10), + "from InvokeHTTP to relationship retry")); + } +}; + +TEST_CASE("Verify InvokeHTTP GET request", "[invokehttp]") { + HttpGetResponder http_handler; + VerifyHTTPGet harness; + std::filesystem::path test_file_path; + std::string key_dir; + SECTION("Insecure") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestHTTPGet.yml"; + } + SECTION("Secure") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestHTTPGetSecure.yml"; + key_dir = TEST_RESOURCES; + } + harness.run("http://localhost:0/", test_file_path.string(), key_dir, &http_handler); +} + +TEST_CASE("Verify InvokeHTTP GET request with retry", "[invokehttp]") { + RetryHttpGetResponder http_handler; + VerifyRetryHTTPGet harness; + std::filesystem::path test_file_path; + std::string key_dir; + SECTION("Insecure") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestHTTPGet.yml"; + } + SECTION("Secure") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestHTTPGetSecure.yml"; + key_dir = TEST_RESOURCES; + } + harness.run("http://localhost:0/", test_file_path.string(), key_dir, &http_handler); +} + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/standard-processors/tests/integration/VerifyInvokeHTTPPostTest.cpp b/extensions/standard-processors/tests/integration/VerifyInvokeHTTPPostTest.cpp new file mode 100644 index 0000000000..4cedc2d008 --- /dev/null +++ b/extensions/standard-processors/tests/integration/VerifyInvokeHTTPPostTest.cpp @@ -0,0 +1,207 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "VerifyInvokeHTTP.h" + +#include "integration/HTTPHandlers.h" +#include "unit/TestUtils.h" +#include "unit/Catch.h" + +namespace org::apache::nifi::minifi::test { + +class VerifyInvokeHTTPOKResponse : public VerifyInvokeHTTP { + public: + void runAssertions() override { + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(std::chrono::seconds(6), + "key:invokehttp.status.code value:201", + "response code 201")); + } +}; + +class VerifyInvokeHTTPOK200Response : public VerifyInvokeHTTP { + public: + void runAssertions() override { + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(std::chrono::seconds(6), + "key:invokehttp.status.code value:200", + "response code 200")); + } +}; + +class VerifyInvokeHTTPRedirectResponse : public VerifyInvokeHTTP { + public: + void setupFlow(const std::optional& flow_yml_path) override { + VerifyInvokeHTTP::setupFlow(flow_yml_path); + setProperty(minifi::processors::InvokeHTTP::FollowRedirects, "false"); + } + + void runAssertions() override { + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(std::chrono::seconds(6), + "key:invokehttp.status.code value:301", + "response code 301")); + } +}; + +class VerifyCouldNotConnectInvokeHTTP : public VerifyInvokeHTTP { + public: + void runAssertions() override { + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(std::chrono::seconds(6), "key:invoke_http value:failure")); + } +}; + +class VerifyNoRetryInvokeHTTP : public VerifyInvokeHTTP { + public: + void runAssertions() override { + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(std::chrono::seconds(6), + "key:invokehttp.status.message value:HTTP/1.1 404 Not Found", + "isSuccess: false, response code 404")); + } +}; + +class VerifyRetryInvokeHTTP : public VerifyInvokeHTTP { + public: + void runAssertions() override { + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(std::chrono::seconds(6), + "key:invokehttp.status.message value:HTTP/1.1 501 Not Implemented", + "isSuccess: false, response code 501")); + } +}; + +class VerifyRWTimeoutInvokeHTTP : public VerifyInvokeHTTP { + public: + void runAssertions() override { + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(std::chrono::seconds(6), + "key:invoke_http value:failure", + "limit (1000ms) reached, terminating connection")); + } +}; + +TEST_CASE("Verify InvokeHTTP POST request with unreachable remote endpoint", "[invokehttp]") { + // Stop civet server to simulate + // unreachable remote end point + InvokeHTTPCouldNotConnectHandler handler; + VerifyCouldNotConnectInvokeHTTP harness; + std::filesystem::path test_file_path; + std::string key_dir; + SECTION("Secure") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestInvokeHTTPPostSecure.yml"; + harness.setKeyDir(TEST_RESOURCES); + } + SECTION("Insecure") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestInvokeHTTPPost.yml"; + } + harness.setUrl("http://localhost:0/", &handler); + harness.setupFlow(test_file_path); + harness.shutdownBeforeFlowController(); + harness.startFlowController(); + harness.runAssertions(); + harness.stopFlowController(); +} + +TEST_CASE("Verify InvokeHTTP POST request with 201 OK response", "[invokehttp]") { + InvokeHTTPResponseOKHandler handler; + VerifyInvokeHTTPOKResponse harness; + std::filesystem::path test_file_path; + std::string key_dir; + SECTION("Secure") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestInvokeHTTPPostSecure.yml"; + harness.setKeyDir(TEST_RESOURCES); + } + SECTION("Insecure") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestInvokeHTTPPost.yml"; + } + harness.run("http://localhost:0/", test_file_path.string(), key_dir, &handler); +} + +TEST_CASE("Verify InvokeHTTP POST request with 200 OK response", "[invokehttp]") { + InvokeHTTPRedirectHandler handler; + VerifyInvokeHTTPOK200Response harness; + std::filesystem::path test_file_path; + std::string key_dir; + SECTION("Secure") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestInvokeHTTPPostSecure.yml"; + harness.setKeyDir(TEST_RESOURCES); + } + SECTION("Insecure") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestInvokeHTTPPost.yml"; + } + harness.run("http://localhost:0/", test_file_path.string(), key_dir, &handler); +} + +TEST_CASE("Verify InvokeHTTP POST request with 301 redirect response", "[invokehttp]") { + InvokeHTTPRedirectHandler handler; + VerifyInvokeHTTPRedirectResponse harness; + std::filesystem::path test_file_path; + std::string key_dir; + SECTION("Secure") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestInvokeHTTPPostSecure.yml"; + harness.setKeyDir(TEST_RESOURCES); + } + SECTION("Insecure") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestInvokeHTTPPost.yml"; + } + harness.run("http://localhost:0/", test_file_path.string(), key_dir, &handler); +} + +TEST_CASE("Verify InvokeHTTP POST request with 404 not found response", "[invokehttp]") { + InvokeHTTPResponse404Handler handler; + VerifyNoRetryInvokeHTTP harness; + std::filesystem::path test_file_path; + std::string key_dir; + SECTION("Secure") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestInvokeHTTPPostSecure.yml"; + harness.setKeyDir(TEST_RESOURCES); + } + SECTION("Insecure") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestInvokeHTTPPost.yml"; + } + harness.run("http://localhost:0/", test_file_path.string(), key_dir, &handler); +} + +TEST_CASE("Verify InvokeHTTP POST request with 501 not implemented response", "[invokehttp]") { + InvokeHTTPResponse501Handler handler; + VerifyRetryInvokeHTTP harness; + std::filesystem::path test_file_path; + std::string key_dir; + SECTION("Secure") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestInvokeHTTPPostSecure.yml"; + harness.setKeyDir(TEST_RESOURCES); + } + SECTION("Insecure") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestInvokeHTTPPost.yml"; + } + harness.run("http://localhost:0/", test_file_path.string(), key_dir, &handler); +} + +TEST_CASE("Verify InvokeHTTP POST request with timeout failure", "[invokehttp]") { + TimeoutingHTTPHandler handler({std::chrono::seconds(2)}); + VerifyRWTimeoutInvokeHTTP harness; + std::filesystem::path test_file_path; + std::string key_dir; + SECTION("Secure") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestInvokeHTTPPostSecure.yml"; + harness.setKeyDir(TEST_RESOURCES); + } + SECTION("Insecure") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestInvokeHTTPPost.yml"; + } + harness.run("http://localhost:0/", test_file_path.string(), key_dir, &handler); +} + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/standard-processors/tests/unit/AdaptiveConfigurationTests.cpp b/extensions/standard-processors/tests/unit/AdaptiveConfigurationTests.cpp index 0863db41ad..4854240f53 100644 --- a/extensions/standard-processors/tests/unit/AdaptiveConfigurationTests.cpp +++ b/extensions/standard-processors/tests/unit/AdaptiveConfigurationTests.cpp @@ -16,11 +16,11 @@ * limitations under the License. */ -#include "TestBase.h" -#include "Catch.h" -#include "ConfigurationTestController.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/ConfigurationTestController.h" #include "core/flow/AdaptiveConfiguration.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" using namespace std::literals::chrono_literals; @@ -89,7 +89,7 @@ TEST_CASE("Adaptive configuration logs json parse errors") { REQUIRE_THROWS(config.getRootFromPayload(json_config)); - REQUIRE(utils::verifyLogLinePresenceInPollTime(0s, "[debug] Could not parse configuration as json, trying yaml")); - REQUIRE(utils::verifyLogLinePresenceInPollTime(0s, "[error] Configuration file is not valid json: Invalid encoding in string. (38)")); - REQUIRE(utils::verifyLogLinePresenceInPollTime(0s, "[error] Configuration file is not valid yaml: yaml-cpp: error at line 3, column 27: end of map flow not found")); + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(0s, "[debug] Could not parse configuration as json, trying yaml")); + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(0s, "[error] Configuration file is not valid json: Invalid encoding in string. (38)")); + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(0s, "[error] Configuration file is not valid yaml: yaml-cpp: error at line 3, column 27: end of map flow not found")); } diff --git a/extensions/standard-processors/tests/unit/AppendHostInfoTests.cpp b/extensions/standard-processors/tests/unit/AppendHostInfoTests.cpp index c0786d88d9..b02f00c191 100644 --- a/extensions/standard-processors/tests/unit/AppendHostInfoTests.cpp +++ b/extensions/standard-processors/tests/unit/AppendHostInfoTests.cpp @@ -16,8 +16,8 @@ * limitations under the License. */ -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "AppendHostInfo.h" #include "LogAttribute.h" diff --git a/extensions/standard-processors/tests/unit/AttributeRollingWindowTests.cpp b/extensions/standard-processors/tests/unit/AttributeRollingWindowTests.cpp index 4aaa6662fa..a6c71a0100 100644 --- a/extensions/standard-processors/tests/unit/AttributeRollingWindowTests.cpp +++ b/extensions/standard-processors/tests/unit/AttributeRollingWindowTests.cpp @@ -17,9 +17,10 @@ #include #include -#include "Catch.h" +#include "unit/Catch.h" #include "AttributeRollingWindow.h" -#include "SingleProcessorTestController.h" +#include "unit/SingleProcessorTestController.h" +#include "core/FlowFile.h" namespace org::apache::nifi::minifi::test { using AttributeRollingWindow = processors::AttributeRollingWindow; diff --git a/extensions/standard-processors/tests/unit/AttributesToJSONTests.cpp b/extensions/standard-processors/tests/unit/AttributesToJSONTests.cpp index e127d9c3c1..630dcb1696 100644 --- a/extensions/standard-processors/tests/unit/AttributesToJSONTests.cpp +++ b/extensions/standard-processors/tests/unit/AttributesToJSONTests.cpp @@ -20,9 +20,9 @@ #include "rapidjson/writer.h" #include "rapidjson/stringbuffer.h" -#include "TestBase.h" -#include "Catch.h" -#include "utils/TestUtils.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/TestUtils.h" #include "AttributesToJSON.h" #include "GetFile.h" #include "PutFile.h" @@ -61,7 +61,7 @@ class AttributesToJSONTestFixture { update_attribute_->setDynamicProperty("other_attribute", "other_value"); update_attribute_->setDynamicProperty("empty_attribute", ""); - utils::putFileToDir(dir_, TEST_FILE_NAME, TEST_FILE_CONTENT); + minifi::test::utils::putFileToDir(dir_, TEST_FILE_NAME, TEST_FILE_CONTENT); } static void assertJSONAttributesFromLog(const std::unordered_map>& expected_attributes) { diff --git a/extensions/standard-processors/tests/unit/ClassLoaderTests.cpp b/extensions/standard-processors/tests/unit/ClassLoaderTests.cpp index dea4303ee4..097ca9c61d 100644 --- a/extensions/standard-processors/tests/unit/ClassLoaderTests.cpp +++ b/extensions/standard-processors/tests/unit/ClassLoaderTests.cpp @@ -15,8 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/ClassLoader.h" TEST_CASE("TestLoader", "[TestLoader]") { diff --git a/extensions/standard-processors/tests/unit/ConfigurationTests.cpp b/extensions/standard-processors/tests/unit/ConfigurationTests.cpp index 426cc37ad0..7b44c048f6 100644 --- a/extensions/standard-processors/tests/unit/ConfigurationTests.cpp +++ b/extensions/standard-processors/tests/unit/ConfigurationTests.cpp @@ -15,8 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "properties/Configuration.h" #include "Environment.h" diff --git a/extensions/standard-processors/tests/unit/ControllerServiceTests.cpp b/extensions/standard-processors/tests/unit/ControllerServiceTests.cpp index d0dd87b330..fc44ba005f 100644 --- a/extensions/standard-processors/tests/unit/ControllerServiceTests.cpp +++ b/extensions/standard-processors/tests/unit/ControllerServiceTests.cpp @@ -19,8 +19,8 @@ #include #include #include "FlowController.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "properties/Configure.h" #include "GetFile.h" #include "core/Core.h" diff --git a/extensions/standard-processors/tests/unit/DefragmentTextTests.cpp b/extensions/standard-processors/tests/unit/DefragmentTextTests.cpp index a675f3264f..120ed11100 100644 --- a/extensions/standard-processors/tests/unit/DefragmentTextTests.cpp +++ b/extensions/standard-processors/tests/unit/DefragmentTextTests.cpp @@ -17,10 +17,10 @@ */ #include -#include "TestBase.h" -#include "Catch.h" -#include "WriteToFlowFileTestProcessor.h" -#include "ReadFromFlowFileTestProcessor.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/WriteToFlowFileTestProcessor.h" +#include "unit/ReadFromFlowFileTestProcessor.h" #include "UpdateAttribute.h" #include "DefragmentText.h" #include "TextFragmentUtils.h" diff --git a/extensions/standard-processors/tests/unit/ExecuteProcessTests.cpp b/extensions/standard-processors/tests/unit/ExecuteProcessTests.cpp index b6784cbdcb..4a3527490b 100644 --- a/extensions/standard-processors/tests/unit/ExecuteProcessTests.cpp +++ b/extensions/standard-processors/tests/unit/ExecuteProcessTests.cpp @@ -16,9 +16,9 @@ */ #include -#include "Catch.h" +#include "unit/Catch.h" #include "processors/ExecuteProcess.h" -#include "SingleProcessorTestController.h" +#include "unit/SingleProcessorTestController.h" #include "utils/file/FileUtils.h" using namespace std::literals::chrono_literals; diff --git a/extensions/standard-processors/tests/unit/ExtractTextTests.cpp b/extensions/standard-processors/tests/unit/ExtractTextTests.cpp index 8e12aaeb07..197f7d33ed 100644 --- a/extensions/standard-processors/tests/unit/ExtractTextTests.cpp +++ b/extensions/standard-processors/tests/unit/ExtractTextTests.cpp @@ -23,12 +23,12 @@ #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/Core.h" #include "unit/ProvenanceTestHelper.h" #include "repository/VolatileContentRepository.h" -#include "utils/TestUtils.h" +#include "unit/TestUtils.h" #include "core/FlowFile.h" #include "core/Processor.h" @@ -217,7 +217,7 @@ TEST_CASE("Test usage of ExtractText in regex mode with large regex matches", "[ plan->setProperty(log_attribute_processor, org::apache::nifi::minifi::processors::LogAttribute::AttributesToLog, TEST_ATTR); std::string additional_long_string(100'000, '.'); - utils::putFileToDir(dir, TEST_FILE, "Speed limit 80" + additional_long_string); + minifi::test::utils::putFileToDir(dir, TEST_FILE, "Speed limit 80" + additional_long_string); test_controller.runSession(plan); diff --git a/extensions/standard-processors/tests/unit/FetchFileTests.cpp b/extensions/standard-processors/tests/unit/FetchFileTests.cpp index ccb04374f1..4cd1e42698 100644 --- a/extensions/standard-processors/tests/unit/FetchFileTests.cpp +++ b/extensions/standard-processors/tests/unit/FetchFileTests.cpp @@ -21,14 +21,13 @@ #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/Property.h" #include "core/Processor.h" #include "processors/FetchFile.h" -#include "utils/TestUtils.h" -#include "utils/IntegrationTestUtils.h" -#include "SingleProcessorTestController.h" +#include "unit/TestUtils.h" +#include "unit/SingleProcessorTestController.h" using namespace std::literals::chrono_literals; @@ -67,8 +66,8 @@ FetchFileTestFixture::FetchFileTestFixture() attributes_ = {{"absolute.path", input_dir_.string()}, {"filename", input_file_name_.string()}}; - utils::putFileToDir(input_dir_, input_file_name_, file_content_); - utils::putFileToDir(input_dir_, permission_denied_file_name_, file_content_); + minifi::test::utils::putFileToDir(input_dir_, input_file_name_, file_content_); + minifi::test::utils::putFileToDir(input_dir_, permission_denied_file_name_, file_content_); std::filesystem::permissions(input_dir_ / permission_denied_file_name_, static_cast(0)); } @@ -95,7 +94,7 @@ TEST_CASE_METHOD(FetchFileTestFixture, "Test fetching file with default but non- auto file_contents = result.at(minifi::processors::FetchFile::NotFound); REQUIRE(file_contents.size() == 1); REQUIRE(test_controller_->plan->getContent(file_contents[0]).empty()); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; REQUIRE(verifyLogLinePresenceInPollTime(1s, "[error] File to fetch was not found")); } @@ -106,7 +105,7 @@ TEST_CASE_METHOD(FetchFileTestFixture, "FileToFetch property set to a non-existe auto file_contents = result.at(minifi::processors::FetchFile::NotFound); REQUIRE(file_contents.size() == 1); REQUIRE(test_controller_->plan->getContent(file_contents[0]).empty()); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; REQUIRE(verifyLogLinePresenceInPollTime(1s, "[info] File to fetch was not found")); } @@ -118,7 +117,7 @@ TEST_CASE_METHOD(FetchFileTestFixture, "Permission denied to read file", "[testF auto file_contents = result.at(minifi::processors::FetchFile::PermissionDenied); REQUIRE(file_contents.size() == 1); REQUIRE(test_controller_->plan->getContent(file_contents[0]).empty()); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; REQUIRE(verifyLogLinePresenceInPollTime(1s, "[warning] Read permission denied for file")); } #endif @@ -133,7 +132,7 @@ TEST_CASE_METHOD(FetchFileTestFixture, "Test fetching file with default file pat TEST_CASE_METHOD(FetchFileTestFixture, "Test fetching file from a custom path", "[testFetchFile]") { REQUIRE(0 == utils::file::FileUtils::create_dir(input_dir_ / "sub")); - utils::putFileToDir(input_dir_ / "sub", input_file_name_, file_content_); + minifi::test::utils::putFileToDir(input_dir_ / "sub", input_file_name_, file_content_); auto file_path = input_dir_ / "sub" / input_file_name_; fetch_file_processor_->setProperty(org::apache::nifi::minifi::processors::FetchFile::FileToFetch, file_path.string()); const auto result = test_controller_->trigger("", attributes_); @@ -150,7 +149,7 @@ TEST_CASE_METHOD(FetchFileTestFixture, "Flow scheduling fails due to missing mov TEST_CASE_METHOD(FetchFileTestFixture, "Flow fails due to move conflict", "[testFetchFile]") { auto move_dir = test_controller_->createTempDirectory(); - utils::putFileToDir(move_dir, input_file_name_, "old content"); + minifi::test::utils::putFileToDir(move_dir, input_file_name_, "old content"); fetch_file_processor_->setProperty(org::apache::nifi::minifi::processors::FetchFile::CompletionStrategy, "Move File"); fetch_file_processor_->setProperty(org::apache::nifi::minifi::processors::FetchFile::MoveDestinationDirectory, move_dir.string()); fetch_file_processor_->setProperty(org::apache::nifi::minifi::processors::FetchFile::MoveConflictStrategy, "Fail"); @@ -166,7 +165,7 @@ TEST_CASE_METHOD(FetchFileTestFixture, "Flow fails due to move conflict", "[test TEST_CASE_METHOD(FetchFileTestFixture, "Move specific properties are ignored when completion strategy is not move file", "[testFetchFile]") { auto move_dir = test_controller_->createTempDirectory(); - utils::putFileToDir(move_dir, input_file_name_, "old content"); + minifi::test::utils::putFileToDir(move_dir, input_file_name_, "old content"); fetch_file_processor_->setProperty(org::apache::nifi::minifi::processors::FetchFile::MoveDestinationDirectory, move_dir.string()); fetch_file_processor_->setProperty(org::apache::nifi::minifi::processors::FetchFile::MoveConflictStrategy, "Fail"); const auto result = test_controller_->trigger("", attributes_); @@ -177,7 +176,7 @@ TEST_CASE_METHOD(FetchFileTestFixture, "Move specific properties are ignored whe TEST_CASE_METHOD(FetchFileTestFixture, "Move destination conflict is resolved with replace file", "[testFetchFile]") { auto move_dir = test_controller_->createTempDirectory(); - utils::putFileToDir(move_dir, input_file_name_, "old content"); + minifi::test::utils::putFileToDir(move_dir, input_file_name_, "old content"); fetch_file_processor_->setProperty(org::apache::nifi::minifi::processors::FetchFile::CompletionStrategy, "Move File"); fetch_file_processor_->setProperty(org::apache::nifi::minifi::processors::FetchFile::MoveDestinationDirectory, move_dir.string()); fetch_file_processor_->setProperty(org::apache::nifi::minifi::processors::FetchFile::MoveConflictStrategy, "Replace File"); @@ -193,7 +192,7 @@ TEST_CASE_METHOD(FetchFileTestFixture, "Move destination conflict is resolved wi TEST_CASE_METHOD(FetchFileTestFixture, "Move destination conflict is resolved with renaming file to a new random filename", "[testFetchFile]") { auto move_dir = test_controller_->createTempDirectory(); - utils::putFileToDir(move_dir, input_file_name_, "old content"); + minifi::test::utils::putFileToDir(move_dir, input_file_name_, "old content"); fetch_file_processor_->setProperty(org::apache::nifi::minifi::processors::FetchFile::CompletionStrategy, "Move File"); fetch_file_processor_->setProperty(org::apache::nifi::minifi::processors::FetchFile::MoveDestinationDirectory, move_dir.string()); fetch_file_processor_->setProperty(org::apache::nifi::minifi::processors::FetchFile::MoveConflictStrategy, "Rename"); @@ -210,7 +209,7 @@ TEST_CASE_METHOD(FetchFileTestFixture, "Move destination conflict is resolved wi TEST_CASE_METHOD(FetchFileTestFixture, "Move destination conflict is resolved with deleting the new file and keeping the old one", "[testFetchFile]") { auto move_dir = test_controller_->createTempDirectory(); - utils::putFileToDir(move_dir, input_file_name_, "old content"); + minifi::test::utils::putFileToDir(move_dir, input_file_name_, "old content"); fetch_file_processor_->setProperty(org::apache::nifi::minifi::processors::FetchFile::CompletionStrategy, "Move File"); fetch_file_processor_->setProperty(org::apache::nifi::minifi::processors::FetchFile::MoveDestinationDirectory, move_dir.string()); fetch_file_processor_->setProperty(org::apache::nifi::minifi::processors::FetchFile::MoveConflictStrategy, "Keep Existing"); @@ -264,7 +263,7 @@ TEST_CASE_METHOD(FetchFileTestFixture, "Move completion strategy failure due to REQUIRE(file_contents.size() == 1); REQUIRE(test_controller_->plan->getContent(file_contents[0]) == file_content_); REQUIRE(utils::file::FileUtils::exists(input_dir_ / input_file_name_)); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; REQUIRE(verifyLogLinePresenceInPollTime(1s, "completion strategy failed")); utils::file::FileUtils::set_permissions(move_dir, 0644); } diff --git a/extensions/standard-processors/tests/unit/FlowJsonTests.cpp b/extensions/standard-processors/tests/unit/FlowJsonTests.cpp index 71c04bf21f..03c8acea84 100644 --- a/extensions/standard-processors/tests/unit/FlowJsonTests.cpp +++ b/extensions/standard-processors/tests/unit/FlowJsonTests.cpp @@ -23,9 +23,9 @@ #include "core/ProcessGroup.h" #include "core/yaml/YamlConfiguration.h" #include "TailFile.h" -#include "Catch.h" +#include "unit/Catch.h" #include "utils/StringUtils.h" -#include "ConfigurationTestController.h" +#include "unit/ConfigurationTestController.h" #include "Funnel.h" using namespace std::literals::chrono_literals; diff --git a/extensions/standard-processors/tests/unit/GenerateFlowFileTests.cpp b/extensions/standard-processors/tests/unit/GenerateFlowFileTests.cpp index fcb46a6155..65978048cd 100644 --- a/extensions/standard-processors/tests/unit/GenerateFlowFileTests.cpp +++ b/extensions/standard-processors/tests/unit/GenerateFlowFileTests.cpp @@ -19,9 +19,9 @@ #include #include -#include "TestBase.h" -#include "SingleProcessorTestController.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/SingleProcessorTestController.h" +#include "unit/Catch.h" #include "GenerateFlowFile.h" using minifi::processors::GenerateFlowFile; diff --git a/extensions/standard-processors/tests/unit/GetFileTests.cpp b/extensions/standard-processors/tests/unit/GetFileTests.cpp index 71c10b1dbe..17c2625c6a 100644 --- a/extensions/standard-processors/tests/unit/GetFileTests.cpp +++ b/extensions/standard-processors/tests/unit/GetFileTests.cpp @@ -20,15 +20,14 @@ #include #include -#include "TestBase.h" -#include "SingleProcessorTestController.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/SingleProcessorTestController.h" +#include "unit/Catch.h" #include "LogAttribute.h" #include "GetFile.h" #include "utils/file/FileUtils.h" -#include "utils/TestUtils.h" +#include "unit/TestUtils.h" #include "unit/ProvenanceTestHelper.h" -#include "Utils.h" using namespace std::literals::chrono_literals; @@ -71,9 +70,9 @@ GetFileTestController::GetFileTestController() auto log_attr = test_plan_->addProcessor("LogAttribute", "Log", core::Relationship("success", "description"), true); test_plan_->setProperty(log_attr, minifi::processors::LogAttribute::FlowFilesToLog, "0"); - utils::putFileToDir(temp_dir_, input_file_name_, "The quick brown fox jumps over the lazy dog\n"); - utils::putFileToDir(temp_dir_, large_input_file_name_, "The quick brown fox jumps over the lazy dog who is 2 legit to quit\n"); - utils::putFileToDir(temp_dir_, hidden_input_file_name_, "But noone has ever seen it\n"); + minifi::test::utils::putFileToDir(temp_dir_, input_file_name_, "The quick brown fox jumps over the lazy dog\n"); + minifi::test::utils::putFileToDir(temp_dir_, large_input_file_name_, "The quick brown fox jumps over the lazy dog who is 2 legit to quit\n"); + minifi::test::utils::putFileToDir(temp_dir_, hidden_input_file_name_, "But noone has ever seen it\n"); #ifdef WIN32 const auto hide_file_err = minifi::test::utils::hide_file(getFullPath(hidden_input_file_name_)); @@ -171,7 +170,7 @@ TEST_CASE("Check if subdirectories are ignored or not if Recurse property is set auto subdir_path = test_controller.getFullPath("subdir"); utils::file::FileUtils::create_dir(subdir_path); - utils::putFileToDir(subdir_path, "subfile.txt", "Some content in a subfile\n"); + minifi::test::utils::putFileToDir(subdir_path, "subfile.txt", "Some content in a subfile\n"); SECTION("File in subdirectory is ignored when Recurse property set to false") { test_controller.setProperty(minifi::processors::GetFile::Recurse, "false"); @@ -275,7 +274,7 @@ TEST_CASE("GetFile sets attributes correctly") { get_file->setProperty(GetFile::Directory, dir.string()); SECTION("File in subdirectory of input directory") { std::filesystem::create_directories(dir / "a" / "b"); - utils::putFileToDir(dir / "a" / "b", "alpha.txt", "The quick brown fox jumps over the lazy dog\n"); + minifi::test::utils::putFileToDir(dir / "a" / "b", "alpha.txt", "The quick brown fox jumps over the lazy dog\n"); auto result = test_controller.trigger(); REQUIRE((result.contains(GetFile::Success) && result.at(GetFile::Success).size() == 1)); auto flow_file = result.at(GetFile::Success)[0]; @@ -284,7 +283,7 @@ TEST_CASE("GetFile sets attributes correctly") { CHECK(flow_file->getAttribute(minifi::core::SpecialFlowAttribute::FILENAME) == "alpha.txt"); } SECTION("File directly in input directory") { - utils::putFileToDir(dir, "beta.txt", "The quick brown fox jumps over the lazy dog\n"); + minifi::test::utils::putFileToDir(dir, "beta.txt", "The quick brown fox jumps over the lazy dog\n"); auto result = test_controller.trigger(); REQUIRE((result.contains(GetFile::Success) && result.at(GetFile::Success).size() == 1)); auto flow_file = result.at(GetFile::Success)[0]; diff --git a/extensions/standard-processors/tests/unit/GetTCPTests.cpp b/extensions/standard-processors/tests/unit/GetTCPTests.cpp index 750ecad213..067492755e 100644 --- a/extensions/standard-processors/tests/unit/GetTCPTests.cpp +++ b/extensions/standard-processors/tests/unit/GetTCPTests.cpp @@ -16,10 +16,10 @@ */ #include -#include "Catch.h" +#include "unit/Catch.h" #include "processors/GetTCP.h" -#include "SingleProcessorTestController.h" -#include "Utils.h" +#include "unit/SingleProcessorTestController.h" +#include "unit/TestUtils.h" #include "utils/net/AsioCoro.h" #include "utils/net/AsioSocketUtils.h" #include "controllers/SSLContextService.h" @@ -170,7 +170,7 @@ TEST_CASE("GetTCP test with delimiter", "[GetTCP]") { tcp_test_server.queueMessage("Hello\n"); tcp_test_server.run(); - REQUIRE(minifi::utils::verifyEventHappenedInPollTime(250ms, [&] { return tcp_test_server.getPort() != 0; }, 20ms)); + REQUIRE(utils::verifyEventHappenedInPollTime(250ms, [&] { return tcp_test_server.getPort() != 0; }, 20ms)); REQUIRE(get_tcp->setProperty(GetTCP::EndpointList, fmt::format("localhost:{}", tcp_test_server.getPort()))); controller.plan->scheduleProcessor(get_tcp); @@ -203,7 +203,7 @@ TEST_CASE("GetTCP test with too large message", "[GetTCP]") { tcp_test_server.queueMessage("abcdefghijklmnopqrstuvwxyz\rBye\r"); tcp_test_server.run(); - REQUIRE(minifi::utils::verifyEventHappenedInPollTime(250ms, [&] { return tcp_test_server.getPort() != 0; }, 20ms)); + REQUIRE(utils::verifyEventHappenedInPollTime(250ms, [&] { return tcp_test_server.getPort() != 0; }, 20ms)); REQUIRE(get_tcp->setProperty(GetTCP::EndpointList, fmt::format("localhost:{}", tcp_test_server.getPort()))); controller.plan->scheduleProcessor(get_tcp); @@ -247,7 +247,7 @@ TEST_CASE("GetTCP test multiple endpoints", "[GetTCP]") { server_2.queueMessage("012345678901234567890\nAuf Wiedersehen\n"); server_2.run(); - REQUIRE(minifi::utils::verifyEventHappenedInPollTime(250ms, [&] { return server_1.getPort() != 0 && server_2.getPort() != 0; }, 20ms)); + REQUIRE(utils::verifyEventHappenedInPollTime(250ms, [&] { return server_1.getPort() != 0 && server_2.getPort() != 0; }, 20ms)); REQUIRE(get_tcp->setProperty(GetTCP::EndpointList, fmt::format("localhost:{},localhost:{}", server_1.getPort(), server_2.getPort()))); controller.plan->scheduleProcessor(get_tcp); @@ -291,7 +291,7 @@ TEST_CASE("GetTCP max queue and max batch size test", "[GetTCP]") { server.run(); - REQUIRE(minifi::utils::verifyEventHappenedInPollTime(250ms, [&] { return server.getPort() != 0; }, 20ms)); + REQUIRE(utils::verifyEventHappenedInPollTime(250ms, [&] { return server.getPort() != 0; }, 20ms)); REQUIRE(get_tcp->setProperty(GetTCP::EndpointList, fmt::format("localhost:{}", server.getPort()))); controller.plan->scheduleProcessor(get_tcp); diff --git a/extensions/standard-processors/tests/unit/HashContentTest.cpp b/extensions/standard-processors/tests/unit/HashContentTest.cpp index 0066de79ff..fd26d038c8 100644 --- a/extensions/standard-processors/tests/unit/HashContentTest.cpp +++ b/extensions/standard-processors/tests/unit/HashContentTest.cpp @@ -15,19 +15,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#ifdef OPENSSL_SUPPORT - #include #include #include #include #include -#include "TestBase.h" -#include "Catch.h" -#include "SingleProcessorTestController.h" -#include "TestUtils.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/SingleProcessorTestController.h" +#include "unit/TestUtils.h" #include "unit/ProvenanceTestHelper.h" #include "core/Processor.h" @@ -178,4 +175,3 @@ TEST_CASE("Invalid hash algorithm throws in onSchedule", "[HashContent]") { } } // namespace org::apache::nifi::minifi::processors::test -#endif // OPENSSL_SUPPORT diff --git a/extensions/standard-processors/tests/unit/JoltTransformJSONTests.cpp b/extensions/standard-processors/tests/unit/JoltTransformJSONTests.cpp index 7ac82d7112..84305f5778 100644 --- a/extensions/standard-processors/tests/unit/JoltTransformJSONTests.cpp +++ b/extensions/standard-processors/tests/unit/JoltTransformJSONTests.cpp @@ -16,12 +16,11 @@ * limitations under the License. */ -#include "TestBase.h" -#include "Catch.h" -#include "TestUtils.h" -#include "Utils.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/TestUtils.h" #include "processors/JoltTransformJSON.h" -#include "SingleProcessorTestController.h" +#include "unit/SingleProcessorTestController.h" #include "rapidjson/error/en.h" namespace org::apache::nifi::minifi::test { diff --git a/extensions/standard-processors/tests/unit/ListFileTests.cpp b/extensions/standard-processors/tests/unit/ListFileTests.cpp index 98eaa259b7..845a4d5d76 100644 --- a/extensions/standard-processors/tests/unit/ListFileTests.cpp +++ b/extensions/standard-processors/tests/unit/ListFileTests.cpp @@ -18,17 +18,15 @@ #include #include -#include "TestBase.h" -#include "SingleProcessorTestController.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/SingleProcessorTestController.h" +#include "unit/Catch.h" #include "core/Property.h" #include "core/Processor.h" #include "processors/LogAttribute.h" #include "processors/ListFile.h" -#include "utils/TestUtils.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" #include "utils/file/PathUtils.h" -#include "Utils.h" using namespace std::literals::chrono_literals; @@ -44,7 +42,7 @@ inline char get_separator() { } #endif -using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; +using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; class ListFileTestFixture { public: @@ -76,13 +74,13 @@ ListFileTestFixture::ListFileTestFixture() auto log_attribute = plan_->addProcessor("LogAttribute", "logAttribute", core::Relationship("success", "description"), true); plan_->setProperty(log_attribute, minifi::processors::LogAttribute::FlowFilesToLog, "0"); - hidden_file_path_ = utils::putFileToDir(input_dir_, ".hidden_file.txt", "hidden"); - standard_file_abs_path_ = utils::putFileToDir(input_dir_, "standard_file.log", "test"); - empty_file_abs_path_ = utils::putFileToDir(input_dir_, "empty_file.txt", ""); + hidden_file_path_ = minifi::test::utils::putFileToDir(input_dir_, ".hidden_file.txt", "hidden"); + standard_file_abs_path_ = minifi::test::utils::putFileToDir(input_dir_, "standard_file.log", "test"); + empty_file_abs_path_ = minifi::test::utils::putFileToDir(input_dir_, "empty_file.txt", ""); utils::file::FileUtils::create_dir(input_dir_ / "first_subdir"); - first_sub_file_abs_path_ = utils::putFileToDir(input_dir_ / "first_subdir", "sub_file_one.txt", "the"); + first_sub_file_abs_path_ = minifi::test::utils::putFileToDir(input_dir_ / "first_subdir", "sub_file_one.txt", "the"); utils::file::FileUtils::create_dir(input_dir_ / "second_subdir"); - second_sub_file_abs_path_ = utils::putFileToDir(input_dir_ / "second_subdir", "sub_file_two.txt", "some_other_content"); + second_sub_file_abs_path_ = minifi::test::utils::putFileToDir(input_dir_ / "second_subdir", "sub_file_two.txt", "some_other_content"); auto last_write_time = *utils::file::FileUtils::last_write_time(standard_file_abs_path_); utils::file::FileUtils::set_last_write_time(empty_file_abs_path_, last_write_time - 1h); @@ -236,7 +234,7 @@ TEST_CASE("ListFile sets attributes correctly") { list_file->setProperty(ListFile::InputDirectory, dir.string()); SECTION("File in subdirectory of input directory") { std::filesystem::create_directories(dir / "a" / "b"); - utils::putFileToDir(dir / "a" / "b", "alpha.txt", "The quick brown fox jumps over the lazy dog\n"); + minifi::test::utils::putFileToDir(dir / "a" / "b", "alpha.txt", "The quick brown fox jumps over the lazy dog\n"); auto result = test_controller.trigger(); REQUIRE((result.contains(ListFile::Success) && result.at(ListFile::Success).size() == 1)); auto flow_file = result.at(ListFile::Success)[0]; @@ -245,7 +243,7 @@ TEST_CASE("ListFile sets attributes correctly") { CHECK(flow_file->getAttribute(minifi::core::SpecialFlowAttribute::FILENAME) == "alpha.txt"); } SECTION("File directly in input directory") { - utils::putFileToDir(dir, "beta.txt", "The quick brown fox jumps over the lazy dog\n"); + minifi::test::utils::putFileToDir(dir, "beta.txt", "The quick brown fox jumps over the lazy dog\n"); auto result = test_controller.trigger(); REQUIRE((result.contains(ListFile::Success) && result.at(ListFile::Success).size() == 1)); auto flow_file = result.at(ListFile::Success)[0]; @@ -266,12 +264,12 @@ TEST_CASE("If a second file with the same modification time shows up later, then const auto common_timestamp = std::chrono::file_clock::now(); - const auto file_one = utils::putFileToDir(input_dir, "file_one.txt", "When I was one, I had just begun."); + const auto file_one = minifi::test::utils::putFileToDir(input_dir, "file_one.txt", "When I was one, I had just begun."); std::filesystem::last_write_time(file_one, common_timestamp); const auto result_one = test_controller.trigger(); CHECK(result_one.at(ListFile::Success).size() == 1); - const auto file_two = utils::putFileToDir(input_dir, "file_two.txt", "When I was two, I was nearly new."); + const auto file_two = minifi::test::utils::putFileToDir(input_dir, "file_two.txt", "When I was two, I was nearly new."); std::filesystem::last_write_time(file_two, common_timestamp); const auto result_two = test_controller.trigger(); CHECK(result_two.at(ListFile::Success).size() == 1); diff --git a/extensions/standard-processors/tests/unit/ListenSyslogTests.cpp b/extensions/standard-processors/tests/unit/ListenSyslogTests.cpp index 565df98d8b..5314a87e9c 100644 --- a/extensions/standard-processors/tests/unit/ListenSyslogTests.cpp +++ b/extensions/standard-processors/tests/unit/ListenSyslogTests.cpp @@ -16,10 +16,10 @@ */ -#include "Catch.h" +#include "unit/Catch.h" #include "ListenSyslog.h" -#include "SingleProcessorTestController.h" -#include "Utils.h" +#include "unit/SingleProcessorTestController.h" +#include "unit/TestUtils.h" #include "controllers/SSLContextService.h" #include "range/v3/algorithm/contains.hpp" diff --git a/extensions/standard-processors/tests/unit/ListenTcpTests.cpp b/extensions/standard-processors/tests/unit/ListenTcpTests.cpp index e65b3fd62d..f63027fc68 100644 --- a/extensions/standard-processors/tests/unit/ListenTcpTests.cpp +++ b/extensions/standard-processors/tests/unit/ListenTcpTests.cpp @@ -16,20 +16,18 @@ */ #include -#include "Catch.h" +#include "unit/Catch.h" #include "processors/ListenTCP.h" -#include "SingleProcessorTestController.h" -#include "Utils.h" +#include "unit/SingleProcessorTestController.h" #include "controllers/SSLContextService.h" #include "range/v3/algorithm/contains.hpp" -#include "utils/IntegrationTestUtils.h" -#include "utils/StringUtils.h" #include "catch2/generators/catch_generators.hpp" +#include "unit/TestUtils.h" using ListenTCP = org::apache::nifi::minifi::processors::ListenTCP; using namespace std::literals::chrono_literals; -using org::apache::nifi::minifi::utils::verifyLogLineVariantPresenceInPollTime; +using org::apache::nifi::minifi::test::utils::verifyLogLineVariantPresenceInPollTime; namespace org::apache::nifi::minifi::test { diff --git a/extensions/standard-processors/tests/unit/ListenUDPTests.cpp b/extensions/standard-processors/tests/unit/ListenUDPTests.cpp index 2005d26ff3..0a9c6490bb 100644 --- a/extensions/standard-processors/tests/unit/ListenUDPTests.cpp +++ b/extensions/standard-processors/tests/unit/ListenUDPTests.cpp @@ -16,12 +16,12 @@ */ #include -#include "Catch.h" +#include "unit/Catch.h" #include "processors/ListenUDP.h" -#include "SingleProcessorTestController.h" -#include "Utils.h" +#include "unit/SingleProcessorTestController.h" #include "controllers/SSLContextService.h" #include "range/v3/algorithm/contains.hpp" +#include "unit/TestUtils.h" using ListenUDP = org::apache::nifi::minifi::processors::ListenUDP; diff --git a/extensions/standard-processors/tests/unit/ManifestTests.cpp b/extensions/standard-processors/tests/unit/ManifestTests.cpp index ed6fc18906..4c484746d0 100644 --- a/extensions/standard-processors/tests/unit/ManifestTests.cpp +++ b/extensions/standard-processors/tests/unit/ManifestTests.cpp @@ -21,8 +21,8 @@ #include #include "core/state/nodes/DeviceInformation.h" #include "core/state/nodes/AgentInformation.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "range/v3/algorithm/contains.hpp" #include "range/v3/algorithm/find_if.hpp" diff --git a/extensions/standard-processors/tests/unit/ProcessGroupTestUtils.h b/extensions/standard-processors/tests/unit/ProcessGroupTestUtils.h index 7298ff0794..8b1d49885c 100644 --- a/extensions/standard-processors/tests/unit/ProcessGroupTestUtils.h +++ b/extensions/standard-processors/tests/unit/ProcessGroupTestUtils.h @@ -23,9 +23,9 @@ #include #include -#include "TestBase.h" -#include "Catch.h" -#include "Utils.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/TestUtils.h" struct Lines { std::vector lines; @@ -278,7 +278,7 @@ auto findByName(const std::set& set, const std::string& name) -> decltype(Res void assertFailure(const Conn& expected, ConnectionFailure failure) { auto assertMessage = [](const std::string& message) { - REQUIRE(utils::verifyLogLinePresenceInPollTime(std::chrono::seconds{1}, message)); + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(std::chrono::seconds{1}, message)); }; switch (failure) { diff --git a/extensions/standard-processors/tests/unit/ProcessorTests.cpp b/extensions/standard-processors/tests/unit/ProcessorTests.cpp index 08998e70de..6c721dfc4f 100644 --- a/extensions/standard-processors/tests/unit/ProcessorTests.cpp +++ b/extensions/standard-processors/tests/unit/ProcessorTests.cpp @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#define EXTENSION_LIST "*minifi-*,!*http-curl*,!*coap*,!*splunk*,!*elastic*,!*grafana-loki*" // NOLINT(cppcoreguidelines-macro-usage) +#define EXTENSION_LIST "*minifi-*,!*coap*,!*splunk*,!*elastic*,!*grafana-loki*" // NOLINT(cppcoreguidelines-macro-usage) #include #include @@ -30,8 +30,8 @@ #include #endif /* WIN32 */ -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "catch2/matchers/catch_matchers_string.hpp" #include "LogAttribute.h" #include "GetFile.h" @@ -47,8 +47,7 @@ #include "core/Resource.h" #include "utils/gsl.h" #include "utils/PropertyErrors.h" -#include "utils/IntegrationTestUtils.h" -#include "Utils.h" +#include "unit/TestUtils.h" #include "io/BufferStream.h" #include "fmt/format.h" @@ -524,7 +523,7 @@ TEST_CASE("TestEmptyContent", "[emptyContent]") { plan->runNextProcessor(); - REQUIRE(utils::verifyLogLinePresenceInPollTime(std::chrono::seconds{0}, "did not create a ResourceClaim")); + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(std::chrono::seconds{0}, "did not create a ResourceClaim")); LogTestController::getInstance().reset(); } @@ -536,7 +535,7 @@ TEST_CASE("TestEmptyContent", "[emptyContent]") { * @param portVal port value to search in the corresponding log message * @param hasException dictates if a failure should occur */ -void testRPGBypass(const std::string &host, const std::string &port, const std::string &portVal = "-1", bool hasException = true) { +void testRPGBypass(const std::string &host, const std::string &port, bool has_error = true) { TestController testController; LogTestController::getInstance().setTrace(); LogTestController::getInstance().setTrace(); @@ -556,23 +555,14 @@ void testRPGBypass(const std::string &host, const std::string &port, const std:: auto node = std::make_shared(rpg.get()); auto context = std::make_shared(node, nullptr, repo, repo, content_repo); auto psf = std::make_shared(context); - if (hasException) { - auto expected_error = "Site2Site Protocol: HTTPClient not resolvable. No peers configured or any port specific hostname and port -- cannot schedule"; - try { - rpg->onSchedule(*context, *psf); - } catch (std::exception &e) { - REQUIRE(expected_error == std::string(e.what())); - } - std::stringstream search_expr; - search_expr << " " << host << "/" << port << "/" << portVal << " -- configuration values after eval of configuration options"; - REQUIRE(LogTestController::getInstance().contains(search_expr.str())); + if (has_error) { + rpg->onSchedule(*context, *psf); + auto expected_error = "No peers selected during scheduling"; + REQUIRE(LogTestController::getInstance().contains(expected_error)); } LogTestController::getInstance().reset(); } -/** - * Since there is no curl loaded in this test folder, we will have is_http_disabled be true. - */ TEST_CASE("TestRPGNoSettings", "[TestRPG1]") { testRPGBypass("", ""); } @@ -594,7 +584,7 @@ TEST_CASE("TestRPGWithoutHostInvalidPort", "[TestRPG5]") { } TEST_CASE("TestRPGValid", "[TestRPG6]") { - testRPGBypass("", "8080", "8080", false); + testRPGBypass("", "8080", false); } namespace { @@ -680,7 +670,7 @@ TEST_CASE_METHOD(ProcessorWithIncomingConnectionTest, "A Processor detects corre REQUIRE_FALSE(processor_->isWorkAvailable()); const auto penalty_has_expired = [flow_file] { return !flow_file->isPenalized(); }; - REQUIRE(utils::verifyEventHappenedInPollTime(std::chrono::seconds{1}, penalty_has_expired, std::chrono::milliseconds{10})); + REQUIRE(minifi::test::utils::verifyEventHappenedInPollTime(std::chrono::seconds{1}, penalty_has_expired, std::chrono::milliseconds{10})); REQUIRE(processor_->isWorkAvailable()); } } @@ -691,7 +681,7 @@ TEST_CASE_METHOD(ProcessorWithIncomingConnectionTest, "A failed and re-penalized session_->penalize(penalized_flow_file); incoming_connection_->put(penalized_flow_file); const auto penalty_has_expired = [penalized_flow_file] { return !penalized_flow_file->isPenalized(); }; - REQUIRE(utils::verifyEventHappenedInPollTime(std::chrono::seconds{1}, penalty_has_expired, std::chrono::milliseconds{10})); + REQUIRE(minifi::test::utils::verifyEventHappenedInPollTime(std::chrono::seconds{1}, penalty_has_expired, std::chrono::milliseconds{10})); const auto flow_file_1 = session_->create(); incoming_connection_->put(flow_file_1); @@ -724,7 +714,7 @@ TEST_CASE_METHOD(ProcessorWithIncomingConnectionTest, "A failed and re-penalized REQUIRE(next_flow_file_4 != penalized_flow_file); REQUIRE(next_flow_file_4 == flow_file_3); - REQUIRE(utils::verifyEventHappenedInPollTime(std::chrono::seconds{1}, penalty_has_expired, std::chrono::milliseconds{10})); + REQUIRE(minifi::test::utils::verifyEventHappenedInPollTime(std::chrono::seconds{1}, penalty_has_expired, std::chrono::milliseconds{10})); REQUIRE(incoming_connection_->isWorkAvailable()); const auto next_flow_file_5 = incoming_connection_->poll(expired_flow_files); REQUIRE(next_flow_file_5 == penalized_flow_file); diff --git a/extensions/standard-processors/tests/unit/PutFileTests.cpp b/extensions/standard-processors/tests/unit/PutFileTests.cpp index da30d491d2..653cc12311 100644 --- a/extensions/standard-processors/tests/unit/PutFileTests.cpp +++ b/extensions/standard-processors/tests/unit/PutFileTests.cpp @@ -23,9 +23,9 @@ #include #include "utils/file/FileUtils.h" -#include "TestBase.h" -#include "Catch.h" -#include "TestUtils.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/TestUtils.h" #include "processors/LogAttribute.h" #include "processors/GetFile.h" #include "processors/PutFile.h" diff --git a/extensions/standard-processors/tests/unit/PutTCPTests.cpp b/extensions/standard-processors/tests/unit/PutTCPTests.cpp index f377e09479..b4b2ff512b 100644 --- a/extensions/standard-processors/tests/unit/PutTCPTests.cpp +++ b/extensions/standard-processors/tests/unit/PutTCPTests.cpp @@ -20,19 +20,19 @@ #include #include -#include "SingleProcessorTestController.h" -#include "Catch.h" +#include "unit/SingleProcessorTestController.h" +#include "unit/Catch.h" #include "PutTCP.h" #include "controllers/SSLContextService.h" #include "core/ProcessSession.h" #include "utils/net/TcpServer.h" #include "utils/net/AsioCoro.h" #include "utils/expected.h" -#include "utils/StringUtils.h" -#include "IntegrationTestUtils.h" +#include "unit/TestUtils.h" using namespace std::literals::chrono_literals; -using org::apache::nifi::minifi::utils::verifyLogLineVariantPresenceInPollTime; +using org::apache::nifi::minifi::test::utils::verifyLogLineVariantPresenceInPollTime; +using org::apache::nifi::minifi::test::utils::verifyEventHappenedInPollTime; namespace org::apache::nifi::minifi::processors { @@ -238,7 +238,7 @@ class PutTCPTestFixture { gsl_Expects(!cancellable_server && !server_thread_.joinable()); cancellable_server = std::make_unique(std::nullopt, 0, core::logging::LoggerFactory::getLogger(), std::move(ssl_server_options), true, "\n"); server_thread_ = std::thread([this]() { cancellable_server->run(); }); - REQUIRE(utils::verifyEventHappenedInPollTime(250ms, [this] { return cancellable_server->getPort() != 0; }, 20ms)); + REQUIRE(verifyEventHappenedInPollTime(250ms, [this] { return cancellable_server->getPort() != 0; }, 20ms)); return cancellable_server->getPort(); } diff --git a/extensions/standard-processors/tests/unit/PutUDPTests.cpp b/extensions/standard-processors/tests/unit/PutUDPTests.cpp index c2cbbae209..d17106fc81 100644 --- a/extensions/standard-processors/tests/unit/PutUDPTests.cpp +++ b/extensions/standard-processors/tests/unit/PutUDPTests.cpp @@ -20,8 +20,8 @@ #include #include #include -#include "SingleProcessorTestController.h" -#include "Catch.h" +#include "unit/SingleProcessorTestController.h" +#include "unit/Catch.h" #include "PutUDP.h" #include "core/ProcessContext.h" #include "utils/net/UdpServer.h" diff --git a/extensions/standard-processors/tests/unit/ReplaceTextTests.cpp b/extensions/standard-processors/tests/unit/ReplaceTextTests.cpp index d3dfb3c3b6..9092262ace 100644 --- a/extensions/standard-processors/tests/unit/ReplaceTextTests.cpp +++ b/extensions/standard-processors/tests/unit/ReplaceTextTests.cpp @@ -18,8 +18,8 @@ #include "GenerateFlowFile.h" #include "LogAttribute.h" #include "ReplaceText.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" namespace org::apache::nifi::minifi::processors { diff --git a/extensions/standard-processors/tests/unit/RetryFlowFileTests.cpp b/extensions/standard-processors/tests/unit/RetryFlowFileTests.cpp index 6bc8108587..53a1afc18b 100644 --- a/extensions/standard-processors/tests/unit/RetryFlowFileTests.cpp +++ b/extensions/standard-processors/tests/unit/RetryFlowFileTests.cpp @@ -22,8 +22,8 @@ #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "processors/GenerateFlowFile.h" #include "processors/UpdateAttribute.h" @@ -31,7 +31,7 @@ #include "processors/PutFile.h" #include "processors/LogAttribute.h" #include "utils/file/FileUtils.h" -#include "utils/TestUtils.h" +#include "unit/TestUtils.h" namespace { using std::optional; diff --git a/extensions/standard-processors/tests/unit/RollingWindowTests.cpp b/extensions/standard-processors/tests/unit/RollingWindowTests.cpp index d7c7346210..b17b4a89bf 100644 --- a/extensions/standard-processors/tests/unit/RollingWindowTests.cpp +++ b/extensions/standard-processors/tests/unit/RollingWindowTests.cpp @@ -16,7 +16,7 @@ */ #include -#include "Catch.h" +#include "unit/Catch.h" #include "RollingWindow.h" #include "range/v3/view/zip.hpp" #include "range/v3/algorithm/contains.hpp" diff --git a/extensions/standard-processors/tests/unit/RouteTextTests.cpp b/extensions/standard-processors/tests/unit/RouteTextTests.cpp index 914925539b..3db31e44c4 100644 --- a/extensions/standard-processors/tests/unit/RouteTextTests.cpp +++ b/extensions/standard-processors/tests/unit/RouteTextTests.cpp @@ -17,8 +17,8 @@ */ #include "FlowFileRecord.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "processors/RouteText.h" namespace processors = minifi::processors; diff --git a/extensions/standard-processors/tests/unit/SplitTextTests.cpp b/extensions/standard-processors/tests/unit/SplitTextTests.cpp index 638f09a5c8..83fbf2612e 100644 --- a/extensions/standard-processors/tests/unit/SplitTextTests.cpp +++ b/extensions/standard-processors/tests/unit/SplitTextTests.cpp @@ -17,10 +17,10 @@ */ #include "FlowFileRecord.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "processors/SplitText.h" -#include "SingleProcessorTestController.h" +#include "unit/SingleProcessorTestController.h" #include "io/BufferStream.h" namespace org::apache::nifi::minifi::test { diff --git a/extensions/standard-processors/tests/unit/TailFileTests.cpp b/extensions/standard-processors/tests/unit/TailFileTests.cpp index 455c9d25f2..8dd5d91e90 100644 --- a/extensions/standard-processors/tests/unit/TailFileTests.cpp +++ b/extensions/standard-processors/tests/unit/TailFileTests.cpp @@ -28,8 +28,8 @@ #include #include #include "FlowController.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/Core.h" #include "core/FlowFile.h" #include "utils/file/FileUtils.h" @@ -42,9 +42,9 @@ #include "core/Resource.h" #include "TailFile.h" #include "LogAttribute.h" -#include "utils/TestUtils.h" +#include "unit/TestUtils.h" #include "utils/StringUtils.h" -#include "SingleProcessorTestController.h" +#include "unit/SingleProcessorTestController.h" using namespace std::literals::chrono_literals; diff --git a/extensions/standard-processors/tests/unit/UpdateAttributeTests.cpp b/extensions/standard-processors/tests/unit/UpdateAttributeTests.cpp index b5d23f3abf..0647b1c530 100644 --- a/extensions/standard-processors/tests/unit/UpdateAttributeTests.cpp +++ b/extensions/standard-processors/tests/unit/UpdateAttributeTests.cpp @@ -15,9 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "TestBase.h" -#include "Catch.h" #include + +#include "unit/TestBase.h" +#include "unit/Catch.h" + #include "LogAttribute.h" #include "UpdateAttribute.h" #include "GenerateFlowFile.h" diff --git a/extensions/standard-processors/tests/unit/YamlConfigurationTests.cpp b/extensions/standard-processors/tests/unit/YamlConfigurationTests.cpp index d1c9beec2c..caa0002f3b 100644 --- a/extensions/standard-processors/tests/unit/YamlConfigurationTests.cpp +++ b/extensions/standard-processors/tests/unit/YamlConfigurationTests.cpp @@ -24,11 +24,11 @@ #include "core/RepositoryFactory.h" #include "core/yaml/YamlConfiguration.h" #include "TailFile.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/StringUtils.h" -#include "ConfigurationTestController.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/ConfigurationTestController.h" +#include "unit/TestUtils.h" using namespace std::literals::chrono_literals; @@ -793,7 +793,7 @@ TEST_CASE("Configuration is not valid yaml", "[YamlConfiguration]") { ConfigurationTestController test_controller; core::YamlConfiguration yaml_config(test_controller.getContext()); REQUIRE_THROWS(yaml_config.getRootFromPayload("}")); - REQUIRE(utils::verifyLogLinePresenceInPollTime(0s, "Configuration is not valid yaml")); + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(0s, "Configuration is not valid yaml")); } namespace { diff --git a/extensions/standard-processors/tests/unit/YamlConnectionParserTest.cpp b/extensions/standard-processors/tests/unit/YamlConnectionParserTest.cpp index 0444c71e69..0103996454 100644 --- a/extensions/standard-processors/tests/unit/YamlConnectionParserTest.cpp +++ b/extensions/standard-processors/tests/unit/YamlConnectionParserTest.cpp @@ -20,9 +20,9 @@ #include "core/yaml/YamlConfiguration.h" #include "TailFile.h" -#include "TestBase.h" -#include "Catch.h" -#include "utils/TestUtils.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/TestUtils.h" #include "core/yaml/YamlNode.h" using namespace std::literals::chrono_literals; @@ -80,8 +80,8 @@ TEST_CASE("Connections components are parsed from yaml", "[YamlConfiguration]") REQUIRE(231 == yaml_connection_parser.getSwapThreshold()); } SECTION("Source and destination names and uuids are read") { - const utils::Identifier expected_source_id = utils::generateUUID(); - const utils::Identifier expected_destination_id = utils::generateUUID(); + const utils::Identifier expected_source_id = minifi::test::utils::generateUUID(); + const utils::Identifier expected_destination_id = minifi::test::utils::generateUUID(); std::string serialized_yaml; parent.addProcessor(std::make_unique("TailFile_1", expected_source_id)); parent.addProcessor(std::make_unique("TailFile_2", expected_destination_id)); diff --git a/extensions/standard-processors/tests/unit/YamlProcessGroupParserTests.cpp b/extensions/standard-processors/tests/unit/YamlProcessGroupParserTests.cpp index f3dd7dfea0..fb5d6df757 100644 --- a/extensions/standard-processors/tests/unit/YamlProcessGroupParserTests.cpp +++ b/extensions/standard-processors/tests/unit/YamlProcessGroupParserTests.cpp @@ -15,11 +15,10 @@ * limitations under the License. */ -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "YamlConfiguration.h" -#include "Utils.h" -#include "IntegrationTestUtils.h" +#include "unit/TestUtils.h" #include "ProcessGroupTestUtils.h" static core::YamlConfiguration config{core::ConfigurationContext{ diff --git a/extensions/systemd/tests/CMakeLists.txt b/extensions/systemd/tests/CMakeLists.txt index ec689d9e32..77160a591b 100644 --- a/extensions/systemd/tests/CMakeLists.txt +++ b/extensions/systemd/tests/CMakeLists.txt @@ -22,12 +22,11 @@ function(add_systemd_test testfile) add_minifi_executable(${TEST_TARGET} "${testfile}") target_include_directories(${TEST_TARGET} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/systemd/") target_include_directories(${TEST_TARGET} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") - target_include_directories(${TEST_TARGET} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test") target_link_libraries(${TEST_TARGET} minifi-systemd) target_link_libraries(${TEST_TARGET} minifi-standard-processors) createTests("${TEST_TARGET}") add_test(NAME ${TEST_TARGET} COMMAND "${TEST_TARGET}" WORKING_DIRECTORY "${TEST_DIR}") - target_link_libraries(${TEST_TARGET} Catch2WithMain) + target_link_libraries(${TEST_TARGET} Catch2::Catch2WithMain) endfunction() add_systemd_test("ConsumeJournaldTest.cpp") diff --git a/extensions/systemd/tests/ConsumeJournaldTest.cpp b/extensions/systemd/tests/ConsumeJournaldTest.cpp index 2a49bc28d0..b612a281fe 100644 --- a/extensions/systemd/tests/ConsumeJournaldTest.cpp +++ b/extensions/systemd/tests/ConsumeJournaldTest.cpp @@ -20,13 +20,13 @@ #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "ConsumeJournald.h" #include "libwrapper/LibWrapper.h" #include "utils/gsl.h" #include "utils/StringUtils.h" -#include "Utils.h" +#include "unit/TestUtils.h" namespace minifi = org::apache::nifi::minifi; namespace utils = minifi::utils; diff --git a/extensions/windows-event-log/tests/BookmarkTests.cpp b/extensions/windows-event-log/tests/BookmarkTests.cpp index 6d0b762554..a6151ee23d 100644 --- a/extensions/windows-event-log/tests/BookmarkTests.cpp +++ b/extensions/windows-event-log/tests/BookmarkTests.cpp @@ -20,8 +20,8 @@ #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/gsl.h" #include "wel/UniqueEvtHandle.h" #include "CWELTestUtils.h" diff --git a/extensions/windows-event-log/tests/CMakeLists.txt b/extensions/windows-event-log/tests/CMakeLists.txt index 68487dcf33..1939cd3553 100644 --- a/extensions/windows-event-log/tests/CMakeLists.txt +++ b/extensions/windows-event-log/tests/CMakeLists.txt @@ -32,12 +32,11 @@ FOREACH(testfile ${WEL_TESTS}) add_minifi_executable("${testfilename}" "${testfile}") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/windows-event-log/") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test/") target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include/") createTests("${testfilename}") target_link_libraries(${testfilename} minifi-wel) target_link_libraries(${testfilename} minifi-standard-processors) - target_link_libraries (${testfilename} ${LIBMINIFI} Catch2WithMain) + target_link_libraries (${testfilename} ${LIBMINIFI} Catch2::Catch2WithMain) MATH(EXPR WEL_TEST_COUNT "${WEL_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/extensions/windows-event-log/tests/CWELCustomProviderTests.cpp b/extensions/windows-event-log/tests/CWELCustomProviderTests.cpp index 9a3a61e4c4..88eb38ab3c 100644 --- a/extensions/windows-event-log/tests/CWELCustomProviderTests.cpp +++ b/extensions/windows-event-log/tests/CWELCustomProviderTests.cpp @@ -14,25 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG - #include "ConsumeWindowsEventLog.h" #include "core/ConfigurableComponent.h" #include "processors/LogAttribute.h" #include "processors/PutFile.h" -#include "TestBase.h" -#include "Catch.h" -#include "utils/TestUtils.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/TestUtils.h" #include "utils/file/FileUtils.h" #include "wel/UniqueEvtHandle.h" -#include "IntegrationTestUtils.h" #include "rapidjson/document.h" #include "CWELTestUtils.h" -#include "Utils.h" // generated from the manifest file "custom-provider/unit-test-provider.man" // using the command "mc -um unit-test-provider.man" @@ -122,7 +117,7 @@ class CustomProviderController : public OutputFormatTestController { private: bool checkNewEventAvailable() { - return org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime(std::chrono::seconds{5}, [&] { + return org::apache::nifi::utils::verifyEventHappenedInPollTime(std::chrono::seconds{5}, [&] { return advanceBookmark(bookmark_, channel_, query_); }); } diff --git a/extensions/windows-event-log/tests/CWELTestUtils.h b/extensions/windows-event-log/tests/CWELTestUtils.h index d3aa867afd..65cd398168 100644 --- a/extensions/windows-event-log/tests/CWELTestUtils.h +++ b/extensions/windows-event-log/tests/CWELTestUtils.h @@ -26,10 +26,10 @@ #include "ConsumeWindowsEventLog.h" #include "processors/PutFile.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/StringUtils.h" -#include "utils/TestUtils.h" +#include "unit/TestUtils.h" #include "utils/UnicodeConversion.h" #include "utils/file/FileUtils.h" #include "utils/gsl.h" diff --git a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp index b0dfd1d42e..31e40a83df 100644 --- a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp +++ b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp @@ -20,12 +20,12 @@ #include "core/ConfigurableComponent.h" #include "processors/LogAttribute.h" #include "processors/PutFile.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/file/FileUtils.h" #include "CWELTestUtils.h" -#include "Utils.h" +#include "unit/TestUtils.h" #include "../wel/UniqueEvtHandle.h" #include "../wel/JSONUtils.h" #include "utils/Deleters.h" diff --git a/extensions/windows-event-log/tests/LookupCacherTests.cpp b/extensions/windows-event-log/tests/LookupCacherTests.cpp index 1bcd410587..3da8c5fb83 100644 --- a/extensions/windows-event-log/tests/LookupCacherTests.cpp +++ b/extensions/windows-event-log/tests/LookupCacherTests.cpp @@ -17,7 +17,7 @@ #include -#include "Catch.h" +#include "unit/Catch.h" #include "wel/LookupCacher.h" namespace wel = org::apache::nifi::minifi::wel; diff --git a/extensions/windows-event-log/tests/MetadataWalkerTests.cpp b/extensions/windows-event-log/tests/MetadataWalkerTests.cpp index 796a8c252c..300b8009a6 100644 --- a/extensions/windows-event-log/tests/MetadataWalkerTests.cpp +++ b/extensions/windows-event-log/tests/MetadataWalkerTests.cpp @@ -19,8 +19,8 @@ #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/Core.h" #include "utils/OsUtils.h" #include "wel/MetadataWalker.h" diff --git a/fedora.sh b/fedora.sh index 7eaf4de946..e64e13e7a1 100644 --- a/fedora.sh +++ b/fedora.sh @@ -38,9 +38,8 @@ build_deps(){ wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm sudo yum -y install epel-release-latest-7.noarch.rpm - COMMAND="sudo yum -y install libuuid libuuid-devel patch" + COMMAND="sudo yum -y install libuuid libuuid-devel patch perl bzip2-devel" INSTALLED=() - INSTALLED+=("bzip2-devel") for option in "${OPTIONS[@]}" ; do option_value="${!option}" if [ "$option_value" = "${TRUE}" ]; then @@ -80,8 +79,6 @@ build_deps(){ INSTALLED+=("xz-devel") elif [ "$FOUND_VALUE" = "libssh2" ]; then INSTALLED+=("libssh2-devel") - elif [ "$FOUND_VALUE" = "opensslbuild" ]; then - INSTALLED+=("perl") fi fi done diff --git a/libminifi/CMakeLists.txt b/libminifi/CMakeLists.txt index f2817790ae..2eaaab406a 100644 --- a/libminifi/CMakeLists.txt +++ b/libminifi/CMakeLists.txt @@ -40,14 +40,13 @@ else() set(SOCKET_SOURCES "src/io/posix/*.cpp") endif() -if (MINIFI_OPENSSL) - set(TLS_SOURCES "src/utils/tls/*.cpp" "src/io/tls/*.cpp") -endif() +set(TLS_SOURCES "src/utils/tls/*.cpp" "src/io/tls/*.cpp") + +file(GLOB SOURCES "src/agent/*.cpp" "src/properties/*.cpp" "src/utils/file/*.cpp" "src/sitetosite/*.cpp" "src/core/logging/*.cpp" "src/core/logging/internal/*.cpp" "src/core/logging/alert/*.cpp" "src/core/state/*.cpp" "src/core/state/nodes/*.cpp" "src/c2/protocols/*.cpp" "src/c2/triggers/*.cpp" "src/c2/*.cpp" "src/io/*.cpp" ${SOCKET_SOURCES} ${TLS_SOURCES} "src/core/controller/*.cpp" "src/controllers/*.cpp" "src/controllers/keyvalue/*.cpp" "src/core/*.cpp" "src/core/repository/*.cpp" "src/core/flow/*.cpp" "src/core/json/*.cpp" "src/core/yaml/*.cpp" "src/core/reporting/*.cpp" "src/core/extension/*.cpp" "src/serialization/*.cpp" "src/provenance/*.cpp" "src/utils/*.cpp" "src/utils/crypto/*.cpp" "src/utils/crypto/ciphers/*.cpp" "src/utils/crypto/property_encryption/*.cpp" "src/*.cpp" "src/utils/net/*.cpp" "src/http/*.cpp") -file(GLOB SOURCES "src/agent/*.cpp" "src/properties/*.cpp" "src/utils/file/*.cpp" "src/sitetosite/*.cpp" "src/core/logging/*.cpp" "src/core/logging/internal/*.cpp" "src/core/logging/alert/*.cpp" "src/core/state/*.cpp" "src/core/state/nodes/*.cpp" "src/c2/protocols/*.cpp" "src/c2/triggers/*.cpp" "src/c2/*.cpp" "src/io/*.cpp" ${SOCKET_SOURCES} ${TLS_SOURCES} "src/core/controller/*.cpp" "src/controllers/*.cpp" "src/controllers/keyvalue/*.cpp" "src/core/*.cpp" "src/core/repository/*.cpp" "src/core/flow/*.cpp" "src/core/json/*.cpp" "src/core/yaml/*.cpp" "src/core/reporting/*.cpp" "src/core/extension/*.cpp" "src/serialization/*.cpp" "src/provenance/*.cpp" "src/utils/*.cpp" "src/utils/crypto/*.cpp" "src/utils/crypto/ciphers/*.cpp" "src/utils/crypto/property_encryption/*.cpp" "src/*.cpp" "src/utils/net/*.cpp") -# manually add this as it might not yet be present when this executes list(APPEND SOURCES "${CMAKE_CURRENT_BINARY_DIR}/agent_version.cpp") + if(WIN32) include(FindMessageCompiler) find_package(MessageCompiler REQUIRED) @@ -78,17 +77,17 @@ endif() include(RangeV3) include(Asio) include(MagicEnum) -list(APPEND LIBMINIFI_LIBRARIES yaml-cpp ZLIB::ZLIB concurrentqueue RapidJSON spdlog Threads::Threads gsl-lite libsodium range-v3 expected-lite date::date date::tz asio magic_enum) +list(APPEND LIBMINIFI_LIBRARIES yaml-cpp ZLIB::ZLIB concurrentqueue RapidJSON spdlog::spdlog Threads::Threads gsl-lite libsodium range-v3 expected-lite date::date date::tz asio magic_enum OpenSSL::Crypto OpenSSL::SSL CURL::libcurl RapidJSON fmt::fmt) if(NOT WIN32) list(APPEND LIBMINIFI_LIBRARIES OSSP::libuuid++) endif() -if (MINIFI_OPENSSL) - list(APPEND LIBMINIFI_LIBRARIES OpenSSL::Crypto OpenSSL::SSL) -endif() if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9) list(APPEND LIBMINIFI_LIBRARIES stdc++fs) endif() target_link_libraries(core-minifi ${CMAKE_DL_LIBS} ${LIBMINIFI_LIBRARIES}) +if (APPLE) + target_link_libraries(core-minifi "-framework CoreFoundation -framework SystemConfiguration") +endif() if (UNIX AND (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7")) target_link_libraries(core-minifi "-latomic") endif() diff --git a/libminifi/include/RemoteProcessorGroupPort.h b/libminifi/include/RemoteProcessorGroupPort.h index b98be93e38..837ce67274 100644 --- a/libminifi/include/RemoteProcessorGroupPort.h +++ b/libminifi/include/RemoteProcessorGroupPort.h @@ -26,7 +26,7 @@ #include #include -#include "utils/BaseHTTPClient.h" +#include "http/BaseHTTPClient.h" #include "concurrentqueue.h" #include "FlowFileRecord.h" #include "core/Processor.h" @@ -163,7 +163,7 @@ class RemoteProcessorGroupPort : public core::Processor { void setURL(std::string val) { auto urls = utils::string::split(val, ","); for (const auto& url : urls) { - utils::URL parsed_url{utils::string::trim(url)}; + http::URL parsed_url{utils::string::trim(url)}; if (parsed_url.isValid()) { logger_->log_debug("Parsed RPG URL '{}' -> '{}'", url, parsed_url.hostPort()); nifi_instances_.push_back({parsed_url.host(), parsed_url.port(), parsed_url.protocol()}); @@ -177,10 +177,10 @@ class RemoteProcessorGroupPort : public core::Processor { return nifi_instances_; } - void setHTTPProxy(const utils::HTTPProxy &proxy) { + void setHTTPProxy(const http::HTTPProxy &proxy) { this->proxy_ = proxy; } - utils::HTTPProxy getHTTPProxy() { + http::HTTPProxy getHTTPProxy() { return this->proxy_; } // refresh remoteSite2SiteInfo via nifi rest api @@ -232,7 +232,7 @@ class RemoteProcessorGroupPort : public core::Processor { std::vector nifi_instances_; // http proxy - utils::HTTPProxy proxy_; + http::HTTPProxy proxy_; bool bypass_rest_api_; diff --git a/extensions/http-curl/protocols/RESTSender.h b/libminifi/include/c2/protocols/RESTSender.h similarity index 91% rename from extensions/http-curl/protocols/RESTSender.h rename to libminifi/include/c2/protocols/RESTSender.h index d09b7cf828..01f0619d27 100644 --- a/extensions/http-curl/protocols/RESTSender.h +++ b/libminifi/include/c2/protocols/RESTSender.h @@ -24,7 +24,7 @@ #include "c2/C2Protocol.h" #include "c2/protocols/RESTProtocol.h" #include "controllers/SSLContextService.h" -#include "../client/HTTPClient.h" +#include "http/HTTPClient.h" #include "utils/Enum.h" namespace org::apache::nifi::minifi::c2 { @@ -46,7 +46,7 @@ class RESTSender : public RESTProtocol, public C2Protocol { explicit RESTSender(std::string_view name, const utils::Identifier &uuid = utils::Identifier()); - EXTENSIONAPI static constexpr const char* Description = "Encapsulates the restful protocol that is built upon C2Protocol."; + MINIFIAPI static constexpr const char* Description = "Encapsulates the restful protocol that is built upon C2Protocol."; C2Payload consumePayload(const std::string &url, const C2Payload &payload, Direction direction, bool async) override; @@ -68,7 +68,7 @@ class RESTSender : public RESTProtocol, public C2Protocol { * @param type type of HTTP request * @param url HTTP url */ - void setSecurityContext(extensions::curl::HTTPClient &client, utils::HttpRequestMethod type, const std::string &url); + void setSecurityContext(http::HTTPClient &client, http::HttpRequestMethod type, const std::string &url); std::shared_ptr ssl_context_service_; diff --git a/libminifi/include/controllers/SSLContextService.h b/libminifi/include/controllers/SSLContextService.h index 50eb6e20e7..0169a949bc 100644 --- a/libminifi/include/controllers/SSLContextService.h +++ b/libminifi/include/controllers/SSLContextService.h @@ -25,12 +25,10 @@ #include #endif -#ifdef OPENSSL_SUPPORT #include #include #include #include -#endif #include #include @@ -52,24 +50,16 @@ namespace org::apache::nifi::minifi::controllers { class SSLContext { public: -#ifdef OPENSSL_SUPPORT SSLContext(SSL_CTX *context) // NOLINT : context_(context) { } -#else - SSLContext(void*) {} // NOLINT -#endif ~SSLContext() { -#ifdef OPENSSL_SUPPORT if (context_) { SSL_CTX_free(context_); } -#endif } protected: -#ifdef OPENSSL_SUPPORT SSL_CTX *context_; -#endif }; /** @@ -160,7 +150,6 @@ class SSLContextService : public core::controller::ControllerService { return false; } -#ifdef OPENSSL_SUPPORT void setMinTlsVersion(long min_version) { // NOLINT(runtime/int) long due to SSL lib API minimum_tls_version_ = min_version; } @@ -169,7 +158,6 @@ class SSLContextService : public core::controller::ControllerService { maximum_tls_version_ = max_version; } bool configure_ssl_context(SSL_CTX *ctx); -#endif void onEnable() override; @@ -264,7 +252,6 @@ class SSLContextService : public core::controller::ControllerService { utils::tls::ExtendedKeyUsage client_cert_key_usage_; #endif // WIN32 -#ifdef OPENSSL_SUPPORT static std::string getLatestOpenSSLErrorString() { unsigned long err = ERR_peek_last_error(); // NOLINT if (err == 0U) { @@ -274,14 +261,12 @@ class SSLContextService : public core::controller::ControllerService { ERR_error_string_n(err, buf, sizeof(buf)); return buf; } -#endif static bool isFileTypeP12(const std::filesystem::path& filename) { return utils::string::endsWith(filename.string(), "p12", false); } private: -#ifdef OPENSSL_SUPPORT bool addP12CertificateToSSLContext(SSL_CTX* ctx) const; bool addPemCertificateToSSLContext(SSL_CTX* ctx) const; bool addClientCertificateFromSystemStoreToSSLContext(SSL_CTX* ctx) const; @@ -298,7 +283,6 @@ class SSLContextService : public core::controller::ControllerService { #endif // WIN32 long minimum_tls_version_ = -1; // NOLINT(runtime/int) long due to SSL lib API long maximum_tls_version_ = -1; // NOLINT(runtime/int) long due to SSL lib API -#endif // OPENSSL_SUPPORT void verifyCertificateExpiration(); diff --git a/libminifi/include/core/ProcessGroup.h b/libminifi/include/core/ProcessGroup.h index b43fe77900..dc0773a229 100644 --- a/libminifi/include/core/ProcessGroup.h +++ b/libminifi/include/core/ProcessGroup.h @@ -39,7 +39,7 @@ #include "controller/ControllerServiceNode.h" #include "controller/ControllerServiceMap.h" #include "utils/Id.h" -#include "utils/BaseHTTPClient.h" +#include "http/BaseHTTPClient.h" #include "utils/CallBackTimer.h" #include "range/v3/algorithm/find_if.hpp" @@ -120,7 +120,7 @@ class ProcessGroup : public CoreComponent { void setHttpProxyPort(int port) { proxy_.port = port; } - utils::HTTPProxy getHTTPProxy() { + http::HTTPProxy getHTTPProxy() { return proxy_; } // Set Processor yield period in MilliSecond @@ -249,7 +249,7 @@ class ProcessGroup : public CoreComponent { // Transmitting std::atomic transmitting_; // http proxy - utils::HTTPProxy proxy_; + http::HTTPProxy proxy_; std::string transport_protocol_; // controller services diff --git a/libminifi/include/utils/BaseHTTPClient.h b/libminifi/include/http/BaseHTTPClient.h similarity index 94% rename from libminifi/include/utils/BaseHTTPClient.h rename to libminifi/include/http/BaseHTTPClient.h index d5ff2700ce..005cf27a0e 100644 --- a/libminifi/include/utils/BaseHTTPClient.h +++ b/libminifi/include/http/BaseHTTPClient.h @@ -25,12 +25,12 @@ #include #include -#include "ByteArrayCallback.h" +#include "utils/ByteArrayCallback.h" #include "controllers/SSLContextService.h" #include "core/Deprecated.h" #include "utils/gsl.h" -namespace org::apache::nifi::minifi::utils { +namespace org::apache::nifi::minifi::http { struct HTTPProxy { std::string host; @@ -51,7 +51,7 @@ class HTTPUploadCallback { virtual void close() = 0; }; -class HTTPUploadByteArrayInputCallback : public HTTPUploadCallback, public ByteInputCallback { +class HTTPUploadByteArrayInputCallback : public HTTPUploadCallback, public utils::ByteInputCallback { public: using ByteInputCallback::ByteInputCallback; @@ -60,7 +60,7 @@ class HTTPUploadByteArrayInputCallback : public HTTPUploadCallback, public ByteI size_t size() override { return getBufferSize(); } void requestStop() override { stop = true; } - void close() override { ByteInputCallback::close(); } + void close() override { utils::ByteInputCallback::close(); } std::atomic stop = false; std::atomic pos = 0; @@ -82,7 +82,7 @@ class HTTPUploadStreamContentsCallback : public HTTPUploadCallback { std::shared_ptr logger_ = core::logging::LoggerFactory::getLogger(); }; -class HTTPReadCallback : public ByteOutputCallback { +class HTTPReadCallback : public utils::ByteOutputCallback { public: using ByteOutputCallback::ByteOutputCallback; @@ -219,7 +219,7 @@ class BaseHTTPClient { virtual void setPeerVerification(bool peer_verification) = 0; virtual void setHostVerification(bool host_verification) = 0; - virtual void setHTTPProxy(const utils::HTTPProxy &proxy) = 0; + virtual void setHTTPProxy(const HTTPProxy &proxy) = 0; virtual void setBasicAuth(const std::string& username, const std::string& password) = 0; virtual void clearBasicAuth() = 0; @@ -232,7 +232,7 @@ class BaseHTTPClient { virtual const std::map& getResponseHeaderMap() = 0; }; -std::string get_token(utils::BaseHTTPClient *client, const std::string& username, const std::string& password); +std::string get_token(BaseHTTPClient *client, const std::string& username, const std::string& password); class URL { public: @@ -253,4 +253,4 @@ class URL { std::shared_ptr logger_ = core::logging::LoggerFactory::getLogger(); }; -} // namespace org::apache::nifi::minifi::utils +} // namespace org::apache::nifi::minifi::http diff --git a/extensions/http-curl/client/HTTPCallback.h b/libminifi/include/http/HTTPCallback.h similarity index 96% rename from extensions/http-curl/client/HTTPCallback.h rename to libminifi/include/http/HTTPCallback.h index 13752d1000..3ea6765d02 100644 --- a/extensions/http-curl/client/HTTPCallback.h +++ b/libminifi/include/http/HTTPCallback.h @@ -27,13 +27,13 @@ #include "core/logging/LoggerConfiguration.h" #include "utils/ByteArrayCallback.h" -#include "utils/BaseHTTPClient.h" +#include "http/BaseHTTPClient.h" -namespace org::apache::nifi::minifi::extensions::curl { +namespace org::apache::nifi::minifi::http { /** * The original class here was deadlock-prone, undocumented and was a smorgasbord of multithreading primitives used inconsistently. - * This is a rewrite based on the contract inferred from this class's usage in curl::HTTPClient + * This is a rewrite based on the contract inferred from this class's usage in http::HTTPClient * through HTTPStream and the non-buggy part of the behaviour of the original class. * Based on these: * - this class provides a mechanism through which chunks of data can be inserted on a producer thread, while a @@ -50,7 +50,7 @@ namespace org::apache::nifi::minifi::extensions::curl { * - because of this, all functions that request data at a specific offset are implicit seeks and potentially modify * the current buffer */ -class HttpStreamingCallback final : public utils::HTTPUploadByteArrayInputCallback { +class HttpStreamingCallback final : public HTTPUploadByteArrayInputCallback { public: void close() override { logger_->log_trace("close() called"); @@ -208,4 +208,4 @@ class HttpStreamingCallback final : public utils::HTTPUploadByteArrayInputCallba std::byte* ptr_{nullptr}; }; -} // namespace org::apache::nifi::minifi::extensions::curl +} // namespace org::apache::nifi::minifi::http diff --git a/extensions/http-curl/client/HTTPClient.h b/libminifi/include/http/HTTPClient.h similarity index 78% rename from extensions/http-curl/client/HTTPClient.h rename to libminifi/include/http/HTTPClient.h index adcc00cbef..dc41571c25 100644 --- a/extensions/http-curl/client/HTTPClient.h +++ b/libminifi/include/http/HTTPClient.h @@ -16,7 +16,7 @@ */ #pragma once -#include "utils/BaseHTTPClient.h" +#include "http/BaseHTTPClient.h" #ifdef WIN32 #pragma comment(lib, "wldap32.lib" ) #pragma comment(lib, "crypt32.lib" ) @@ -44,7 +44,7 @@ #include "core/logging/Logger.h" #include "core/logging/LoggerConfiguration.h" -namespace org::apache::nifi::minifi::extensions::curl { +namespace org::apache::nifi::minifi::http { struct KeepAliveProbeData { std::chrono::seconds keep_alive_delay; @@ -53,7 +53,7 @@ struct KeepAliveProbeData { struct HTTPResponseData { std::vector response_body; - utils::HTTPHeaderResponse header_response; + HTTPHeaderResponse header_response; char* response_content_type{nullptr}; int64_t response_code{0}; @@ -65,7 +65,7 @@ struct HTTPResponseData { } }; -class HTTPClient : public utils::BaseHTTPClient, public core::Connectable { +class HTTPClient : public BaseHTTPClient, public core::Connectable { public: HTTPClient(); @@ -78,30 +78,30 @@ class HTTPClient : public utils::BaseHTTPClient, public core::Connectable { ~HTTPClient() override; - EXTENSIONAPI static constexpr auto Properties = std::array{}; - EXTENSIONAPI static constexpr bool SupportsDynamicProperties = false; - EXTENSIONAPI static constexpr bool SupportsDynamicRelationships = false; + MINIFIAPI static constexpr auto Properties = std::array{}; + MINIFIAPI static constexpr bool SupportsDynamicProperties = false; + MINIFIAPI static constexpr bool SupportsDynamicRelationships = false; static int debug_callback(CURL *handle, curl_infotype type, char *data, size_t size, void *userptr); void setVerbose(bool use_stderr) override; - void addFormPart(const std::string& content_type, const std::string& name, std::unique_ptr form_callback, const std::optional& filename); + void addFormPart(const std::string& content_type, const std::string& name, std::unique_ptr form_callback, const std::optional& filename); void forceClose(); - void initialize(utils::HttpRequestMethod method, std::string url, std::shared_ptr ssl_context_service) override; + void initialize(http::HttpRequestMethod method, std::string url, std::shared_ptr ssl_context_service) override; void setConnectionTimeout(std::chrono::milliseconds timeout) override; void setReadTimeout(std::chrono::milliseconds timeout) override; - void setUploadCallback(std::unique_ptr callback) override; + void setUploadCallback(std::unique_ptr callback) override; - void setReadCallback(std::unique_ptr callback); + void setReadCallback(std::unique_ptr callback); - utils::HTTPUploadCallback* getUploadCallback() const { return write_callback_.get(); } - utils::HTTPReadCallback* getReadCallback() const { return read_callback_.get(); } + HTTPUploadCallback* getUploadCallback() const { return write_callback_.get(); } + HTTPReadCallback* getReadCallback() const { return read_callback_.get(); } void setContentType(std::string content_type) override; @@ -119,7 +119,7 @@ class HTTPClient : public utils::BaseHTTPClient, public core::Connectable { const std::vector& getResponseBody() override; - void set_request_method(utils::HttpRequestMethod method) override; + void set_request_method(http::HttpRequestMethod method) override; void setPeerVerification(bool peer_verification) override; void setHostVerification(bool host_verification) override; @@ -127,9 +127,9 @@ class HTTPClient : public utils::BaseHTTPClient, public core::Connectable { void setBasicAuth(const std::string& username, const std::string& password) override; void clearBasicAuth() override; - bool setSpecificSSLVersion(utils::SSLVersion specific_version) override; + bool setSpecificSSLVersion(SSLVersion specific_version) override; - bool setMinimumSSLVersion(utils::SSLVersion minimum_version) override; + bool setMinimumSSLVersion(SSLVersion minimum_version) override; void setKeepAliveProbe(std::optional probe_data); @@ -187,7 +187,7 @@ class HTTPClient : public utils::BaseHTTPClient, public core::Connectable { void setPostSize(size_t size); - void setHTTPProxy(const utils::HTTPProxy &proxy) override; + void setHTTPProxy(const HTTPProxy &proxy) override; static bool isValidHttpHeaderField(std::string_view field_name); static std::string replaceInvalidCharactersInHttpHeaderFieldName(std::string field_name); @@ -209,7 +209,6 @@ class HTTPClient : public utils::BaseHTTPClient, public core::Connectable { Progress progress_; protected: -#ifdef OPENSSL_SUPPORT static CURLcode configure_ssl_context(CURL* /*curl*/, void *ctx, void *param) { gsl_Expects(ctx); gsl_Expects(param); @@ -219,21 +218,16 @@ class HTTPClient : public utils::BaseHTTPClient, public core::Connectable { } return CURLE_OK; } -#else - static CURLcode configure_ssl_context(CURL*, void*, void*) { - return CURLE_FAILED_INIT; - } -#endif void configure_secure_connection(); std::chrono::milliseconds getAbsoluteTimeout() const { return 3*read_timeout_; } - utils::HTTPReadCallback content_{std::numeric_limits::max()}; + HTTPReadCallback content_{std::numeric_limits::max()}; std::shared_ptr ssl_context_service_; std::string url_; - std::optional method_; + std::optional method_; std::chrono::milliseconds connect_timeout_{std::chrono::seconds(30)}; std::chrono::milliseconds read_timeout_{std::chrono::seconds(30)}; @@ -249,11 +243,11 @@ class HTTPClient : public utils::BaseHTTPClient, public core::Connectable { std::unique_ptr http_session_; std::unique_ptr form_; - std::unique_ptr read_callback_; - std::unique_ptr write_callback_; - std::unique_ptr form_callback_; + std::unique_ptr read_callback_; + std::unique_ptr write_callback_; + std::unique_ptr form_callback_; std::shared_ptr logger_{core::logging::LoggerFactory::getLogger()}; }; -} // namespace org::apache::nifi::minifi::extensions::curl +} // namespace org::apache::nifi::minifi::http diff --git a/extensions/http-curl/client/HTTPStream.h b/libminifi/include/http/HTTPStream.h similarity index 97% rename from extensions/http-curl/client/HTTPStream.h rename to libminifi/include/http/HTTPStream.h index 9e90eb0c63..61033f2ef6 100644 --- a/extensions/http-curl/client/HTTPStream.h +++ b/libminifi/include/http/HTTPStream.h @@ -27,7 +27,7 @@ #include "io/BaseStream.h" #include "HTTPClient.h" -namespace org::apache::nifi::minifi::extensions::curl { +namespace org::apache::nifi::minifi::http { class HttpStream : public io::BaseStream { public: @@ -149,4 +149,4 @@ class HttpStream : public io::BaseStream { private: std::shared_ptr logger_ = core::logging::LoggerFactory::getLogger(); }; -} // namespace org::apache::nifi::minifi::extensions::curl +} // namespace org::apache::nifi::minifi::http diff --git a/extensions/http-curl/sitetosite/HTTPProtocol.h b/libminifi/include/sitetosite/HTTPProtocol.h similarity index 85% rename from extensions/http-curl/sitetosite/HTTPProtocol.h rename to libminifi/include/sitetosite/HTTPProtocol.h index 4532c20db0..769275870d 100644 --- a/extensions/http-curl/sitetosite/HTTPProtocol.h +++ b/libminifi/include/sitetosite/HTTPProtocol.h @@ -31,7 +31,7 @@ #include "sitetosite/Peer.h" #include "utils/Id.h" -namespace org::apache::nifi::minifi::extensions::curl { +namespace org::apache::nifi::minifi::sitetosite { class HttpSiteToSiteClient : public sitetosite::SiteToSiteClient { static constexpr char const* PROTOCOL_VERSION_HEADER = "x-nifi-site-to-site-protocol-version"; @@ -51,9 +51,9 @@ class HttpSiteToSiteClient : public sitetosite::SiteToSiteClient { } ~HttpSiteToSiteClient() override = default; - EXTENSIONAPI static constexpr auto Properties = std::array{}; - EXTENSIONAPI static constexpr bool SupportsDynamicProperties = false; - EXTENSIONAPI static constexpr bool SupportsDynamicRelationships = false; + MINIFIAPI static constexpr auto Properties = std::array{}; + MINIFIAPI static constexpr bool SupportsDynamicProperties = false; + MINIFIAPI static constexpr bool SupportsDynamicRelationships = false; void setPeer(std::unique_ptr peer) override { peer_ = std::move(peer); @@ -102,7 +102,7 @@ class HttpSiteToSiteClient : public sitetosite::SiteToSiteClient { * @param transaction transaction against which we are performing our sends * @return HTTP Client shared pointer. */ - std::shared_ptr openConnectionForSending(const std::shared_ptr &transaction); + std::shared_ptr openConnectionForSending(const std::shared_ptr &transaction); /** * Creates a connection for receiving, returning an HTTP client that is structured and configured @@ -110,7 +110,7 @@ class HttpSiteToSiteClient : public sitetosite::SiteToSiteClient { * @param transaction transaction against which we are performing our reads * @return HTTP Client shared pointer. */ - std::shared_ptr openConnectionForReceive(const std::shared_ptr &transaction); + std::shared_ptr openConnectionForReceive(const std::shared_ptr &transaction); std::string getBaseURI() { std::string uri = ssl_context_service_ != nullptr ? "https://" : "http://"; @@ -125,8 +125,8 @@ class HttpSiteToSiteClient : public sitetosite::SiteToSiteClient { static std::optional parseTransactionId(const std::string &uri); - std::unique_ptr create_http_client(const std::string &uri, utils::HttpRequestMethod method = utils::HttpRequestMethod::POST, bool setPropertyHeaders = false) { - std::unique_ptr http_client_ = std::make_unique(uri, ssl_context_service_); + std::unique_ptr create_http_client(const std::string &uri, http::HttpRequestMethod method = http::HttpRequestMethod::POST, bool setPropertyHeaders = false) { + std::unique_ptr http_client_ = std::make_unique(uri, ssl_context_service_); http_client_->initialize(method, uri, ssl_context_service_); if (setPropertyHeaders) { if (_currentVersion >= 5) { @@ -155,4 +155,4 @@ class HttpSiteToSiteClient : public sitetosite::SiteToSiteClient { static std::shared_ptr id_generator_; }; -} // namespace org::apache::nifi::minifi::extensions::curl +} // namespace org::apache::nifi::minifi::sitetosite diff --git a/extensions/http-curl/sitetosite/HTTPTransaction.h b/libminifi/include/sitetosite/HTTPTransaction.h similarity index 87% rename from extensions/http-curl/sitetosite/HTTPTransaction.h rename to libminifi/include/sitetosite/HTTPTransaction.h index 2365a6b210..faa6129a61 100644 --- a/extensions/http-curl/sitetosite/HTTPTransaction.h +++ b/libminifi/include/sitetosite/HTTPTransaction.h @@ -24,9 +24,9 @@ #include "io/CRCStream.h" #include "sitetosite/SiteToSiteClient.h" #include "sitetosite/Peer.h" -#include "HTTPStream.h" +#include "http/HTTPStream.h" -namespace org::apache::nifi::minifi::extensions::curl { +namespace org::apache::nifi::minifi::sitetosite { /** * Purpose: HTTP Transaction is an implementation that exposes the site to site client. @@ -40,7 +40,7 @@ class HttpTransaction : public sitetosite::Transaction { } ~HttpTransaction() { - auto stream = dynamic_cast(dynamic_cast(crcStream.getstream())->getStream() ); + auto stream = dynamic_cast(dynamic_cast(crcStream.getstream())->getStream() ); if (stream) stream->forceClose(); } @@ -60,4 +60,4 @@ class HttpTransaction : public sitetosite::Transaction { std::string transaction_url_; }; -} // namespace org::apache::nifi::minifi::extensions::curl +} // namespace org::apache::nifi::minifi::sitetosite diff --git a/libminifi/include/sitetosite/Peer.h b/libminifi/include/sitetosite/Peer.h index 2b4992119b..d1ced0f139 100644 --- a/libminifi/include/sitetosite/Peer.h +++ b/libminifi/include/sitetosite/Peer.h @@ -32,7 +32,7 @@ #include "io/BaseStream.h" #include "io/BufferStream.h" #include "properties/Configure.h" -#include "utils/BaseHTTPClient.h" +#include "http/BaseHTTPClient.h" #include "utils/TimeUtil.h" #include "io/NetworkPrioritizer.h" @@ -264,10 +264,10 @@ class SiteToSitePeer : public org::apache::nifi::minifi::io::BaseStream { std::chrono::milliseconds getTimeout() { return timeout_; } - void setHTTPProxy(const utils::HTTPProxy &proxy) { + void setHTTPProxy(const http::HTTPProxy &proxy) { this->proxy_ = proxy; } - utils::HTTPProxy getHTTPProxy() { + http::HTTPProxy getHTTPProxy() { return this->proxy_; } @@ -327,7 +327,7 @@ class SiteToSitePeer : public org::apache::nifi::minifi::io::BaseStream { io::NetworkInterface local_network_interface_; - utils::HTTPProxy proxy_; + http::HTTPProxy proxy_; // Mutex for protection std::mutex mutex_; diff --git a/extensions/http-curl/sitetosite/PeersEntity.h b/libminifi/include/sitetosite/PeersEntity.h similarity index 100% rename from extensions/http-curl/sitetosite/PeersEntity.h rename to libminifi/include/sitetosite/PeersEntity.h diff --git a/libminifi/include/sitetosite/SiteToSite.h b/libminifi/include/sitetosite/SiteToSite.h index 65db0cb92e..97491a59fd 100644 --- a/libminifi/include/sitetosite/SiteToSite.h +++ b/libminifi/include/sitetosite/SiteToSite.h @@ -27,7 +27,7 @@ #include "properties/Configure.h" #include "io/CRCStream.h" #include "utils/Id.h" -#include "utils/BaseHTTPClient.h" +#include "http/BaseHTTPClient.h" #include "utils/Export.h" namespace org::apache::nifi::minifi::sitetosite { @@ -364,10 +364,10 @@ class SiteToSiteClientConfiguration { std::string getInterface() const { return local_network_interface_; } - void setHTTPProxy(const utils::HTTPProxy &proxy) { + void setHTTPProxy(const http::HTTPProxy &proxy) { proxy_ = proxy; } - utils::HTTPProxy getHTTPProxy() const { + http::HTTPProxy getHTTPProxy() const { return this->proxy_; } @@ -384,7 +384,7 @@ class SiteToSiteClientConfiguration { std::shared_ptr ssl_service_; - utils::HTTPProxy proxy_; + http::HTTPProxy proxy_; }; #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic pop diff --git a/libminifi/include/utils/IntegrationTestUtils.h b/libminifi/include/utils/IntegrationTestUtils.h deleted file mode 100644 index be9fe45443..0000000000 --- a/libminifi/include/utils/IntegrationTestUtils.h +++ /dev/null @@ -1,62 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include - -#include "../../../libminifi/test/TestBase.h" - -namespace org::apache::nifi::minifi::utils { - -template -bool verifyEventHappenedInPollTime( - const std::chrono::duration& wait_duration, - Fun&& check, - std::chrono::microseconds check_interval = std::chrono::milliseconds(100)) { - std::chrono::steady_clock::time_point wait_end = std::chrono::steady_clock::now() + wait_duration; - do { - if (std::forward(check)()) { - return true; - } - std::this_thread::sleep_for(check_interval); - } while (std::chrono::steady_clock::now() < wait_end); - return false; -} - -template -bool verifyLogLinePresenceInPollTime(const std::chrono::duration& wait_duration, String&&... patterns) { - auto check = [&patterns...] { - const std::string logs = LogTestController::getInstance().getLogs(); - return ((logs.find(patterns) != std::string::npos) && ...); - }; - return verifyEventHappenedInPollTime(wait_duration, check); -} - -template -bool verifyLogLineVariantPresenceInPollTime(const std::chrono::duration& wait_duration, String&&... patterns) { - auto check = [&patterns...] { - const std::string logs = LogTestController::getInstance().getLogs(); - return ((logs.find(patterns) != std::string::npos) || ...); - }; - return verifyEventHappenedInPollTime(wait_duration, check); -} - -} // namespace org::apache::nifi::minifi::utils diff --git a/libminifi/include/utils/TestUtils.h b/libminifi/include/utils/TestUtils.h deleted file mode 100644 index 35431164e6..0000000000 --- a/libminifi/include/utils/TestUtils.h +++ /dev/null @@ -1,117 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "utils/file/FileUtils.h" -#include "utils/Environment.h" -#include "utils/Id.h" -#include "utils/TimeUtil.h" - -#ifdef WIN32 -#include "date/tz.h" -#endif - -namespace org::apache::nifi::minifi::utils { - -std::filesystem::path putFileToDir(const std::filesystem::path& dir_path, const std::filesystem::path& file_name, const std::string& content) { - auto file_path = dir_path/file_name; - std::ofstream out_file(file_path, std::ios::binary | std::ios::out); - if (out_file.is_open()) { - out_file << content; - } - return file_path; -} - -std::string getFileContent(const std::filesystem::path& file_name) { - std::ifstream file_handle(file_name, std::ios::binary | std::ios::in); - assert(file_handle.is_open()); - std::string file_content{ (std::istreambuf_iterator(file_handle)), (std::istreambuf_iterator()) }; - return file_content; -} - -void makeFileOrDirectoryNotWritable(const std::filesystem::path& file_name); - -void makeFileOrDirectoryWritable(const std::filesystem::path& file_name); - -Identifier generateUUID() { - // TODO(hunyadi): Will make the Id generator manage lifetime using a unique_ptr and return a raw ptr on access - static std::shared_ptr id_generator = utils::IdGenerator::getIdGenerator(); - return id_generator->generate(); -} - -class ManualClock : public timeutils::SteadyClock { - public: - [[nodiscard]] std::chrono::milliseconds timeSinceEpoch() const override { - std::lock_guard lock(mtx_); - return time_; - } - - [[nodiscard]] std::chrono::time_point now() const override { - return std::chrono::steady_clock::time_point{timeSinceEpoch()}; - } - - void advance(std::chrono::milliseconds elapsed_time) { - if (elapsed_time.count() < 0) { - throw std::logic_error("A steady clock can only be advanced forward"); - } - std::lock_guard lock(mtx_); - time_ += elapsed_time; - for (auto* cv : cvs_) { - cv->notify_all(); - } - } - - bool wait_until(std::condition_variable& cv, std::unique_lock& lck, std::chrono::milliseconds time, const std::function& pred) override { - std::chrono::milliseconds now; - { - std::unique_lock lock(mtx_); - now = time_; - cvs_.insert(&cv); - } - cv.wait_for(lck, time - now, [&] { - now = timeSinceEpoch(); - return now >= time || pred(); - }); - { - std::unique_lock lock(mtx_); - cvs_.erase(&cv); - } - return pred(); - } - - private: - mutable std::mutex mtx_; - std::unordered_set cvs_; - std::chrono::milliseconds time_{0}; -}; - -#ifdef WIN32 -// The tzdata location is set as a global variable in date-tz library -// We need to set it from from libminifi to effect calls made from libminifi (on Windows) -void dateSetInstall(const std::string& install); -#endif - -} // namespace org::apache::nifi::minifi::utils diff --git a/libminifi/include/utils/TimeUtil.h b/libminifi/include/utils/TimeUtil.h index 72f828431a..4159f089a6 100644 --- a/libminifi/include/utils/TimeUtil.h +++ b/libminifi/include/utils/TimeUtil.h @@ -281,4 +281,10 @@ inline date::local_seconds roundToNextSecond(date::local_seconds tp) { return std::chrono::floor(tp) + std::chrono::seconds(1); } +#ifdef WIN32 +// The tzdata location is set as a global variable in date-tz library +// We need to set it from from libminifi to effect calls made from libminifi (on Windows) +void dateSetInstall(const std::string& install); +#endif + } // namespace org::apache::nifi::minifi::utils::timeutils diff --git a/libminifi/include/utils/tls/CertificateUtils.h b/libminifi/include/utils/tls/CertificateUtils.h index 3c4a8ff6f2..4b7eb9b898 100644 --- a/libminifi/include/utils/tls/CertificateUtils.h +++ b/libminifi/include/utils/tls/CertificateUtils.h @@ -15,7 +15,6 @@ * limitations under the License. */ #pragma once -#ifdef OPENSSL_SUPPORT #include #include @@ -113,5 +112,3 @@ std::error_code processP12Certificate(const std::filesystem::path& cert_file, co std::error_code processPEMCertificate(const std::filesystem::path& cert_file, const std::optional& passphrase, const CertHandler& handler); } // namespace org::apache::nifi::minifi::utils::tls - -#endif // OPENSSL_SUPPORT diff --git a/libminifi/include/utils/tls/ExtendedKeyUsage.h b/libminifi/include/utils/tls/ExtendedKeyUsage.h index 089433ab04..0af711bbe5 100644 --- a/libminifi/include/utils/tls/ExtendedKeyUsage.h +++ b/libminifi/include/utils/tls/ExtendedKeyUsage.h @@ -15,7 +15,6 @@ * limitations under the License. */ #pragma once -#ifdef OPENSSL_SUPPORT #include #include @@ -51,5 +50,3 @@ class ExtendedKeyUsage { }; } // namespace org::apache::nifi::minifi::utils::tls - -#endif // OPENSSL_SUPPORT diff --git a/libminifi/src/FlowController.cpp b/libminifi/src/FlowController.cpp index dc0136a6a3..8e5739d9df 100644 --- a/libminifi/src/FlowController.cpp +++ b/libminifi/src/FlowController.cpp @@ -37,7 +37,7 @@ #include "core/Connectable.h" #include "utils/file/PathUtils.h" #include "utils/file/FileSystem.h" -#include "utils/BaseHTTPClient.h" +#include "http/BaseHTTPClient.h" #include "io/NetworkPrioritizer.h" #include "io/FileStream.h" #include "core/ClassLoader.h" diff --git a/libminifi/src/RemoteProcessorGroupPort.cpp b/libminifi/src/RemoteProcessorGroupPort.cpp index b415a7e702..aee1f7c390 100644 --- a/libminifi/src/RemoteProcessorGroupPort.cpp +++ b/libminifi/src/RemoteProcessorGroupPort.cpp @@ -36,7 +36,7 @@ #include "core/logging/Logger.h" #include "core/ProcessContext.h" #include "core/ProcessorNode.h" -#include "utils/BaseHTTPClient.h" +#include "http/BaseHTTPClient.h" #undef GetObject // windows.h #defines GetObject = GetObjectA or GetObjectW, which conflicts with rapidjson @@ -144,26 +144,6 @@ void RemoteProcessorGroupPort::onSchedule(core::ProcessContext& context, core::P if (!peers_.empty()) peer_index_ = 0; } - /** - * If at this point we have no peers and HTTP support is disabled this means - * we must rely on the configured host/port - */ - if (peers_.empty() && is_http_disabled()) { - std::string host; - std::string portStr; - int configured_port = -1; - // place hostname/port into the log message if we have it - context.getProperty(hostName, host); - context.getProperty(port, portStr); - if (!host.empty() && !portStr.empty() && !portStr.empty() && core::Property::StringToInt(portStr, configured_port)) { - nifi_instances_.push_back({ host, configured_port, "" }); - bypass_rest_api_ = true; - } else { - // we cannot proceed, so log error and throw an exception - logger_->log_error("{}/{}/{} -- configuration values after eval of configuration options", host, portStr, configured_port); - throw(Exception(SITE2SITE_EXCEPTION, "HTTPClient not resolvable. No peers configured or any port specific hostname and port -- cannot schedule")); - } - } // populate the site2site protocol for load balancing between them if (!peers_.empty()) { auto count = peers_.size(); @@ -266,7 +246,7 @@ std::pair RemoteProcessorGroupPort::refreshRemoteSite2SiteInfo configure_->get(Configure::nifi_rest_api_password, this->rest_password_); std::string token; - std::unique_ptr client; + std::unique_ptr client; if (!rest_user_name_.empty()) { std::stringstream loginUrl; loginUrl << protocol << host; @@ -281,14 +261,14 @@ std::pair RemoteProcessorGroupPort::refreshRemoteSite2SiteInfo logger_->log_error("Could not locate HTTPClient. You do not have cURL support!"); return std::make_pair("", -1); } - client = std::unique_ptr(dynamic_cast(client_ptr)); - client->initialize(utils::HttpRequestMethod::GET, loginUrl.str(), ssl_service); + client = std::unique_ptr(dynamic_cast(client_ptr)); + client->initialize(http::HttpRequestMethod::GET, loginUrl.str(), ssl_service); // use a connection timeout. if this times out we will simply attempt re-connection // so no need for configuration parameter that isn't already defined in Processor client->setConnectionTimeout(10s); client->setReadTimeout(idle_timeout_); - token = utils::get_token(client.get(), this->rest_user_name_, this->rest_password_); + token = http::get_token(client.get(), this->rest_user_name_, this->rest_password_); logger_->log_debug("Token from NiFi REST Api endpoint {}, {}", loginUrl.str(), token); if (token.empty()) return std::make_pair("", -1); @@ -300,8 +280,8 @@ std::pair RemoteProcessorGroupPort::refreshRemoteSite2SiteInfo return std::make_pair("", -1); } int siteTosite_port_ = -1; - client = std::unique_ptr(dynamic_cast(client_ptr)); - client->initialize(utils::HttpRequestMethod::GET, fullUrl.str(), ssl_service); + client = std::unique_ptr(dynamic_cast(client_ptr)); + client->initialize(http::HttpRequestMethod::GET, fullUrl.str(), ssl_service); // use a connection timeout. if this times out we will simply attempt re-connection // so no need for configuration parameter that isn't already defined in Processor client->setConnectionTimeout(10s); diff --git a/libminifi/src/c2/C2Agent.cpp b/libminifi/src/c2/C2Agent.cpp index e94efa049b..318f3e95ec 100644 --- a/libminifi/src/c2/C2Agent.cpp +++ b/libminifi/src/c2/C2Agent.cpp @@ -37,7 +37,7 @@ #include "utils/file/FileUtils.h" #include "utils/file/FileManager.h" #include "utils/file/FileSystem.h" -#include "utils/BaseHTTPClient.h" +#include "http/BaseHTTPClient.h" #include "utils/Environment.h" #include "utils/Monitors.h" #include "utils/StringUtils.h" @@ -796,7 +796,7 @@ std::optional C2Agent::resolveFlowUrl(const std::string& url) const base += url; return base; } else if (configuration_->get(Configuration::nifi_c2_rest_url, "c2.rest.url", base)) { - utils::URL base_url{utils::string::trim(base)}; + http::URL base_url{utils::string::trim(base)}; if (base_url.isValid()) { return base_url.hostPort() + "/c2/api/" + url; } @@ -815,7 +815,7 @@ std::optional C2Agent::resolveUrl(const std::string& url) const { logger_->log_error("Missing C2 REST URL"); return std::nullopt; } - utils::URL base_url{utils::string::trim(base)}; + http::URL base_url{utils::string::trim(base)}; if (base_url.isValid()) { return base_url.hostPort() + url; } diff --git a/extensions/http-curl/protocols/RESTSender.cpp b/libminifi/src/c2/protocols/RESTSender.cpp similarity index 92% rename from extensions/http-curl/protocols/RESTSender.cpp rename to libminifi/src/c2/protocols/RESTSender.cpp index 6cf8f89b7c..98642b009f 100644 --- a/extensions/http-curl/protocols/RESTSender.cpp +++ b/libminifi/src/c2/protocols/RESTSender.cpp @@ -15,7 +15,7 @@ * limitations under the License. */ -#include "RESTSender.h" +#include "c2/protocols/RESTSender.h" #include #include @@ -93,7 +93,7 @@ C2Payload RESTSender::consumePayload(const C2Payload &payload, Direction directi void RESTSender::update(const std::shared_ptr &) { } -void RESTSender::setSecurityContext(extensions::curl::HTTPClient &client, utils::HttpRequestMethod type, const std::string &url) { +void RESTSender::setSecurityContext(http::HTTPClient &client, http::HttpRequestMethod type, const std::string &url) { // only use the SSL Context if we have a secure URL. auto generatedService = std::make_shared("Service", configuration_); generatedService->onEnable(); @@ -107,11 +107,11 @@ C2Payload RESTSender::sendPayload(const std::string& url, const Direction direct } // Client declared last to make sure callbacks are still available when client is destructed - extensions::curl::HTTPClient client(url, ssl_context_service_); - client.setKeepAliveProbe(extensions::curl::KeepAliveProbeData{2s, 2s}); + http::HTTPClient client(url, ssl_context_service_); + client.setKeepAliveProbe(http::KeepAliveProbeData{2s, 2s}); client.setConnectionTimeout(2s); - auto setUpHttpRequest = [&](utils::HttpRequestMethod http_method) { + auto setUpHttpRequest = [&](http::HttpRequestMethod http_method) { client.set_request_method(http_method); if (url.find("https://") == 0) { if (!ssl_context_service_) { @@ -122,7 +122,7 @@ C2Payload RESTSender::sendPayload(const std::string& url, const Direction direct } }; if (direction == Direction::TRANSMIT) { - setUpHttpRequest(utils::HttpRequestMethod::POST); + setUpHttpRequest(http::HttpRequestMethod::POST); if (payload.getOperation() == Operation::transfer) { // treat nested payloads as files for (const auto& file : payload.getNestedPayloads()) { @@ -130,12 +130,12 @@ C2Payload RESTSender::sendPayload(const std::string& url, const Direction direct if (filename.empty()) { throw std::logic_error("Missing filename"); } - auto file_cb = std::make_unique(); + auto file_cb = std::make_unique(); file_cb->write(file.getRawDataAsString()); client.addFormPart("application/octet-stream", "file", std::move(file_cb), filename); } } else { - auto data_input = std::make_unique(); + auto data_input = std::make_unique(); if (data && req_encoding_ == RequestEncoding::gzip) { io::BufferStream compressed_payload; bool compression_success = [&] { @@ -163,11 +163,11 @@ C2Payload RESTSender::sendPayload(const std::string& url, const Direction direct } else { // we do not need to set the upload callback // since we are not uploading anything on a get - setUpHttpRequest(utils::HttpRequestMethod::GET); + setUpHttpRequest(http::HttpRequestMethod::GET); } if (payload.getOperation() == Operation::transfer) { - auto read = std::make_unique(std::numeric_limits::max()); + auto read = std::make_unique(std::numeric_limits::max()); client.setReadCallback(std::move(read)); if (accepted_formats && !accepted_formats->empty()) { client.setRequestHeader("Accept", utils::string::join(", ", accepted_formats.value())); diff --git a/libminifi/src/controllers/SSLContextService.cpp b/libminifi/src/controllers/SSLContextService.cpp index 718078c512..820e9401b8 100644 --- a/libminifi/src/controllers/SSLContextService.cpp +++ b/libminifi/src/controllers/SSLContextService.cpp @@ -17,7 +17,6 @@ #include "controllers/SSLContextService.h" -#ifdef OPENSSL_SUPPORT #include #include #include @@ -27,7 +26,6 @@ #pragma comment(lib, "Ws2_32.lib") #include #endif // WIN32 -#endif // OPENSSL_SUPPORT #include #include @@ -87,7 +85,6 @@ void SSLContextService::initialize() { initialized_ = true; } -#ifdef OPENSSL_SUPPORT bool SSLContextService::configure_ssl_context(SSL_CTX *ctx) { if (!certificate_.empty()) { if (isFileTypeP12(certificate_)) { @@ -355,7 +352,6 @@ bool SSLContextService::useServerCertificate(PCCERT_CONTEXT certificate, ServerC return cb(std::move(x509_cert)); } #endif // WIN32 -#endif // OPENSSL_SUPPORT /** * If OpenSSL is not installed we may still continue operations. Nullptr will @@ -363,7 +359,6 @@ bool SSLContextService::useServerCertificate(PCCERT_CONTEXT certificate, ServerC * recoverable. */ std::unique_ptr SSLContextService::createSSLContext() { -#ifdef OPENSSL_SUPPORT SSL_library_init(); const SSL_METHOD *method = nullptr; @@ -382,9 +377,6 @@ std::unique_ptr SSLContextService::createSSLContext() { } return std::make_unique(ctx); -#else - return nullptr; -#endif } const std::filesystem::path &SSLContextService::getCertificateFile() const { @@ -523,9 +515,9 @@ void SSLContextService::verifyCertificateExpiration() { if (end_date.value() < std::chrono::system_clock::now()) { logger_->log_error("Certificate in '{}' expired at {}", cert_file, end_date_str); } else if (auto diff = end_date.value() - std::chrono::system_clock::now(); diff < std::chrono::weeks{2}) { - logger_->log_error("Certificate in '{}' will expire at {}", cert_file, end_date_str); + logger_->log_warn("Certificate in '{}' will expire at {}", cert_file, end_date_str); } else { - logger_->log_error("Certificate in '{}' will expire at {}", cert_file, end_date_str); + logger_->log_debug("Certificate in '{}' will expire at {}", cert_file, end_date_str); } } else { logger_->log_error("Could not determine expiration date for certificate in '{}'", cert_file); diff --git a/libminifi/src/core/logging/alert/AlertSink.cpp b/libminifi/src/core/logging/alert/AlertSink.cpp index 34e7eba948..89508b2203 100644 --- a/libminifi/src/core/logging/alert/AlertSink.cpp +++ b/libminifi/src/core/logging/alert/AlertSink.cpp @@ -18,7 +18,7 @@ #include "core/logging/alert/AlertSink.h" #include "core/TypedValues.h" #include "core/ClassLoader.h" -#include "utils/BaseHTTPClient.h" +#include "http/BaseHTTPClient.h" #include "utils/Hash.h" #include "core/logging/Utils.h" #include "controllers/SSLContextService.h" @@ -194,12 +194,12 @@ void AlertSink::send(Services& services) { return; } - auto client = core::ClassLoader::getDefaultClassLoader().instantiate("HTTPClient", "HTTPClient"); + auto client = core::ClassLoader::getDefaultClassLoader().instantiate("HTTPClient", "HTTPClient"); if (!client) { logger_->log_error("Could not instantiate a HTTPClient object"); return; } - client->initialize(utils::HttpRequestMethod::PUT, config_.url, services.ssl_service); + client->initialize(http::HttpRequestMethod::PUT, config_.url, services.ssl_service); rapidjson::Document doc(rapidjson::kObjectType); std::string agent_id = services.agent_id->getAgentIdentifier(); @@ -212,7 +212,7 @@ void AlertSink::send(Services& services) { rapidjson::Writer writer(buffer); doc.Accept(writer); - auto data_input = std::make_unique(); + auto data_input = std::make_unique(); data_input->write(std::string(buffer.GetString(), buffer.GetSize())); client->setUploadCallback(std::move(data_input)); client->setContentType("application/json"); diff --git a/libminifi/src/utils/BaseHTTPClient.cpp b/libminifi/src/http/BaseHTTPClient.cpp similarity index 96% rename from libminifi/src/utils/BaseHTTPClient.cpp rename to libminifi/src/http/BaseHTTPClient.cpp index 96721d2b1b..10b55fb509 100644 --- a/libminifi/src/utils/BaseHTTPClient.cpp +++ b/libminifi/src/http/BaseHTTPClient.cpp @@ -15,10 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include "http/BaseHTTPClient.h" + #include #include -#include "utils/BaseHTTPClient.h" #include "utils/StringUtils.h" namespace minifi = org::apache::nifi::minifi; @@ -55,9 +56,9 @@ std::optional parsePortNumber(const std::string& port_string) { } // namespace -namespace org::apache::nifi::minifi::utils { +namespace org::apache::nifi::minifi::http { -std::string get_token(utils::BaseHTTPClient* client, const std::string& username, const std::string& password) { +std::string get_token(BaseHTTPClient* client, const std::string& username, const std::string& password) { if (nullptr == client) { return ""; } @@ -272,4 +273,4 @@ size_t HTTPUploadStreamContentsCallback::setPosition(int64_t offset) { return HTTPRequestResponse::SEEKFUNC_OK; } -} // namespace org::apache::nifi::minifi::utils +} // namespace org::apache::nifi::minifi::http diff --git a/extensions/http-curl/client/HTTPClient.cpp b/libminifi/src/http/HTTPClient.cpp similarity index 89% rename from extensions/http-curl/client/HTTPClient.cpp rename to libminifi/src/http/HTTPClient.cpp index 0fda8841fb..df478a3ebd 100644 --- a/extensions/http-curl/client/HTTPClient.cpp +++ b/libminifi/src/http/HTTPClient.cpp @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "HTTPClient.h" +#include "http/HTTPClient.h" #include #include @@ -33,7 +33,7 @@ using namespace std::literals::chrono_literals; -namespace org::apache::nifi::minifi::extensions::curl { +namespace org::apache::nifi::minifi::http { HTTPClient::HTTPClient(std::string url, std::shared_ptr ssl_context_service) : core::Connectable("HTTPClient"), @@ -52,7 +52,7 @@ HTTPClient::HTTPClient() http_session_.reset(curl_easy_init()); } -void HTTPClient::addFormPart(const std::string& content_type, const std::string& name, std::unique_ptr form_callback, const std::optional& filename) { +void HTTPClient::addFormPart(const std::string& content_type, const std::string& name, std::unique_ptr form_callback, const std::optional& filename) { if (!form_) { form_.reset(curl_mime_init(http_session_.get())); } @@ -64,7 +64,7 @@ void HTTPClient::addFormPart(const std::string& content_type, const std::string& } curl_mime_name(part, name.c_str()); curl_mime_data_cb(part, gsl::narrow(form_callback_->size()), - &utils::HTTPRequestResponse::send_write, nullptr, nullptr, static_cast(form_callback_.get())); + &HTTPRequestResponse::send_write, nullptr, nullptr, static_cast(form_callback_.get())); } HTTPClient::~HTTPClient() { @@ -110,7 +110,7 @@ bool isSecure(const std::string& url) { } } // namespace -void HTTPClient::initialize(utils::HttpRequestMethod method, std::string url, std::shared_ptr ssl_context_service) { +void HTTPClient::initialize(http::HttpRequestMethod method, std::string url, std::shared_ptr ssl_context_service) { set_request_method(method); if (ssl_context_service) { ssl_context_service_ = std::move(ssl_context_service); @@ -146,26 +146,25 @@ void HTTPClient::clearBasicAuth() { curl_easy_setopt(http_session_.get(), CURLOPT_PASSWORD, nullptr); } -bool HTTPClient::setSpecificSSLVersion(utils::SSLVersion specific_version) { -#ifdef OPENSSL_SUPPORT +bool HTTPClient::setSpecificSSLVersion(SSLVersion specific_version) { if (ssl_context_service_) { switch (specific_version) { - case utils::SSLVersion::TLSv1_0: { + case SSLVersion::TLSv1_0: { ssl_context_service_->setMinTlsVersion(TLS1_VERSION); ssl_context_service_->setMaxTlsVersion(TLS1_VERSION); break; } - case utils::SSLVersion::TLSv1_1: { + case SSLVersion::TLSv1_1: { ssl_context_service_->setMinTlsVersion(TLS1_1_VERSION); ssl_context_service_->setMaxTlsVersion(TLS1_1_VERSION); break; } - case utils::SSLVersion::TLSv1_2: { + case SSLVersion::TLSv1_2: { ssl_context_service_->setMinTlsVersion(TLS1_2_VERSION); ssl_context_service_->setMaxTlsVersion(TLS1_2_VERSION); break; } - case utils::SSLVersion::TLSv1_3: { + case SSLVersion::TLSv1_3: { ssl_context_service_->setMinTlsVersion(TLS1_3_VERSION); ssl_context_service_->setMaxTlsVersion(TLS1_3_VERSION); break; @@ -173,64 +172,57 @@ bool HTTPClient::setSpecificSSLVersion(utils::SSLVersion specific_version) { default: break; } } -#endif -#if CURL_AT_LEAST_VERSION(7, 54, 0) // bitwise or of different enum types is deprecated in C++20, but the curl api explicitly supports ORing one of CURL_SSLVERSION and one of CURL_SSLVERSION_MAX switch (specific_version) { - case utils::SSLVersion::TLSv1_0: + case SSLVersion::TLSv1_0: return CURLE_OK == curl_easy_setopt(http_session_.get(), CURLOPT_SSLVERSION, static_cast(CURL_SSLVERSION_TLSv1_0) | static_cast(CURL_SSLVERSION_MAX_TLSv1_0)); - case utils::SSLVersion::TLSv1_1: + case SSLVersion::TLSv1_1: return CURLE_OK == curl_easy_setopt(http_session_.get(), CURLOPT_SSLVERSION, static_cast(CURL_SSLVERSION_TLSv1_1) | static_cast(CURL_SSLVERSION_MAX_TLSv1_1)); - case utils::SSLVersion::TLSv1_2: + case SSLVersion::TLSv1_2: return CURLE_OK == curl_easy_setopt(http_session_.get(), CURLOPT_SSLVERSION, static_cast(CURL_SSLVERSION_TLSv1_2) | static_cast(CURL_SSLVERSION_MAX_TLSv1_2)); - case utils::SSLVersion::TLSv1_3: + case SSLVersion::TLSv1_3: return CURLE_OK == curl_easy_setopt(http_session_.get(), CURLOPT_SSLVERSION, static_cast(CURL_SSLVERSION_TLSv1_3) | static_cast(CURL_SSLVERSION_MAX_TLSv1_3)); default: return false; } -#else - return false; -#endif } // If not set, the default will be TLS 1.0, see https://curl.haxx.se/libcurl/c/CURLOPT_SSLVERSION.html -bool HTTPClient::setMinimumSSLVersion(utils::SSLVersion minimum_version) { -#ifdef OPENSSL_SUPPORT +bool HTTPClient::setMinimumSSLVersion(SSLVersion minimum_version) { if (ssl_context_service_) { switch (minimum_version) { - case utils::SSLVersion::TLSv1_0: { + case SSLVersion::TLSv1_0: { ssl_context_service_->setMinTlsVersion(TLS1_VERSION); break; } - case utils::SSLVersion::TLSv1_1: { + case SSLVersion::TLSv1_1: { ssl_context_service_->setMinTlsVersion(TLS1_1_VERSION); break; } - case utils::SSLVersion::TLSv1_2: { + case SSLVersion::TLSv1_2: { ssl_context_service_->setMinTlsVersion(TLS1_2_VERSION); break; } - case utils::SSLVersion::TLSv1_3: { + case SSLVersion::TLSv1_3: { ssl_context_service_->setMinTlsVersion(TLS1_3_VERSION); break; } default: break; } } -#endif CURLcode ret = CURLE_UNKNOWN_OPTION; switch (minimum_version) { - case utils::SSLVersion::TLSv1_0: + case SSLVersion::TLSv1_0: ret = curl_easy_setopt(http_session_.get(), CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_0); break; - case utils::SSLVersion::TLSv1_1: + case SSLVersion::TLSv1_1: ret = curl_easy_setopt(http_session_.get(), CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_1); break; - case utils::SSLVersion::TLSv1_2: + case SSLVersion::TLSv1_2: ret = curl_easy_setopt(http_session_.get(), CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2); break; - case utils::SSLVersion::TLSv1_3: + case SSLVersion::TLSv1_3: ret = curl_easy_setopt(http_session_.get(), CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_3); break; } @@ -264,22 +256,22 @@ void HTTPClient::setReadTimeout(std::chrono::milliseconds timeout) { read_timeout_ = timeout; } -void HTTPClient::setReadCallback(std::unique_ptr callback) { +void HTTPClient::setReadCallback(std::unique_ptr callback) { read_callback_ = std::move(callback); - curl_easy_setopt(http_session_.get(), CURLOPT_WRITEFUNCTION, &utils::HTTPRequestResponse::receiveWrite); + curl_easy_setopt(http_session_.get(), CURLOPT_WRITEFUNCTION, &HTTPRequestResponse::receiveWrite); curl_easy_setopt(http_session_.get(), CURLOPT_WRITEDATA, static_cast(read_callback_.get())); } -void HTTPClient::setUploadCallback(std::unique_ptr callback) { +void HTTPClient::setUploadCallback(std::unique_ptr callback) { logger_->log_debug("Setting callback for {}", url_); write_callback_ = std::move(callback); - if (method_ == utils::HttpRequestMethod::PUT) { + if (method_ == http::HttpRequestMethod::PUT) { curl_easy_setopt(http_session_.get(), CURLOPT_INFILESIZE_LARGE, (curl_off_t) write_callback_->size()); } - curl_easy_setopt(http_session_.get(), CURLOPT_READFUNCTION, &utils::HTTPRequestResponse::send_write); + curl_easy_setopt(http_session_.get(), CURLOPT_READFUNCTION, &HTTPRequestResponse::send_write); curl_easy_setopt(http_session_.get(), CURLOPT_READDATA, static_cast(write_callback_.get())); curl_easy_setopt(http_session_.get(), CURLOPT_SEEKDATA, static_cast(write_callback_.get())); - curl_easy_setopt(http_session_.get(), CURLOPT_SEEKFUNCTION, &utils::HTTPRequestResponse::seek_callback); + curl_easy_setopt(http_session_.get(), CURLOPT_SEEKFUNCTION, &HTTPRequestResponse::seek_callback); } void HTTPClient::setContentType(std::string content_type) { @@ -306,7 +298,7 @@ void HTTPClient::setPostSize(size_t size) { } } -void HTTPClient::setHTTPProxy(const utils::HTTPProxy &proxy) { +void HTTPClient::setHTTPProxy(const HTTPProxy &proxy) { if (!proxy.host.empty()) { curl_easy_setopt(http_session_.get(), CURLOPT_PROXY, proxy.host.c_str()); curl_easy_setopt(http_session_.get(), CURLOPT_PROXYPORT, proxy.port); @@ -375,11 +367,11 @@ bool HTTPClient::submit() { curl_easy_setopt(http_session_.get(), CURLOPT_URL, url_.c_str()); logger_->log_debug("Submitting to {}", url_); if (read_callback_ == nullptr) { - curl_easy_setopt(http_session_.get(), CURLOPT_WRITEFUNCTION, &utils::HTTPRequestResponse::receiveWrite); + curl_easy_setopt(http_session_.get(), CURLOPT_WRITEFUNCTION, &HTTPRequestResponse::receiveWrite); curl_easy_setopt(http_session_.get(), CURLOPT_WRITEDATA, static_cast(&content_)); } - curl_easy_setopt(http_session_.get(), CURLOPT_HEADERFUNCTION, &utils::HTTPHeaderResponse::receive_headers); + curl_easy_setopt(http_session_.get(), CURLOPT_HEADERFUNCTION, &HTTPHeaderResponse::receive_headers); curl_easy_setopt(http_session_.get(), CURLOPT_HEADERDATA, static_cast(&response_data_.header_response)); if (form_ != nullptr) { @@ -425,24 +417,24 @@ const std::vector &HTTPClient::getResponseBody() { return response_data_.response_body; } -void HTTPClient::set_request_method(utils::HttpRequestMethod method) { +void HTTPClient::set_request_method(http::HttpRequestMethod method) { if (method_ == method) return; method_ = method; switch (*method_) { - case utils::HttpRequestMethod::POST: + case http::HttpRequestMethod::POST: curl_easy_setopt(http_session_.get(), CURLOPT_POST, 1L); curl_easy_setopt(http_session_.get(), CURLOPT_CUSTOMREQUEST, nullptr); break; - case utils::HttpRequestMethod::HEAD: + case http::HttpRequestMethod::HEAD: curl_easy_setopt(http_session_.get(), CURLOPT_NOBODY, 1L); curl_easy_setopt(http_session_.get(), CURLOPT_CUSTOMREQUEST, nullptr); break; - case utils::HttpRequestMethod::GET: + case http::HttpRequestMethod::GET: curl_easy_setopt(http_session_.get(), CURLOPT_HTTPGET, 1L); curl_easy_setopt(http_session_.get(), CURLOPT_CUSTOMREQUEST, nullptr); break; - case utils::HttpRequestMethod::PUT: + case http::HttpRequestMethod::PUT: curl_easy_setopt(http_session_.get(), CURLOPT_UPLOAD, 1L); curl_easy_setopt(http_session_.get(), CURLOPT_CUSTOMREQUEST, nullptr); break; @@ -478,7 +470,6 @@ int HTTPClient::onProgress(void *clientp, curl_off_t /*dltotal*/, curl_off_t dln } void HTTPClient::configure_secure_connection() { -#ifdef OPENSSL_SUPPORT if (ssl_context_service_) { curl_easy_setopt(http_session_.get(), CURLOPT_SSL_CTX_FUNCTION, &configure_ssl_context); curl_easy_setopt(http_session_.get(), CURLOPT_SSL_CTX_DATA, static_cast(ssl_context_service_.get())); @@ -500,7 +491,6 @@ void HTTPClient::configure_secure_connection() { curl_easy_setopt(http_session_.get(), CURLOPT_CAINFO, nullptr); curl_easy_setopt(http_session_.get(), CURLOPT_CAPATH, nullptr); } -#endif } void HTTPClient::setInterface(const std::string &ifc) { @@ -552,4 +542,4 @@ void HTTPClient::CurlMimeFree::operator()(curl_mime* curl_mime) const { REGISTER_RESOURCE(HTTPClient, InternalResource); -} // namespace org::apache::nifi::minifi::extensions::curl +} // namespace org::apache::nifi::minifi::http diff --git a/extensions/http-curl/client/HTTPStream.cpp b/libminifi/src/http/HTTPStream.cpp similarity index 91% rename from extensions/http-curl/client/HTTPStream.cpp rename to libminifi/src/http/HTTPStream.cpp index ca8b53b10e..cd989af9bd 100644 --- a/extensions/http-curl/client/HTTPStream.cpp +++ b/libminifi/src/http/HTTPStream.cpp @@ -16,17 +16,17 @@ * limitations under the License. */ -#include "HTTPStream.h" +#include "http/HTTPStream.h" #include #include #include -#include "HTTPCallback.h" +#include "http/HTTPCallback.h" #include "io/validation.h" #include "utils/gsl.h" -namespace org::apache::nifi::minifi::extensions::curl { +namespace org::apache::nifi::minifi::http { HttpStream::HttpStream(std::shared_ptr client) : http_client_(std::move(client)) { @@ -78,7 +78,7 @@ size_t HttpStream::read(std::span buf) { if (!started_) { std::lock_guard lock(mutex_); if (!started_) { - auto read_callback = std::make_unique(66560, true); + auto read_callback = std::make_unique(66560, true); http_client_future_ = std::async(std::launch::async, submit_read_client, http_client_, read_callback.get()); http_client_->setReadCallback(std::move(read_callback)); started_ = true; @@ -90,5 +90,5 @@ size_t HttpStream::read(std::span buf) { } } -} // namespace org::apache::nifi::minifi::extensions::curl +} // namespace org::apache::nifi::minifi::http diff --git a/extensions/http-curl/sitetosite/HTTPProtocol.cpp b/libminifi/src/sitetosite/HTTPProtocol.cpp similarity index 89% rename from extensions/http-curl/sitetosite/HTTPProtocol.cpp rename to libminifi/src/sitetosite/HTTPProtocol.cpp index a8b72057ce..600ff8a825 100644 --- a/extensions/http-curl/sitetosite/HTTPProtocol.cpp +++ b/libminifi/src/sitetosite/HTTPProtocol.cpp @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "HTTPProtocol.h" +#include "sitetosite/HTTPProtocol.h" #include #include @@ -24,13 +24,13 @@ #include #include -#include "PeersEntity.h" +#include "sitetosite/PeersEntity.h" #include "io/CRCStream.h" #include "sitetosite/Peer.h" #include "io/validation.h" #include "core/Resource.h" -namespace org::apache::nifi::minifi::extensions::curl { +namespace org::apache::nifi::minifi::sitetosite { std::shared_ptr HttpSiteToSiteClient::id_generator_ = utils::IdGenerator::getIdGenerator(); @@ -47,7 +47,7 @@ std::shared_ptr HttpSiteToSiteClient::createTransaction std::string dir_str = direction == sitetosite::SEND ? "input-ports" : "output-ports"; std::stringstream uri; uri << getBaseURI() << "data-transfer/" << dir_str << "/" << getPortId().to_string() << "/transactions"; - auto client = create_http_client(uri.str(), utils::HttpRequestMethod::POST); + auto client = create_http_client(uri.str(), http::HttpRequestMethod::POST); client->setRequestHeader(PROTOCOL_VERSION_HEADER, "1"); client->setConnectionTimeout(std::chrono::milliseconds(5000)); client->setContentType("application/json"); @@ -55,7 +55,7 @@ std::shared_ptr HttpSiteToSiteClient::createTransaction client->setRequestHeader("Transfer-Encoding", "chunked"); client->setPostFields(""); client->submit(); - auto http_stream = dynamic_cast(peer_->getStream()); + auto http_stream = dynamic_cast(peer_->getStream()); if (peer_->getStream() != nullptr) logger_->log_debug("Closing {}", http_stream->getClientRef()->getURL()); if (client->getResponseCode() == 201) { @@ -74,7 +74,7 @@ std::shared_ptr HttpSiteToSiteClient::createTransaction if (!transactionId) return nullptr; transaction->setTransactionId(transactionId.value()); - std::shared_ptr transaction_client; + std::shared_ptr transaction_client; if (transaction->getDirection() == sitetosite::SEND) { transaction_client = openConnectionForSending(transaction); } else { @@ -84,7 +84,7 @@ std::shared_ptr HttpSiteToSiteClient::createTransaction } transaction_client->setRequestHeader(PROTOCOL_VERSION_HEADER, "1"); - peer_->setStream(std::unique_ptr(new HttpStream(transaction_client))); + peer_->setStream(std::unique_ptr(new http::HttpStream(transaction_client))); logger_->log_debug("Created transaction id -{}-", transaction->getUUID().to_string()); known_transactions_[transaction->getUUID()] = transaction; return transaction; @@ -102,7 +102,7 @@ std::shared_ptr HttpSiteToSiteClient::createTransaction int HttpSiteToSiteClient::readResponse(const std::shared_ptr &transaction, sitetosite::RespondCode &code, std::string &message) { if (current_code == sitetosite::FINISH_TRANSACTION) { if (transaction->getDirection() == sitetosite::SEND) { - auto stream = dynamic_cast(peer_->getStream()); + auto stream = dynamic_cast(peer_->getStream()); if (!stream) throw std::runtime_error("Invalid HTTPStream"); stream->close(); @@ -121,7 +121,7 @@ int HttpSiteToSiteClient::readResponse(const std::shared_ptrgetDirection() == sitetosite::RECEIVE) { if (transaction->getState() == sitetosite::TRANSACTION_STARTED || transaction->getState() == sitetosite::DATA_EXCHANGED) { if (current_code == sitetosite::CONFIRM_TRANSACTION && transaction->getState() == sitetosite::DATA_EXCHANGED) { - auto stream = dynamic_cast(peer_->getStream()); + auto stream = dynamic_cast(peer_->getStream()); if (!stream->isFinished()) { logger_->log_debug("confirm read for {}, but not finished ", transaction->getUUIDStr()); if (stream->waitForDataAvailable()) { @@ -132,7 +132,7 @@ int HttpSiteToSiteClient::readResponse(const std::shared_ptr(peer_->getStream()); + auto stream = dynamic_cast(peer_->getStream()); if (stream->isFinished()) { logger_->log_debug("Finished {} ", transaction->getUUIDStr()); code = sitetosite::FINISH_TRANSACTION; @@ -179,7 +179,7 @@ bool HttpSiteToSiteClient::getPeerList(std::vector &peer std::stringstream uri; uri << getBaseURI() << "site-to-site/peers"; - auto client = create_http_client(uri.str(), utils::HttpRequestMethod::GET); + auto client = create_http_client(uri.str(), http::HttpRequestMethod::GET); client->setRequestHeader(PROTOCOL_VERSION_HEADER, "1"); @@ -193,20 +193,20 @@ bool HttpSiteToSiteClient::getPeerList(std::vector &peer return false; } -std::shared_ptr HttpSiteToSiteClient::openConnectionForSending(const std::shared_ptr &transaction) { +std::shared_ptr HttpSiteToSiteClient::openConnectionForSending(const std::shared_ptr &transaction) { std::stringstream uri; uri << transaction->getTransactionUrl() << "/flow-files"; - std::shared_ptr client = create_http_client(uri.str(), utils::HttpRequestMethod::POST); + std::shared_ptr client = create_http_client(uri.str(), http::HttpRequestMethod::POST); client->setContentType("application/octet-stream"); client->setRequestHeader("Accept", "text/plain"); client->setRequestHeader("Transfer-Encoding", "chunked"); return client; } -std::shared_ptr HttpSiteToSiteClient::openConnectionForReceive(const std::shared_ptr &transaction) { +std::shared_ptr HttpSiteToSiteClient::openConnectionForReceive(const std::shared_ptr &transaction) { std::stringstream uri; uri << transaction->getTransactionUrl() << "/flow-files"; - std::shared_ptr client = create_http_client(uri.str(), utils::HttpRequestMethod::GET); + std::shared_ptr client = create_http_client(uri.str(), http::HttpRequestMethod::GET); return client; } @@ -269,7 +269,7 @@ void HttpSiteToSiteClient::closeTransaction(const utils::Identifier &transaction uri << "&checksum=" << transaction->getCRC(); } - auto client = create_http_client(uri.str(), utils::HttpRequestMethod::DELETE); + auto client = create_http_client(uri.str(), http::HttpRequestMethod::DELETE); client->setRequestHeader(PROTOCOL_VERSION_HEADER, "1"); @@ -303,4 +303,4 @@ void HttpSiteToSiteClient::deleteTransaction(const utils::Identifier& transactio REGISTER_RESOURCE_AS(HttpSiteToSiteClient, InternalResource, ("HttpSiteToSiteClient", "HttpProtocol")); -} // namespace org::apache::nifi::minifi::extensions::curl +} // namespace org::apache::nifi::minifi::sitetosite diff --git a/libminifi/src/utils/TestUtils.cpp b/libminifi/src/utils/TestUtils.cpp deleted file mode 100644 index ebf63d1d9b..0000000000 --- a/libminifi/src/utils/TestUtils.cpp +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "utils/TestUtils.h" - -#include - -#ifdef WIN32 -#include -#include -#endif - -#include "utils/gsl.h" - -#ifdef WIN32 -namespace { - -void setAclOnFileOrDirectory(std::string file_name, DWORD perms, ACCESS_MODE perm_options) { - PSECURITY_DESCRIPTOR security_descriptor = nullptr; - const auto security_descriptor_deleter = gsl::finally([&security_descriptor] { if (security_descriptor) { LocalFree((HLOCAL) security_descriptor); } }); - - PACL old_acl = nullptr; // GetNamedSecurityInfo will set this to a non-owning pointer to a field inside security_descriptor: no need to free it - if (GetNamedSecurityInfo(file_name.c_str(), SE_FILE_OBJECT, DACL_SECURITY_INFORMATION, NULL, NULL, &old_acl, NULL, &security_descriptor) != ERROR_SUCCESS) { - throw std::runtime_error("Could not get security info for file: " + file_name); - } - - char trustee_name[] = "Everyone"; - EXPLICIT_ACCESS explicit_access = { - .grfAccessPermissions = perms, - .grfAccessMode = perm_options, - .grfInheritance = CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE, - .Trustee = { .TrusteeForm = TRUSTEE_IS_NAME, .ptstrName = trustee_name } - }; - - PACL new_acl = nullptr; - const auto new_acl_deleter = gsl::finally([&new_acl] { if (new_acl) { LocalFree((HLOCAL) new_acl); } }); - - if (SetEntriesInAcl(1, &explicit_access, old_acl, &new_acl) != ERROR_SUCCESS) { - throw std::runtime_error("Could not create new ACL for file: " + file_name); - } - - if (SetNamedSecurityInfo(file_name.data(), SE_FILE_OBJECT, DACL_SECURITY_INFORMATION, NULL, NULL, new_acl, NULL) != ERROR_SUCCESS) { - throw std::runtime_error("Could not set the new ACL for file: " + file_name); - } -} - -} // namespace -#endif - -namespace org::apache::nifi::minifi::utils { - -#ifdef WIN32 -// If minifi is not installed through the MSI installer, then TZDATA might be missing -// date::set_install can point to the TZDATA location, but it has to be called from each library/executable that wants to use timezones -void dateSetInstall(const std::string& install) { - date::set_install(install); -} -#endif - -void makeFileOrDirectoryNotWritable(const std::filesystem::path& file_name) { -#ifdef WIN32 - setAclOnFileOrDirectory(file_name.string(), FILE_GENERIC_WRITE, DENY_ACCESS); -#else - std::filesystem::permissions(file_name, std::filesystem::perms::owner_write, std::filesystem::perm_options::remove); -#endif -} - -void makeFileOrDirectoryWritable(const std::filesystem::path& file_name) { -#ifdef WIN32 - setAclOnFileOrDirectory(file_name.string(), FILE_GENERIC_WRITE, GRANT_ACCESS); -#else - std::filesystem::permissions(file_name, std::filesystem::perms::owner_write, std::filesystem::perm_options::add); -#endif -} - -} // namespace org::apache::nifi::minifi::utils diff --git a/libminifi/src/utils/TimeUtil.cpp b/libminifi/src/utils/TimeUtil.cpp index 0ae167e62c..4e9be88dac 100644 --- a/libminifi/src/utils/TimeUtil.cpp +++ b/libminifi/src/utils/TimeUtil.cpp @@ -18,6 +18,10 @@ #include "utils/TimeUtil.h" #include "range/v3/algorithm/contains.hpp" +#ifdef WIN32 +#include "date/tz.h" +#endif + namespace org::apache::nifi::minifi::utils::timeutils { using namespace std::literals::chrono_literals; @@ -67,4 +71,12 @@ std::optional parseRfc3339(const std::str return date::sys_days(date_part) + time_part - offset; } +#ifdef WIN32 +// If minifi is not installed through the MSI installer, then TZDATA might be missing +// date::set_install can point to the TZDATA location, but it has to be called from each library/executable that wants to use timezones +void dateSetInstall(const std::string& install) { + date::set_install(install); +} +#endif + } // namespace org::apache::nifi::minifi::utils::timeutils diff --git a/libminifi/src/utils/tls/CertificateUtils.cpp b/libminifi/src/utils/tls/CertificateUtils.cpp index 5b8763a3ab..0e941ea2b7 100644 --- a/libminifi/src/utils/tls/CertificateUtils.cpp +++ b/libminifi/src/utils/tls/CertificateUtils.cpp @@ -14,8 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifdef OPENSSL_SUPPORT - #include "utils/tls/CertificateUtils.h" #include "openssl/rsa.h" @@ -49,9 +47,9 @@ std::string ssl_error_category::message(int value) const { if (err == 0) { return ""; } - char buf[4096]; - ERR_error_string_n(err, buf, sizeof(buf)); - return buf; + std::array buf{}; + ERR_error_string_n(err, buf.data(), buf.size()); + return buf.data(); } std::error_code get_last_ssl_error_code() { @@ -300,5 +298,3 @@ std::error_code processPEMCertificate(const std::filesystem::path& cert_file, co } } // namespace org::apache::nifi::minifi::utils::tls - -#endif // OPENSSL_SUPPORT diff --git a/libminifi/src/utils/tls/ExtendedKeyUsage.cpp b/libminifi/src/utils/tls/ExtendedKeyUsage.cpp index 611441a3ec..e33fe5b79e 100644 --- a/libminifi/src/utils/tls/ExtendedKeyUsage.cpp +++ b/libminifi/src/utils/tls/ExtendedKeyUsage.cpp @@ -14,8 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifdef OPENSSL_SUPPORT - #include "utils/tls/ExtendedKeyUsage.h" #include @@ -94,5 +92,3 @@ bool operator!=(const ExtendedKeyUsage& left, const ExtendedKeyUsage& right) { } } // namespace org::apache::nifi::minifi::utils::tls - -#endif // OPENSSL_SUPPORT diff --git a/libminifi/test/.device_id b/libminifi/test/.device_id deleted file mode 100644 index 500a690abe..0000000000 --- a/libminifi/test/.device_id +++ /dev/null @@ -1 +0,0 @@ -2291398467776739051 \ No newline at end of file diff --git a/libminifi/test/CPPLINT.cfg b/libminifi/test/CPPLINT.cfg index 3eafe4ee69..aee39bdc42 100644 --- a/libminifi/test/CPPLINT.cfg +++ b/libminifi/test/CPPLINT.cfg @@ -1,3 +1,2 @@ filter=-build/namespaces,-whitespace/braces -exclude_files=Server.cpp exclude_files=TestBase.cpp diff --git a/libminifi/test/Server.cpp b/libminifi/test/Server.cpp deleted file mode 100644 index 39efb12b78..0000000000 --- a/libminifi/test/Server.cpp +++ /dev/null @@ -1,537 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* A simple server in the internet domain using TCP - The port number is passed as an argument */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include // std::ifstream -#include - -#define DEFAULT_NIFI_SERVER_PORT 9000 -#define DEFAULT_REPORT_INTERVAL 1000 // 1 sec -#define MAX_READ_TIMEOUT 30000 // 30 seconds - -// FlowControl Protocol Msg Type -typedef enum { - REGISTER_REQ, // Device Register Request from device to server which contain device serial number, current running flow YAML version - REGISTER_RESP, // Device Register Respond from server to device, may contain new flow.yml from server ask device to apply and also device report interval - REPORT_REQ, // Period Device Report from device to server which contain device serial number, current running flow YAML name/version and other period report info - REPORT_RESP, // Report Respond from server to device, may ask device to update flow YAML or processor property - MAX_FLOW_CONTROL_MSG_TYPE -} FlowControlMsgType; - -// FlowControl Protocol Msg Type String -static const char *FlowControlMsgTypeStr[MAX_FLOW_CONTROL_MSG_TYPE] = { "REGISTER_REQ", "REGISTER_RESP", "REPORT_REQ", "REPORT_RESP" }; - -// Flow Control Msg Type to String -inline const char *FlowControlMsgTypeToStr(FlowControlMsgType type) { - if (type < MAX_FLOW_CONTROL_MSG_TYPE) - return FlowControlMsgTypeStr[type]; - else - return NULL; -} - -// FlowControll Protocol Msg ID (Some Messages are fix length, Some are variable length (TLV) -typedef enum { - //Fix length 8 bytes: client to server in register request, required field - FLOW_SERIAL_NUMBER, - // Flow YAML name TLV: client to server in register request and report request, required field - FLOW_YAML_NAME, - // Flow YAML content, TLV: server to client in register respond, option field in case server want to ask client to load YAML from server - FLOW_YAML_CONTENT, - // Fix length, 4 bytes Report interval in msec: server to client in register respond, option field - REPORT_INTERVAL, - // Processor Name TLV: server to client in report respond, option field in case server want to ask client to update processor property - PROCESSOR_NAME, - // Processor Property Name TLV: server to client in report respond, option field in case server want to ask client to update processor property - PROPERTY_NAME, - // Processor Property Value TLV: server to client in report respond, option field in case server want to ask client to update processor property - PROPERTY_VALUE, - // Report Blob TLV: client to server in report request, option field in case client want to pickyback the report blob in report request to server - REPORT_BLOB, - MAX_FLOW_MSG_ID -} FlowControlMsgID; - -// FlowControl Protocol Msg ID String -static const char *FlowControlMsgIDStr[MAX_FLOW_MSG_ID] = { "FLOW_SERIAL_NUMBER", "FLOW_YAML_NAME", "FLOW_YAML_CONTENT", "REPORT_INTERVAL", "PROCESSOR_NAME" - "PROPERTY_NAME", "PROPERTY_VALUE", "REPORT_BLOB" }; - -#define TYPE_HDR_LEN 4 // Fix Hdr Type -#define TLV_HDR_LEN 8 // Type 4 bytes and Len 4 bytes - -// FlowControl Protocol Msg Len -inline int FlowControlMsgIDEncodingLen(FlowControlMsgID id, int payLoadLen) { - if (id == FLOW_SERIAL_NUMBER) - return (TYPE_HDR_LEN + 8); - else if (id == REPORT_INTERVAL) - return (TYPE_HDR_LEN + 4); - else if (id < MAX_FLOW_MSG_ID) - return (TLV_HDR_LEN + payLoadLen); - else - return -1; -} - -// Flow Control Msg Id to String -inline const char *FlowControlMsgIdToStr(FlowControlMsgID id) { - if (id < MAX_FLOW_MSG_ID) - return FlowControlMsgIDStr[id]; - else - return NULL; -} - -// Flow Control Respond status code -typedef enum { - RESP_SUCCESS, - RESP_TRIGGER_REGISTER, // Server respond to client report to re trigger register - RESP_START_FLOW_CONTROLLER, // Server respond to client to start flow controller - RESP_STOP_FLOW_CONTROLLER, // Server respond to client to stop flow controller - RESP_FAILURE, - MAX_RESP_CODE -} FlowControlRespCode; - -// FlowControl Resp Code str -static const char *FlowControlRespCodeStr[MAX_RESP_CODE] = { "RESP_SUCCESS", "RESP_TRIGGER_REGISTER", "RESP_START_FLOW_CONTROLLER", "RESP_STOP_FLOW_CONTROLLER", "RESP_FAILURE" }; - -// Flow Control Resp Code to String -inline const char *FlowControlRespCodeToStr(FlowControlRespCode code) { - if (code < MAX_RESP_CODE) - return FlowControlRespCodeStr[code]; - else - return NULL; -} - -// Common FlowControlProtocol Header -typedef struct { - uint32_t msgType; // Msg Type - uint32_t seqNumber; // Seq Number to match Req with Resp - uint32_t status; // Resp Code, see FlowControlRespCode - uint32_t payloadLen; // Msg Payload length -} FlowControlProtocolHeader; - -// encode uint32_t -uint8_t *encode(uint8_t *buf, uint32_t value) { - *buf++ = (value & 0xFF000000) >> 24; - *buf++ = (value & 0x00FF0000) >> 16; - *buf++ = (value & 0x0000FF00) >> 8; - *buf++ = (value & 0x000000FF); - return buf; -} - -// encode uint32_t -uint8_t *decode(uint8_t *buf, uint32_t &value) { - value = ((buf[0] << 24) | (buf[1] << 16) | (buf[2] << 8) | (buf[3])); - return (buf + 4); -} - -// encode byte array -uint8_t *encode(uint8_t *buf, uint8_t *bufArray, int size) { - memcpy(buf, bufArray, size); - buf += size; - return buf; -} - -// encode std::string -uint8_t *encode(uint8_t *buf, std::string value) { - // add the \0 for size - buf = encode(buf, value.size() + 1); - buf = encode(buf, (uint8_t *) value.c_str(), value.size() + 1); - return buf; -} - -int sendData(int socket, uint8_t *buf, int buflen) { - int ret = 0, bytes = 0; - - while (bytes < buflen) { - ret = send(socket, buf + bytes, buflen - bytes, 0); - //check for errors - if (ret == -1) { - return ret; - } - bytes += ret; - } - - return bytes; -} - -void error(const char *msg) { - perror(msg); - exit(1); -} - -/* readline - read a '\n' terminated line from socket fd - into buffer bufptr of size len. The line in the - buffer is terminated with '\0'. - It returns -1 in case of error or if - the capacity of the buffer is exceeded. - It returns 0 if EOF is encountered before reading '\n'. - */ -int readline(int fd, char *bufptr, size_t len) { - /* Note that this function is very tricky. It uses the - static variables bp, cnt, and b to establish a local buffer. - The recv call requests large chunks of data (the size of the buffer). - Then if the recv call reads more than one line, the overflow - remains in the buffer and it is made available to the next call - to readline. - Notice also that this routine reads up to '\n' and overwrites - it with '\0'. Thus if the line is really terminated with - "\r\n", the '\r' will remain unchanged. - */ - char *bufx = bufptr; - static char *bp; - static int cnt = 0; - static char b[4096]; - char c; - - while (--len > 0) { - if (--cnt <= 0) { - cnt = recv(fd, b, sizeof(b), 0); - if (cnt < 0) { - if ( errno == EINTR) { - len++; /* the while will decrement */ - continue; - } - return -1; - } - if (cnt == 0) - return 0; - bp = b; - } - c = *bp++; - *bufptr++ = c; - if (c == '\n') { - *bufptr = '\0'; - return bufptr - bufx; - } - } - return -1; -} - -int readData(int socket, uint8_t *buf, int buflen) { - int sendSize = buflen; - int status; - - while (buflen) { -#ifndef __MACH__ - status = read(socket, buf, buflen); -#else - status = recv(socket, buf, buflen, 0); -#endif - if (status <= 0) { - return status; - } - buflen -= status; - buf += status; - } - - return sendSize; -} - -int readHdr(int socket, FlowControlProtocolHeader *hdr) { - uint8_t buffer[sizeof(FlowControlProtocolHeader)]; - - uint8_t *data = buffer; - - int status = readData(socket, buffer, sizeof(FlowControlProtocolHeader)); - if (status <= 0) - return status; - - uint32_t value; - data = decode(data, value); - hdr->msgType = value; - - data = decode(data, value); - hdr->seqNumber = value; - - data = decode(data, value); - hdr->status = value; - - data = decode(data, value); - hdr->payloadLen = value; - - return sizeof(FlowControlProtocolHeader); -} - -int readYAML(char **ymlContent) { - std::ifstream is("conf/flowServer.yml", std::ifstream::binary); - if (is) { - // get length of file: - is.seekg(0, is.end); - int length = is.tellg(); - is.seekg(0, is.beg); - - char * buffer = new char[length]; - - printf("Reading %s len %d\n", "conf/flowServer.yml", length); - // read data as a block: - is.read(buffer, length); - - is.close(); - - // ...buffer contains the entire file... - *ymlContent = buffer; - - return length; - } - return 0; -} - -static int sockfd = 0, newsockfd = 0; -void sigHandler(int signal) { - if (signal == SIGINT || signal == SIGTERM) { - close(newsockfd); - close(sockfd); - exit(1); - } -} - -int main(int argc, char *argv[]) { - int portno; - socklen_t clilen; - struct sockaddr_in serv_addr, cli_addr; - char buffer[4096]; - int flag = 0; - int number = 0; - - int n; - if (argc < 2) { - fprintf(stderr, "ERROR, no port provided\n"); - exit(1); - } - - if (signal(SIGINT, sigHandler) == SIG_ERR || signal(SIGTERM, sigHandler) == SIG_ERR) { - - return -1; - } - sockfd = socket(AF_INET, SOCK_STREAM, 0); - if (sockfd < 0) - error("ERROR opening socket"); - bzero((char *) &serv_addr, sizeof(serv_addr)); - portno = atoi(argv[1]); - serv_addr.sin_family = AF_INET; - serv_addr.sin_addr.s_addr = INADDR_ANY; - serv_addr.sin_port = htons(portno); - if (bind(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) - error("ERROR on binding"); - listen(sockfd, 5); - if (portno == DEFAULT_NIFI_SERVER_PORT) { - while (true) { - clilen = sizeof(cli_addr); - newsockfd = accept(sockfd, (struct sockaddr *) &cli_addr, &clilen); - if (newsockfd < 0) { - error("ERROR on accept"); - break; - } - // process request - FlowControlProtocolHeader hdr; - int status = readHdr(newsockfd, &hdr); - if (status > 0) { - printf("Flow Control Protocol receive MsgType %s\n", FlowControlMsgTypeToStr((FlowControlMsgType) hdr.msgType)); - printf("Flow Control Protocol receive Seq Num %d\n", hdr.seqNumber); - printf("Flow Control Protocol receive Resp Code %s\n", FlowControlRespCodeToStr((FlowControlRespCode) hdr.status)); - printf("Flow Control Protocol receive Payload len %d\n", hdr.payloadLen); - if (((FlowControlMsgType) hdr.msgType) == REGISTER_REQ) { - printf("Flow Control Protocol Register Req receive\n"); - uint8_t *payload = new uint8_t[hdr.payloadLen]; - uint8_t *payloadPtr = payload; - status = readData(newsockfd, payload, hdr.payloadLen); - while (status > 0 && payloadPtr < (payload + hdr.payloadLen)) { - uint32_t msgID = 0xFFFFFFFF; - payloadPtr = decode(payloadPtr, msgID); - if (((FlowControlMsgID) msgID) == FLOW_SERIAL_NUMBER) { - // Fixed 8 bytes - uint8_t seqNum[8]; - memcpy(seqNum, payloadPtr, 8); - printf("Flow Control Protocol Register Req receive serial num\n"); - payloadPtr += 8; - } else if (((FlowControlMsgID) msgID) == FLOW_YAML_NAME) { - uint32_t len; - payloadPtr = decode(payloadPtr, len); - printf("Flow Control Protocol receive YAML name length %d\n", len); - std::string flowName = (const char *) payloadPtr; - payloadPtr += len; - printf("Flow Control Protocol receive YAML name %s\n", flowName.c_str()); - } else { - break; - } - } - delete[] payload; - // Send Register Respond - // Calculate the total payload msg size - char *ymlContent; - uint32_t yamlLen = readYAML(&ymlContent); - uint32_t payloadSize = FlowControlMsgIDEncodingLen(REPORT_INTERVAL, 0); - if (yamlLen > 0) - payloadSize += FlowControlMsgIDEncodingLen(FLOW_YAML_CONTENT, yamlLen); - - uint32_t size = sizeof(FlowControlProtocolHeader) + payloadSize; - uint8_t *data = new uint8_t[size]; - uint8_t *start = data; - - // encode the HDR - hdr.msgType = REGISTER_RESP; - hdr.payloadLen = payloadSize; - hdr.status = RESP_SUCCESS; - data = encode(data, hdr.msgType); - data = encode(data, hdr.seqNumber); - data = encode(data, hdr.status); - data = encode(data, hdr.payloadLen); - - // encode the report interval - data = encode(data, REPORT_INTERVAL); - data = encode(data, DEFAULT_REPORT_INTERVAL); - - // encode the YAML content - if (yamlLen > 0) { - data = encode(data, FLOW_YAML_CONTENT); - data = encode(data, yamlLen); - data = encode(data, (uint8_t *) ymlContent, yamlLen); - delete[] ymlContent; - } - - // send it - status = sendData(newsockfd, start, size); - delete[] start; - } else if (((FlowControlMsgType) hdr.msgType) == REPORT_REQ) { - printf("Flow Control Protocol Report Req receive\n"); - uint8_t *payload = new uint8_t[hdr.payloadLen]; - uint8_t *payloadPtr = payload; - status = readData(newsockfd, payload, hdr.payloadLen); - while (status > 0 && payloadPtr < (payload + hdr.payloadLen)) { - uint32_t msgID = 0xFFFFFFFF; - payloadPtr = decode(payloadPtr, msgID); - if (((FlowControlMsgID) msgID) == FLOW_YAML_NAME) { - uint32_t len; - payloadPtr = decode(payloadPtr, len); - printf("Flow Control Protocol receive YAML name length %d\n", len); - std::string flowName = (const char *) payloadPtr; - payloadPtr += len; - printf("Flow Control Protocol receive YAML name %s\n", flowName.c_str()); - } else { - break; - } - } - delete[] payload; - // Send Register Respond - // Calculate the total payload msg size - std::string processor = "RealTimeDataCollector"; - std::string propertyName1 = "real Time Message ID"; - std::string propertyValue1 = "41"; - std::string propertyName2 = "Batch Message ID"; - std::string propertyValue2 = "172,30,48"; - if (flag == 0) { - propertyName1 = "Real Time Message ID"; - propertyValue1 = "41"; - propertyName2 = "Batch Message ID"; - propertyValue2 = "172,48"; - flag = 1; - } else if (flag == 1) { - propertyName1 = "Real Time Message ID"; - propertyValue1 = "172,48"; - propertyName2 = "Batch Message ID"; - propertyValue2 = "41"; - flag = 0; - } - uint32_t payloadSize = FlowControlMsgIDEncodingLen(PROCESSOR_NAME, processor.size() + 1); - payloadSize += FlowControlMsgIDEncodingLen(PROPERTY_NAME, propertyName1.size() + 1); - payloadSize += FlowControlMsgIDEncodingLen(PROPERTY_VALUE, propertyValue1.size() + 1); - payloadSize += FlowControlMsgIDEncodingLen(PROPERTY_NAME, propertyName2.size() + 1); - payloadSize += FlowControlMsgIDEncodingLen(PROPERTY_VALUE, propertyValue2.size() + 1); - - uint32_t size = sizeof(FlowControlProtocolHeader) + payloadSize; - uint8_t *data = new uint8_t[size]; - uint8_t *start = data; - - // encode the HDR - hdr.msgType = REPORT_RESP; - hdr.payloadLen = payloadSize; - hdr.status = RESP_SUCCESS; - - if (number >= 10 && number < 20) { - // After 10 second report, stop the flow controller for 10 second - hdr.status = RESP_STOP_FLOW_CONTROLLER; - } else if (number == 20) { - // restart the flow controller after 10 second - hdr.status = RESP_START_FLOW_CONTROLLER; - } else if (number == 30) { - // retrigger register - hdr.status = RESP_TRIGGER_REGISTER; - number = 0; - } - - number++; - - data = encode(data, hdr.msgType); - data = encode(data, hdr.seqNumber); - data = encode(data, hdr.status); - data = encode(data, hdr.payloadLen); - - // encode the processorName - data = encode(data, PROCESSOR_NAME); - data = encode(data, processor); - - // encode the propertyName and value TLV - data = encode(data, PROPERTY_NAME); - data = encode(data, propertyName1); - data = encode(data, PROPERTY_VALUE); - data = encode(data, propertyValue1); - data = encode(data, PROPERTY_NAME); - data = encode(data, propertyName2); - data = encode(data, PROPERTY_VALUE); - data = encode(data, propertyValue2); - // send it - status = sendData(newsockfd, start, size); - delete[] start; - } - } - close(newsockfd); - } - close(sockfd); - } else { - clilen = sizeof(cli_addr); - newsockfd = accept(sockfd, (struct sockaddr *) &cli_addr, &clilen); - if (newsockfd < 0) - error("ERROR on accept"); - while (1) { - bzero(buffer, 4096); - n = readline(newsockfd, buffer, 4095); - if (n <= 0) { - close(newsockfd); - newsockfd = accept(sockfd, (struct sockaddr *) &cli_addr, &clilen); - continue; - } - printf("%s", buffer); - } - close(newsockfd); - close(sockfd); - } - return 0; -} diff --git a/libminifi/test/SingleProcessorTestController.h b/libminifi/test/SingleProcessorTestController.h deleted file mode 100644 index 86149b3060..0000000000 --- a/libminifi/test/SingleProcessorTestController.h +++ /dev/null @@ -1,144 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once -#include -#include -#include -#include -#include -#include -#include -#include "TestBase.h" -#include "FlowFileRecord.h" -#include "core/Processor.h" -#include "range/v3/algorithm/all_of.hpp" - -using namespace std::literals::chrono_literals; - -namespace org::apache::nifi::minifi::test { -struct InputFlowFileData { - std::string_view content; - std::unordered_map attributes = {}; -}; - - -using ProcessorTriggerResult = std::unordered_map>>; - -class SingleProcessorTestController : public TestController { - public: - explicit SingleProcessorTestController(const std::shared_ptr& processor) - : processor_{plan->addProcessor(processor, processor->getName())} - {} - - auto trigger() { - plan->runProcessor(processor_); - std::unordered_map>> result; - for (const auto& [relationship, connection]: outgoing_connections_) { - std::set> expired_flow_files; - std::vector> output_flow_files; - while (connection->isWorkAvailable()) { - auto output_flow_file = connection->poll(expired_flow_files); - assert(expired_flow_files.empty()); - if (!output_flow_file) continue; - output_flow_files.push_back(std::move(output_flow_file)); - } - result.insert_or_assign(relationship, std::move(output_flow_files)); - } - return result; - } - - auto trigger(InputFlowFileData&& input_flow_file_data) { - input_->put(createFlowFile(input_flow_file_data.content, std::move(input_flow_file_data.attributes))); - return trigger(); - } - - auto trigger(const std::string_view input_flow_file_content, std::unordered_map input_flow_file_attributes = {}) { - return trigger({input_flow_file_content, std::move(input_flow_file_attributes)}); - } - - auto trigger(std::vector&& input_flow_file_datas) { - for (auto& input_flow_file_data : std::move(input_flow_file_datas)) { - input_->put(createFlowFile(input_flow_file_data.content, std::move(input_flow_file_data.attributes))); - } - return trigger(); - } - - bool triggerUntil(const std::unordered_map& expected_quantities, - ProcessorTriggerResult& result, - const std::chrono::milliseconds max_duration, - const std::chrono::milliseconds wait_time = 50ms) { - auto start_time = std::chrono::steady_clock::now(); - while (std::chrono::steady_clock::now() < start_time + max_duration) { - for (auto& [relationship, flow_files] : trigger()) { - result[relationship].insert(result[relationship].end(), flow_files.begin(), flow_files.end()); - } - if (ranges::all_of(expected_quantities, [&result](const auto& kv) { - const auto& [relationship, expected_quantity] = kv; - return result[relationship].size() >= expected_quantity; - })) { - return true; - } - std::this_thread::sleep_for(wait_time); - } - return false; - } - - core::Relationship addDynamicRelationship(std::string name) { - auto relationship = core::Relationship{std::move(name), ""}; - outgoing_connections_.insert_or_assign(relationship, plan->addConnection(processor_, relationship, nullptr)); - return relationship; - } - - private: - std::shared_ptr createFlowFile(const std::string_view content, std::unordered_map attributes) { - const auto flow_file = std::make_shared(); - for (auto& attr : std::move(attributes)) { - flow_file->setAttribute(attr.first, std::move(attr.second)); - } - auto content_session = plan->getContentRepo()->createSession(); - auto claim = content_session->create(); - auto stream = content_session->write(claim); - stream->write(reinterpret_cast(content.data()), content.size()); - flow_file->setResourceClaim(claim); - flow_file->setSize(stream->size()); - flow_file->setOffset(0); - - stream->close(); - content_session->commit(); - return flow_file; - } - - public: - std::shared_ptr plan = createPlan(); - - protected: - core::Processor& getProcessor() const { return *processor_; } - - private: - std::shared_ptr processor_; - std::unordered_map outgoing_connections_{[this] { - std::unordered_map result; - for (const auto& relationship: processor_->getSupportedRelationships()) { - result.insert_or_assign(relationship, plan->addConnection(processor_, relationship, nullptr)); - } - return result; - }()}; - Connection* input_ = plan->addConnection(nullptr, core::Relationship{"success", "success"}, processor_); -}; - -} // namespace org::apache::nifi::minifi::test diff --git a/libminifi/test/flow-tests/CMakeLists.txt b/libminifi/test/flow-tests/CMakeLists.txt index 092f0e8df0..9111ffdefc 100644 --- a/libminifi/test/flow-tests/CMakeLists.txt +++ b/libminifi/test/flow-tests/CMakeLists.txt @@ -24,7 +24,6 @@ SET(FLOW_TEST_COUNT 0) FOREACH(testfile ${FLOW_TESTS}) get_filename_component(testfilename "${testfile}" NAME_WE) add_minifi_executable("${testfilename}" "${testfile}") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test/") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/") createTests("${testfilename}") target_link_libraries(${testfilename} minifi-standard-processors) @@ -32,7 +31,7 @@ FOREACH(testfile ${FLOW_TESTS}) target_link_libraries(${testfilename} minifi-rocksdb-repos) endif() - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) MATH(EXPR FLOW_TEST_COUNT "${FLOW_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}") ENDFOREACH() diff --git a/libminifi/test/flow-tests/CustomProcessors.h b/libminifi/test/flow-tests/CustomProcessors.h index c3bb54fa30..41f43da251 100644 --- a/libminifi/test/flow-tests/CustomProcessors.h +++ b/libminifi/test/flow-tests/CustomProcessors.h @@ -25,8 +25,8 @@ #include #include "core/Processor.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "processors/GenerateFlowFile.h" #include "core/PropertyDefinition.h" #include "core/PropertyDefinitionBuilder.h" @@ -100,10 +100,10 @@ class TestProcessor : public core::Processor, public ProcessorWithStatistics { void onSchedule(core::ProcessContext& context, core::ProcessSessionFactory&) override { int apple; bool appleSuccess = context.getProperty(AppleProbability, apple); - assert(appleSuccess); + REQUIRE(appleSuccess); int banana; bool bananaSuccess = context.getProperty(BananaProbability, banana); - assert(bananaSuccess); + REQUIRE(bananaSuccess); apple_probability_ = apple; banana_probability_ = banana; } diff --git a/libminifi/test/flow-tests/CycleTest.cpp b/libminifi/test/flow-tests/CycleTest.cpp index 1be43c2939..c6064ef8e8 100644 --- a/libminifi/test/flow-tests/CycleTest.cpp +++ b/libminifi/test/flow-tests/CycleTest.cpp @@ -15,10 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG #include "CustomProcessors.h" -#include "TestControllerWithFlow.h" +#include "unit/TestControllerWithFlow.h" // A flow with structure: // @@ -118,7 +116,7 @@ TEST_CASE("Flow with two long cycle", "[FlowWithCycle]") { auto procB = static_cast(root->findProcessorByName("B")); int tryCount = 0; - while (tryCount++ < 10 && !(procA->trigger_count.load() >= 10 && procB->trigger_count.load() >= 10)) { + while (tryCount++ < 10 && (procA->trigger_count.load() < 10 || procB->trigger_count.load() < 10)) { std::this_thread::sleep_for(std::chrono::milliseconds{1000}); } diff --git a/libminifi/test/flow-tests/FlowControllerTests.cpp b/libminifi/test/flow-tests/FlowControllerTests.cpp index 1ddad0eda8..633029f89a 100644 --- a/libminifi/test/flow-tests/FlowControllerTests.cpp +++ b/libminifi/test/flow-tests/FlowControllerTests.cpp @@ -15,8 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG #include #include #include @@ -29,13 +27,13 @@ #include "FlowFileRecord.h" #include "provenance/Provenance.h" #include "properties/Configure.h" -#include "../unit/ProvenanceTestHelper.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/ProvenanceTestHelper.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "CustomProcessors.h" -#include "TestControllerWithFlow.h" -#include "EmptyFlow.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestControllerWithFlow.h" +#include "unit/EmptyFlow.h" +#include "unit/TestUtils.h" using namespace std::literals::chrono_literals; @@ -295,6 +293,6 @@ Controller Services: [] // manually destroy the controller controller.controller_.reset(); - REQUIRE(utils::verifyLogLinePresenceInPollTime(0s, "Destroying FlowController")); + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(0s, "Destroying FlowController")); REQUIRE(LogTestController::getInstance().countOccurrences("Destroying scheduling agent") == 3); } diff --git a/libminifi/test/flow-tests/LoopTest.cpp b/libminifi/test/flow-tests/LoopTest.cpp index 0d2fd0b226..b2209d2587 100644 --- a/libminifi/test/flow-tests/LoopTest.cpp +++ b/libminifi/test/flow-tests/LoopTest.cpp @@ -15,10 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG #include "CustomProcessors.h" -#include "TestControllerWithFlow.h" +#include "unit/TestControllerWithFlow.h" // A flow with structure: // [Generator] ---> [A] ---| diff --git a/libminifi/test/flow-tests/MultiLoopTest.cpp b/libminifi/test/flow-tests/MultiLoopTest.cpp index a64dabbe4e..850ece7044 100644 --- a/libminifi/test/flow-tests/MultiLoopTest.cpp +++ b/libminifi/test/flow-tests/MultiLoopTest.cpp @@ -15,10 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG #include "CustomProcessors.h" -#include "TestControllerWithFlow.h" +#include "unit/TestControllerWithFlow.h" // A flow with structure: // v-----| diff --git a/libminifi/test/flow-tests/SessionTests.cpp b/libminifi/test/flow-tests/SessionTests.cpp index b722e93b11..c0609db2bd 100644 --- a/libminifi/test/flow-tests/SessionTests.cpp +++ b/libminifi/test/flow-tests/SessionTests.cpp @@ -18,8 +18,8 @@ #include #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/repository/VolatileFlowFileRepository.h" #include "../../extensions/rocksdb-repos/DatabaseContentRepository.h" #include "../../extensions/rocksdb-repos/FlowFileRepository.h" diff --git a/libminifi/test/flow-tests/TestControllerWithFlow.h b/libminifi/test/flow-tests/TestControllerWithFlow.h deleted file mode 100644 index 7dd05c3c6e..0000000000 --- a/libminifi/test/flow-tests/TestControllerWithFlow.h +++ /dev/null @@ -1,102 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include - -#include "FlowController.h" -#include "unit/ProvenanceTestHelper.h" -#include "core/yaml/YamlConfiguration.h" -#include "repository/VolatileContentRepository.h" -#include "CustomProcessors.h" - -class TestControllerWithFlow: public TestController { - public: - explicit TestControllerWithFlow(const char* yamlConfigContent, bool setup_flow = true) { - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - - home_ = createTempDirectory(); - - yaml_path_ = home_ / "config.yml"; - std::ofstream{yaml_path_} << yamlConfigContent; - - configuration_ = std::make_shared(); - configuration_->setHome(home_.string()); - configuration_->set(minifi::Configure::nifi_flow_configuration_file, yaml_path_.string()); - configuration_->set(minifi::Configure::nifi_c2_enable, "true"); - - if (setup_flow) { - setupFlow(); - } - } - - void setupFlow() { - std::shared_ptr prov_repo = std::make_shared(); - std::shared_ptr ff_repo = std::make_shared(); - std::shared_ptr content_repo = std::make_shared(); - - REQUIRE(content_repo->initialize(configuration_)); - - auto flow = std::make_shared(core::ConfigurationContext{ - .flow_file_repo = ff_repo, - .content_repo = content_repo, - .configuration = configuration_, - .path = yaml_path_.string(), - .filesystem = std::make_shared(), - .sensitive_properties_encryptor = utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}} - }); - auto root = flow->getRoot(); - root_ = root.get(); - std::vector> repo_metric_sources{prov_repo, ff_repo, content_repo}; - auto metrics_publisher_store = std::make_unique(configuration_, repo_metric_sources, flow); - metrics_publisher_store_ = metrics_publisher_store.get(); - controller_ = std::make_shared(prov_repo, ff_repo, configuration_, std::move(flow), content_repo, std::move(metrics_publisher_store)); - controller_->load(std::move(root)); - } - - void startFlow() { - controller_->start(); - } - - ~TestControllerWithFlow() { - if (controller_) { - controller_->stop(); - } - LogTestController::getInstance().reset(); - } - - std::filesystem::path home_; - std::filesystem::path yaml_path_; - std::shared_ptr configuration_; - std::shared_ptr controller_; - core::ProcessGroup* root_{nullptr}; - minifi::state::MetricsPublisherStore* metrics_publisher_store_{nullptr}; -}; diff --git a/extensions/http-curl/tests/AbsoluteTimeoutTest.cpp b/libminifi/test/integration/AbsoluteTimeoutTest.cpp similarity index 77% rename from extensions/http-curl/tests/AbsoluteTimeoutTest.cpp rename to libminifi/test/integration/AbsoluteTimeoutTest.cpp index 4eac7b1ebf..af7ca01d74 100644 --- a/extensions/http-curl/tests/AbsoluteTimeoutTest.cpp +++ b/libminifi/test/integration/AbsoluteTimeoutTest.cpp @@ -15,16 +15,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#undef NDEBUG #include -#include "TestBase.h" -#include "tests/TestServer.h" -#include "HTTPHandlers.h" +#include "unit/TestBase.h" +#include "integration/TestServer.h" +#include "integration/HTTPHandlers.h" +#include "unit/Catch.h" +#include "core/Processor.h" using namespace std::literals::chrono_literals; -int main() { +namespace org::apache::nifi::minifi::test { + +TEST_CASE("TestAbsoluteTimeout", "[httptest]") { TestController controller; std::string port = "12324"; @@ -42,5 +45,7 @@ int main() { plan->runNextProcessor(); - assert(LogTestController::getInstance().contains("HTTP operation timed out, with absolute timeout 3000ms")); + REQUIRE(LogTestController::getInstance().contains("HTTP operation timed out, with absolute timeout 3000ms")); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/unit/AlertTests.cpp b/libminifi/test/integration/AlertTests.cpp similarity index 90% rename from extensions/http-curl/tests/unit/AlertTests.cpp rename to libminifi/test/integration/AlertTests.cpp index 5a573fedf6..57eb3b49d3 100644 --- a/extensions/http-curl/tests/unit/AlertTests.cpp +++ b/libminifi/test/integration/AlertTests.cpp @@ -15,17 +15,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG -#include "TestBase.h" -#include "tests/ServerAwareHandler.h" +#include "unit/TestBase.h" #include "CivetServer.h" -#include "tests/TestServer.h" -#include "tests/HTTPIntegrationBase.h" +#include "integration/ServerAwareHandler.h" +#include "integration/TestServer.h" +#include "integration/HTTPIntegrationBase.h" #include "rapidjson/document.h" -#include "EmptyFlow.h" -#include "Utils.h" -#include "TestUtils.h" +#include "unit/EmptyFlow.h" +#include "unit/TestUtils.h" + +namespace org::apache::nifi::minifi::test { class AlertHandler : public ServerAwareHandler { public: @@ -54,7 +53,7 @@ class AlertHandler : public ServerAwareHandler { } std::string agent_id_; - utils::ConditionConcurrentQueue> alerts_; + minifi::utils::ConditionConcurrentQueue> alerts_; }; class VerifyAlerts : public HTTPIntegrationBase { @@ -69,8 +68,8 @@ class VerifyAlerts : public HTTPIntegrationBase { }; TEST_CASE("Alert system forwards logs") { - auto clock = std::make_shared(); - utils::timeutils::setClock(clock); + auto clock = std::make_shared(); + minifi::utils::timeutils::setClock(clock); TempDirectory dir; auto flow_config_file = dir.getPath() / "config.yml"; @@ -144,3 +143,5 @@ TEST_CASE("Alert system forwards logs") { harness.run(flow_config_file.string(), dir.getPath()); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2ClearCoreComponentStateTest.cpp b/libminifi/test/integration/C2ClearCoreComponentStateTest.cpp similarity index 80% rename from extensions/http-curl/tests/C2ClearCoreComponentStateTest.cpp rename to libminifi/test/integration/C2ClearCoreComponentStateTest.cpp index f43204c77e..db03035a8f 100644 --- a/extensions/http-curl/tests/C2ClearCoreComponentStateTest.cpp +++ b/libminifi/test/integration/C2ClearCoreComponentStateTest.cpp @@ -16,24 +16,26 @@ * limitations under the License. */ -#undef NDEBUG #include -#include "TestBase.h" -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" +#include "unit/TestBase.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" #include "processors/TailFile.h" #include "state/ProcessorController.h" #include "utils/file/FileUtils.h" -#include "utils/TestUtils.h" +#include "unit/TestUtils.h" +#include "unit/Catch.h" using namespace std::literals::chrono_literals; +namespace org::apache::nifi::minifi::test { + class VerifyC2ClearCoreComponentState : public VerifyC2Base { public: explicit VerifyC2ClearCoreComponentState(const std::atomic_bool& component_cleared_successfully) : component_cleared_successfully_(component_cleared_successfully) { auto temp_dir = testController.createTempDirectory(); - test_file_1_ = minifi::utils::putFileToDir(temp_dir, "test1.txt", "foo\n"); - test_file_2_ = minifi::utils::putFileToDir(temp_dir, "test2.txt", "foobar\n"); + test_file_1_ = utils::putFileToDir(temp_dir, "test1.txt", "foo\n"); + test_file_2_ = utils::putFileToDir(temp_dir, "test2.txt", "foobar\n"); } void testSetup() override { @@ -47,8 +49,8 @@ class VerifyC2ClearCoreComponentState : public VerifyC2Base { } void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; - assert(verifyEventHappenedInPollTime(40s, [&] { return component_cleared_successfully_.load(); }, 1s)); + using org::apache::nifi::minifi::test::utils::verifyEventHappenedInPollTime; + REQUIRE(verifyEventHappenedInPollTime(40s, [&] { return component_cleared_successfully_.load(); }, 1s)); } [[nodiscard]] std::filesystem::path getFile1Location() const { @@ -85,11 +87,11 @@ class ClearCoreComponentStateHandler: public HeartbeatHandler { } void handleHeartbeat(const rapidjson::Document&, struct mg_connection * conn) override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; switch (flow_state_) { case FlowState::STARTED: - assert(verifyLogLinePresenceInPollTime(10s, "ProcessSession committed for TailFile1")); - assert(verifyLogLinePresenceInPollTime(10s, "ProcessSession committed for TailFile2")); + REQUIRE(verifyLogLinePresenceInPollTime(10s, "ProcessSession committed for TailFile1")); + REQUIRE(verifyLogLinePresenceInPollTime(10s, "ProcessSession committed for TailFile2")); sendHeartbeatResponse("DESCRIBE", "corecomponentstate", "889345", conn); flow_state_ = FlowState::FIRST_DESCRIBE_SENT; break; @@ -112,17 +114,17 @@ class ClearCoreComponentStateHandler: public HeartbeatHandler { void handleAcknowledge(const rapidjson::Document& root) override { switch (flow_state_) { case FlowState::FIRST_DESCRIBE_SENT: { - assert(root.HasMember("corecomponentstate")); + REQUIRE(root.HasMember("corecomponentstate")); auto assertExpectedTailFileState = [&](const char* uuid, const char* name, const char* position) { - assert(root["corecomponentstate"].HasMember(uuid)); + REQUIRE(root["corecomponentstate"].HasMember(uuid)); const auto& tf = root["corecomponentstate"][uuid]; - assert(tf.HasMember("file.0.name")); - assert(std::string(tf["file.0.name"].GetString()) == name); - assert(tf.HasMember("file.0.position")); - assert(std::string(tf["file.0.position"].GetString()) == position); - assert(tf.HasMember("file.0.current")); - assert(strlen(tf["file.0.current"].GetString()) > 0U); + REQUIRE(tf.HasMember("file.0.name")); + REQUIRE(std::string(tf["file.0.name"].GetString()) == name); + REQUIRE(tf.HasMember("file.0.position")); + REQUIRE(std::string(tf["file.0.position"].GetString()) == position); + REQUIRE(tf.HasMember("file.0.current")); + REQUIRE(strlen(tf["file.0.current"].GetString()) > 0U); }; assertExpectedTailFileState("2438e3c8-015a-1000-79ca-83af40ec1993", "test1.txt", "4"); @@ -130,8 +132,8 @@ class ClearCoreComponentStateHandler: public HeartbeatHandler { last_read_time_1_ = std::string(root["corecomponentstate"]["2438e3c8-015a-1000-79ca-83af40ec1993"]["file.0.last_read_time"].GetString()); last_read_time_2_ = std::string(root["corecomponentstate"]["2438e3c8-015a-1000-79ca-83af40ec1994"]["file.0.last_read_time"].GetString()); - assert(!last_read_time_1_.empty()); - assert(!last_read_time_2_.empty()); + REQUIRE(!last_read_time_1_.empty()); + REQUIRE(!last_read_time_2_.empty()); flow_state_ = FlowState::FIRST_DESCRIBE_ACK; break; } @@ -141,7 +143,7 @@ class ClearCoreComponentStateHandler: public HeartbeatHandler { const std::string tailing_file_pattern = "[debug] Tailing file " + file_1_location_.string(); const std::string tail_file_committed_pattern = "[trace] ProcessSession committed for TailFile1"; const std::vector patterns = {tailing_file_pattern, tailing_file_pattern, tail_file_committed_pattern}; - return utils::string::matchesSequence(log_contents, patterns); + return minifi::utils::string::matchesSequence(log_contents, patterns); }; if (tail_file_ran_again_checker()) { flow_state_ = FlowState::CLEAR_SENT_ACK; @@ -190,13 +192,13 @@ class ClearCoreComponentStateHandler: public HeartbeatHandler { std::filesystem::path file_1_location_; }; -int main(int argc, char **argv) { +TEST_CASE("C2ClearCoreComponentState", "[c2test]") { std::atomic_bool component_cleared_successfully{false}; - const cmd_args args = parse_cmdline_args(argc, argv, "api/heartbeat"); VerifyC2ClearCoreComponentState harness(component_cleared_successfully); - harness.setKeyDir(args.key_dir); ClearCoreComponentStateHandler handler(component_cleared_successfully, harness.getConfiguration(), harness.getFile1Location()); - harness.setUrl(args.url, &handler); - harness.run(args.test_file); - return 0; + harness.setUrl("http://localhost:0/api/heartbeat", &handler); + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestC2DescribeCoreComponentState.yml"; + harness.run(test_file_path); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2CompressTest.cpp b/libminifi/test/integration/C2CompressTest.cpp similarity index 86% rename from extensions/http-curl/tests/C2CompressTest.cpp rename to libminifi/test/integration/C2CompressTest.cpp index feccdf9aac..2cf809e104 100644 --- a/extensions/http-curl/tests/C2CompressTest.cpp +++ b/libminifi/test/integration/C2CompressTest.cpp @@ -15,28 +15,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG - -#include "TestBase.h" +#include "unit/TestBase.h" #include "c2/C2Agent.h" #include "c2/HeartbeatLogger.h" #include "protocols/RESTProtocol.h" #include "protocols/RESTSender.h" -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" #include "range/v3/action/sort.hpp" #include "range/v3/action/unique.hpp" #include "range/v3/range/conversion.hpp" #include "range/v3/view/filter.hpp" #include "range/v3/view/split.hpp" #include "range/v3/view/transform.hpp" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" #include "utils/StringUtils.h" #include "utils/span.h" #include "properties/Configuration.h" #include "io/ZlibStream.h" +#include "unit/Catch.h" + +namespace org::apache::nifi::minifi::test { class CompressedHeartbeatHandler : public HeartbeatHandler { protected: @@ -51,9 +51,9 @@ class CompressedHeartbeatHandler : public HeartbeatHandler { { minifi::io::ZlibDecompressStream decompressor(gsl::make_not_null(&output)); auto ret = decompressor.write(as_bytes(std::span(payload))); - assert(ret == payload.size()); + REQUIRE(ret == payload.size()); } - auto str_span = utils::as_span(output.getBuffer()); + auto str_span = minifi::utils::as_span(output.getBuffer()); return {str_span.data(), str_span.size()}; } @@ -72,7 +72,7 @@ class VerifyCompressedHeartbeat : public VerifyC2Base { } void runAssertions() override { - assert(utils::verifyEventHappenedInPollTime(std::chrono::seconds(10), verify_)); + REQUIRE(utils::verifyEventHappenedInPollTime(std::chrono::seconds(10), verify_)); } void configureC2() override { @@ -89,7 +89,7 @@ class VerifyCompressedHeartbeat : public VerifyC2Base { std::function verify_; }; -int main() { +TEST_CASE("C2CompressTest", "[c2test]") { VerifyCompressedHeartbeat harness; CompressedHeartbeatHandler responder(harness.getConfiguration()); harness.setVerifier([&] () -> bool { @@ -98,3 +98,5 @@ int main() { harness.setUrl("https://localhost:0/heartbeat", &responder); harness.run(); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2ConfigEncryption.cpp b/libminifi/test/integration/C2ConfigEncryption.cpp similarity index 60% rename from extensions/http-curl/tests/C2ConfigEncryption.cpp rename to libminifi/test/integration/C2ConfigEncryption.cpp index 4f4a7c19e0..25edaead40 100644 --- a/extensions/http-curl/tests/C2ConfigEncryption.cpp +++ b/libminifi/test/integration/C2ConfigEncryption.cpp @@ -15,43 +15,52 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG #include #include -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" -#include "utils/IntegrationTestUtils.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" +#include "unit/TestUtils.h" #include "utils/crypto/EncryptionProvider.h" +#include "unit/Catch.h" using namespace std::literals::chrono_literals; -int main(int argc, char **argv) { - const cmd_args args = parse_cmdline_args(argc, argv, "update"); +namespace org::apache::nifi::minifi::test { + +TEST_CASE("C2ConfigEncryption", "[c2test]") { + std::filesystem::path test_file_path; + SECTION("Yaml config") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "decrypted.config.yml"; + } + SECTION("Json config") { + test_file_path = std::filesystem::path(TEST_RESOURCES) / "decrypted.config.json"; + } TestController controller; // copy config file to temporary location as it will get overridden auto home_path = controller.createTempDirectory(); auto live_config_file = home_path / "config.yml"; - utils::file::copy_file(args.test_file, live_config_file); - // the C2 server will update the flow with the contents of args.test_file + minifi::utils::file::copy_file(test_file_path, live_config_file); + // the C2 server will update the flow with the contents of test_file_path // which will be encrypted and persisted to the temporary live_config_file - C2UpdateHandler handler(args.test_file); + C2UpdateHandler handler(test_file_path.string()); VerifyC2Update harness(10s); harness.getConfiguration()->set(minifi::Configure::nifi_flow_configuration_encrypt, "true"); - harness.setKeyDir(args.key_dir); - harness.setUrl(args.url, &handler); + harness.setKeyDir(TEST_RESOURCES); + harness.setUrl("https://localhost:0/update", &handler); handler.setC2RestResponse(harness.getC2RestUrl(), "configuration", "true"); - harness.run(live_config_file, args.key_dir); + harness.run(live_config_file, TEST_RESOURCES); - auto encryptor = utils::crypto::EncryptionProvider::create(args.key_dir); - assert(encryptor); + auto encryptor = minifi::utils::crypto::EncryptionProvider::create(TEST_RESOURCES); + REQUIRE(encryptor); std::ifstream encrypted_file{live_config_file}; std::string decrypted_config = encryptor->decrypt(std::string(std::istreambuf_iterator(encrypted_file), {})); - std::ifstream expected_file{args.test_file + ".reformatted"}; + std::ifstream expected_file{test_file_path.string() + ".reformatted"}; std::string expected_config{std::istreambuf_iterator(expected_file), {}}; - assert(decrypted_config == expected_config); + REQUIRE(decrypted_config == expected_config); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2DebugBundleTest.cpp b/libminifi/test/integration/C2DebugBundleTest.cpp similarity index 87% rename from extensions/http-curl/tests/C2DebugBundleTest.cpp rename to libminifi/test/integration/C2DebugBundleTest.cpp index dfb0197614..a6d0482d6c 100644 --- a/extensions/http-curl/tests/C2DebugBundleTest.cpp +++ b/libminifi/test/integration/C2DebugBundleTest.cpp @@ -15,22 +15,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG - -#include "TestBase.h" +#include "unit/TestBase.h" #include "c2/C2Agent.h" #include "protocols/RESTProtocol.h" #include "protocols/RESTSender.h" -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" #include "io/ArchiveStream.h" -#include "EmptyFlow.h" +#include "unit/EmptyFlow.h" #include "properties/Configuration.h" +#include "unit/Catch.h" using namespace std::literals::chrono_literals; +namespace org::apache::nifi::minifi::test { + class VerifyDebugInfo : public VerifyC2Base { public: explicit VerifyDebugInfo(std::function verify): verify_(std::move(verify)) {} @@ -43,7 +43,7 @@ class VerifyDebugInfo : public VerifyC2Base { } void runAssertions() override { - assert(utils::verifyEventHappenedInPollTime(std::chrono::seconds(300), verify_)); + REQUIRE(utils::verifyEventHappenedInPollTime(std::chrono::seconds(300), verify_)); } void configureC2() override { @@ -63,7 +63,7 @@ class C2DebugBundleHandler : public ServerAwareHandler { form_handler.field_get = field_get; form_handler.user_data = &file_content; mg_handle_form_request(conn, &form_handler); - assert(file_content); + REQUIRE(file_content); { std::lock_guard lock(mtx_); bundles_.push_back(std::move(*file_content)); @@ -139,7 +139,7 @@ class C2HeartbeatHandler : public ServerAwareHandler { static std::string properties_file = "some.dummy.content = here\n"; static std::string flow_config_file = empty_flow; -int main() { +TEST_CASE("C2DebugBundleTest", "[c2test]") { TestController controller; C2HeartbeatHandler heartbeat_handler; @@ -147,7 +147,7 @@ int main() { C2DebugBundleHandler bundle_handler; std::filesystem::path home_dir = controller.createTempDirectory(); - utils::file::PathUtils::create_dir(home_dir / "conf"); + minifi::utils::file::PathUtils::create_dir(home_dir / "conf"); std::ofstream{home_dir / "conf/minifi.properties", std::ios::binary} << properties_file; std::ofstream{home_dir / "conf/config.yml", std::ios::binary} << flow_config_file; @@ -156,25 +156,25 @@ int main() { return false; } auto bundles = bundle_handler.getBundles(); - assert(bundles.size() == 1); + REQUIRE(bundles.size() == 1); // verify the bundle auto bundle_stream = std::make_shared(); bundle_stream->write(reinterpret_cast(bundles[0].data()), bundles[0].length()); auto archive_provider = core::ClassLoader::getDefaultClassLoader().instantiate( "ArchiveStreamProvider", "ArchiveStreamProvider"); - assert(archive_provider); + REQUIRE(archive_provider); auto decompressor = archive_provider->createReadStream(bundle_stream); - assert(decompressor); + REQUIRE(decompressor); std::map archive_content; while (auto info = decompressor->nextEntry()) { std::string file_content; file_content.resize(info->size); - assert(decompressor->read(as_writable_bytes(std::span(file_content))) == + REQUIRE(decompressor->read(as_writable_bytes(std::span(file_content))) == file_content.length()); archive_content[info->filename] = std::move(file_content); } - assert(archive_content["minifi.properties"] == properties_file); - assert(archive_content["config.yml"] == flow_config_file); + REQUIRE(archive_content["minifi.properties"] == properties_file); + REQUIRE(archive_content["config.yml"] == flow_config_file); auto log_gz = archive_content["minifi.log.gz"]; auto log_stream = std::make_shared(); { @@ -184,8 +184,8 @@ int main() { std::string log_text; log_text.resize(log_stream->size()); log_stream->read(as_writable_bytes(std::span(log_text))); - assert(log_text.find("Tis but a scratch") != std::string::npos); - assert(archive_content["manifest.json"].find("minifi-archive-extensions") != std::string::npos); + REQUIRE(log_text.find("Tis but a scratch") != std::string::npos); + REQUIRE(archive_content["manifest.json"].find("minifi-archive-extensions") != std::string::npos); return true; }); @@ -202,3 +202,5 @@ int main() { harness.run((home_dir / "conf/config.yml").string()); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2DescribeCoreComponentStateTest.cpp b/libminifi/test/integration/C2DescribeCoreComponentStateTest.cpp similarity index 77% rename from extensions/http-curl/tests/C2DescribeCoreComponentStateTest.cpp rename to libminifi/test/integration/C2DescribeCoreComponentStateTest.cpp index e3cc2556c3..8cfe8bf724 100644 --- a/extensions/http-curl/tests/C2DescribeCoreComponentStateTest.cpp +++ b/libminifi/test/integration/C2DescribeCoreComponentStateTest.cpp @@ -15,15 +15,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG #include -#include "TestBase.h" -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" +#include "unit/TestBase.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" #include "processors/TailFile.h" #include "state/ProcessorController.h" #include "utils/file/FileUtils.h" +#include "unit/Catch.h" + +namespace org::apache::nifi::minifi::test { class VerifyC2DescribeCoreComponentState : public VerifyC2Describe { public: @@ -72,17 +73,17 @@ class DescribeCoreComponentStateHandler: public HeartbeatHandler { } void handleAcknowledge(const rapidjson::Document& root) override { - assert(root.HasMember("corecomponentstate")); + REQUIRE(root.HasMember("corecomponentstate")); auto assertExpectedTailFileState = [&](const char* uuid, const char* name, const char* position) { - assert(root["corecomponentstate"].HasMember(uuid)); + REQUIRE(root["corecomponentstate"].HasMember(uuid)); const auto& tf = root["corecomponentstate"][uuid]; - assert(tf.HasMember("file.0.name")); - assert(std::string(tf["file.0.name"].GetString()) == name); - assert(tf.HasMember("file.0.position")); - assert(std::string(tf["file.0.position"].GetString()) == position); - assert(tf.HasMember("file.0.current")); - assert(strlen(tf["file.0.current"].GetString()) > 0U); + REQUIRE(tf.HasMember("file.0.name")); + REQUIRE(std::string(tf["file.0.name"].GetString()) == name); + REQUIRE(tf.HasMember("file.0.position")); + REQUIRE(std::string(tf["file.0.position"].GetString()) == position); + REQUIRE(tf.HasMember("file.0.current")); + REQUIRE(strlen(tf["file.0.current"].GetString()) > 0U); }; assertExpectedTailFileState("2438e3c8-015a-1000-79ca-83af40ec1993", "test1.txt", "4"); @@ -94,13 +95,14 @@ class DescribeCoreComponentStateHandler: public HeartbeatHandler { std::atomic_bool& verified_; }; -int main(int argc, char **argv) { +TEST_CASE("C2DescribeCoreComponentStateTest", "[c2test]") { std::atomic_bool verified{false}; - const cmd_args args = parse_cmdline_args(argc, argv, "api/heartbeat"); VerifyC2DescribeCoreComponentState harness(verified); - harness.setKeyDir(args.key_dir); + harness.setKeyDir(TEST_RESOURCES); DescribeCoreComponentStateHandler handler(harness.getConfiguration(), verified); - harness.setUrl(args.url, &handler); - harness.run(args.test_file); - return 0; + harness.setUrl("https://localhost:0/api/heartbeat", &handler); + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestC2DescribeCoreComponentState.yml"; + harness.run(test_file_path); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2DescribeManifestTest.cpp b/libminifi/test/integration/C2DescribeManifestTest.cpp similarity index 78% rename from extensions/http-curl/tests/C2DescribeManifestTest.cpp rename to libminifi/test/integration/C2DescribeManifestTest.cpp index 294eb8db3f..7f59f2a94c 100644 --- a/extensions/http-curl/tests/C2DescribeManifestTest.cpp +++ b/libminifi/test/integration/C2DescribeManifestTest.cpp @@ -15,13 +15,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG #include -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" #include "properties/Configuration.h" #include "ConfigTestAccessor.h" +#include "unit/Catch.h" + +namespace org::apache::nifi::minifi::test { // from TestHTTPGet.yml constexpr auto invokehttp_uuid = "2438e3c8-015a-1000-79ca-83af40ec1991"; @@ -47,29 +48,31 @@ class DescribeManifestHandler: public HeartbeatHandler { std::atomic& verified_; }; -int main(int argc, char **argv) { - const cmd_args args = parse_cmdline_args(argc, argv, "heartbeat"); +TEST_CASE("C2DescribeManifestTest", "[c2test]") { std::atomic_bool verified{false}; VerifyC2Describe harness(verified); - utils::crypto::Bytes encryption_key = utils::string::from_hex("4024b327fdc987ce3eb43dd1f690b9987e4072e0020e3edf4349ce1ad91a4e38"); - minifi::Decryptor decryptor{utils::crypto::EncryptionProvider{encryption_key}}; + minifi::utils::crypto::Bytes encryption_key = minifi::utils::string::from_hex("4024b327fdc987ce3eb43dd1f690b9987e4072e0020e3edf4349ce1ad91a4e38"); + minifi::Decryptor decryptor{minifi::utils::crypto::EncryptionProvider{encryption_key}}; std::string encrypted_value = "l3WY1V27knTiPa6jVX0jrq4qjmKsySOu||ErntqZpHP1M+6OkA14p5sdnqJhuNHWHDVUU5EyMloTtSytKk9a5xNKo="; harness.setConfiguration(std::make_shared(decryptor)); - harness.setKeyDir(args.key_dir); + harness.setKeyDir(TEST_RESOURCES); DescribeManifestHandler responder(harness.getConfiguration(), verified); auto logger_properties = std::make_shared(); ConfigTestAccessor::call_setLoggerProperties(harness.getConfiguration(), logger_properties); harness.getConfiguration()->set(minifi::Configuration::nifi_rest_api_password, encrypted_value); - harness.getConfiguration()->set(std::string(minifi::Configuration::nifi_rest_api_password) + ".protected", utils::crypto::EncryptionType::name()); + harness.getConfiguration()->set(std::string(minifi::Configuration::nifi_rest_api_password) + ".protected", minifi::utils::crypto::EncryptionType::name()); harness.getConfiguration()->set(minifi::Configuration::nifi_c2_agent_identifier_fallback, "c2_id_fallback"); harness.getConfiguration()->set(minifi::Configuration::nifi_framework_dir, "framework_path"); harness.getConfiguration()->set(minifi::Configuration::nifi_sensitive_props_additional_keys, std::string(minifi::Configuration::nifi_framework_dir) + ", " + std::string(minifi::Configuration::nifi_c2_agent_identifier_fallback)); harness.getConfiguration()->set(minifi::Configuration::nifi_log_appender_rolling_directory, "/var/log/minifi"); - harness.setUrl(args.url, &responder); - harness.run(args.test_file); + harness.setUrl("https://localhost:0/api/heartbeat", &responder); + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestHTTPGet.yml"; + harness.run(test_file_path.string()); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2DescribeMetricsTest.cpp b/libminifi/test/integration/C2DescribeMetricsTest.cpp similarity index 90% rename from extensions/http-curl/tests/C2DescribeMetricsTest.cpp rename to libminifi/test/integration/C2DescribeMetricsTest.cpp index b5458a3b47..692e0e37b2 100644 --- a/extensions/http-curl/tests/C2DescribeMetricsTest.cpp +++ b/libminifi/test/integration/C2DescribeMetricsTest.cpp @@ -15,20 +15,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG #include -#include "TestBase.h" -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" +#include "unit/TestBase.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" #include "processors/TailFile.h" #include "state/ProcessorController.h" #include "utils/file/FileUtils.h" -#include "utils/TestUtils.h" +#include "unit/TestUtils.h" #include "processors/GetTCP.h" #include "utils/StringUtils.h" #include "utils/file/PathUtils.h" +#include "unit/Catch.h" using namespace std::literals::chrono_literals; @@ -48,8 +47,7 @@ class VerifyEmptyC2Metric : public VerifyC2Base { } void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; - assert(verifyEventHappenedInPollTime(40s, [&] { return metrics_found_.load(); }, 1s)); + REQUIRE(utils::verifyEventHappenedInPollTime(40s, [&] { return metrics_found_.load(); }, 1s)); } private: @@ -157,12 +155,9 @@ class MetricsHandler: public HeartbeatHandler { std::atomic_bool& metrics_found_; }; -} // namespace org::apache::nifi::minifi::test - -int main(int argc, char **argv) { +TEST_CASE("C2DescribeMetricsTest", "[c2test]") { std::atomic_bool metrics_found{false}; - const cmd_args args = parse_cmdline_args(argc, argv, "api/heartbeat"); - org::apache::nifi::minifi::test::VerifyEmptyC2Metric harness(metrics_found); + VerifyEmptyC2Metric harness(metrics_found); harness.getConfiguration()->set("nifi.c2.root.class.definitions", "metrics"); harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.name", "metrics"); harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics", "processormetrics,systemmetrics"); @@ -170,9 +165,10 @@ int main(int argc, char **argv) { harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics.processormetrics.classes", "GetFileMetrics,GetTCPMetrics"); harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics.systemmetrics.name", "SystemMetrics"); harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics.systemmetrics.classes", "QueueMetrics"); - harness.setKeyDir(args.key_dir); - org::apache::nifi::minifi::test::MetricsHandler handler(metrics_found, harness.getConfiguration()); - harness.setUrl(args.url, &handler); - harness.run(args.test_file); - return 0; + MetricsHandler handler(metrics_found, harness.getConfiguration()); + harness.setUrl("http://localhost:0/api/heartbeat", &handler); + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestSameProcessorMetrics.yml"; + harness.run(test_file_path); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2EmptyMetricTest.cpp b/libminifi/test/integration/C2EmptyMetricTest.cpp similarity index 84% rename from extensions/http-curl/tests/C2EmptyMetricTest.cpp rename to libminifi/test/integration/C2EmptyMetricTest.cpp index 6323a48394..52c10661ff 100644 --- a/extensions/http-curl/tests/C2EmptyMetricTest.cpp +++ b/libminifi/test/integration/C2EmptyMetricTest.cpp @@ -15,22 +15,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG #include #include #include -#include "TestBase.h" -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" +#include "unit/TestBase.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" #include "processors/TailFile.h" #include "state/ProcessorController.h" #include "utils/file/FileUtils.h" -#include "utils/TestUtils.h" +#include "unit/TestUtils.h" #include "processors/GetTCP.h" #include "utils/StringUtils.h" #include "utils/file/PathUtils.h" +#include "unit/Catch.h" using namespace std::literals::chrono_literals; @@ -50,8 +49,8 @@ class VerifyEmptyC2Metric : public VerifyC2Base { } void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; - assert(verifyEventHappenedInPollTime(40s, [&] { return metrics_found_.load(); }, 1s)); + using org::apache::nifi::minifi::test::utils::verifyEventHappenedInPollTime; + REQUIRE(verifyEventHappenedInPollTime(40s, [&] { return metrics_found_.load(); }, 1s)); } private: @@ -95,20 +94,18 @@ class MetricsHandler: public HeartbeatHandler { std::atomic_bool& metrics_found_; }; -} // namespace org::apache::nifi::minifi::test - -int main(int argc, char **argv) { +TEST_CASE("C2EmptyMetricTest", "[c2test]") { std::atomic_bool metrics_found{false}; - const cmd_args args = parse_cmdline_args(argc, argv, "api/heartbeat"); - org::apache::nifi::minifi::test::VerifyEmptyC2Metric harness(metrics_found); + VerifyEmptyC2Metric harness(metrics_found); harness.getConfiguration()->set("nifi.c2.root.class.definitions", "metrics"); harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.name", "metrics"); harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics", "loadmetrics"); harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics.loadmetrics.name", "LoadMetrics"); harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics.loadmetrics.classes", "QueueMetrics,RepositoryMetrics"); - harness.setKeyDir(args.key_dir); - org::apache::nifi::minifi::test::MetricsHandler handler(metrics_found, harness.getConfiguration()); - harness.setUrl(args.url, &handler); - harness.run(args.test_file); - return 0; + MetricsHandler handler(metrics_found, harness.getConfiguration()); + harness.setUrl("http://localhost:0/api/heartbeat", &handler); + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestEmpty.yml"; + harness.run(test_file_path); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2FailedUpdateTest.cpp b/libminifi/test/integration/C2FailedUpdateTest.cpp similarity index 60% rename from extensions/http-curl/tests/C2FailedUpdateTest.cpp rename to libminifi/test/integration/C2FailedUpdateTest.cpp index a4cd425b81..ed55f5216a 100644 --- a/extensions/http-curl/tests/C2FailedUpdateTest.cpp +++ b/libminifi/test/integration/C2FailedUpdateTest.cpp @@ -15,21 +15,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" -#include "utils/IntegrationTestUtils.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" +#include "unit/TestUtils.h" +#include "unit/Catch.h" using namespace std::literals::chrono_literals; -int main(int argc, char **argv) { - const cmd_args args = parse_cmdline_args(argc, argv, "update"); - C2FailedUpdateHandler handler(args.bad_test_file); +namespace org::apache::nifi::minifi::test { + +TEST_CASE("C2FailedUpdateTest", "[c2test]") { + const auto bad_test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestBad.yml"; + C2FailedUpdateHandler handler(bad_test_file_path.string()); VerifyC2FailedUpdate harness(10s); - harness.setKeyDir(args.key_dir); - harness.setUrl(args.url, &handler); + harness.setKeyDir(TEST_RESOURCES); + harness.setUrl("http://localhost:0/update", &handler); handler.setC2RestResponse(harness.getC2RestUrl(), "configuration"); - harness.run(args.test_file); - return 0; + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestHTTPGet.yml"; + harness.run(test_file_path); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2FetchFlowIfMissingTest.cpp b/libminifi/test/integration/C2FetchFlowIfMissingTest.cpp similarity index 67% rename from extensions/http-curl/tests/C2FetchFlowIfMissingTest.cpp rename to libminifi/test/integration/C2FetchFlowIfMissingTest.cpp index 9fb1236a6c..a6a7065ad0 100644 --- a/extensions/http-curl/tests/C2FetchFlowIfMissingTest.cpp +++ b/libminifi/test/integration/C2FetchFlowIfMissingTest.cpp @@ -15,30 +15,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" -#include "utils/IntegrationTestUtils.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" +#include "unit/TestUtils.h" #include "utils/file/PathUtils.h" +#include "unit/Catch.h" using namespace std::literals::chrono_literals; -int main(int argc, char **argv) { +namespace org::apache::nifi::minifi::test { + +TEST_CASE("C2FetchFlowIfMissingTest", "[c2test]") { TestController controller; auto minifi_home = controller.createTempDirectory(); - const cmd_args args = parse_cmdline_args(argc, argv); - C2FlowProvider handler(args.test_file); + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestEmpty.yml"; + C2FlowProvider handler(test_file_path.string()); VerifyFlowFetched harness(10s); - harness.setKeyDir(args.key_dir); - harness.setUrl(args.url, &handler); + harness.setKeyDir(TEST_RESOURCES); + harness.setUrl("https://localhost:0/", &handler); harness.setFlowUrl(harness.getC2RestUrl()); harness.run(minifi_home / "config.yml"); // check existence of the config file - assert(std::ifstream{minifi_home / "config.yml"}); - - return 0; + REQUIRE(std::ifstream{minifi_home / "config.yml"}); } +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2JstackTest.cpp b/libminifi/test/integration/C2JstackTest.cpp similarity index 74% rename from extensions/http-curl/tests/C2JstackTest.cpp rename to libminifi/test/integration/C2JstackTest.cpp index 6ccfd987b9..b34792444d 100644 --- a/extensions/http-curl/tests/C2JstackTest.cpp +++ b/libminifi/test/integration/C2JstackTest.cpp @@ -15,14 +15,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG #include #include -#include "TestBase.h" -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestBase.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" +#include "unit/TestUtils.h" +#include "unit/Catch.h" + +namespace org::apache::nifi::minifi::test { class DescribeJstackHandler : public HeartbeatHandler { public: @@ -36,20 +37,21 @@ class DescribeJstackHandler : public HeartbeatHandler { } void handleAcknowledge(const rapidjson::Document& root) override { - assert(root.HasMember("Flowcontroller threadpool #0")); + REQUIRE(root.HasMember("Flowcontroller threadpool #0")); acknowledgement_received_.store(true); } protected: std::atomic_bool& acknowledgement_received_; }; -int main(int argc, char **argv) { - const cmd_args args = parse_cmdline_args(argc, argv, "heartbeat"); +TEST_CASE("C2JstackTest", "[c2test]") { std::atomic_bool acknowledgement_received{ false }; VerifyC2Describe harness{acknowledgement_received}; - harness.setKeyDir(args.key_dir); + harness.setKeyDir(TEST_RESOURCES); DescribeJstackHandler responder{acknowledgement_received, harness.getConfiguration()}; - harness.setUrl(args.url, &responder); - harness.run(args.test_file); - return 0; + harness.setUrl("https://localhost:0/heartbeat", &responder); + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestHTTPGet.yml"; + harness.run(test_file_path); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2LogHeartbeatTest.cpp b/libminifi/test/integration/C2LogHeartbeatTest.cpp similarity index 79% rename from extensions/http-curl/tests/C2LogHeartbeatTest.cpp rename to libminifi/test/integration/C2LogHeartbeatTest.cpp index 3c34dbe893..71755cc0d8 100644 --- a/extensions/http-curl/tests/C2LogHeartbeatTest.cpp +++ b/libminifi/test/integration/C2LogHeartbeatTest.cpp @@ -15,26 +15,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG - -#include "TestBase.h" +#include "unit/TestBase.h" #include "c2/C2Agent.h" #include "c2/HeartbeatLogger.h" #include "protocols/RESTProtocol.h" #include "protocols/RESTSender.h" -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" #include "range/v3/action/sort.hpp" #include "range/v3/action/unique.hpp" #include "range/v3/range/conversion.hpp" #include "range/v3/view/filter.hpp" #include "range/v3/view/split.hpp" #include "range/v3/view/transform.hpp" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" #include "utils/StringUtils.h" #include "properties/Configuration.h" +#include "unit/Catch.h" + +namespace org::apache::nifi::minifi::test { class VerifyLogC2Heartbeat : public VerifyC2Base { public: @@ -48,31 +48,32 @@ class VerifyLogC2Heartbeat : public VerifyC2Base { } void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; - assert(verifyLogLinePresenceInPollTime( + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime( std::chrono::milliseconds(wait_time_), "\"operation\": \"heartbeat\"")); const auto log = LogTestController::getInstance().getLogs(); auto types_in_heartbeat = log | ranges::views::split('\n') | ranges::views::transform([](auto&& rng) { return rng | ranges::to; }) - | ranges::views::filter([](auto&& line) { return utils::string::startsWith(line, " \"type\":"); }) + | ranges::views::filter([](auto&& line) { return minifi::utils::string::startsWith(line, " \"type\":"); }) | ranges::to>; const auto num_types = types_in_heartbeat.size(); types_in_heartbeat |= ranges::actions::sort | ranges::actions::unique; const auto num_distinct_types = types_in_heartbeat.size(); - assert(num_types == num_distinct_types); + REQUIRE(num_types == num_distinct_types); } void configureC2() override { VerifyC2Base::configureC2(); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_agent_heartbeat_reporter_classes, "HeartbeatLogger"); + configuration->set(minifi::Configuration::nifi_c2_agent_heartbeat_reporter_classes, "HeartbeatLogger"); } }; -int main() { +TEST_CASE("C2LogHeartbeatTest", "[c2test]") { VerifyLogC2Heartbeat harness; HeartbeatHandler responder(harness.getConfiguration()); harness.setUrl("https://localhost:0/heartbeat", &responder); harness.run(); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2MetricsTest.cpp b/libminifi/test/integration/C2MetricsTest.cpp similarity index 92% rename from extensions/http-curl/tests/C2MetricsTest.cpp rename to libminifi/test/integration/C2MetricsTest.cpp index 20decf0138..4d356fc2bb 100644 --- a/extensions/http-curl/tests/C2MetricsTest.cpp +++ b/libminifi/test/integration/C2MetricsTest.cpp @@ -15,22 +15,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG #include #include #include -#include "TestBase.h" -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" +#include "unit/TestBase.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" #include "processors/TailFile.h" #include "state/ProcessorController.h" #include "utils/file/FileUtils.h" -#include "utils/TestUtils.h" +#include "unit/TestUtils.h" #include "processors/GetTCP.h" #include "utils/StringUtils.h" #include "utils/file/PathUtils.h" +#include "unit/Catch.h" using namespace std::literals::chrono_literals; @@ -50,8 +49,8 @@ class VerifyC2Metrics : public VerifyC2Base { } void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; - assert(verifyEventHappenedInPollTime(40s, [&] { return metrics_updated_successfully_.load(); }, 1s)); + using org::apache::nifi::minifi::test::utils::verifyEventHappenedInPollTime; + REQUIRE(verifyEventHappenedInPollTime(40s, [&] { return metrics_updated_successfully_.load(); }, 1s)); } private: @@ -192,12 +191,9 @@ class MetricsHandler: public HeartbeatHandler { std::string replacement_config_; }; -} // namespace org::apache::nifi::minifi::test - -int main(int argc, char **argv) { +TEST_CASE("C2MetricsTest", "[c2test]") { std::atomic_bool metrics_updated_successfully{false}; - const cmd_args args = parse_cmdline_args(argc, argv, "api/heartbeat"); - org::apache::nifi::minifi::test::VerifyC2Metrics harness(metrics_updated_successfully); + VerifyC2Metrics harness(metrics_updated_successfully); harness.getConfiguration()->set("nifi.c2.root.class.definitions", "metrics"); harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.name", "metrics"); harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics", "runtimemetrics,loadmetrics,processorMetrics"); @@ -207,11 +203,12 @@ int main(int argc, char **argv) { harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics.loadmetrics.classes", "QueueMetrics,RepositoryMetrics"); harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics.processorMetrics.name", "ProcessorMetrics"); harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics.processorMetrics.classes", "processorMetrics/GetTCP.*"); - harness.setKeyDir(args.key_dir); - auto replacement_path = args.test_file; + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestC2Metrics.yml"; + auto replacement_path = test_file_path.string(); minifi::utils::string::replaceAll(replacement_path, "TestC2Metrics", "TestC2MetricsUpdate"); - org::apache::nifi::minifi::test::MetricsHandler handler(metrics_updated_successfully, harness.getConfiguration(), replacement_path); - harness.setUrl(args.url, &handler); - harness.run(args.test_file); - return 0; + MetricsHandler handler(metrics_updated_successfully, harness.getConfiguration(), replacement_path); + harness.setUrl("https://localhost:0/api/heartbeat", &handler); + harness.run(test_file_path); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2MultipleCommandsTest.cpp b/libminifi/test/integration/C2MultipleCommandsTest.cpp similarity index 80% rename from extensions/http-curl/tests/C2MultipleCommandsTest.cpp rename to libminifi/test/integration/C2MultipleCommandsTest.cpp index bb82b72708..d8d335edf6 100644 --- a/extensions/http-curl/tests/C2MultipleCommandsTest.cpp +++ b/libminifi/test/integration/C2MultipleCommandsTest.cpp @@ -15,15 +15,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG #include #include #include -#include "TestBase.h" -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" +#include "unit/TestBase.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" +#include "unit/Catch.h" + +using namespace std::literals::chrono_literals; + +namespace org::apache::nifi::minifi::test { class AckAuditor { public: @@ -44,9 +47,7 @@ class AckAuditor { void verifyAck(const rapidjson::Document& root) { std::lock_guard guard(verify_ack_mutex_); - if (ack_verifiers_.empty()) { - assert(false); - } + REQUIRE(!ack_verifiers_.empty()); ack_verifiers_[next_verifier_index_](root); ++next_verifier_index_; @@ -76,7 +77,7 @@ class MultipleC2CommandHandler: public HeartbeatHandler { verifyJsonHasAgentManifest(root); }); ack_auditor_.addVerifier([](const rapidjson::Document& root) { - assert(root.HasMember("corecomponentstate")); + REQUIRE(root.HasMember("corecomponentstate")); }); sendHeartbeatResponse(operations, conn); } @@ -106,24 +107,25 @@ class VerifyC2MultipleCommands : public VerifyC2Base { } void configureFullHeartbeat() override { - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_full_heartbeat, "false"); + configuration->set(minifi::Configuration::nifi_c2_full_heartbeat, "false"); } void runAssertions() override { - assert(utils::verifyEventHappenedInPollTime(10s, [&] {return ack_auditor_.isAcknowledged("889345");})); - assert(utils::verifyEventHappenedInPollTime(10s, [&] {return ack_auditor_.isAcknowledged("889346");})); + REQUIRE(utils::verifyEventHappenedInPollTime(10s, [&] {return ack_auditor_.isAcknowledged("889345");})); + REQUIRE(utils::verifyEventHappenedInPollTime(10s, [&] {return ack_auditor_.isAcknowledged("889346");})); } private: AckAuditor& ack_auditor_; }; -int main(int argc, char **argv) { - const cmd_args args = parse_cmdline_args(argc, argv, "heartbeat"); +TEST_CASE("C2MultipleCommandsTest", "[c2test]") { AckAuditor ack_auditor; VerifyC2MultipleCommands harness(ack_auditor); - harness.setKeyDir(args.key_dir); MultipleC2CommandHandler responder(ack_auditor, harness.getConfiguration()); - harness.setUrl(args.url, &responder); - harness.run(args.test_file); + harness.setUrl("https://localhost:0/heartbeat", &responder); + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestC2DescribeCoreComponentState.yml"; + harness.run(test_file_path); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2NullConfiguration.cpp b/libminifi/test/integration/C2NullConfiguration.cpp similarity index 79% rename from extensions/http-curl/tests/C2NullConfiguration.cpp rename to libminifi/test/integration/C2NullConfiguration.cpp index 4f04e2a7f6..ad36cc807d 100644 --- a/extensions/http-curl/tests/C2NullConfiguration.cpp +++ b/libminifi/test/integration/C2NullConfiguration.cpp @@ -15,24 +15,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG -#include #include #include #include -#include "InvokeHTTP.h" -#include "TestBase.h" +#include "processors/InvokeHTTP.h" +#include "unit/TestBase.h" #include "core/logging/Logger.h" #include "core/ProcessGroup.h" -#include "TestServer.h" +#include "integration/TestServer.h" #include "protocols/RESTReceiver.h" #include "c2/C2Agent.h" #include "processors/LogAttribute.h" -#include "HTTPIntegrationBase.h" -#include "utils/IntegrationTestUtils.h" +#include "integration/HTTPIntegrationBase.h" +#include "unit/TestUtils.h" +#include "unit/Catch.h" -namespace org::apache::nifi::minifi { +namespace org::apache::nifi::minifi::test { class VerifyC2Server : public HTTPIntegrationBase { public: @@ -42,7 +40,7 @@ class VerifyC2Server : public HTTPIntegrationBase { } void testSetup() override { - LogTestController::getInstance().setDebug(); + LogTestController::getInstance().setDebug(); LogTestController::getInstance().setDebug(); LogTestController::getInstance().setDebug(); LogTestController::getInstance().setDebug(); @@ -56,18 +54,18 @@ class VerifyC2Server : public HTTPIntegrationBase { } void runAssertions() override { - assert(utils::verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), "C2Agent] [error] Could not instantiate null", "Class is RESTSender")); } void queryRootProcessGroup(std::shared_ptr pg) override { auto* const proc = pg->findProcessorByName("invoke"); - assert(proc != nullptr); + REQUIRE(proc != nullptr); const auto* const inv = dynamic_cast(proc); - assert(inv != nullptr); + REQUIRE(inv != nullptr); std::string url; inv->getProperty(processors::InvokeHTTP::URL, url); @@ -91,15 +89,11 @@ class VerifyC2Server : public HTTPIntegrationBase { TestController testController; }; -} // namespace org::apache::nifi::minifi - -int main(int argc, char **argv) { - const cmd_args args = parse_cmdline_args(argc, argv); - const bool isSecure = args.isUrlSecure(); - - org::apache::nifi::minifi::VerifyC2Server harness(isSecure); - harness.setKeyDir(args.key_dir); - harness.run(args.test_file); - return 0; +TEST_CASE("C2NullConfiguration", "[c2test]") { + VerifyC2Server harness(true); + harness.setKeyDir(TEST_RESOURCES); + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestNull.yml"; + harness.run(test_file_path); } +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2PauseResumeTest.cpp b/libminifi/test/integration/C2PauseResumeTest.cpp similarity index 80% rename from extensions/http-curl/tests/C2PauseResumeTest.cpp rename to libminifi/test/integration/C2PauseResumeTest.cpp index 02eb5b0743..7e819403f1 100644 --- a/extensions/http-curl/tests/C2PauseResumeTest.cpp +++ b/libminifi/test/integration/C2PauseResumeTest.cpp @@ -15,20 +15,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG - #include -#include "TestBase.h" -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" -#include "InvokeHTTP.h" -#include "TestServer.h" +#include "unit/TestBase.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" +#include "processors/InvokeHTTP.h" +#include "integration/TestServer.h" #include "core/yaml/YamlConfiguration.h" #include "FlowController.h" #include "properties/Configure.h" #include "integration/IntegrationBase.h" +#include "unit/Catch.h" +#include "unit/ProvenanceTestHelper.h" + +namespace org::apache::nifi::minifi::test { class VerifyC2PauseResume : public VerifyC2Base { public: @@ -42,12 +43,11 @@ class VerifyC2PauseResume : public VerifyC2Base { void configureC2() override { VerifyC2Base::configureC2(); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_agent_heartbeat_period, "500"); + configuration->set(minifi::Configuration::nifi_c2_agent_heartbeat_period, "500"); } void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; - assert(verifyEventHappenedInPollTime(std::chrono::seconds(20), [&] { return flow_resumed_successfully_.load(); })); + REQUIRE(utils::verifyEventHappenedInPollTime(std::chrono::seconds(20), [&] { return flow_resumed_successfully_.load(); })); } private: @@ -63,7 +63,7 @@ class PauseResumeHandler: public HeartbeatHandler { : HeartbeatHandler(std::move(configuration)), flow_resumed_successfully_(flow_resumed_successfully) { } bool handleGet(CivetServer* /*server*/, struct mg_connection* conn) override { - assert(flow_state_ != FlowState::PAUSED); + REQUIRE(flow_state_ != FlowState::PAUSED); ++get_invoke_count_; if (flow_state_ == FlowState::RESUMED) { flow_resumed_successfully_ = true; @@ -114,18 +114,16 @@ class PauseResumeHandler: public HeartbeatHandler { std::atomic_bool& flow_resumed_successfully_; }; -int main(int argc, char **argv) { - const cmd_args args = parse_cmdline_args(argc, argv, "heartbeat"); +TEST_CASE("C2PauseResumeTest", "[c2test]") { std::atomic_bool flow_resumed_successfully{false}; VerifyC2PauseResume harness{flow_resumed_successfully}; - harness.setKeyDir(args.key_dir); PauseResumeHandler responder{flow_resumed_successfully, harness.getConfiguration()}; std::shared_ptr test_repo = std::make_shared(); std::shared_ptr test_flow_repo = std::make_shared(); std::shared_ptr configuration = std::make_shared(); - configuration->set(minifi::Configure::nifi_default_directory, args.key_dir); - configuration->set(minifi::Configure::nifi_flow_configuration_file, args.test_file); + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "C2PauseResumeTest.yml"; + configuration->set(minifi::Configure::nifi_flow_configuration_file, test_file_path.string()); std::shared_ptr content_repo = std::make_shared(); content_repo->initialize(configuration); @@ -134,9 +132,9 @@ int main(int argc, char **argv) { .flow_file_repo = test_repo, .content_repo = content_repo, .configuration = configuration, - .path = args.test_file, - .filesystem = std::make_shared(), - .sensitive_properties_encryptor = utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}} + .path = test_file_path, + .filesystem = std::make_shared(), + .sensitive_properties_encryptor = minifi::utils::crypto::EncryptionProvider{minifi::utils::crypto::XSalsa20Cipher{minifi::utils::crypto::XSalsa20Cipher::generateKey()}} }); std::vector> repo_metric_sources{test_repo, test_flow_repo, content_repo}; auto metrics_publisher_store = std::make_unique(configuration, repo_metric_sources, yaml_ptr); @@ -147,16 +145,16 @@ int main(int argc, char **argv) { .flow_file_repo = test_repo, .content_repo = content_repo, .configuration = configuration, - .path = args.test_file, - .filesystem = std::make_shared(), - .sensitive_properties_encryptor = utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}} + .path = test_file_path, + .filesystem = std::make_shared(), + .sensitive_properties_encryptor = minifi::utils::crypto::EncryptionProvider{minifi::utils::crypto::XSalsa20Cipher{minifi::utils::crypto::XSalsa20Cipher::generateKey()}} }); auto root = yaml_config.getRoot(); const auto proc = root->findProcessorByName("invoke"); - assert(proc != nullptr); + REQUIRE(proc != nullptr); const auto inv = dynamic_cast(proc); - assert(inv != nullptr); + REQUIRE(inv != nullptr); std::string url; inv->getProperty(minifi::processors::InvokeHTTP::URL, url); std::string port; @@ -166,6 +164,8 @@ int main(int argc, char **argv) { minifi::utils::parse_http_components(url, port, scheme, path); server = std::make_unique(port, path, &responder); - harness.setUrl(args.url, &responder); - harness.run(args.test_file); + harness.setUrl("http://localhost:0/heartbeat", &responder); + harness.run(test_file_path); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2PropertiesUpdateTests.cpp b/libminifi/test/integration/C2PropertiesUpdateTests.cpp similarity index 76% rename from extensions/http-curl/tests/C2PropertiesUpdateTests.cpp rename to libminifi/test/integration/C2PropertiesUpdateTests.cpp index 52de9a7d38..74326571b6 100644 --- a/extensions/http-curl/tests/C2PropertiesUpdateTests.cpp +++ b/libminifi/test/integration/C2PropertiesUpdateTests.cpp @@ -15,22 +15,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG #include -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" #include "utils/gsl.h" -#include "utils/IntegrationTestUtils.h" -#include "EmptyFlow.h" +#include "unit/TestUtils.h" +#include "unit/EmptyFlow.h" #include "fmt/format.h" #include "spdlog/sinks/stdout_sinks.h" #include "spdlog/sinks/ostream_sink.h" #include "spdlog/sinks/dist_sink.h" -#include "LogUtils.h" +#include "unit/LogUtils.h" #include "properties/PropertiesFile.h" #include "ConfigTestAccessor.h" +#include "unit/Catch.h" + +namespace org::apache::nifi::minifi::test { struct PropertyChange { std::string name; @@ -74,7 +75,7 @@ class C2HeartbeatHandler : public ServerAwareHandler { "operationid" : "79", "name": "properties", "args": {)" + - utils::string::join(", ", fields) + minifi::utils::string::join(", ", fields) + R"(} }] })"; @@ -120,16 +121,16 @@ using namespace std::literals::chrono_literals; struct DummyClass1 {}; struct DummyClass2 {}; -namespace test { +namespace dummy { struct DummyClass3 {}; -} // namespace test +} // namespace dummy -int main() { +TEST_CASE("C2PropertiesUpdateTests", "[c2test]") { TempDirectory tmp_dir; std::filesystem::path home_dir = tmp_dir.getPath(); - utils::file::PathUtils::create_dir(home_dir / "conf"); + minifi::utils::file::PathUtils::create_dir(home_dir / "conf"); std::ofstream{home_dir / "conf/minifi.properties"} << properties_file; std::ofstream{home_dir / "conf/minifi-log.properties"} << log_properties_file; std::ofstream{home_dir / "conf/config.yml"} << empty_flow; @@ -147,7 +148,7 @@ int main() { auto logger1 = core::logging::LoggerFactory::getLogger(); auto logger2 = core::logging::LoggerFactory::getLogger(); - auto logger3 = core::logging::LoggerFactory::getLogger(); + auto logger3 = core::logging::LoggerFactory::getLogger(); { // verify initial log levels, none of these should be logged @@ -155,21 +156,21 @@ int main() { logger2->log_debug("DummyClass2::before"); logger3->log_debug("DummyClass3::before"); - assert(!log_test_controller->contains("DummyClass1::before", 0s)); - assert(!log_test_controller->contains("DummyClass2::before", 0s)); - assert(!log_test_controller->contains("DummyClass3::before", 0s)); + REQUIRE(!log_test_controller->contains("DummyClass1::before", 0s)); + REQUIRE(!log_test_controller->contains("DummyClass2::before", 0s)); + REQUIRE(!log_test_controller->contains("DummyClass3::before", 0s)); } // On msvc, the passed lambda can't capture a reference to the object under construction, so we need to late-init harness. VerifyPropertyUpdate harness; harness = VerifyPropertyUpdate([&] { - assert(utils::verifyEventHappenedInPollTime(10s, [&] {return ack_handler.isAcknowledged("79");})); - assert(utils::verifyEventHappenedInPollTime(10s, [&] { + REQUIRE(utils::verifyEventHappenedInPollTime(10s, [&] {return ack_handler.isAcknowledged("79");})); + REQUIRE(utils::verifyEventHappenedInPollTime(10s, [&] { return ack_handler.getApplyCount("FULLY_APPLIED") == 1; })); // Updating the same property will result in a no operation response - assert(utils::verifyEventHappenedInPollTime(10s, [&] { + REQUIRE(utils::verifyEventHappenedInPollTime(10s, [&] { return ack_handler.getApplyCount("NO_OPERATION") > 0; })); @@ -177,12 +178,12 @@ int main() { hb_handler.setProperties({{minifi::Configuration::nifi_c2_rest_heartbeat_minimize_updates, "banana", true}, {minifi::Configuration::minifi_disk_space_watchdog_enable, "true", true}}); // Due to 1 invalid value the result will be partially applied - assert(utils::verifyEventHappenedInPollTime(10s, [&] { + REQUIRE(utils::verifyEventHappenedInPollTime(10s, [&] { return ack_handler.getApplyCount("PARTIALLY_APPLIED") == 1; })); // Repeating the previous update request results in 1 no operation and 1 failure which results in not applied response - assert(utils::verifyEventHappenedInPollTime(10s, [&] { + REQUIRE(utils::verifyEventHappenedInPollTime(10s, [&] { return ack_handler.getApplyCount("NOT_APPLIED") > 0 && harness.getRestartRequestedCount() == 2; })); @@ -194,23 +195,23 @@ int main() { logger2->log_debug("DummyClass2::after"); // this should still not log logger3->log_debug("DummyClass3::after"); } - assert(log_test_controller->contains("DummyClass1::after", 0s)); - assert(!log_test_controller->contains("DummyClass2::after", 0s)); - assert(log_test_controller->contains("DummyClass3::after", 0s)); + REQUIRE(log_test_controller->contains("DummyClass1::after", 0s)); + REQUIRE(!log_test_controller->contains("DummyClass2::after", 0s)); + REQUIRE(log_test_controller->contains("DummyClass3::after", 0s)); { minifi::PropertiesFile minifi_properties(std::ifstream{home_dir / "conf/minifi.properties"}); - assert(!minifi_properties.hasValue("nifi.dummy.property")); - assert(minifi_properties.getValue("nifi.property.one") == "bush"); - assert(minifi_properties.getValue("nifi.property.two") == "ring"); - assert(!minifi_properties.hasValue(minifi::Configuration::nifi_c2_rest_heartbeat_minimize_updates)); - assert(minifi_properties.getValue(minifi::Configuration::minifi_disk_space_watchdog_enable) == "true"); + REQUIRE(!minifi_properties.hasValue("nifi.dummy.property")); + REQUIRE(minifi_properties.getValue("nifi.property.one") == "bush"); + REQUIRE(minifi_properties.getValue("nifi.property.two") == "ring"); + REQUIRE(!minifi_properties.hasValue(minifi::Configuration::nifi_c2_rest_heartbeat_minimize_updates)); + REQUIRE(minifi_properties.getValue(minifi::Configuration::minifi_disk_space_watchdog_enable) == "true"); } { minifi::PropertiesFile minifi_log_properties(std::ifstream{home_dir / "conf/minifi-log.properties"}); - assert(!minifi_log_properties.hasValue("logger.test")); - assert(minifi_log_properties.getValue("logger.DummyClass1") == "DEBUG,ostream"); + REQUIRE(!minifi_log_properties.hasValue("logger.org::apache::nifi::minifi::test::dummy")); + REQUIRE(minifi_log_properties.getValue("logger.org::apache::nifi::minifi::test::DummyClass1") == "DEBUG,ostream"); } }); @@ -226,9 +227,11 @@ int main() { {"nifi.dummy.property", "banana", false}, {"nifi.property.one", "bush", std::nullopt}, // default persist = true {"nifi.property.two", "ring", true}, - {"nifi.log.logger.test", "DEBUG,ostream", false}, - {"nifi.log.logger.DummyClass1", "DEBUG,ostream", true} + {"nifi.log.logger.org::apache::nifi::minifi::test::dummy", "DEBUG,ostream", false}, + {"nifi.log.logger.org::apache::nifi::minifi::test::DummyClass1", "DEBUG,ostream", true} }); harness.run((home_dir / "conf/config.yml").string()); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2RequestClassTest.cpp b/libminifi/test/integration/C2RequestClassTest.cpp similarity index 81% rename from extensions/http-curl/tests/C2RequestClassTest.cpp rename to libminifi/test/integration/C2RequestClassTest.cpp index 8e87d37685..b00b0a815f 100644 --- a/extensions/http-curl/tests/C2RequestClassTest.cpp +++ b/libminifi/test/integration/C2RequestClassTest.cpp @@ -15,18 +15,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG #include #include #include -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" -#include "utils/IntegrationTestUtils.h" -#include "CivetStream.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" +#include "unit/TestUtils.h" +#include "integration/CivetStream.h" #include "StreamPipe.h" #include "properties/Configuration.h" +#include "unit/Catch.h" + +namespace org::apache::nifi::minifi::test { class C2HeartbeatHandler : public ServerAwareHandler { public: @@ -76,21 +77,21 @@ class VerifyC2ClassRequest : public VerifyC2Base { explicit VerifyC2ClassRequest(std::function verify) : verify_(std::move(verify)) {} void configureC2() override { - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_agent_protocol_class, "RESTSender"); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_enable, "true"); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_agent_heartbeat_period, "100"); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_root_classes, "DeviceInfoNode,AgentInformation,FlowInformation"); + configuration->set(minifi::Configuration::nifi_c2_agent_protocol_class, "RESTSender"); + configuration->set(minifi::Configuration::nifi_c2_enable, "true"); + configuration->set(minifi::Configuration::nifi_c2_agent_heartbeat_period, "100"); + configuration->set(minifi::Configuration::nifi_c2_root_classes, "DeviceInfoNode,AgentInformation,FlowInformation"); } void runAssertions() override { - assert(utils::verifyEventHappenedInPollTime(std::chrono::seconds(10), verify_)); + REQUIRE(utils::verifyEventHappenedInPollTime(std::chrono::seconds(10), verify_)); } private: std::function verify_; }; -int main() { +TEST_CASE("C2RequestClassTest", "[c2test]") { const std::string class_update_id = "321"; C2HeartbeatHandler heartbeat_handler(R"({ "requested_operations": [{ @@ -113,3 +114,5 @@ int main() { harness.run(); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2SameProcessorMetrics.cpp b/libminifi/test/integration/C2SameProcessorMetrics.cpp similarity index 84% rename from extensions/http-curl/tests/C2SameProcessorMetrics.cpp rename to libminifi/test/integration/C2SameProcessorMetrics.cpp index a7144582f6..e4f93cb091 100644 --- a/extensions/http-curl/tests/C2SameProcessorMetrics.cpp +++ b/libminifi/test/integration/C2SameProcessorMetrics.cpp @@ -15,20 +15,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG #include -#include "TestBase.h" -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" +#include "unit/TestBase.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" #include "processors/TailFile.h" #include "state/ProcessorController.h" #include "utils/file/FileUtils.h" -#include "utils/TestUtils.h" +#include "unit/TestUtils.h" #include "processors/GetTCP.h" #include "utils/StringUtils.h" #include "utils/file/PathUtils.h" +#include "unit/Catch.h" using namespace std::literals::chrono_literals; @@ -48,8 +47,8 @@ class VerifyEmptyC2Metric : public VerifyC2Base { } void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; - assert(verifyEventHappenedInPollTime(40s, [&] { return metrics_found_.load(); }, 1s)); + using org::apache::nifi::minifi::test::utils::verifyEventHappenedInPollTime; + REQUIRE(verifyEventHappenedInPollTime(40s, [&] { return metrics_found_.load(); }, 1s)); } private: @@ -95,20 +94,18 @@ class MetricsHandler: public HeartbeatHandler { std::atomic_bool& metrics_found_; }; -} // namespace org::apache::nifi::minifi::test - -int main(int argc, char **argv) { +TEST_CASE("Test support for setting metrics for multiple processors of the same type in a flow", "[c2test]") { std::atomic_bool metrics_found{false}; - const cmd_args args = parse_cmdline_args(argc, argv, "api/heartbeat"); - org::apache::nifi::minifi::test::VerifyEmptyC2Metric harness(metrics_found); + VerifyEmptyC2Metric harness(metrics_found); harness.getConfiguration()->set("nifi.c2.root.class.definitions", "metrics"); harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.name", "metrics"); harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics", "processormetrics"); harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics.processormetrics.name", "ProcessorMetrics"); harness.getConfiguration()->set("nifi.c2.root.class.definitions.metrics.metrics.processormetrics.classes", "GetFileMetrics,GetTCPMetrics"); - harness.setKeyDir(args.key_dir); - org::apache::nifi::minifi::test::MetricsHandler handler(metrics_found, harness.getConfiguration()); - harness.setUrl(args.url, &handler); - harness.run(args.test_file); - return 0; + MetricsHandler handler(metrics_found, harness.getConfiguration()); + harness.setUrl("http://localhost:0/api/heartbeat", &handler); + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestSameProcessorMetrics.yml"; + harness.run(test_file_path); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2UpdateAssetTest.cpp b/libminifi/test/integration/C2UpdateAssetTest.cpp similarity index 90% rename from extensions/http-curl/tests/C2UpdateAssetTest.cpp rename to libminifi/test/integration/C2UpdateAssetTest.cpp index f5bb97669e..9b279d0b5a 100644 --- a/extensions/http-curl/tests/C2UpdateAssetTest.cpp +++ b/libminifi/test/integration/C2UpdateAssetTest.cpp @@ -15,18 +15,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG #include #include #include #include -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" -#include "utils/IntegrationTestUtils.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" +#include "unit/TestUtils.h" #include "utils/Environment.h" #include "utils/file/FileUtils.h" +#include "unit/Catch.h" + +namespace org::apache::nifi::minifi::test { class FileProvider : public ServerAwareHandler { public: @@ -79,7 +80,7 @@ class VerifyC2AssetUpdate : public VerifyC2Base { } void runAssertions() override { - assert(utils::verifyEventHappenedInPollTime(std::chrono::seconds(10), verify_)); + REQUIRE(utils::verifyEventHappenedInPollTime(std::chrono::seconds(10), verify_)); } void setVerifier(std::function verify) { @@ -97,7 +98,7 @@ struct AssetUpdateOperation { std::optional details; }; -int main() { +TEST_CASE("Test update asset C2 command", "[c2test]") { TestController controller; // setup minifi home @@ -216,12 +217,12 @@ int main() { if (auto res = ack_handler.getState(op.id)) { if (res->state != op.state) { controller.getLogger()->log_error("Operation '{}' in expected to return '{}', got '{}'", op.id, op.state, res->state); - assert(false); + REQUIRE(false); } if (op.details) { if (res->details.find(op.details.value()) == std::string::npos) { controller.getLogger()->log_error("In operation '{}' failed to find '{}' in ack details '{}'", op.id, op.details.value(), res->details); - assert(false); + REQUIRE(false); } } } else { @@ -244,20 +245,22 @@ int main() { // this op failed no file made on the disk continue; } - expected_files[(asset_dir / op.args["file"]).string()] = utils::string::endsWith(op.args["url"], "A.txt") ? file_A : file_B; + expected_files[(asset_dir / op.args["file"]).string()] = minifi::utils::string::endsWith(op.args["url"], "A.txt") ? file_A : file_B; } - size_t file_count = utils::file::list_dir_all(asset_dir.string(), controller.getLogger()).size(); + size_t file_count = minifi::utils::file::list_dir_all(asset_dir.string(), controller.getLogger()).size(); if (file_count != expected_files.size()) { controller.getLogger()->log_error("Expected {} files, got {}", expected_files.size(), file_count); - assert(false); + REQUIRE(false); } for (auto& [path, content] : expected_files) { - if (utils::file::get_content(path) != content) { + if (minifi::utils::file::get_content(path) != content) { controller.getLogger()->log_error("File content mismatch at '{}'", path); - assert(false); + REQUIRE(false); } } std::filesystem::current_path(minifi::utils::file::get_executable_dir()); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2UpdateTest.cpp b/libminifi/test/integration/C2UpdateTest.cpp similarity index 66% rename from extensions/http-curl/tests/C2UpdateTest.cpp rename to libminifi/test/integration/C2UpdateTest.cpp index d82989c24f..66072dfbfe 100644 --- a/extensions/http-curl/tests/C2UpdateTest.cpp +++ b/libminifi/test/integration/C2UpdateTest.cpp @@ -15,29 +15,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" #include "utils/gsl.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" using namespace std::literals::chrono_literals; -int main(int argc, char **argv) { - const cmd_args args = parse_cmdline_args(argc, argv, "update"); - C2UpdateHandler handler(args.test_file); +namespace org::apache::nifi::minifi::test { + +TEST_CASE("Test update configuration C2 command", "[c2test]") { + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestHTTPGet.yml"; + C2UpdateHandler handler(test_file_path.string()); VerifyC2Update harness(10s); - harness.setKeyDir(args.key_dir); - harness.setUrl(args.url, &handler); + harness.setKeyDir(TEST_RESOURCES); + harness.setUrl("https://localhost:0/update", &handler); handler.setC2RestResponse(harness.getC2RestUrl(), "configuration"); const auto start = std::chrono::steady_clock::now(); - harness.run(args.test_file); + harness.run(test_file_path); const auto then = std::chrono::steady_clock::now(); const auto seconds = std::chrono::duration_cast(then - start).count(); - assert(handler.getCallCount() <= gsl::narrow(seconds + 1)); - return 0; + REQUIRE(handler.getCallCount() <= gsl::narrow(seconds + 1)); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2VerifyHeartbeatAndStop.cpp b/libminifi/test/integration/C2VerifyHeartbeatAndStop.cpp similarity index 69% rename from extensions/http-curl/tests/C2VerifyHeartbeatAndStop.cpp rename to libminifi/test/integration/C2VerifyHeartbeatAndStop.cpp index a4791793fa..3eb4eb7101 100644 --- a/extensions/http-curl/tests/C2VerifyHeartbeatAndStop.cpp +++ b/libminifi/test/integration/C2VerifyHeartbeatAndStop.cpp @@ -15,17 +15,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG -#include "TestBase.h" +#include "unit/TestBase.h" #include "c2/C2Agent.h" #include "protocols/RESTProtocol.h" #include "protocols/RESTSender.h" #include "protocols/RESTReceiver.h" -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" -#include "utils/IntegrationTestUtils.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" +#include "unit/TestUtils.h" #include "properties/Configuration.h" +#include "unit/Catch.h" + +namespace org::apache::nifi::minifi::test { class VerifyC2Heartbeat : public VerifyC2Base { public: @@ -38,8 +39,8 @@ class VerifyC2Heartbeat : public VerifyC2Base { } void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; - assert(verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; + REQUIRE(verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), "Received Ack from Server", "C2Agent] [debug] Stopping component 2438e3c8-015a-1000-79ca-83af40ec1991", "C2Agent] [debug] Stopping component FlowController")); @@ -50,13 +51,18 @@ class VerifyC2Heartbeat : public VerifyC2Base { } }; -int main(int argc, char **argv) { - const cmd_args args = parse_cmdline_args(argc, argv, "heartbeat"); +TEST_CASE("Verify C2 heartbeat and stop operation", "[c2test]") { VerifyC2Heartbeat harness; - harness.setKeyDir(args.key_dir); StoppingHeartbeatHandler responder(harness.getConfiguration()); - harness.setUrl(args.url, &responder); - harness.run(args.test_file); - - return 0; + SECTION("Secure") { + harness.setKeyDir(TEST_RESOURCES); + harness.setUrl("https://localhost:0/heartbeat", &responder); + } + SECTION("Insecure") { + harness.setUrl("http://localhost:0/heartbeat", &responder); + } + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "C2VerifyHeartbeatAndStopSecure.yml"; + harness.run(test_file_path); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2VerifyLightweightHeartbeatAndStop.cpp b/libminifi/test/integration/C2VerifyLightweightHeartbeatAndStop.cpp similarity index 73% rename from extensions/http-curl/tests/C2VerifyLightweightHeartbeatAndStop.cpp rename to libminifi/test/integration/C2VerifyLightweightHeartbeatAndStop.cpp index 5b2d26fe4d..dae07f1fed 100644 --- a/extensions/http-curl/tests/C2VerifyLightweightHeartbeatAndStop.cpp +++ b/libminifi/test/integration/C2VerifyLightweightHeartbeatAndStop.cpp @@ -15,19 +15,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG #include -#include "TestBase.h" +#include "unit/TestBase.h" #include "c2/C2Agent.h" #include "protocols/RESTProtocol.h" #include "protocols/RESTSender.h" #include "protocols/RESTReceiver.h" -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" -#include "utils/IntegrationTestUtils.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" +#include "unit/TestUtils.h" #include "properties/Configuration.h" +#include "unit/Catch.h" + +namespace org::apache::nifi::minifi::test { class LightWeightC2Handler : public StoppingHeartbeatHandler { public: @@ -40,8 +41,8 @@ class LightWeightC2Handler : public StoppingHeartbeatHandler { if (calls_ == 0) { verifyJsonHasAgentManifest(root); } else { - assert(root.HasMember("agentInfo")); - assert(!root["agentInfo"].HasMember("agentManifest")); + REQUIRE(root.HasMember("agentInfo")); + REQUIRE(!root["agentInfo"].HasMember("agentManifest")); } calls_++; } @@ -62,8 +63,7 @@ class VerifyLightWeightC2Heartbeat : public VerifyC2Base { } void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; - assert(verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), "Received Ack from Server", "C2Agent] [debug] Stopping component 2438e3c8-015a-1000-79ca-83af40ec1991", "C2Agent] [debug] Stopping component FlowController")); @@ -74,13 +74,18 @@ class VerifyLightWeightC2Heartbeat : public VerifyC2Base { } }; -int main(int argc, char **argv) { - const cmd_args args = parse_cmdline_args(argc, argv, "heartbeat"); +TEST_CASE("Verify C2 lightweight heartbeat and stop operation", "[c2test]") { VerifyLightWeightC2Heartbeat harness; - harness.setKeyDir(args.key_dir); LightWeightC2Handler responder(harness.getConfiguration()); - harness.setUrl(args.url, &responder); - harness.run(args.test_file); - - return 0; + SECTION("Secure") { + harness.setKeyDir(TEST_RESOURCES); + harness.setUrl("https://localhost:0/heartbeat", &responder); + } + SECTION("Insecure") { + harness.setUrl("http://localhost:0/heartbeat", &responder); + } + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "C2VerifyHeartbeatAndStopSecure.yml"; + harness.run(test_file_path); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2VerifyResourceConsumptionInHeartbeat.cpp b/libminifi/test/integration/C2VerifyResourceConsumptionInHeartbeat.cpp similarity index 63% rename from extensions/http-curl/tests/C2VerifyResourceConsumptionInHeartbeat.cpp rename to libminifi/test/integration/C2VerifyResourceConsumptionInHeartbeat.cpp index 8302d6175f..bda77d6f86 100644 --- a/extensions/http-curl/tests/C2VerifyResourceConsumptionInHeartbeat.cpp +++ b/libminifi/test/integration/C2VerifyResourceConsumptionInHeartbeat.cpp @@ -15,20 +15,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG #include #include -#include "TestBase.h" +#include "unit/TestBase.h" #include "c2/C2Agent.h" #include "protocols/RESTProtocol.h" #include "protocols/RESTSender.h" #include "protocols/RESTReceiver.h" -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" -#include "utils/IntegrationTestUtils.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" +#include "unit/TestUtils.h" #include "properties/Configuration.h" +#include "unit/Catch.h" + +namespace org::apache::nifi::minifi::test { class ResourceConsumptionInHeartbeatHandler : public HeartbeatHandler { public: @@ -48,55 +49,55 @@ class ResourceConsumptionInHeartbeatHandler : public HeartbeatHandler { protected: static void verifySystemResourceConsumption(const rapidjson::Document& root, bool firstCall) { - assert(root.HasMember("deviceInfo")); + REQUIRE(root.HasMember("deviceInfo")); auto& device_info = root["deviceInfo"]; - assert(device_info.HasMember("systemInfo")); + REQUIRE(device_info.HasMember("systemInfo")); auto& system_info = device_info["systemInfo"]; - assert(system_info.HasMember("vCores")); - assert(system_info["vCores"].GetUint() > 0); + REQUIRE(system_info.HasMember("vCores")); + REQUIRE(system_info["vCores"].GetUint() > 0); - assert(system_info.HasMember("physicalMem")); - assert(system_info["physicalMem"].GetUint64() > 0); + REQUIRE(system_info.HasMember("physicalMem")); + REQUIRE(system_info["physicalMem"].GetUint64() > 0); - assert(system_info.HasMember("memoryUsage")); - assert(system_info["memoryUsage"].GetUint64() > 0); + REQUIRE(system_info.HasMember("memoryUsage")); + REQUIRE(system_info["memoryUsage"].GetUint64() > 0); - assert(system_info.HasMember("cpuUtilization")); + REQUIRE(system_info.HasMember("cpuUtilization")); if (!firstCall) { - assert(system_info["cpuUtilization"].GetDouble() >= 0.0); - assert(system_info["cpuUtilization"].GetDouble() <= 1.0); + REQUIRE(system_info["cpuUtilization"].GetDouble() >= 0.0); + REQUIRE(system_info["cpuUtilization"].GetDouble() <= 1.0); } - assert(system_info.HasMember("machineArch")); - assert(system_info["machineArch"].GetStringLength() > 0); + REQUIRE(system_info.HasMember("machineArch")); + REQUIRE(system_info["machineArch"].GetStringLength() > 0); #ifndef WIN32 - assert(system_info.HasMember("cpuLoadAverage")); - assert(system_info["cpuLoadAverage"].GetDouble() >= 0.0); + REQUIRE(system_info.HasMember("cpuLoadAverage")); + REQUIRE(system_info["cpuLoadAverage"].GetDouble() >= 0.0); #endif } static void verifyProcessResourceConsumption(const rapidjson::Document& root, bool firstCall) { - assert(root.HasMember("agentInfo")); + REQUIRE(root.HasMember("agentInfo")); auto& agent_info = root["agentInfo"]; - assert(agent_info.HasMember("status")); + REQUIRE(agent_info.HasMember("status")); auto& status = agent_info["status"]; - assert(status.HasMember("resourceConsumption")); + REQUIRE(status.HasMember("resourceConsumption")); auto& resource_consumption = status["resourceConsumption"]; - assert(resource_consumption.HasMember("memoryUsage")); - assert(resource_consumption["memoryUsage"].GetUint64() > 0); + REQUIRE(resource_consumption.HasMember("memoryUsage")); + REQUIRE(resource_consumption["memoryUsage"].GetUint64() > 0); - assert(resource_consumption.HasMember("cpuUtilization")); + REQUIRE(resource_consumption.HasMember("cpuUtilization")); auto& cpu_utilization = resource_consumption["cpuUtilization"]; - assert(cpu_utilization.IsDouble()); + REQUIRE(cpu_utilization.IsDouble()); if (!firstCall) { - assert(cpu_utilization.GetDouble() >= 0.0); - assert(cpu_utilization.GetDouble() <= 1.0); + REQUIRE(cpu_utilization.GetDouble() >= 0.0); + REQUIRE(cpu_utilization.GetDouble() <= 1.0); } } @@ -115,8 +116,7 @@ class VerifyResourceConsumptionInHeartbeat : public VerifyC2Base { } void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; - assert(verifyEventHappenedInPollTime(std::chrono::milliseconds(7000), event_to_wait_for_)); + REQUIRE(utils::verifyEventHappenedInPollTime(std::chrono::milliseconds(7000), event_to_wait_for_)); } void configureFullHeartbeat() override { @@ -130,17 +130,17 @@ class VerifyResourceConsumptionInHeartbeat : public VerifyC2Base { std::function event_to_wait_for_; }; -int main(int argc, char **argv) { - const cmd_args args = parse_cmdline_args(argc, argv, "heartbeat"); +TEST_CASE("Verify resource consumption in C2 heartbeat", "[c2test]") { VerifyResourceConsumptionInHeartbeat harness; ResourceConsumptionInHeartbeatHandler responder(harness.getConfiguration()); auto event_to_wait_for = [&responder] { return responder.getNumberOfHandledHeartBeats() >= 3; }; - harness.setUrl(args.url, &responder); + harness.setUrl("http://localhost:0/heartbeat", &responder); harness.setEventToWaitFor(event_to_wait_for); - harness.run(args.test_file); - - return 0; + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "C2VerifyHeartbeatAndStop.yml"; + harness.run(test_file_path); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/C2VerifyServeResults.cpp b/libminifi/test/integration/C2VerifyServeResults.cpp similarity index 66% rename from extensions/http-curl/tests/C2VerifyServeResults.cpp rename to libminifi/test/integration/C2VerifyServeResults.cpp index 8242dcb79f..a4c51847a9 100644 --- a/extensions/http-curl/tests/C2VerifyServeResults.cpp +++ b/libminifi/test/integration/C2VerifyServeResults.cpp @@ -15,20 +15,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG #include #include #include #include #include "processors/InvokeHTTP.h" -#include "TestBase.h" +#include "unit/TestBase.h" #include "core/ProcessGroup.h" #include "properties/Configure.h" -#include "TestServer.h" -#include "HTTPIntegrationBase.h" -#include "utils/IntegrationTestUtils.h" +#include "integration/TestServer.h" +#include "integration/HTTPIntegrationBase.h" +#include "unit/TestUtils.h" #include "properties/Configuration.h" +#include "unit/Catch.h" + +namespace org::apache::nifi::minifi::test { class VerifyC2Server : public HTTPIntegrationBase { public: @@ -52,19 +53,18 @@ class VerifyC2Server : public HTTPIntegrationBase { } void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; - assert(verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), "Import offset 0", "Outputting success and response")); } void queryRootProcessGroup(std::shared_ptr pg) override { auto proc = pg->findProcessorByName("invoke"); - assert(proc != nullptr); + REQUIRE(proc != nullptr); auto inv = dynamic_cast(proc); - assert(inv != nullptr); + REQUIRE(inv != nullptr); std::string url; inv->getProperty(minifi::processors::InvokeHTTP::URL, url); @@ -72,12 +72,12 @@ class VerifyC2Server : public HTTPIntegrationBase { std::string scheme; std::string path; minifi::utils::parse_http_components(url, port, scheme, path); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_enable, "true"); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_agent_class, "test"); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_agent_heartbeat_reporter_classes, "RESTReceiver"); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_agent_protocol_class, "RESTSender"); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_rest_listener_port, port); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_agent_heartbeat_period, "10"); + configuration->set(minifi::Configuration::nifi_c2_enable, "true"); + configuration->set(minifi::Configuration::nifi_c2_agent_class, "test"); + configuration->set(minifi::Configuration::nifi_c2_agent_heartbeat_reporter_classes, "RESTReceiver"); + configuration->set(minifi::Configuration::nifi_c2_agent_protocol_class, "RESTSender"); + configuration->set(minifi::Configuration::nifi_c2_rest_listener_port, port); + configuration->set(minifi::Configuration::nifi_c2_agent_heartbeat_period, "10"); } protected: @@ -86,12 +86,11 @@ class VerifyC2Server : public HTTPIntegrationBase { TestController testController; }; -int main(int argc, char **argv) { - const cmd_args args = parse_cmdline_args(argc, argv); - +TEST_CASE("C2VerifyServeResults", "[c2test]") { VerifyC2Server harness; - harness.setKeyDir(args.key_dir); - harness.run(args.test_file); - - return 0; + harness.setKeyDir(TEST_RESOURCES); + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "C2VerifyServeResults.yml"; + harness.run(test_file_path); } + +} // namespace org::apache::nifi::minifi::test diff --git a/libminifi/test/integration/CMakeLists.txt b/libminifi/test/integration/CMakeLists.txt new file mode 100644 index 0000000000..c52f40cf9e --- /dev/null +++ b/libminifi/test/integration/CMakeLists.txt @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +GETSOURCEFILES(INTEGRATION_TESTS "${TEST_DIR}/integration/") + +SET(TESTS_WITH_LIBARCHIVE_DEPENDENCY C2DebugBundleTest) +SET(INT_TEST_COUNT 0) +FOREACH(testfile ${INTEGRATION_TESTS}) + get_filename_component(testfilename "${testfile}" NAME_WE) + add_minifi_executable("${testfilename}" "${TEST_DIR}/integration/${testfile}") + createIntegrationTests("${testfilename}") + target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/") + target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/") + target_link_libraries(${testfilename} minifi-standard-processors) + target_link_libraries(${testfilename} Catch2::Catch2WithMain Threads::Threads) + target_compile_definitions(${testfilename} PRIVATE TEST_RESOURCES="${TEST_RESOURCES}") + if(NOT ${testfilename} IN_LIST TESTS_WITH_LIBARCHIVE_DEPENDENCY) + add_test(NAME "${testfilename}" COMMAND "${testfilename}") + endif() + MATH(EXPR INT_TEST_COUNT "${INT_TEST_COUNT}+1") +ENDFOREACH() + +FOREACH(testfilename ${TESTS_WITH_LIBARCHIVE_DEPENDENCY}) + if (ENABLE_LIBARCHIVE) + add_test(NAME "${testfilename}" COMMAND "${testfilename}") + endif() +ENDFOREACH() + +target_link_libraries(OnScheduleErrorHandlingTests minifi-test-processors) + +message("-- Finished building ${INT_TEST_COUNT} integration test file(s)...") diff --git a/extensions/http-curl/tests/ConfigTestAccessor.h b/libminifi/test/integration/ConfigTestAccessor.h similarity index 100% rename from extensions/http-curl/tests/ConfigTestAccessor.h rename to libminifi/test/integration/ConfigTestAccessor.h diff --git a/extensions/http-curl/tests/ControllerServiceIntegrationTests.cpp b/libminifi/test/integration/ControllerServiceIntegrationTests.cpp similarity index 75% rename from extensions/http-curl/tests/ControllerServiceIntegrationTests.cpp rename to libminifi/test/integration/ControllerServiceIntegrationTests.cpp index fe36a2e308..5ae76f56a5 100644 --- a/extensions/http-curl/tests/ControllerServiceIntegrationTests.cpp +++ b/libminifi/test/integration/ControllerServiceIntegrationTests.cpp @@ -15,10 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - #define EXTENSION_LIST "*minifi-*,!*jni*" // NOLINT(cppcoreguidelines-macro-usage) -#undef NDEBUG #include #include #include @@ -38,7 +36,10 @@ #include "unit/MockClasses.h" #include "unit/ProvenanceTestHelper.h" #include "integration/IntegrationBase.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" +#include "unit/Catch.h" + +namespace org::apache::nifi::minifi::test { REGISTER_RESOURCE(MockControllerService, ControllerService); REGISTER_RESOURCE(MockProcessor, Processor); @@ -47,33 +48,32 @@ void waitToVerifyProcessor() { std::this_thread::sleep_for(std::chrono::seconds(2)); } -int main(int argc, char **argv) { - using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; - const cmd_args args = parse_cmdline_args(argc, argv); - +TEST_CASE("ControllerServiceIntegrationTests", "[controller]") { + using org::apache::nifi::minifi::test::utils::verifyEventHappenedInPollTime; std::shared_ptr configuration = std::make_shared(); std::shared_ptr test_repo = std::make_shared(); std::shared_ptr test_flow_repo = std::make_shared(); - configuration->set(minifi::Configure::nifi_flow_configuration_file, args.test_file); + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestControllerServices.yml"; + configuration->set(minifi::Configure::nifi_flow_configuration_file, test_file_path.string()); std::string priv_key_file = "cn.ckey.pem"; std::string passphrase = "cn.pass"; - configuration->set(minifi::Configure::nifi_security_client_certificate, args.test_file); + configuration->set(minifi::Configure::nifi_security_client_certificate, test_file_path.string()); configuration->set(minifi::Configure::nifi_security_client_private_key, priv_key_file); configuration->set(minifi::Configure::nifi_security_client_pass_phrase, passphrase); - configuration->set(minifi::Configure::nifi_default_directory, args.key_dir); + configuration->set(minifi::Configure::nifi_default_directory, TEST_RESOURCES); std::shared_ptr content_repo = std::make_shared(); content_repo->initialize(configuration); - auto encryption_key = utils::string::from_hex("e4bce4be67f417ed2530038626da57da7725ff8c0b519b692e4311e4d4fe8a28"); + auto encryption_key = minifi::utils::string::from_hex("e4bce4be67f417ed2530038626da57da7725ff8c0b519b692e4311e4d4fe8a28"); std::unique_ptr yaml_ptr = std::make_unique(core::ConfigurationContext{ .flow_file_repo = test_repo, .content_repo = content_repo, .configuration = configuration, - .path = args.test_file, - .filesystem = std::make_shared(), - .sensitive_properties_encryptor = utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{encryption_key}} + .path = test_file_path, + .filesystem = std::make_shared(), + .sensitive_properties_encryptor = minifi::utils::crypto::EncryptionProvider{minifi::utils::crypto::XSalsa20Cipher{encryption_key}} }); const auto controller = std::make_shared(test_repo, test_flow_repo, configuration, std::move(yaml_ptr), content_repo); @@ -84,21 +84,21 @@ int main(int argc, char **argv) { .flow_file_repo = test_repo, .content_repo = content_repo, .configuration = configuration, - .path = args.test_file, - .filesystem = std::make_shared(), - .sensitive_properties_encryptor = utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{encryption_key}} + .path = test_file_path, + .filesystem = std::make_shared(), + .sensitive_properties_encryptor = minifi::utils::crypto::EncryptionProvider{minifi::utils::crypto::XSalsa20Cipher{encryption_key}} }); auto pg = yaml_config.getRoot(); auto provider = std::make_shared(map, std::make_shared()); std::shared_ptr mockNode = pg->findControllerService("MockItLikeIts1995"); - assert(mockNode != nullptr); + REQUIRE(mockNode != nullptr); mockNode->enable(); std::vector > linkedNodes = mockNode->getLinkedControllerServices(); - assert(linkedNodes.size() == 1); + REQUIRE(linkedNodes.size() == 1); std::shared_ptr notexistNode = pg->findControllerService("MockItLikeItsWrong"); - assert(notexistNode == nullptr); + REQUIRE(notexistNode == nullptr); std::shared_ptr ssl_client_cont = nullptr; std::shared_ptr ssl_client = nullptr; @@ -108,14 +108,14 @@ int main(int argc, char **argv) { controller->start(); ssl_client_cont = controller->getControllerServiceNode("SSLClientServiceTest"); ssl_client_cont->enable(); - assert(ssl_client_cont != nullptr); - assert(ssl_client_cont->getControllerServiceImplementation() != nullptr); + REQUIRE(ssl_client_cont != nullptr); + REQUIRE(ssl_client_cont->getControllerServiceImplementation() != nullptr); ssl_client = std::static_pointer_cast(ssl_client_cont->getControllerServiceImplementation()); } - assert(!ssl_client->getCACertificate().empty()); + REQUIRE(!ssl_client->getCACertificate().empty()); // now let's disable one of the controller services. std::shared_ptr cs_id = controller->getControllerServiceNode("ID"); - assert(cs_id != nullptr); + REQUIRE(cs_id != nullptr); // TODO(adebreceni): MINIFICPP-1992 // const auto checkCsIdEnabledMatchesDisabledFlag = [&cs_id] { return !disabled == cs_id->enabled(); }; // { @@ -124,20 +124,21 @@ int main(int argc, char **argv) { // disabled = false; // } // std::shared_ptr mock_cont = controller->getControllerServiceNode("MockItLikeIts1995"); -// assert(verifyEventHappenedInPollTime(std::chrono::seconds(4), checkCsIdEnabledMatchesDisabledFlag)); +// REQUIRE(verifyEventHappenedInPollTime(std::chrono::seconds(4), checkCsIdEnabledMatchesDisabledFlag)); // { // std::lock_guard lock(control_mutex); // controller->disableReferencingServices(mock_cont); // disabled = true; // } -// assert(verifyEventHappenedInPollTime(std::chrono::seconds(2), checkCsIdEnabledMatchesDisabledFlag)); +// REQUIRE(verifyEventHappenedInPollTime(std::chrono::seconds(2), checkCsIdEnabledMatchesDisabledFlag)); // { // std::lock_guard lock(control_mutex); // controller->enableReferencingServices(mock_cont); // disabled = false; // } -// assert(verifyEventHappenedInPollTime(std::chrono::seconds(2), checkCsIdEnabledMatchesDisabledFlag)); +// REQUIRE(verifyEventHappenedInPollTime(std::chrono::seconds(2), checkCsIdEnabledMatchesDisabledFlag)); controller->waitUnload(60s); - return 0; } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/unit/HTTPClientTests.cpp b/libminifi/test/integration/HTTPClientTests.cpp similarity index 84% rename from extensions/http-curl/tests/unit/HTTPClientTests.cpp rename to libminifi/test/integration/HTTPClientTests.cpp index 5d46957875..688beb7349 100644 --- a/extensions/http-curl/tests/unit/HTTPClientTests.cpp +++ b/libminifi/test/integration/HTTPClientTests.cpp @@ -19,17 +19,17 @@ #include #include #include -#include "TestBase.h" -#include "Catch.h" -#include "client/HTTPClient.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "http/HTTPClient.h" #include "CivetServer.h" -#include "ConnectionCountingServer.h" +#include "integration/ConnectionCountingServer.h" #include "magic_enum.hpp" -#include "utils/BaseHTTPClient.h" +#include "http/BaseHTTPClient.h" using namespace std::literals::chrono_literals; -namespace org::apache::nifi::minifi::extensions::curl::testing { +namespace org::apache::nifi::minifi::http::test { TEST_CASE("HTTPClientTestChunkedResponse", "[basic]") { LogTestController::getInstance().setDebug(); @@ -82,7 +82,7 @@ TEST_CASE("HTTPClientTestChunkedResponse", "[basic]") { const std::string port = std::to_string(vec.at(0)); HTTPClient client; - client.initialize(utils::HttpRequestMethod::GET, "http://localhost:" + port + "/testytesttest", nullptr); + client.initialize(http::HttpRequestMethod::GET, "http://localhost:" + port + "/testytesttest", nullptr); REQUIRE(client.submit()); @@ -122,20 +122,20 @@ TEST_CASE("HTTPClient replaceInvalidCharactersInHttpHeaderFieldName test") { TEST_CASE("HTTPClient should be reusable", "[basic]") { LogTestController::getInstance().setDebug(); - ConnectionCountingServer keep_alive_server_; + minifi::test::ConnectionCountingServer keep_alive_server_; HTTPClient client; client.setKeepAliveProbe(KeepAliveProbeData{2s, 2s}); - client.initialize(utils::HttpRequestMethod::GET, "http://localhost:" + keep_alive_server_.getPort() + "/method", nullptr); + client.initialize(http::HttpRequestMethod::GET, "http://localhost:" + keep_alive_server_.getPort() + "/method", nullptr); - std::vector methods = { - utils::HttpRequestMethod::GET, utils::HttpRequestMethod::GET, utils::HttpRequestMethod::PUT, utils::HttpRequestMethod::GET, - utils::HttpRequestMethod::GET, utils::HttpRequestMethod::POST, utils::HttpRequestMethod::POST, utils::HttpRequestMethod::PUT }; + std::vector methods = { + http::HttpRequestMethod::GET, http::HttpRequestMethod::GET, http::HttpRequestMethod::PUT, http::HttpRequestMethod::GET, + http::HttpRequestMethod::GET, http::HttpRequestMethod::POST, http::HttpRequestMethod::POST, http::HttpRequestMethod::PUT }; uint64_t request_number = 0; for (const auto& method: methods) { client.set_request_method(method); - if (method != utils::HttpRequestMethod::GET) { - auto callback = std::make_unique(); + if (method != http::HttpRequestMethod::GET) { + auto callback = std::make_unique(); std::string content = R"({ "content": "a" })"; callback->write(content); client.setRequestHeader("Content-Length", std::to_string(content.size())); @@ -162,9 +162,9 @@ TEST_CASE("HTTPClient should be reusable", "[basic]") { #ifdef __linux__ TEST_CASE("SSL without SSLContextService", "[HTTPClient]") { HTTPClient client; - client.initialize(utils::HttpRequestMethod::GET, "https://apache.org", nullptr); + client.initialize(http::HttpRequestMethod::GET, "https://apache.org", nullptr); REQUIRE(client.submit()); } #endif -} // namespace org::apache::nifi::minifi::extensions::curl::testing +} // namespace org::apache::nifi::minifi::http::test diff --git a/extensions/http-curl/tests/HTTPSiteToSiteTests.cpp b/libminifi/test/integration/HTTPSiteToSiteTests.cpp similarity index 75% rename from extensions/http-curl/tests/HTTPSiteToSiteTests.cpp rename to libminifi/test/integration/HTTPSiteToSiteTests.cpp index 8f2b2406e1..c8a44f87a2 100644 --- a/extensions/http-curl/tests/HTTPSiteToSiteTests.cpp +++ b/libminifi/test/integration/HTTPSiteToSiteTests.cpp @@ -15,50 +15,50 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG -#include #include #include #include #include #include #include "sitetosite/HTTPProtocol.h" -#include "InvokeHTTP.h" -#include "TestBase.h" +#include "processors/InvokeHTTP.h" +#include "unit/TestBase.h" #include "FlowController.h" #include "RemoteProcessorGroupPort.h" #include "core/ConfigurableComponent.h" -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" -#include "client/HTTPStream.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" +#include "http/HTTPStream.h" #include "properties/Configuration.h" +#include "unit/Catch.h" using namespace std::literals::chrono_literals; +namespace org::apache::nifi::minifi::test { + class SiteToSiteTestHarness : public HTTPIntegrationBase { public: - explicit SiteToSiteTestHarness(bool isSecure, std::chrono::seconds waitTime = 2s) - : HTTPIntegrationBase(waitTime), isSecure(isSecure) { + explicit SiteToSiteTestHarness(std::chrono::seconds waitTime = 2s) + : HTTPIntegrationBase(waitTime) { dir = testController.createTempDirectory(); } void testSetup() override { LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); LogTestController::getInstance().setTrace(); LogTestController::getInstance().setInfo(); LogTestController::getInstance().setDebug(); - LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); std::fstream file; file.open(dir / "tstFile.ext", std::ios::out); file << "tempFile"; file.close(); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_enable, "false"); + configuration->set(minifi::Configuration::nifi_c2_enable, "false"); } void runAssertions() override { @@ -67,7 +67,6 @@ class SiteToSiteTestHarness : public HTTPIntegrationBase { } protected: - bool isSecure; std::filesystem::path dir; TestController testController; }; @@ -87,13 +86,13 @@ struct test_profile { bool invalid_checksum{false}; }; -void run_variance(const std::string& test_file_location, bool isSecure, const std::string& url, const struct test_profile &profile) { - SiteToSiteTestHarness harness(isSecure); +void run_variance(const std::string& test_file_location, const std::string& url, const struct test_profile &profile) { + SiteToSiteTestHarness harness; std::string in_port = "471deef6-2a6e-4a7d-912a-81cc17e3a204"; std::string out_port = "471deef6-2a6e-4a7d-912a-81cc17e3a203"; - auto responder = std::make_unique(isSecure); + auto responder = std::make_unique(false); auto *transaction_response = new TransactionResponder(url, in_port, true, profile.transaction_url_broken, profile.empty_transaction_url); @@ -160,61 +159,51 @@ void run_variance(const std::string& test_file_location, bool isSecure, const st std::stringstream assertStr; if (profile.allFalse()) { assertStr << "Site2Site transaction " << transaction_id << " peer finished transaction"; - assert(LogTestController::getInstance().contains(assertStr.str())); + REQUIRE(LogTestController::getInstance().contains(assertStr.str())); } else if (profile.empty_transaction_url) { - assert(LogTestController::getInstance().contains("Location is empty")); + REQUIRE(LogTestController::getInstance().contains("Location is empty")); } else if (profile.transaction_url_broken) { - assert(LogTestController::getInstance().contains("Could not create transaction, intent is ohstuff")); + REQUIRE(LogTestController::getInstance().contains("Could not create transaction, intent is ohstuff")); } else if (profile.invalid_checksum) { assertStr << "Site2Site transaction " << transaction_id << " peer confirm transaction with CRC Imawrongchecksumshortandstout"; - assert(LogTestController::getInstance().contains(assertStr.str())); + REQUIRE(LogTestController::getInstance().contains(assertStr.str())); assertStr.str(std::string()); assertStr << "Site2Site transaction " << transaction_id << " CRC not matched"; - assert(LogTestController::getInstance().contains(assertStr.str())); + REQUIRE(LogTestController::getInstance().contains(assertStr.str())); assertStr.str(std::string()); assertStr << "Site2Site delete transaction " << transaction_id; - assert(LogTestController::getInstance().contains(assertStr.str())); + REQUIRE(LogTestController::getInstance().contains(assertStr.str())); } else { assertStr << "Site2Site transaction " << transaction_id << " peer unknown respond code 254"; - assert(LogTestController::getInstance().contains(assertStr.str())); + REQUIRE(LogTestController::getInstance().contains(assertStr.str())); } LogTestController::getInstance().reset(); } -int main(int argc, char **argv) { - transaction_id = 0; - transaction_id_output = 0; - const cmd_args args = parse_cmdline_args_with_url(argc, argv); - const bool isSecure = args.isUrlSecure(); +TEST_CASE("Test site to site with HTTP", "[s2s]") { + test_profile profile; - { - struct test_profile profile; - run_variance(args.test_file, isSecure, args.url, profile); + SECTION("Default test profile") { } - { - struct test_profile profile; + SECTION("Flow url broken") { profile.flow_url_broken = true; - run_variance(args.test_file, isSecure, args.url, profile); } - { - struct test_profile profile; + SECTION("Empty transaction url") { profile.empty_transaction_url = true; - run_variance(args.test_file, isSecure, args.url, profile); } - { - struct test_profile profile; + SECTION("Transaction url broken") { profile.transaction_url_broken = true; - run_variance(args.test_file, isSecure, args.url, profile); } - { - struct test_profile profile; + SECTION("Invalid checksum") { profile.invalid_checksum = true; - run_variance(args.test_file, isSecure, args.url, profile); } - return 0; + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestHTTPSiteToSite.yml"; + run_variance(test_file_path.string(), parseUrl("http://localhost:8099/nifi-api"), profile); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/HttpPostIntegrationTest.cpp b/libminifi/test/integration/HttpPostIntegrationTest.cpp similarity index 69% rename from extensions/http-curl/tests/HttpPostIntegrationTest.cpp rename to libminifi/test/integration/HttpPostIntegrationTest.cpp index d6d03e8858..b07b8e0c26 100644 --- a/extensions/http-curl/tests/HttpPostIntegrationTest.cpp +++ b/libminifi/test/integration/HttpPostIntegrationTest.cpp @@ -15,25 +15,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG -#include #include #include #include -#include "InvokeHTTP.h" +#include "processors/InvokeHTTP.h" #include "processors/ListenHTTP.h" #include "processors/LogAttribute.h" -#include "TestBase.h" +#include "unit/TestBase.h" #include "core/logging/Logger.h" #include "core/ProcessGroup.h" #include "FlowController.h" -#include "HTTPIntegrationBase.h" -#include "utils/IntegrationTestUtils.h" +#include "integration/HTTPIntegrationBase.h" +#include "unit/TestUtils.h" #include "properties/Configuration.h" +#include "unit/Catch.h" using namespace std::literals::chrono_literals; +namespace org::apache::nifi::minifi::test { + class HttpTestHarness : public HTTPIntegrationBase { public: HttpTestHarness() { @@ -46,7 +46,7 @@ class HttpTestHarness : public HTTPIntegrationBase { LogTestController::getInstance().setDebug(); LogTestController::getInstance().setDebug(); LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setDebug(); + LogTestController::getInstance().setDebug(); LogTestController::getInstance().setDebug(); LogTestController::getInstance().setDebug(); LogTestController::getInstance().setDebug(); @@ -59,8 +59,8 @@ class HttpTestHarness : public HTTPIntegrationBase { file.open(test_file_, std::ios::out); file << "tempFile"; file.close(); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_flow_engine_threads, "8"); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_enable, "false"); + configuration->set(minifi::Configuration::nifi_flow_engine_threads, "8"); + configuration->set(minifi::Configuration::nifi_c2_enable, "false"); } void cleanup() override { @@ -69,11 +69,10 @@ class HttpTestHarness : public HTTPIntegrationBase { } void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; - assert(verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), "curl performed", "Size:1024 Offset:0")); - assert(false == verifyLogLinePresenceInPollTime(std::chrono::milliseconds(200), "Size:0 Offset:0")); + REQUIRE_FALSE(minifi::test::utils::verifyLogLinePresenceInPollTime(std::chrono::milliseconds(200), "Size:0 Offset:0")); } protected: @@ -82,11 +81,19 @@ class HttpTestHarness : public HTTPIntegrationBase { TestController test_controller_; }; -int main(int argc, char **argv) { - const cmd_args args = parse_cmdline_args(argc, argv); - +TEST_CASE("Test HTTP client POST request", "[httptest]") { HttpTestHarness harness; - harness.setKeyDir(args.key_dir); - harness.run(args.test_file); - return 0; + harness.setKeyDir(TEST_RESOURCES); + SECTION("Without chunked encoding") { + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestHTTPPost.yml"; + harness.run(test_file_path); + } +#ifndef __APPLE__ + SECTION("With chunked encoding") { + const auto test_file_path = std::filesystem::path(TEST_RESOURCES) / "TestHTTPPostChunkedEncoding.yml"; + harness.run(test_file_path); + } +#endif } + +} // namespace org::apache::nifi::minifi::test diff --git a/libminifi/test/integration/IntegrationBase.h b/libminifi/test/integration/IntegrationBase.h deleted file mode 100644 index e7be12fd08..0000000000 --- a/libminifi/test/integration/IntegrationBase.h +++ /dev/null @@ -1,297 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#define DEFAULT_WAITTIME_MSECS 10000 - -#include -#include -#include -#include -#include -#include - -#include "core/logging/Logger.h" -#include "core/ProcessGroup.h" -#include "core/yaml/YamlConfiguration.h" -#include "FlowController.h" -#include "properties/Configure.h" -#include "../unit/ProvenanceTestHelper.h" -#include "RemoteProcessorGroupPort.h" -#include "core/ConfigurationFactory.h" -#include "core/ConfigurableComponent.h" -#include "controllers/SSLContextService.h" -#include "HTTPUtils.h" -#include "utils/FifoExecutor.h" -#include "core/state/MetricsPublisherFactory.h" -#include "c2/C2Utils.h" -#include "utils/net/DNS.h" -#include "utils/crypto/ciphers/XSalsa20.h" - -namespace minifi = org::apache::nifi::minifi; -namespace core = minifi::core; -namespace utils = minifi::utils; - -class IntegrationBase { - public: - explicit IntegrationBase(std::chrono::milliseconds waitTime = std::chrono::milliseconds(DEFAULT_WAITTIME_MSECS)); - IntegrationBase(const IntegrationBase&) = delete; - IntegrationBase(IntegrationBase&& other) noexcept - :configuration{std::move(other.configuration)}, - flowController_{std::move(other.flowController_)}, - wait_time_{other.wait_time_}, - port{std::move(other.port)}, - scheme{std::move(other.scheme)}, - key_dir{std::move(other.key_dir)}, - state_dir{std::move(other.state_dir)}, - restart_requested_count_{other.restart_requested_count_.load()} - {} - IntegrationBase& operator=(const IntegrationBase&) = delete; - IntegrationBase& operator=(IntegrationBase&& other) noexcept { - if (&other == this) return *this; - configuration = std::move(other.configuration); - flowController_ = std::move(other.flowController_); - wait_time_ = other.wait_time_; - port = std::move(other.port); - scheme = std::move(other.scheme); - key_dir = std::move(other.key_dir); - state_dir = std::move(other.state_dir); - restart_requested_count_ = other.restart_requested_count_.load(); - return *this; - } - virtual ~IntegrationBase() = default; - - virtual void run(const std::optional& test_file_location = {}, const std::optional& home_path = {}); - - void setKeyDir(const std::string& key_dir) { - this->key_dir = key_dir; - configureSecurity(); - } - - virtual void testSetup() = 0; - - virtual void shutdownBeforeFlowController() { - } - - const std::shared_ptr& getConfiguration() const { - return configuration; - } - - void setConfiguration(std::shared_ptr configuration) { - this->configuration = std::move(configuration); - } - - virtual void cleanup() { - if (!state_dir.empty()) { - utils::file::delete_dir(state_dir); - } - } - - virtual void runAssertions() = 0; - - protected: - virtual void configureC2() { - } - - virtual void queryRootProcessGroup(std::shared_ptr /*pg*/) { - } - - virtual void configureFullHeartbeat() { - } - - virtual void updateProperties(minifi::FlowController& /*fc*/) { - } - - void configureSecurity(); - std::shared_ptr configuration; - std::unique_ptr response_node_loader_; - std::unique_ptr flowController_; - std::chrono::milliseconds wait_time_; - std::string port, scheme; - std::string key_dir; - std::filesystem::path state_dir; - std::atomic restart_requested_count_{0}; -}; - -IntegrationBase::IntegrationBase(std::chrono::milliseconds waitTime) - : configuration(std::make_shared()), - wait_time_(waitTime) { -} - -void IntegrationBase::configureSecurity() { - if (!key_dir.empty()) { - configuration->set(minifi::Configure::nifi_security_client_certificate, key_dir + "cn.crt.pem"); - configuration->set(minifi::Configure::nifi_security_client_private_key, key_dir + "cn.ckey.pem"); - configuration->set(minifi::Configure::nifi_security_client_pass_phrase, key_dir + "cn.pass"); - configuration->set(minifi::Configure::nifi_security_client_ca_certificate, key_dir + "nifi-cert.pem"); - configuration->set(minifi::Configure::nifi_default_directory, key_dir); - } -} - -void IntegrationBase::run(const std::optional& test_file_location, const std::optional& home_path) { - using namespace std::literals::chrono_literals; - testSetup(); - - std::shared_ptr test_repo = std::make_shared(); - std::shared_ptr test_flow_repo = std::make_shared(); - - if (test_file_location) { - configuration->set(minifi::Configure::nifi_flow_configuration_file, test_file_location->string()); - } - configuration->set(minifi::Configure::nifi_state_storage_local_class_name, "VolatileMapStateStorage"); - - configureC2(); - configureFullHeartbeat(); - - std::shared_ptr content_repo = std::make_shared(); - content_repo->initialize(configuration); - - std::atomic running = true; - utils::FifoExecutor assertion_runner; - std::future assertions_done; - while (running) { - running = false; // Stop running after this iteration, unless restart is explicitly requested - - bool should_encrypt_flow_config = (configuration->get(minifi::Configure::nifi_flow_configuration_encrypt) - | utils::andThen(utils::string::toBool)).value_or(false); - - std::shared_ptr filesystem; - if (home_path) { - filesystem = std::make_shared( - should_encrypt_flow_config, - utils::crypto::EncryptionProvider::create(*home_path)); - } else { - filesystem = std::make_shared(); - } - - std::optional sensitive_properties_encryptor = [&]() { - if (home_path) { - return utils::crypto::EncryptionProvider::createSensitivePropertiesEncryptor(*home_path); - } else { - auto encryption_key = utils::string::from_hex("e4bce4be67f417ed2530038626da57da7725ff8c0b519b692e4311e4d4fe8a28"); - return utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{encryption_key}}; - } - }(); - - std::string nifi_configuration_class_name = "adaptiveconfiguration"; - configuration->get(minifi::Configure::nifi_configuration_class_name, nifi_configuration_class_name); - - std::shared_ptr flow_config = core::createFlowConfiguration( - core::ConfigurationContext{ - .flow_file_repo = test_repo, - .content_repo = content_repo, - .configuration = configuration, - .path = test_file_location, - .filesystem = filesystem, - .sensitive_properties_encryptor = sensitive_properties_encryptor - }, nifi_configuration_class_name); - - auto controller_service_provider = flow_config->getControllerServiceProvider(); - char state_dir_name_template[] = "/var/tmp/integrationstate.XXXXXX"; - state_dir = utils::file::create_temp_directory(state_dir_name_template); - if (!configuration->get(minifi::Configure::nifi_state_storage_local_path)) { - configuration->set(minifi::Configure::nifi_state_storage_local_path, state_dir.string()); - } - core::ProcessContext::getOrCreateDefaultStateStorage(controller_service_provider.get(), configuration); - - std::shared_ptr pg(flow_config->getRoot()); - queryRootProcessGroup(pg); - - const auto request_restart = [&, this] { - ++restart_requested_count_; - running = true; - }; - - std::vector> repo_metric_sources{test_repo, test_flow_repo, content_repo}; - auto metrics_publisher_store = std::make_unique(configuration, repo_metric_sources, flow_config); - flowController_ = std::make_unique(test_repo, test_flow_repo, configuration, - std::move(flow_config), content_repo, std::move(metrics_publisher_store), filesystem, request_restart); - flowController_->load(); - updateProperties(*flowController_); - flowController_->start(); - - assertions_done = assertion_runner.enqueue([this] { runAssertions(); }); - std::future_status status = std::future_status::ready; - while (!running && (status = assertions_done.wait_for(10ms)) == std::future_status::timeout) { /* wait */ } - if (running && status != std::future_status::timeout) { - // cancel restart, because assertions have finished running - running = false; - } - - if (!running) { - // Only stop servers if we're shutting down - shutdownBeforeFlowController(); - } - flowController_->stop(); - } - - cleanup(); -} - -struct cmd_args { - bool isUrlSecure() const { - // check https prefix - return url.rfind("https://", 0) == 0; - } - - std::string test_file; - std::string key_dir; - std::string bad_test_file; - std::string url; -}; - -cmd_args parse_basic_cmdline_args(int argc, char ** argv) { - cmd_args args; - if (argc > 1) { - args.test_file = argv[1]; - } - if (argc > 2) { - args.key_dir = argv[2]; - } - return args; -} - -cmd_args parse_cmdline_args(int argc, char ** argv, const std::string& uri_path = "") { - cmd_args args = parse_basic_cmdline_args(argc, argv); - if (argc == 2) { - args.url = "http://localhost:0/" + uri_path; - } - if (argc > 2) { - args.url = "https://localhost:0/" + uri_path; - } - if (argc > 3) { - args.bad_test_file = argv[3]; - } - return args; -} - -cmd_args parse_cmdline_args_with_url(int argc, char ** argv) { - cmd_args args = parse_basic_cmdline_args(argc, argv); - if (argc > 3) { - std::string url = argv[3]; -#ifdef WIN32 - if (url.find("localhost") != std::string::npos) { - std::string port, scheme, path; - minifi::utils::parse_http_components(url, port, scheme, path); - url = scheme + "://" + org::apache::nifi::minifi::utils::net::getMyHostName() + ":" + port + path; - } -#endif - args.url = url; - } - return args; -} diff --git a/libminifi/test/integration/OnScheduleErrorHandlingTests.cpp b/libminifi/test/integration/OnScheduleErrorHandlingTests.cpp index 32d95cdb03..dc10474c4e 100644 --- a/libminifi/test/integration/OnScheduleErrorHandlingTests.cpp +++ b/libminifi/test/integration/OnScheduleErrorHandlingTests.cpp @@ -15,27 +15,27 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG -#include -#include "IntegrationBase.h" +#include "integration/IntegrationBase.h" #include "core/logging/Logger.h" #include "core/Scheduling.h" #include "core/state/ProcessorController.h" -#include "../TestBase.h" +#include "unit/TestBase.h" #include "../../../extensions/test-processors/KamikazeProcessor.h" #include "utils/StringUtils.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" +#include "unit/Catch.h" + +namespace org::apache::nifi::minifi::test { /*Verify behavior in case exceptions are thrown in onSchedule or onTrigger functions * KamikazeProcessor is a test processor to trigger errors in these functions */ class KamikazeErrorHandlingTests : public IntegrationBase { public: void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; - assert(verifyEventHappenedInPollTime(wait_time_, [&] { + using minifi::test::utils::verifyEventHappenedInPollTime; + REQUIRE(verifyEventHappenedInPollTime(wait_time_, [&] { const std::string logs = LogTestController::getInstance().getLogs(); - const auto result = utils::string::countOccurrences(logs, minifi::processors::KamikazeProcessor::OnScheduleExceptionStr); + const auto result = minifi::utils::string::countOccurrences(logs, minifi::processors::KamikazeProcessor::OnScheduleExceptionStr); const int occurrences = result.second; return 1 < occurrences; })); @@ -48,7 +48,7 @@ class KamikazeErrorHandlingTests : public IntegrationBase { const bool test_success = verifyEventHappenedInPollTime(std::chrono::milliseconds(wait_time_), [&] { const std::string logs = LogTestController::getInstance().getLogs(); - const auto result = utils::string::countOccurrences(logs, minifi::processors::KamikazeProcessor::OnScheduleExceptionStr); + const auto result = minifi::utils::string::countOccurrences(logs, minifi::processors::KamikazeProcessor::OnScheduleExceptionStr); size_t last_pos = result.first; for (const std::string& msg : must_appear_byorder_msgs) { last_pos = logs.find(msg, last_pos); @@ -58,9 +58,9 @@ class KamikazeErrorHandlingTests : public IntegrationBase { } return true; }); - assert(test_success); + REQUIRE(test_success); - assert(LogTestController::getInstance().getLogs().find(minifi::processors::KamikazeProcessor::OnTriggerLogStr) == std::string::npos); + REQUIRE(LogTestController::getInstance().getLogs().find(minifi::processors::KamikazeProcessor::OnTriggerLogStr) == std::string::npos); } void testSetup() override { @@ -84,28 +84,28 @@ class EventDriverScheduleErrorHandlingTests: public IntegrationBase { * that most probably means a breaking change. */ size_t controllerVecIdx = 0; - fc.executeOnAllComponents([&controllerVecIdx](org::apache::nifi::minifi::state::StateController& component){ + fc.executeOnAllComponents([&controllerVecIdx](minifi::state::StateController& component){ if (controllerVecIdx == 1) { - assert(component.getComponentName() == "FlowController"); + REQUIRE(component.getComponentName() == "FlowController"); } else if (controllerVecIdx == 0) { - assert(component.getComponentName() == "kamikaze"); + REQUIRE(component.getComponentName() == "kamikaze"); - auto process_controller = dynamic_cast(&component); - assert(process_controller != nullptr); + auto process_controller = dynamic_cast(&component); + REQUIRE(process_controller != nullptr); - process_controller->getProcessor().setSchedulingStrategy(org::apache::nifi::minifi::core::SchedulingStrategy::EVENT_DRIVEN); + process_controller->getProcessor().setSchedulingStrategy(minifi::core::SchedulingStrategy::EVENT_DRIVEN); } ++controllerVecIdx; }); // check controller vector size - assert(controllerVecIdx == 2); + REQUIRE(controllerVecIdx == 2); } void runAssertions() override { std::string logs = LogTestController::getInstance().getLogs(); - assert(logs.find("EventDrivenSchedulingAgent cannot schedule processor without incoming connection!") != std::string::npos); + REQUIRE(logs.find("EventDrivenSchedulingAgent cannot schedule processor without incoming connection!") != std::string::npos); } void testSetup() override { @@ -114,19 +114,18 @@ class EventDriverScheduleErrorHandlingTests: public IntegrationBase { } }; -int main(int argc, char **argv) { - std::string test_file_location; - std::string url; - if (argc > 1) { - test_file_location = argv[1]; - } - +TEST_CASE("KamikazeErrorHandlingTests", "[OnScheduleErrorHandlingTests]") { KamikazeErrorHandlingTests harness_kamikaze; + const auto test_file_location = std::filesystem::path(TEST_RESOURCES) / "TestOnScheduleRetry.yml"; harness_kamikaze.run(test_file_location); +} +TEST_CASE("EventDriverScheduleErrorHandlingTests", "[OnScheduleErrorHandlingTests]") { EventDriverScheduleErrorHandlingTests harness_eventdrivenerror; - harness_eventdrivenerror.run(test_file_location); - return 0; + const auto test_file_location = std::filesystem::path(TEST_RESOURCES) / "TestOnScheduleRetry.yml"; + harness_eventdrivenerror.run(test_file_location); } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/SiteToSiteRestTest.cpp b/libminifi/test/integration/SiteToSiteRestTest.cpp similarity index 71% rename from extensions/http-curl/tests/SiteToSiteRestTest.cpp rename to libminifi/test/integration/SiteToSiteRestTest.cpp index b24880b8f0..209571dfba 100644 --- a/extensions/http-curl/tests/SiteToSiteRestTest.cpp +++ b/libminifi/test/integration/SiteToSiteRestTest.cpp @@ -15,22 +15,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG -#include #include #include #include -#include "InvokeHTTP.h" -#include "TestBase.h" +#include "processors/InvokeHTTP.h" +#include "unit/TestBase.h" #include "core/logging/Logger.h" #include "FlowController.h" #include "CivetServer.h" #include "RemoteProcessorGroupPort.h" #include "core/ConfigurableComponent.h" #include "controllers/SSLContextService.h" -#include "HTTPIntegrationBase.h" -#include "utils/IntegrationTestUtils.h" +#include "integration/HTTPIntegrationBase.h" +#include "unit/TestUtils.h" + +namespace org::apache::nifi::minifi::test { class Responder : public ServerAwareHandler { public: @@ -69,7 +68,7 @@ class SiteToSiteTestHarness : public HTTPIntegrationBase { void testSetup() override { LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setDebug(); + LogTestController::getInstance().setDebug(); LogTestController::getInstance().setTrace(); LogTestController::getInstance().setInfo(); LogTestController::getInstance().setDebug(); @@ -87,13 +86,13 @@ class SiteToSiteTestHarness : public HTTPIntegrationBase { } void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using minifi::test::utils::verifyLogLinePresenceInPollTime; if (isSecure) { - assert(verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), "process group remote site2site port 10001, is secure true")); + REQUIRE(verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), "process group remote site2site port 10001, is secure true")); } else { - assert(verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), "process group remote site2site port 10001, is secure false")); + REQUIRE(verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), "process group remote site2site port 10001, is secure false")); } - assert(verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), "ProcessGroup::refreshRemoteSite2SiteInfo -- curl_easy_perform() failed ")); + REQUIRE(verifyLogLinePresenceInPollTime(std::chrono::milliseconds(wait_time_), "ProcessGroup::refreshRemoteSite2SiteInfo -- curl_easy_perform() failed ")); } protected: @@ -103,15 +102,13 @@ class SiteToSiteTestHarness : public HTTPIntegrationBase { TestController test_controller_; }; -int main(int argc, char **argv) { - const cmd_args args = parse_cmdline_args_with_url(argc, argv); - const bool isSecure = args.isUrlSecure(); - - SiteToSiteTestHarness harness(isSecure); - Responder responder(isSecure); - harness.setKeyDir(args.key_dir); - harness.setUrl(args.url, &responder); - harness.run(args.test_file); - - return 0; +TEST_CASE("Test site to site using REST", "[s2s]") { + SiteToSiteTestHarness harness(false); + Responder responder(false); + harness.setKeyDir(TEST_RESOURCES); + harness.setUrl(parseUrl("http://localhost:8077/nifi-api/site-to-site"), &responder); + const auto test_file_location = std::filesystem::path(TEST_RESOURCES) / "TestSite2SiteRest.yml"; + harness.run(test_file_location); } + +} // namespace org::apache::nifi::minifi::test diff --git a/libminifi/test/integration/StateTransactionalityTests.cpp b/libminifi/test/integration/StateTransactionalityTests.cpp index e51df99433..b540135eeb 100644 --- a/libminifi/test/integration/StateTransactionalityTests.cpp +++ b/libminifi/test/integration/StateTransactionalityTests.cpp @@ -14,20 +14,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#undef NDEBUG - #include -#include "IntegrationBase.h" -#include "../StatefulProcessor.h" -#include "../TestBase.h" -#include "utils/IntegrationTestUtils.h" +#include "integration/IntegrationBase.h" +#include "unit/StatefulProcessor.h" +#include "unit/TestBase.h" +#include "unit/TestUtils.h" #include "core/state/ProcessorController.h" +#include "unit/Catch.h" -using org::apache::nifi::minifi::processors::StatefulProcessor; -using org::apache::nifi::minifi::state::ProcessorController; +namespace org::apache::nifi::minifi::test { -namespace { +using minifi::processors::StatefulProcessor; +using minifi::state::ProcessorController; using LogChecker = std::function; struct HookCollection { @@ -63,16 +61,16 @@ class StatefulIntegrationTest : public IntegrationBase { * that most probably means a breaking change. */ size_t controllerVecIdx = 0; - fc.executeOnAllComponents([this, &controllerVecIdx](org::apache::nifi::minifi::state::StateController& component){ + fc.executeOnAllComponents([this, &controllerVecIdx](minifi::state::StateController& component){ if (controllerVecIdx == 1) { - assert(component.getComponentName() == "FlowController"); + REQUIRE(component.getComponentName() == "FlowController"); } else if (controllerVecIdx == 0) { - assert(component.getComponentName() == "statefulProcessor"); + REQUIRE(component.getComponentName() == "statefulProcessor"); // set hooks const auto processController = dynamic_cast(&component); - assert(processController != nullptr); + REQUIRE(processController != nullptr); stateful_processor_ = dynamic_cast(&processController->getProcessor()); - assert(stateful_processor_); + REQUIRE(stateful_processor_); stateful_processor_->setHooks(on_schedule_hook_, on_trigger_hooks_); } @@ -80,12 +78,11 @@ class StatefulIntegrationTest : public IntegrationBase { }); // check controller vector size - assert(controllerVecIdx == 2); + REQUIRE(controllerVecIdx == 2); } void runAssertions() override { - using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; - assert(verifyEventHappenedInPollTime(std::chrono::milliseconds(wait_time_), [&] { + REQUIRE(utils::verifyEventHappenedInPollTime(std::chrono::milliseconds(wait_time_), [&] { return stateful_processor_->hasFinishedHooks() && log_checker_(); })); } @@ -104,16 +101,16 @@ const std::unordered_map exampleState2{{"key3", "value auto standardLogChecker = [] { const std::string logs = LogTestController::getInstance().getLogs(); - const auto errorResult = utils::string::countOccurrences(logs, "[error]"); - const auto warningResult = utils::string::countOccurrences(logs, "[warning]"); + const auto errorResult = minifi::utils::string::countOccurrences(logs, "[error]"); + const auto warningResult = minifi::utils::string::countOccurrences(logs, "[warning]"); return errorResult.second == 0 && warningResult.second == 0; }; auto exceptionRollbackWarnings = [] { const std::string logs = LogTestController::getInstance().getLogs(); - const auto errorResult = utils::string::countOccurrences(logs, "[error]"); - const auto exceptionWarningResult = utils::string::countOccurrences(logs, "[warning] Caught \"Triggering rollback\""); - const auto rollbackWarningResult = utils::string::countOccurrences(logs, "[warning] ProcessSession rollback for statefulProcessor executed"); + const auto errorResult = minifi::utils::string::countOccurrences(logs, "[error]"); + const auto exceptionWarningResult = minifi::utils::string::countOccurrences(logs, "[warning] Caught \"Triggering rollback\""); + const auto rollbackWarningResult = minifi::utils::string::countOccurrences(logs, "[warning] ProcessSession rollback for statefulProcessor executed"); return errorResult.second == 0 && exceptionWarningResult.second == 1 && rollbackWarningResult.second == 1; }; @@ -122,12 +119,12 @@ const std::unordered_map testCasesToHookLists { {}, { [] (core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); + REQUIRE(stateManager.set(exampleState)); }, [] (core::StateManager& stateManager) { std::unordered_map state; - assert(stateManager.get(state)); - assert(state == exampleState); + REQUIRE(stateManager.get(state)); + REQUIRE(state == exampleState); } }, standardLogChecker @@ -136,16 +133,16 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); + REQUIRE(stateManager.set(exampleState)); }, [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState2)); + REQUIRE(stateManager.set(exampleState2)); throw std::runtime_error("Triggering rollback"); }, [](core::StateManager& stateManager) { std::unordered_map state; - assert(stateManager.get(state)); - assert(state == exampleState); + REQUIRE(stateManager.get(state)); + REQUIRE(state == exampleState); } }, exceptionRollbackWarnings @@ -154,8 +151,8 @@ const std::unordered_map testCasesToHookLists { "Get_in_onSchedule_without_previous_state", { [](core::StateManager& stateManager) { std::unordered_map state; - assert(!stateManager.get(state)); - assert(state.empty()); + REQUIRE(!stateManager.get(state)); + REQUIRE(state.empty()); }, {}, standardLogChecker @@ -164,13 +161,13 @@ const std::unordered_map testCasesToHookLists { { "Set_in_onSchedule", { [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); + REQUIRE(stateManager.set(exampleState)); }, { [](core::StateManager& stateManager) { std::unordered_map state; - assert(stateManager.get(state)); - assert(state == exampleState); + REQUIRE(stateManager.get(state)); + REQUIRE(state == exampleState); } }, standardLogChecker @@ -179,15 +176,15 @@ const std::unordered_map testCasesToHookLists { { "Clear_in_onSchedule", { [](core::StateManager& stateManager) { - assert(!stateManager.clear()); - assert(stateManager.set(exampleState)); - assert(stateManager.clear()); + REQUIRE(!stateManager.clear()); + REQUIRE(stateManager.set(exampleState)); + REQUIRE(stateManager.clear()); }, { [](core::StateManager& stateManager) { std::unordered_map state; - assert(!stateManager.get(state)); - assert(state.empty()); + REQUIRE(!stateManager.get(state)); + REQUIRE(state.empty()); } }, standardLogChecker @@ -197,7 +194,7 @@ const std::unordered_map testCasesToHookLists { "Persist_in_onSchedule", { { [](core::StateManager& stateManager) { - assert(stateManager.persist()); + REQUIRE(stateManager.persist()); } }, {}, @@ -209,7 +206,7 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(!stateManager.beginTransaction()); + REQUIRE(!stateManager.beginTransaction()); } }, standardLogChecker @@ -220,8 +217,8 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); - assert(stateManager.commit()); + REQUIRE(stateManager.set(exampleState)); + REQUIRE(stateManager.commit()); } }, standardLogChecker @@ -232,7 +229,7 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.rollback()); + REQUIRE(stateManager.rollback()); } }, standardLogChecker @@ -244,8 +241,8 @@ const std::unordered_map testCasesToHookLists { { [](core::StateManager& stateManager) { std::unordered_map state; - assert(!stateManager.get(state)); - assert(state.empty()); + REQUIRE(!stateManager.get(state)); + REQUIRE(state.empty()); } }, standardLogChecker @@ -256,14 +253,14 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); + REQUIRE(stateManager.set(exampleState)); }, [](core::StateManager& stateManager) { std::unordered_map state; - assert(stateManager.get(state)); - assert(state == exampleState); - assert(stateManager.get(state)); - assert(state == exampleState); + REQUIRE(stateManager.get(state)); + REQUIRE(state == exampleState); + REQUIRE(stateManager.get(state)); + REQUIRE(state == exampleState); } }, standardLogChecker @@ -274,13 +271,13 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); + REQUIRE(stateManager.set(exampleState)); }, [](core::StateManager& stateManager) { std::unordered_map state; - assert(stateManager.get(state)); - assert(state == exampleState); - assert(stateManager.set(exampleState)); + REQUIRE(stateManager.get(state)); + REQUIRE(state == exampleState); + REQUIRE(stateManager.set(exampleState)); } }, standardLogChecker @@ -291,13 +288,13 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); + REQUIRE(stateManager.set(exampleState)); }, [](core::StateManager& stateManager) { std::unordered_map state; - assert(stateManager.get(state)); - assert(state == exampleState); - assert(stateManager.clear()); + REQUIRE(stateManager.get(state)); + REQUIRE(state == exampleState); + REQUIRE(stateManager.clear()); } }, standardLogChecker @@ -308,13 +305,13 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); + REQUIRE(stateManager.set(exampleState)); }, [](core::StateManager& stateManager) { std::unordered_map state; - assert(stateManager.get(state)); - assert(state == exampleState); - assert(stateManager.persist()); + REQUIRE(stateManager.get(state)); + REQUIRE(state == exampleState); + REQUIRE(stateManager.persist()); } }, standardLogChecker @@ -325,10 +322,10 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); + REQUIRE(stateManager.set(exampleState)); std::unordered_map state; - assert(!stateManager.get(state)); - assert(state.empty()); + REQUIRE(!stateManager.get(state)); + REQUIRE(state.empty()); }, }, standardLogChecker @@ -339,8 +336,8 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); - assert(stateManager.set(exampleState)); + REQUIRE(stateManager.set(exampleState)); + REQUIRE(stateManager.set(exampleState)); }, }, standardLogChecker @@ -351,8 +348,8 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); - assert(!stateManager.clear()); + REQUIRE(stateManager.set(exampleState)); + REQUIRE(!stateManager.clear()); }, }, standardLogChecker @@ -363,8 +360,8 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); - assert(stateManager.persist()); + REQUIRE(stateManager.set(exampleState)); + REQUIRE(stateManager.persist()); }, }, standardLogChecker @@ -375,13 +372,13 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); + REQUIRE(stateManager.set(exampleState)); }, [](core::StateManager& stateManager) { - assert(stateManager.clear()); + REQUIRE(stateManager.clear()); std::unordered_map state; - assert(!stateManager.get(state)); - assert(state.empty()); + REQUIRE(!stateManager.get(state)); + REQUIRE(state.empty()); } }, standardLogChecker @@ -392,23 +389,23 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); + REQUIRE(stateManager.set(exampleState)); }, [](core::StateManager& stateManager) { - assert(stateManager.clear()); + REQUIRE(stateManager.clear()); }, [](core::StateManager& stateManager) { std::unordered_map state; - assert(!stateManager.get(state)); - assert(state.empty()); + REQUIRE(!stateManager.get(state)); + REQUIRE(state.empty()); }, [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); + REQUIRE(stateManager.set(exampleState)); }, [](core::StateManager& stateManager) { std::unordered_map state; - assert(stateManager.get(state)); - assert(state == exampleState); + REQUIRE(stateManager.get(state)); + REQUIRE(state == exampleState); } }, standardLogChecker @@ -419,11 +416,11 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); + REQUIRE(stateManager.set(exampleState)); }, [](core::StateManager& stateManager) { - assert(stateManager.clear()); - assert(stateManager.set(exampleState)); + REQUIRE(stateManager.clear()); + REQUIRE(stateManager.set(exampleState)); }, }, standardLogChecker @@ -434,13 +431,13 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); + REQUIRE(stateManager.set(exampleState)); }, [](core::StateManager& stateManager) { - assert(stateManager.clear()); + REQUIRE(stateManager.clear()); }, [](core::StateManager& stateManager) { - assert(!stateManager.clear()); + REQUIRE(!stateManager.clear()); }, }, standardLogChecker @@ -451,13 +448,13 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); + REQUIRE(stateManager.set(exampleState)); }, [](core::StateManager& stateManager) { - assert(stateManager.clear()); + REQUIRE(stateManager.clear()); }, [](core::StateManager& stateManager) { - assert(stateManager.persist()); + REQUIRE(stateManager.persist()); }, }, standardLogChecker @@ -468,15 +465,15 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.persist()); + REQUIRE(stateManager.persist()); }, [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); + REQUIRE(stateManager.set(exampleState)); }, [](core::StateManager& stateManager) { std::unordered_map state; - assert(stateManager.get(state)); - assert(state == exampleState); + REQUIRE(stateManager.get(state)); + REQUIRE(state == exampleState); }, }, standardLogChecker @@ -487,13 +484,13 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.persist()); + REQUIRE(stateManager.persist()); }, [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); + REQUIRE(stateManager.set(exampleState)); }, [](core::StateManager& stateManager) { - assert(stateManager.clear()); + REQUIRE(stateManager.clear()); }, }, standardLogChecker @@ -504,10 +501,10 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.persist()); + REQUIRE(stateManager.persist()); }, [](core::StateManager& stateManager) { - assert(stateManager.persist()); + REQUIRE(stateManager.persist()); }, }, standardLogChecker @@ -528,7 +525,7 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(!stateManager.clear()); + REQUIRE(!stateManager.clear()); }, }, standardLogChecker @@ -539,12 +536,12 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); + REQUIRE(stateManager.set(exampleState)); }, [](core::StateManager& stateManager) { std::unordered_map state; - assert(stateManager.get(state)); - assert(state == exampleState); + REQUIRE(stateManager.get(state)); + REQUIRE(state == exampleState); throw std::runtime_error("Triggering rollback"); }, }, @@ -556,16 +553,16 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.set(exampleState)); + REQUIRE(stateManager.set(exampleState)); }, [](core::StateManager& stateManager) { - assert(stateManager.clear()); + REQUIRE(stateManager.clear()); throw std::runtime_error("Triggering rollback"); }, [](core::StateManager& stateManager) { std::unordered_map state; - assert(stateManager.get(state)); - assert(state == exampleState); + REQUIRE(stateManager.get(state)); + REQUIRE(state == exampleState); }, }, exceptionRollbackWarnings @@ -576,7 +573,7 @@ const std::unordered_map testCasesToHookLists { {}, { [](core::StateManager& stateManager) { - assert(stateManager.persist()); + REQUIRE(stateManager.persist()); throw std::runtime_error("Triggering rollback"); }, }, @@ -584,34 +581,13 @@ const std::unordered_map testCasesToHookLists { } } }; -} // namespace - - -int main(int argc, char** argv) { - if (argc < 2) { - std::cerr << "A test file (*.yml) argument is mandatory, a second argument for test case name is optional\n"; - return EXIT_FAILURE; - } - const std::string testFile = argv[1]; - if (argc == 2) { - // run all tests - for (const auto& test : testCasesToHookLists) { - StatefulIntegrationTest statefulIntegrationTest(test.first, test.second); - statefulIntegrationTest.run(testFile); - } - } else if (argc == 3) { - // run specified test case - const std::string testCase = argv[2]; - auto iter = testCasesToHookLists.find(testCase); - if (iter == testCasesToHookLists.end()) { - std::cerr << "Test case \"" << testCase << "\" cannot be found\n"; - return EXIT_FAILURE; - } - StatefulIntegrationTest statefulIntegrationTest(iter->first, iter->second); - statefulIntegrationTest.run(testFile); - } else { - std::cerr << "Too many arguments\n"; - return EXIT_FAILURE; +TEST_CASE("Test state transactionality", "[statemanagement]") { + for (const auto& test : testCasesToHookLists) { + StatefulIntegrationTest statefulIntegrationTest(test.first, test.second); + const auto test_file_location = std::filesystem::path(TEST_RESOURCES) / "TestStateTransactionality.yml"; + statefulIntegrationTest.run(test_file_location); } } + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/TimeoutHTTPSiteToSiteTests.cpp b/libminifi/test/integration/TimeoutHTTPSiteToSiteTests.cpp similarity index 77% rename from extensions/http-curl/tests/TimeoutHTTPSiteToSiteTests.cpp rename to libminifi/test/integration/TimeoutHTTPSiteToSiteTests.cpp index 25b7ffefe7..2012bd9d16 100644 --- a/extensions/http-curl/tests/TimeoutHTTPSiteToSiteTests.cpp +++ b/libminifi/test/integration/TimeoutHTTPSiteToSiteTests.cpp @@ -15,28 +15,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - #include -#undef NDEBUG -#include #include #include #include #include #include #include "sitetosite/HTTPProtocol.h" -#include "InvokeHTTP.h" -#include "TestBase.h" +#include "processors/InvokeHTTP.h" +#include "unit/TestBase.h" #include "FlowController.h" #include "RemoteProcessorGroupPort.h" #include "core/ConfigurableComponent.h" -#include "HTTPIntegrationBase.h" -#include "HTTPHandlers.h" -#include "client/HTTPStream.h" +#include "integration/HTTPIntegrationBase.h" +#include "integration/HTTPHandlers.h" +#include "http/HTTPStream.h" #include "properties/Configuration.h" +#include "unit/Catch.h" using namespace std::literals::chrono_literals; +namespace org::apache::nifi::minifi::test { + class SiteToSiteTestHarness : public HTTPIntegrationBase { public: explicit SiteToSiteTestHarness(bool isSecure, std::chrono::seconds waitTime = 1s) @@ -46,20 +46,20 @@ class SiteToSiteTestHarness : public HTTPIntegrationBase { void testSetup() override { LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); LogTestController::getInstance().setTrace(); LogTestController::getInstance().setInfo(); LogTestController::getInstance().setDebug(); - LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); std::fstream file; file.open(dir / "tstFile.ext", std::ios::out); file << "tempFile"; file.close(); - configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_enable, "false"); + configuration->set(minifi::Configuration::nifi_c2_enable, "false"); } void runAssertions() override { @@ -137,48 +137,37 @@ void run_timeout_variance(std::string test_file_location, bool isSecure, const s harness.run(test_file_location); - assert(LogTestController::getInstance().contains("limit (200ms) reached, terminating connection")); + REQUIRE(LogTestController::getInstance().contains("limit (200ms) reached, terminating connection")); LogTestController::getInstance().reset(); } -int main(int argc, char **argv) { - transaction_id = 0; - transaction_id_output = 0; - const cmd_args args = parse_cmdline_args_with_url(argc, argv); - const bool isSecure = args.isUrlSecure(); - +TEST_CASE("Test timeout handling in HTTP site to site", "[s2s]") { + timeout_test_profile profile; const auto timeout = std::chrono::milliseconds{500}; - { - timeout_test_profile profile; + SECTION("Test base responder") { profile.base_.set({timeout}); - run_timeout_variance(args.test_file, isSecure, args.url, profile); } - { - timeout_test_profile profile; + SECTION("Test flow file responder") { profile.flow_.set({timeout}); - run_timeout_variance(args.test_file, isSecure, args.url, profile); } - { - timeout_test_profile profile; + SECTION("Test transaction responder") { profile.transaction_.set({timeout}); - run_timeout_variance(args.test_file, isSecure, args.url, profile); } - { - timeout_test_profile profile; + SECTION("Test delete transaction responder") { profile.delete_.set({timeout}); - run_timeout_variance(args.test_file, isSecure, args.url, profile); } - { - timeout_test_profile profile; + SECTION("Test peer responder") { profile.peer_.set({timeout}); - run_timeout_variance(args.test_file, isSecure, args.url, profile); } - return 0; + const auto test_file_location = std::filesystem::path(TEST_RESOURCES) / "TestTimeoutHTTPSiteToSite.yml"; + run_timeout_variance(test_file_location.string(), false, parseUrl("http://localhost:8098/nifi-api"), profile); } + +} // namespace org::apache::nifi::minifi::test diff --git a/libminifi/test/keyvalue-tests/CMakeLists.txt b/libminifi/test/keyvalue-tests/CMakeLists.txt index 2225f3a34d..e954e85ea2 100644 --- a/libminifi/test/keyvalue-tests/CMakeLists.txt +++ b/libminifi/test/keyvalue-tests/CMakeLists.txt @@ -24,9 +24,8 @@ SET(KEYVALUE_INT_TEST_COUNT 0) FOREACH(testfile ${KEYVALUE_TESTS}) get_filename_component(testfilename "${testfile}" NAME_WE) add_minifi_executable("${testfilename}" "${testfile}") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test/") createTests("${testfilename}") - target_link_libraries(${testfilename} minifi-standard-processors Catch2) + target_link_libraries(${testfilename} minifi-standard-processors Catch2::Catch2WithMain) if (ENABLE_ROCKSDB) target_link_libraries(${testfilename} minifi-rocksdb-repos) endif() diff --git a/libminifi/test/keyvalue-tests/PersistentStateStorageTest.cpp b/libminifi/test/keyvalue-tests/PersistentStateStorageTest.cpp index 1243f2ace9..4b0a873a19 100644 --- a/libminifi/test/keyvalue-tests/PersistentStateStorageTest.cpp +++ b/libminifi/test/keyvalue-tests/PersistentStateStorageTest.cpp @@ -21,10 +21,8 @@ #include #include -#include "Catch.h" - -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/Catch.h" +#include "unit/TestBase.h" #include "catch2/catch_session.hpp" #include "core/controller/ControllerService.h" #include "core/ProcessGroup.h" diff --git a/libminifi/test/keyvalue-tests/VolatileMapStateStorageTest.cpp b/libminifi/test/keyvalue-tests/VolatileMapStateStorageTest.cpp index 58c6852cc9..0e178ecee6 100644 --- a/libminifi/test/keyvalue-tests/VolatileMapStateStorageTest.cpp +++ b/libminifi/test/keyvalue-tests/VolatileMapStateStorageTest.cpp @@ -19,8 +19,8 @@ #define CATCH_CONFIG_RUNNER #include #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "catch2/catch_session.hpp" #include "controllers/keyvalue/AutoPersistor.h" #include "controllers/keyvalue/KeyValueStateStorage.h" diff --git a/libminifi/test/libtest/CMakeLists.txt b/libminifi/test/libtest/CMakeLists.txt new file mode 100644 index 0000000000..95ab353f73 --- /dev/null +++ b/libminifi/test/libtest/CMakeLists.txt @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +add_subdirectory(unit) +add_subdirectory(integration) diff --git a/libminifi/test/libtest/integration/CMakeLists.txt b/libminifi/test/libtest/integration/CMakeLists.txt new file mode 100644 index 0000000000..e184e85cf8 --- /dev/null +++ b/libminifi/test/libtest/integration/CMakeLists.txt @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +file(GLOB SOURCES "*.cpp") +add_minifi_library(libminifi-integrationtest STATIC ${SOURCES}) +target_link_libraries(libminifi-integrationtest libminifi-unittest minifi-civet-extensions) +target_include_directories(libminifi-integrationtest BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/test/libtest/") +target_include_directories(libminifi-integrationtest BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/civetweb/") diff --git a/extensions/http-curl/tests/CivetLibrary.h b/libminifi/test/libtest/integration/CivetLibrary.h similarity index 100% rename from extensions/http-curl/tests/CivetLibrary.h rename to libminifi/test/libtest/integration/CivetLibrary.h diff --git a/extensions/http-curl/tests/CivetStream.h b/libminifi/test/libtest/integration/CivetStream.h similarity index 86% rename from extensions/http-curl/tests/CivetStream.h rename to libminifi/test/libtest/integration/CivetStream.h index 94acdd71a3..0aa764a508 100644 --- a/extensions/http-curl/tests/CivetStream.h +++ b/libminifi/test/libtest/integration/CivetStream.h @@ -22,11 +22,7 @@ #include "civetweb.h" #include "utils/gsl.h" -namespace org { -namespace apache { -namespace nifi { -namespace minifi { -namespace io { +namespace org::apache::nifi::minifi::io { class CivetStream : public io::InputStream { public: @@ -53,10 +49,6 @@ class CivetStream : public io::InputStream { struct mg_connection *conn; private: - std::shared_ptr logger_; + std::shared_ptr logger_; }; -} /* namespace io */ -} /* namespace minifi */ -} /* namespace nifi */ -} /* namespace apache */ -} /* namespace org */ +} // namespace org::apache::nifi::minifi::io diff --git a/libminifi/test/libtest/integration/ConnectionCountingServer.cpp b/libminifi/test/libtest/integration/ConnectionCountingServer.cpp new file mode 100644 index 0000000000..41ac99dca8 --- /dev/null +++ b/libminifi/test/libtest/integration/ConnectionCountingServer.cpp @@ -0,0 +1,111 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "ConnectionCountingServer.h" + +#include + +#include "utils/Id.h" + +namespace org::apache::nifi::minifi::test { + +namespace details { + +bool NumberedMethodResponder::handleGet(CivetServer*, struct mg_connection* conn) { + sendNumberedMessage("GET", conn); + return true; +} + +bool NumberedMethodResponder::handlePost(CivetServer*, struct mg_connection* conn) { + sendNumberedMessage("POST", conn); + return true; +} + +bool NumberedMethodResponder::handlePut(CivetServer*, struct mg_connection* conn) { + sendNumberedMessage("PUT", conn); + return true; +} + +bool NumberedMethodResponder::handleHead(CivetServer*, struct mg_connection* conn) { + sendNumberedMessage("HEAD", conn); + return true; +} + +void NumberedMethodResponder::sendNumberedMessage(std::string body, struct mg_connection* conn) { + saveConnectionId(conn); + body.append(std::to_string(response_id_)); + mg_printf(conn, "HTTP/1.1 200 OK\r\n"); + mg_printf(conn, "Content-length: %lu\r\n", body.length()); + mg_printf(conn, "Response-number: %" PRIu64 "\r\n", response_id_); + mg_printf(conn, "\r\n"); + mg_printf(conn, body.data(), body.length()); + ++response_id_; +} + +void NumberedMethodResponder::saveConnectionId(struct mg_connection* conn) { + auto user_connection_data = reinterpret_cast*>(mg_get_user_connection_data(conn)); + assert(user_connection_data); + connections_.emplace(*user_connection_data); +} + +bool ReverseBodyPostHandler::handlePost(CivetServer* /*server*/, struct mg_connection* conn) { + saveConnectionId(conn); + std::vector request_body; + request_body.reserve(2048); + size_t read_size = mg_read(conn, request_body.data(), 2048); + assert(read_size < 2048); + std::string response_body{request_body.begin(), request_body.begin() + read_size}; + std::reverse(std::begin(response_body), std::end(response_body)); + mg_printf(conn, "HTTP/1.1 200 OK\r\n"); + mg_printf(conn, "Content-length: %zu\r\n", read_size); + mg_printf(conn, "\r\n"); + mg_printf(conn, response_body.data(), read_size); + + return true; +} + +void ReverseBodyPostHandler::saveConnectionId(struct mg_connection* conn) { + auto user_connection_data = reinterpret_cast*>(mg_get_user_connection_data(conn)); + connections_.emplace(*user_connection_data); +} + +AddIdToUserConnectionData::AddIdToUserConnectionData() { + init_connection = [](const struct mg_connection*, void** user_connection_data) -> int { + minifi::utils::SmallString<36>* id = new minifi::utils::SmallString<36>(minifi::utils::IdGenerator::getIdGenerator()->generate().to_string()); + *user_connection_data = reinterpret_cast(id); + return 0; + }; + + connection_close = [](const struct mg_connection* conn) -> void { + auto user_connection_data = reinterpret_cast*>(mg_get_user_connection_data(conn)); + delete user_connection_data; + }; +} +} // namespace details + +ConnectionCountingServer::ConnectionCountingServer() { + server_.addHandler("/method", numbered_method_responder_); + server_.addHandler("/reverse", reverse_body_post_handler_); +} + +std::string ConnectionCountingServer::getPort() { + const auto& listening_ports = server_.getListeningPorts(); + assert(!listening_ports.empty()); + return std::to_string(listening_ports[0]); +} + +} // namespace org::apache::nifi::minifi::test diff --git a/libminifi/test/libtest/integration/ConnectionCountingServer.h b/libminifi/test/libtest/integration/ConnectionCountingServer.h new file mode 100644 index 0000000000..76a5f4e673 --- /dev/null +++ b/libminifi/test/libtest/integration/ConnectionCountingServer.h @@ -0,0 +1,88 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include "CivetServer.h" +#include "utils/SmallString.h" + +namespace org::apache::nifi::minifi::test { + +namespace details { + +class NumberedMethodResponder : public CivetHandler { + public: + explicit NumberedMethodResponder(std::set>& connections) : connections_(connections) {} + + bool handleGet(CivetServer*, struct mg_connection* conn) override; + bool handlePost(CivetServer*, struct mg_connection* conn) override; + bool handlePut(CivetServer*, struct mg_connection* conn) override; + bool handleHead(CivetServer*, struct mg_connection* conn) override; + + private: + void sendNumberedMessage(std::string body, struct mg_connection* conn); + void saveConnectionId(struct mg_connection* conn); + + uint64_t response_id_ = 0; + std::set>& connections_; +}; + +class ReverseBodyPostHandler : public CivetHandler { + public: + explicit ReverseBodyPostHandler(std::set>& connections) : connections_(connections) {} + + bool handlePost(CivetServer* /*server*/, struct mg_connection* conn) override; + + private: + void saveConnectionId(struct mg_connection* conn); + + std::set>& connections_; +}; + +struct AddIdToUserConnectionData : public CivetCallbacks { + AddIdToUserConnectionData(); +}; +} // namespace details + +class ConnectionCountingServer { + public: + ConnectionCountingServer(); + + size_t getConnectionCounter() { return connections_.size(); } + + std::string getPort(); + + private: + static inline std::vector options = { + "enable_keep_alive", "yes", + "keep_alive_timeout_ms", "15000", + "num_threads", "1", + "listening_ports", "0"}; + + std::set> connections_; + details::AddIdToUserConnectionData add_id_to_user_connection_data_; + CivetServer server_{options, &add_id_to_user_connection_data_}; + details::ReverseBodyPostHandler reverse_body_post_handler_{connections_}; + details::NumberedMethodResponder numbered_method_responder_{connections_}; +}; + +} // namespace org::apache::nifi::minifi::test diff --git a/libminifi/test/libtest/integration/HTTPHandlers.cpp b/libminifi/test/libtest/integration/HTTPHandlers.cpp new file mode 100644 index 0000000000..eb357b2b99 --- /dev/null +++ b/libminifi/test/libtest/integration/HTTPHandlers.cpp @@ -0,0 +1,558 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "HTTPHandlers.h" + +#include + +#include "CivetStream.h" +#include "io/CRCStream.h" +#include "io/BufferStream.h" +#include "rapidjson/error/en.h" +#include "utils/gsl.h" +#include "agent/build_description.h" +#include "range/v3/algorithm/contains.hpp" +#include "range/v3/view/filter.hpp" +#include "range/v3/view/view.hpp" +#include "utils/net/DNS.h" + +namespace org::apache::nifi::minifi::test { + +bool SiteToSiteLocationResponder::handleGet(CivetServer* /*server*/, struct mg_connection *conn) { + std::string site2site_rest_resp = "{" + "\"revision\": {" + "\"clientId\": \"483d53eb-53ec-4e93-b4d4-1fc3d23dae6f\"" + "}," + "\"controller\": {" + "\"id\": \"fe4a3a42-53b6-4af1-a80d-6fdfe60de97f\"," + "\"name\": \"NiFi Flow\"," + "\"siteToSiteSecure\": "; + site2site_rest_resp += (isSecure ? "true" : "false"); + site2site_rest_resp += "}}"; + mg_printf(conn, "HTTP/1.1 200 OK\r\nContent-Type: " + "text/plain\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", + site2site_rest_resp.length()); + mg_printf(conn, "%s", site2site_rest_resp.c_str()); + return true; +} + +bool PeerResponder::handleGet(CivetServer* /*server*/, struct mg_connection *conn) { +#ifdef WIN32 + std::string hostname = org::apache::nifi::minifi::utils::net::getMyHostName(); +#else + std::string hostname = "localhost"; +#endif + std::string site2site_rest_resp = "{\"peers\" : [{ \"hostname\": \"" + hostname + "\", \"port\": " + port + ", \"secure\": false, \"flowFileCount\" : 0 }] }"; + std::stringstream headers; + headers << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << site2site_rest_resp.length() << "\r\nConnection: close\r\n\r\n"; + mg_printf(conn, "%s", headers.str().c_str()); + mg_printf(conn, "%s", site2site_rest_resp.c_str()); + return true; +} + +bool SiteToSiteBaseResponder::handleGet(CivetServer* /*server*/, struct mg_connection *conn) { + std::string site2site_rest_resp = + "{\"controller\":{\"id\":\"96dab149-0162-1000-7924-ed3122d6ea2b\",\"name\":\"NiFi Flow\",\"comments\":\"\",\"runningCount\":3,\"stoppedCount\":6,\"invalidCount\":1,\"disabledCount\":0,\"inputPortCount\":1,\"outputPortCount\":1,\"remoteSiteListeningPort\":10443,\"siteToSiteSecure\":false,\"instanceId\":\"13881505-0167-1000-be72-aa29341a3e9a\",\"inputPorts\":[{\"id\":\"471deef6-2a6e-4a7d-912a-81cc17e3a204\",\"name\":\"RPGIN\",\"comments\":\"\",\"state\":\"RUNNING\"}],\"outputPorts\":[{\"id\":\"9cf15a63-0166-1000-1b29-027406d96013\",\"name\":\"ddsga\",\"comments\":\"\",\"state\":\"STOPPED\"}]}}"; // NOLINT line length + std::stringstream headers; + headers << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << site2site_rest_resp.length() << "\r\nConnection: close\r\n\r\n"; + mg_printf(conn, "%s", headers.str().c_str()); + mg_printf(conn, "%s", site2site_rest_resp.c_str()); + return true; +} + +TransactionResponder::TransactionResponder(std::string base_url, std::string port_id, bool input_port, bool wrong_uri, bool empty_transaction_uri) + : base_url(std::move(base_url)), + wrong_uri(wrong_uri), + empty_transaction_uri(empty_transaction_uri), + input_port(input_port), + port_id(std::move(port_id)), + flow_files_feed_(nullptr) { + if (input_port) { + transaction_id_str = "fe4a3a42-53b6-4af1-a80d-6fdfe60de96"; + transaction_id_str += std::to_string(transaction_id.load()); + transaction_id++; + } else { + transaction_id_str = "fe4a3a42-53b6-4af1-a80d-6fdfe60de95"; + transaction_id_str += std::to_string(transaction_id_output.load()); + transaction_id_output++; + } +} + +bool TransactionResponder::handlePost(CivetServer* /*server*/, struct mg_connection *conn) { + std::string site2site_rest_resp; + std::stringstream headers; + headers << "HTTP/1.1 201 OK\r\nContent-Type: application/json\r\nContent-Length: " << site2site_rest_resp.length() << "\r\nX-Location-Uri-Intent: "; + if (wrong_uri) + headers << "ohstuff\r\n"; + else + headers << "transaction-url\r\n"; + + std::string port_type; + + if (input_port) + port_type = "input-ports"; + else + port_type = "output-ports"; + if (!empty_transaction_uri) + headers << "locAtion: " << base_url << "/site-to-site/" << port_type << "/" << port_id << "/transactions/" << transaction_id_str << "\r\n"; + headers << "Connection: close\r\n\r\n"; + mg_printf(conn, "%s", headers.str().c_str()); + mg_printf(conn, "%s", site2site_rest_resp.c_str()); + return true; +} + +bool FlowFileResponder::handlePost(CivetServer* /*server*/, struct mg_connection *conn) { + std::string site2site_rest_resp; + std::stringstream headers; + + if (!wrong_uri) { + minifi::io::CivetStream civet_stream(conn); + minifi::io::CRCStream < minifi::io::CivetStream > stream(gsl::make_not_null(&civet_stream)); + uint32_t num_attributes = 0; + uint64_t total_size = 0; + { + const auto read = stream.read(num_attributes); + if (!isServerRunning()) return false; + REQUIRE(read > 0); + total_size += read; + } + + const auto flow = std::make_shared(); + + for (uint32_t i = 0; i < num_attributes; i++) { + std::string name, value; + { + const auto read = stream.read(name, true); + if (!isServerRunning()) return false; + REQUIRE(read > 0); + total_size += read; + } + { + const auto read = stream.read(value, true); + if (!isServerRunning()) return false; + REQUIRE(read > 0); + total_size += read; + } + flow->attributes[name] = value; + } + uint64_t length{}; + { + const auto read = stream.read(length); + if (!isServerRunning()) return false; + REQUIRE(read > 0); + total_size += read; + } + + total_size += length; + flow->data.resize(gsl::narrow(length)); + flow->total_size = total_size; + + { + const auto read = stream.read(flow->data); + if (!isServerRunning()) return false; + (void)read; + REQUIRE(read == length); + } + + if (!invalid_checksum) { + site2site_rest_resp = std::to_string(stream.getCRC()); + flow_files_.enqueue(flow); + } else { + site2site_rest_resp = "Imawrongchecksumshortandstout"; + } + + headers << "HTTP/1.1 202 OK\r\nContent-Type: application/json\r\nContent-Length: " << site2site_rest_resp.length() << "\r\nConnection: close\r\n\r\n"; + } else { + headers << "HTTP/1.1 404\r\nConnection: close\r\n\r\n"; + } + + mg_printf(conn, "%s", headers.str().c_str()); + mg_printf(conn, "%s", site2site_rest_resp.c_str()); + return true; +} + +bool FlowFileResponder::handleGet(CivetServer* /*server*/, struct mg_connection *conn) { + if (flow_files_feed_->size_approx() > 0) { + std::shared_ptr flowobj; + std::vector> flows; + uint64_t total = 0; + + while (flow_files_feed_->try_dequeue(flowobj)) { + flows.push_back(flowobj); + total += flowobj->total_size; + } + mg_printf(conn, + "HTTP/1.1 200 OK\r\n" + "Content-Length: %" PRIu64 "\r\n" + "Content-Type: application/octet-stream\r\n" + "Connection: close\r\n\r\n", + total); + minifi::io::BufferStream serializer; + minifi::io::CRCStream stream(gsl::make_not_null(&serializer)); + for (const auto& flow : flows) { + uint32_t num_attributes = gsl::narrow(flow->attributes.size()); + stream.write(num_attributes); + for (const auto& entry : flow->attributes) { + stream.write(entry.first); + stream.write(entry.second); + } + uint64_t length = flow->data.size(); + stream.write(length); + stream.write(flow->data); + } + } else { + mg_printf(conn, "HTTP/1.1 200 OK\r\nConnection: " + "close\r\nContent-Length: 0\r\n"); + mg_printf(conn, "Content-Type: text/plain\r\n\r\n"); + } + return true; +} + +bool DeleteTransactionResponder::handleDelete(CivetServer* /*server*/, struct mg_connection *conn) { + std::string site2site_rest_resp; + std::stringstream headers; + std::string resp; + CivetServer::getParam(conn, "responseCode", resp); + headers << "HTTP/1.1 " << response_code << "\r\nContent-Type: application/json\r\nContent-Length: " << site2site_rest_resp.length() << "\r\n"; + headers << "Connection: close\r\n\r\n"; + mg_printf(conn, "%s", headers.str().c_str()); + mg_printf(conn, "%s", site2site_rest_resp.c_str()); + return true; +} + +void HeartbeatHandler::sendHeartbeatResponse(const std::vector& operations, struct mg_connection * conn) { + std::string operation_jsons; + for (const auto& c2_operation : operations) { + std::string resp_args; + if (!c2_operation.args.empty()) { + resp_args = ", \"args\": {"; + auto it = c2_operation.args.begin(); + while (it != c2_operation.args.end()) { + resp_args += "\"" + it->first + "\": \"" + it->second + "\""; + ++it; + if (it != c2_operation.args.end()) { + resp_args += ", "; + } + } + resp_args += "}"; + } + + std::string operation_json = "{" + "\"operation\" : \"" + c2_operation.operation + "\"," + "\"operationid\" : \"" + c2_operation.operation_id + "\"," + "\"operand\": \"" + c2_operation.operand + "\"" + + resp_args + "}"; + + if (operation_jsons.empty()) { + operation_jsons += operation_json; + } else { + operation_jsons += ", " + operation_json; + } + } + + std::string heartbeat_response = "{\"operation\" : \"heartbeat\",\"requested_operations\": [ " + operation_jsons + " ]}"; + + mg_printf(conn, "HTTP/1.1 200 OK\r\nContent-Type: " + "text/plain\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", + heartbeat_response.length()); + mg_printf(conn, "%s", heartbeat_response.c_str()); +} + +void HeartbeatHandler::verifyJsonHasAgentManifest(const rapidjson::Document& root, const std::vector& verify_components, const std::vector& disallowed_properties) { + bool found = false; + REQUIRE(root.HasMember("agentInfo")); + REQUIRE(root["agentInfo"].HasMember("agentManifest")); + REQUIRE(root["agentInfo"]["agentManifest"].HasMember("bundles")); + REQUIRE(root["agentInfo"].HasMember("agentManifestHash")); + const std::string manifestHash = root["agentInfo"]["agentManifestHash"].GetString(); + REQUIRE(manifestHash.length() == 128); + + // throws if not a valid hexadecimal hash + const auto hashVec = utils::string::from_hex(manifestHash); + REQUIRE(hashVec.size() == 64); + + for (auto &bundle : root["agentInfo"]["agentManifest"]["bundles"].GetArray()) { + REQUIRE(bundle.HasMember("artifact")); + std::string str = bundle["artifact"].GetString(); + if (str == "minifi-standard-processors") { + std::vector classes; + for (auto &proc : bundle["componentManifest"]["processors"].GetArray()) { + classes.push_back(proc["type"].GetString()); + } + + auto group = minifi::BuildDescription{}.getClassDescriptions(str); + for (const auto& proc : group.processors_) { + REQUIRE(std::find(classes.begin(), classes.end(), proc.full_name_) != std::end(classes)); + (void)proc; + found = true; + } + } + } + REQUIRE(found); + + verifySupportedOperations(root, verify_components, disallowed_properties); +} + +void HeartbeatHandler::verify(struct mg_connection *conn) { + auto post_data = readPayload(conn); + if (!isServerRunning()) { + return; + } + if (!IsNullOrEmpty(post_data)) { + rapidjson::Document root; + rapidjson::ParseResult result = root.Parse(post_data.data(), post_data.size()); + if (!result) { + throw std::runtime_error(fmt::format("JSON parse error: {0}\n JSON data: {1}", std::string(rapidjson::GetParseError_En(result.Code())), post_data)); + } + std::string operation = root["operation"].GetString(); + if (operation == "heartbeat") { + handleHeartbeat(root, conn); + } else if (operation == "acknowledge") { + handleAcknowledge(root); + } else { + throw std::runtime_error("operation not supported " + operation); + } + } +} + +std::set HeartbeatHandler::getOperandsOfProperties(const rapidjson::Value& operation_node) { + std::set operands; + REQUIRE(operation_node.HasMember("properties")); + const auto& properties_node = operation_node["properties"]; + for (auto it = properties_node.MemberBegin(); it != properties_node.MemberEnd(); ++it) { + operands.insert(it->name.GetString()); + } + return operands; +} + +void HeartbeatHandler::verifyMetadata(const rapidjson::Value& operation_node, const std::unordered_map& operand_with_metadata) { + std::unordered_map operand_with_metadata_found; + const auto& properties_node = operation_node["properties"]; + for (auto prop_it = properties_node.MemberBegin(); prop_it != properties_node.MemberEnd(); ++prop_it) { + if (prop_it->value.ObjectEmpty()) { + continue; + } + Metadata metadata_item; + for (auto metadata_it = prop_it->value.MemberBegin(); metadata_it != prop_it->value.MemberEnd(); ++metadata_it) { + std::vector> values; + for (const auto& value : metadata_it->value.GetArray()) { + std::unordered_map value_item; + for (auto value_it = value.MemberBegin(); value_it != value.MemberEnd(); ++value_it) { + value_item.emplace(value_it->name.GetString(), value_it->value.GetString()); + } + values.push_back(value_item); + } + metadata_item.emplace(metadata_it->name.GetString(), values); + } + operand_with_metadata_found.emplace(prop_it->name.GetString(), metadata_item); + } + REQUIRE(operand_with_metadata_found == operand_with_metadata); +} + +void HeartbeatHandler::verifyProperties(const rapidjson::Value& operation_node, minifi::c2::Operation operation, + const std::vector& verify_components, const std::vector& disallowed_properties) { + switch (operation) { + case minifi::c2::Operation::describe: { + verifyOperands(operation_node); + break; + } + case minifi::c2::Operation::update: { + std::vector> config_properties; + const auto prop_reader = [this](const std::string& sensitive_props) { return configuration_->getString(sensitive_props); }; + const auto sensitive_props = minifi::Configuration::getSensitiveProperties(prop_reader); + + auto allowed_not_sensitive_configuration_properties = minifi::Configuration::CONFIGURATION_PROPERTIES | ranges::views::filter([&](const auto& configuration_property) { + const auto& configuration_property_name = configuration_property.first; + return !ranges::contains(sensitive_props, configuration_property_name) && !ranges::contains(disallowed_properties, configuration_property_name); + }); + for (const auto& [property_name, property_validator] : allowed_not_sensitive_configuration_properties) { + std::unordered_map config_property; + config_property.emplace("propertyName", property_name); + if (auto value = configuration_->getRawValue(std::string(property_name))) { + config_property.emplace("propertyValue", *value); + } + config_property.emplace("validator", property_validator->getValidatorName()); + config_properties.push_back(config_property); + } + Metadata metadata; + metadata.emplace("availableProperties", config_properties); + std::unordered_map operand_with_metadata; + operand_with_metadata.emplace("properties", metadata); + verifyOperands(operation_node, operand_with_metadata); + break; + } + case minifi::c2::Operation::transfer: { + verifyOperands(operation_node); + break; + } + case minifi::c2::Operation::clear: { + verifyOperands(operation_node); + break; + } + case minifi::c2::Operation::start: + case minifi::c2::Operation::stop: { + auto operands = getOperandsOfProperties(operation_node); + REQUIRE(operands.find("c2") != operands.end()); + // FlowController is also present, but this handler has no way of knowing its UUID to test it + for (const auto& component : verify_components) { + REQUIRE(operands.find(component) != operands.end()); + } + break; + } + default: + break; + } +} + +void HeartbeatHandler::verifySupportedOperations(const rapidjson::Document& root, const std::vector& verify_components, const std::vector& disallowed_properties) { + auto& agent_manifest = root["agentInfo"]["agentManifest"]; + REQUIRE(agent_manifest.HasMember("supportedOperations")); + + std::set operations; + for (const auto& operation_node : agent_manifest["supportedOperations"].GetArray()) { + REQUIRE(operation_node.HasMember("type")); + operations.insert(operation_node["type"].GetString()); + verifyProperties(operation_node, utils::enumCast(operation_node["type"].GetString(), true), verify_components, disallowed_properties); + } + + REQUIRE(operations == std::set(magic_enum::enum_names().begin(), magic_enum::enum_names().end())); +} + +bool StoppingHeartbeatHandler::handlePost(CivetServer *, struct mg_connection *conn) { + verify(conn); + sendStopOperation(conn); + return true; +} +void StoppingHeartbeatHandler::sendStopOperation(struct mg_connection *conn) { + std::string resp = "{\"operation\" : \"heartbeat\", \"requested_operations\" : [{ \"operationid\" : 41, \"operation\" : \"stop\", \"operand\" : \"2438e3c8-015a-1000-79ca-83af40ec1991\" }, " + "{ \"operationid\" : 42, \"operation\" : \"stop\", \"operand\" : \"FlowController\" } ]}"; + mg_printf(conn, "HTTP/1.1 200 OK\r\nContent-Type: " + "text/plain\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", + resp.length()); + mg_printf(conn, "%s", resp.c_str()); +} + +bool C2FlowProvider::handleGet(CivetServer* /*server*/, struct mg_connection *conn) { + std::ifstream myfile(test_file_location_.c_str(), std::ios::in | std::ios::binary); + if (myfile.good()) { + std::string str((std::istreambuf_iterator(myfile)), (std::istreambuf_iterator())); + mg_printf(conn, "HTTP/1.1 200 OK\r\nContent-Type: " + "text/plain\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", + str.length()); + mg_printf(conn, "%s", str.c_str()); + } else { + mg_printf(conn, "HTTP/1.1 500 Internal Server Error\r\n"); + } + + return true; +} + +bool C2UpdateHandler::handlePost(CivetServer* /*server*/, struct mg_connection *conn) { + calls_++; + if (!response_.empty()) { + mg_printf(conn, "HTTP/1.1 200 OK\r\nContent-Type: " + "text/plain\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", + response_.length()); + mg_printf(conn, "%s", response_.c_str()); + response_.clear(); + } else { + mg_printf(conn, "HTTP/1.1 500 Internal Server Error\r\n"); + } + + return true; +} + +void C2UpdateHandler::setC2RestResponse(const std::string& url, const std::string& name, const std::optional& persist) { + std::string content = "{\"location\": \"" + url + "\""; + if (persist) { + content += ", \"persist\": \"" + *persist + "\""; + } + content += "}"; + response_ = + "{\"operation\" : \"heartbeat\", " + "\"requested_operations\": [ {" + "\"operation\" : \"update\", " + "\"operationid\" : \"8675309\", " + "\"name\": \"" + name + "\", " + "\"content\": " + content + "}]}"; +} + +bool C2FailedUpdateHandler::handlePost(CivetServer *server, struct mg_connection *conn) { + calls_++; + const auto data = readPayload(conn); + + if (data.find("operationState") != std::string::npos) { + REQUIRE(data.find("state\": \"NOT_APPLIED") != std::string::npos); + } + + return C2UpdateHandler::handlePost(server, conn); +} + +void TimeoutingHTTPHandler::respond(struct mg_connection *conn) { + if (!wait_times_.empty() && wait_times_[0] > std::chrono::seconds(0)) { + sleep_for(wait_times_[0]); + } + int chunk_count = std::max(static_cast(wait_times_.size()) - 1, 0); + mg_printf(conn, "HTTP/1.1 201 OK\r\nContent-Type: text/plain\r\nContent-Length: %d\r\nConnection: close\r\n\r\n", chunk_count); + for (int chunkIdx = 0; chunkIdx < chunk_count; ++chunkIdx) { + mg_printf(conn, "a"); + if (wait_times_[chunkIdx + 1].count() > 0) { + sleep_for(wait_times_[chunkIdx + 1]); + } + } +} +bool HttpGetResponder::handleGet(CivetServer* /*server*/, struct mg_connection *conn) { + puts("handle get"); + static const std::string site2site_rest_resp = "hi this is a get test"; + mg_printf(conn, "HTTP/1.1 200 OK\r\nContent-Type: " + "text/plain\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", + site2site_rest_resp.length()); + mg_printf(conn, "%s", site2site_rest_resp.c_str()); + return true; +} + +bool C2AcknowledgeHandler::handlePost(CivetServer* /*server*/, struct mg_connection* conn) { + std::string req = readPayload(conn); + rapidjson::Document root; + root.Parse(req.data(), req.size()); + + std::string result_state; + std::string details; + + if (root.IsObject() && root.HasMember("operationState")) { + if (root["operationState"].IsObject()) { + if (root["operationState"].HasMember("state")) { + result_state = root["operationState"]["state"].GetString(); + std::lock_guard guard(apply_count_mtx_); + ++apply_count_[result_state]; + } + if (root["operationState"].HasMember("details")) { + details = root["operationState"]["details"].GetString(); + } + } + } + if (root.IsObject() && root.HasMember("operationId")) { + std::lock_guard guard(ack_operations_mtx_); + acknowledged_operations_.insert({root["operationId"].GetString(), OpResult{result_state, details}}); + } + + mg_printf(conn, "HTTP/1.1 200 OK\r\nContent-Type: " + "text/plain\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + return true; +} + +} // namespace org::apache::nifi::minifi::test diff --git a/libminifi/test/libtest/integration/HTTPHandlers.h b/libminifi/test/libtest/integration/HTTPHandlers.h new file mode 100644 index 0000000000..a47d2a5061 --- /dev/null +++ b/libminifi/test/libtest/integration/HTTPHandlers.h @@ -0,0 +1,408 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +// #include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "civetweb.h" +#include "CivetServer.h" +#include "concurrentqueue.h" +// #include "CivetStream.h" +// #include "io/CRCStream.h" +#include "rapidjson/document.h" +// #include "rapidjson/error/en.h" +#include "utils/HTTPUtils.h" +#include "ServerAwareHandler.h" +// #include "utils/gsl.h" +// #include "agent/build_description.h" +#include "c2/C2Payload.h" +#include "properties/Configuration.h" +// #include "range/v3/algorithm/contains.hpp" +// #include "range/v3/view/filter.hpp" +// #include "range/v3/view/view.hpp" +// #include "utils/net/DNS.h" +#include "unit/Catch.h" + +namespace org::apache::nifi::minifi::test { + +static std::atomic transaction_id; +static std::atomic transaction_id_output; + +struct FlowObj { + FlowObj() = default; + + FlowObj(FlowObj &&other) noexcept + : total_size(other.total_size), + attributes(std::move(other.attributes)), + data(std::move(other.data)) + { } + + uint64_t total_size{0}; + std::map attributes; + std::vector data; +}; + +class SiteToSiteLocationResponder : public ServerAwareHandler { + public: + explicit SiteToSiteLocationResponder(bool isSecure) + : isSecure(isSecure) { + } + bool handleGet(CivetServer* /*server*/, struct mg_connection *conn) override; + + protected: + bool isSecure; +}; + +class PeerResponder : public ServerAwareHandler { + public: + explicit PeerResponder(std::string base_url) { + std::string scheme; + REQUIRE(minifi::utils::parse_http_components(base_url, port, scheme, path)); + } + + bool handleGet(CivetServer* /*server*/, struct mg_connection *conn) override; + + protected: + std::string base_url; + std::string port; + std::string path; +}; + +class SiteToSiteBaseResponder : public ServerAwareHandler { + public: + explicit SiteToSiteBaseResponder(std::string base_url) + : base_url(std::move(base_url)) { + } + + bool handleGet(CivetServer* /*server*/, struct mg_connection *conn) override; + + protected: + std::string base_url; +}; + +class TransactionResponder : public ServerAwareHandler { + public: + explicit TransactionResponder(std::string base_url, std::string port_id, bool input_port, bool wrong_uri = false, bool empty_transaction_uri = false); + + bool handlePost(CivetServer* /*server*/, struct mg_connection *conn) override; + + void setFeed(moodycamel::ConcurrentQueue> *feed) { + flow_files_feed_ = feed; + } + + std::string getTransactionId() { + return transaction_id_str; + } + + protected: + std::string base_url; + std::string transaction_id_str; + bool wrong_uri; + bool empty_transaction_uri; + bool input_port; + std::string port_id; + moodycamel::ConcurrentQueue> *flow_files_feed_; +}; + +class FlowFileResponder : public ServerAwareHandler { + public: + explicit FlowFileResponder(bool input_port, bool wrong_uri = false, bool invalid_checksum = false) + : wrong_uri(wrong_uri), + input_port(input_port), + invalid_checksum(invalid_checksum), + flow_files_feed_(nullptr) { + } + + moodycamel::ConcurrentQueue> *getFlows() { + return &flow_files_; + } + + void setFeed(moodycamel::ConcurrentQueue> *feed) { + flow_files_feed_ = feed; + } + + bool handlePost(CivetServer* /*server*/, struct mg_connection *conn) override; + bool handleGet(CivetServer* /*server*/, struct mg_connection *conn) override; + + void setFlowUrl(std::string flowUrl) { + base_url = std::move(flowUrl); + } + + protected: + // base url + std::string base_url; + // set the wrong url + bool wrong_uri; + // we are running an input port + bool input_port; + // invalid checksum is returned. + bool invalid_checksum; + moodycamel::ConcurrentQueue> flow_files_; + moodycamel::ConcurrentQueue> *flow_files_feed_; +}; + +class DeleteTransactionResponder : public ServerAwareHandler { + public: + explicit DeleteTransactionResponder(std::string base_url, std::string response_code, int expected_resp_code) + : flow_files_feed_(nullptr), + base_url(std::move(base_url)), + response_code(std::move(response_code)) { + expected_resp_code_str = std::to_string(expected_resp_code); + } + + explicit DeleteTransactionResponder(std::string base_url, std::string response_code, moodycamel::ConcurrentQueue> *feed) + : flow_files_feed_(feed), + base_url(std::move(base_url)), + response_code(std::move(response_code)) { + } + + bool handleDelete(CivetServer* /*server*/, struct mg_connection *conn) override; + + void setFeed(moodycamel::ConcurrentQueue> *feed) { + flow_files_feed_ = feed; + } + + protected: + moodycamel::ConcurrentQueue> *flow_files_feed_; + std::string base_url; + std::string expected_resp_code_str; + std::string response_code; +}; + +class HeartbeatHandler : public ServerAwareHandler { + public: + explicit HeartbeatHandler(std::shared_ptr configuration) : configuration_(std::move(configuration)) {} + + virtual void handleHeartbeat(const rapidjson::Document& root, struct mg_connection *) { + verifyJsonHasAgentManifest(root); + } + + virtual void handleAcknowledge(const rapidjson::Document&) { + } + + bool handlePost(CivetServer *, struct mg_connection *conn) override { + verify(conn); + return true; + } + + protected: + struct C2Operation { + std::string operation; + std::string operand; + std::string operation_id; + std::unordered_map args; + }; + + void sendHeartbeatResponse(const std::string& operation, const std::string& operand, const std::string& operation_id, struct mg_connection* conn, + const std::unordered_map& args = {}) { + sendHeartbeatResponse({{operation, operand, operation_id, args}}, conn); + } + + void sendHeartbeatResponse(const std::vector& operations, struct mg_connection * conn); + void verifyJsonHasAgentManifest(const rapidjson::Document& root, const std::vector& verify_components = {}, const std::vector& disallowed_properties = {}); + void verify(struct mg_connection *conn); + + private: + using Metadata = std::unordered_map>>; + + static std::set getOperandsOfProperties(const rapidjson::Value& operation_node); + static void verifyMetadata(const rapidjson::Value& operation_node, const std::unordered_map& operand_with_metadata); + + template + void verifyOperands(const rapidjson::Value& operation_node, const std::unordered_map& operand_with_metadata = {}) { + auto operands = getOperandsOfProperties(operation_node); + REQUIRE(operands == std::set(magic_enum::enum_names().begin(), magic_enum::enum_names().end())); + verifyMetadata(operation_node, operand_with_metadata); + } + + void verifyProperties(const rapidjson::Value& operation_node, minifi::c2::Operation operation, + const std::vector& verify_components, const std::vector& disallowed_properties); + void verifySupportedOperations(const rapidjson::Document& root, const std::vector& verify_components, const std::vector& disallowed_properties); + + std::shared_ptr configuration_; +}; + +class StoppingHeartbeatHandler : public HeartbeatHandler { + public: + explicit StoppingHeartbeatHandler(std::shared_ptr configuration) : HeartbeatHandler(std::move(configuration)) {} + + bool handlePost(CivetServer *, struct mg_connection *conn) override; + + private: + static void sendStopOperation(struct mg_connection *conn); +}; + +class C2FlowProvider : public ServerAwareHandler { + public: + explicit C2FlowProvider(std::string test_file_location) + : test_file_location_(std::move(test_file_location)) { + } + + bool handleGet(CivetServer* /*server*/, struct mg_connection *conn) override; + + private: + const std::string test_file_location_; +}; + +class C2UpdateHandler : public C2FlowProvider { + public: + using C2FlowProvider::C2FlowProvider; + + bool handlePost(CivetServer* /*server*/, struct mg_connection *conn) override; + void setC2RestResponse(const std::string& url, const std::string& name, const std::optional& persist = {}); + + size_t getCallCount() const { + return calls_; + } + + protected: + std::atomic calls_{0}; + + private: + std::string response_; +}; + +class C2FailedUpdateHandler : public C2UpdateHandler { + public: + explicit C2FailedUpdateHandler(const std::string& test_file_location) : C2UpdateHandler(test_file_location) { + } + + bool handlePost(CivetServer *server, struct mg_connection *conn) override; +}; + +class InvokeHTTPCouldNotConnectHandler : public ServerAwareHandler { +}; + +class InvokeHTTPResponseOKHandler : public ServerAwareHandler { + public: + bool handlePost(CivetServer *, struct mg_connection *conn) override { + mg_printf(conn, "HTTP/1.1 201 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + return true; + } +}; + +class InvokeHTTPRedirectHandler : public ServerAwareHandler { + public: + bool handlePost(CivetServer *, struct mg_connection *conn) override { + mg_printf(conn, "HTTP/1.1 301 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\nLocation: /\r\n\r\n"); + return true; + } +}; + +class InvokeHTTPResponse404Handler : public ServerAwareHandler { + public: + bool handlePost(CivetServer *, struct mg_connection *conn) override { + mg_printf(conn, "HTTP/1.1 404 Not Found\r\nContent-Type: text/plain\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + return true; + } +}; + +class InvokeHTTPResponse501Handler : public ServerAwareHandler { + public: + bool handlePost(CivetServer *, struct mg_connection *conn) override { + mg_printf(conn, "HTTP/1.1 501 Not Implemented\r\nContent-Type: text/plain\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + return true; + } +}; + +class TimeoutingHTTPHandler : public ServerAwareHandler { + public: + explicit TimeoutingHTTPHandler(std::vector wait_times) + : wait_times_(wait_times) { + } + bool handlePost(CivetServer *, struct mg_connection *conn) override { + respond(conn); + return true; + } + bool handleGet(CivetServer *, struct mg_connection *conn) override { + respond(conn); + return true; + } + bool handleDelete(CivetServer *, struct mg_connection *conn) override { + respond(conn); + return true; + } + bool handlePut(CivetServer *, struct mg_connection *conn) override { + respond(conn); + return true; + } + + private: + void respond(struct mg_connection *conn); + + std::vector wait_times_; +}; + +class HttpGetResponder : public ServerAwareHandler { + public: + bool handleGet(CivetServer* /*server*/, struct mg_connection *conn) override; +}; + +class RetryHttpGetResponder : public ServerAwareHandler { + public: + bool handleGet(CivetServer* /*server*/, struct mg_connection *conn) override { + puts("handle get with retry"); + mg_printf(conn, "HTTP/1.1 501 Not Implemented\r\nContent-Type: text/plain\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + return true; + } +}; + +class C2AcknowledgeHandler : public ServerAwareHandler { + struct OpResult { + std::string state; + std::string details; + }; + + public: + bool handlePost(CivetServer* /*server*/, struct mg_connection* conn) override; + + bool isAcknowledged(const std::string& operation_id) const { + std::lock_guard guard(ack_operations_mtx_); + return acknowledged_operations_.count(operation_id) > 0; + } + + std::optional getState(const std::string& operation_id) const { + std::lock_guard guard(ack_operations_mtx_); + if (auto it = acknowledged_operations_.find(operation_id); it != acknowledged_operations_.end()) { + return it->second; + } + return std::nullopt; + } + + uint32_t getApplyCount(const std::string& result_state) const { + std::lock_guard guard(apply_count_mtx_); + return apply_count_.find(result_state) != apply_count_.end() ? apply_count_.at(result_state) : 0; + } + + private: + mutable std::mutex ack_operations_mtx_; + mutable std::mutex apply_count_mtx_; + std::unordered_map acknowledged_operations_; + std::unordered_map apply_count_; +}; + +} // namespace org::apache::nifi::minifi::test diff --git a/libminifi/test/libtest/integration/HTTPIntegrationBase.cpp b/libminifi/test/libtest/integration/HTTPIntegrationBase.cpp new file mode 100644 index 0000000000..c6439f9bb8 --- /dev/null +++ b/libminifi/test/libtest/integration/HTTPIntegrationBase.cpp @@ -0,0 +1,94 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "HTTPIntegrationBase.h" + +namespace org::apache::nifi::minifi::test { + +int log_message(const struct mg_connection* /*conn*/, const char *message) { + puts(message); + return 1; +} + +int ssl_enable(void* /*ssl_context*/, void* /*user_data*/) { + return 0; +} + +std::string HTTPIntegrationBase::getWebPort() { + std::string ret_val = port; + if (ret_val.back() == 's') { + ret_val = ret_val.substr(0, ret_val.size() - 1); + } + return ret_val; +} + +std::string HTTPIntegrationBase::getC2RestUrl() const { + std::string c2_rest_url; + configuration->get(org::apache::nifi::minifi::Configuration::nifi_c2_rest_url, c2_rest_url); + return c2_rest_url; +} + +void HTTPIntegrationBase::setUrl(const std::string &url, ServerAwareHandler *handler) { + std::string url_port, url_scheme, url_path; + minifi::utils::parse_http_components(url, url_port, url_scheme, url_path); + if (server) { + if (url_port != "0" && url_port != port) { + throw std::logic_error("Inconsistent port requirements"); + } + if (url_scheme != scheme) { + throw std::logic_error("Inconsistent scheme requirements"); + } + server->addHandler(url_path, handler); + return; + } + // initialize server + scheme = url_scheme; + port = url_port; + CivetCallbacks callback{}; + if (scheme == "https" && !key_dir.empty()) { + auto cert = key_dir / "nifi-cert.pem"; + callback.init_ssl = ssl_enable; + port += "s"; + callback.log_message = log_message; + server = std::make_unique(port, url_path, handler, &callback, cert.string(), cert.string()); + } else { + server = std::make_unique(port, url_path, handler); + } + bool secure{false}; + if (port == "0" || port == "0s") { + secure = (port == "0s"); + port = std::to_string(server->getListeningPorts()[0]); + if (secure) { + port += "s"; + } + } + std::string c2_url = std::string("http") + (secure ? "s" : "") + "://localhost:" + getWebPort() + url_path; + configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_rest_url, c2_url); + configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_rest_url_ack, c2_url); +} + +void HTTPIntegrationBase::setC2Url(const std::string &heartbeat_path, const std::string &acknowledge_path) { + if (port.empty()) { + throw std::logic_error("Port is not yet initialized"); + } + bool secure = port.back() == 's'; + std::string base = std::string("http") + (secure ? "s" : "") + "://localhost:" + getWebPort(); + configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_rest_url, base + heartbeat_path); + configuration->set(org::apache::nifi::minifi::Configuration::nifi_c2_rest_url_ack, base + acknowledge_path); +} + +} // namespace org::apache::nifi::minifi::test diff --git a/libminifi/test/libtest/integration/HTTPIntegrationBase.h b/libminifi/test/libtest/integration/HTTPIntegrationBase.h new file mode 100644 index 0000000000..43e71c5735 --- /dev/null +++ b/libminifi/test/libtest/integration/HTTPIntegrationBase.h @@ -0,0 +1,197 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include + +#include "CivetServer.h" +#include "IntegrationBase.h" +#include "c2/C2Agent.h" +#include "c2/protocols/RESTSender.h" +#include "ServerAwareHandler.h" +#include "unit/TestBase.h" +#include "unit/TestUtils.h" +#include "TestServer.h" +#include "properties/Configuration.h" +#include "unit/Catch.h" + +namespace org::apache::nifi::minifi::test { + +int log_message(const struct mg_connection* /*conn*/, const char *message); +int ssl_enable(void* /*ssl_context*/, void* /*user_data*/); + +class HTTPIntegrationBase : public IntegrationBase { + public: + explicit HTTPIntegrationBase(std::chrono::milliseconds waitTime = std::chrono::milliseconds(DEFAULT_WAITTIME_MSECS)) + : IntegrationBase(waitTime), + server(nullptr) { + } + HTTPIntegrationBase(const HTTPIntegrationBase&) = delete; + HTTPIntegrationBase(HTTPIntegrationBase&&) = default; + HTTPIntegrationBase& operator=(const HTTPIntegrationBase&) = delete; + HTTPIntegrationBase& operator=(HTTPIntegrationBase&&) = default; + + virtual void setUrl(const std::string &url, ServerAwareHandler *handler); + + void setC2Url(const std::string& heartbeat_path, const std::string& acknowledge_path); + + void shutdownBeforeFlowController() override { + server.reset(); + } + + std::string getWebPort(); + std::string getC2RestUrl() const; + + protected: + std::unique_ptr server; +}; + +class VerifyC2Base : public HTTPIntegrationBase { + public: + using HTTPIntegrationBase::HTTPIntegrationBase; + void testSetup() override { + LogTestController::getInstance().setDebug(); + LogTestController::getInstance().setDebug(); + } + + void configureC2() override { + configuration->set(minifi::Configuration::nifi_c2_agent_protocol_class, "RESTSender"); + configuration->set(minifi::Configuration::nifi_c2_enable, "true"); + configuration->set(minifi::Configuration::nifi_c2_agent_class, "test"); + configuration->set(minifi::Configuration::nifi_c2_agent_heartbeat_period, "1000"); + configuration->set(minifi::Configuration::nifi_c2_root_classes, "DeviceInfoNode,AgentInformation,FlowInformation"); + } + + void cleanup() override { + LogTestController::getInstance().reset(); + HTTPIntegrationBase::cleanup(); + } +}; + +class VerifyC2Describe : public VerifyC2Base { + public: + explicit VerifyC2Describe(std::atomic& verified) + : verified_(verified) { + } + + void testSetup() override { + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setDebug(); + LogTestController::getInstance().setInfo(); + VerifyC2Base::testSetup(); + } + + void configureFullHeartbeat() override { + configuration->set(minifi::Configuration::nifi_c2_full_heartbeat, "false"); + } + + void runAssertions() override { + REQUIRE(utils::verifyEventHappenedInPollTime(std::chrono::milliseconds(wait_time_), [&] { return verified_.load(); })); + } + + protected: + std::atomic& verified_; +}; + +class VerifyC2Update : public HTTPIntegrationBase { + public: + explicit VerifyC2Update(std::chrono::milliseconds waitTime) + : HTTPIntegrationBase(waitTime) { + } + + void testSetup() override { + LogTestController::getInstance().setInfo(); + LogTestController::getInstance().setDebug(); + LogTestController::getInstance().setDebug(); + LogTestController::getInstance().setDebug(); + } + + void configureC2() override { + configuration->set(minifi::Configuration::nifi_c2_agent_protocol_class, "RESTSender"); + configuration->set(minifi::Configuration::nifi_c2_enable, "true"); + configuration->set(minifi::Configuration::nifi_c2_agent_class, "test"); + configuration->set(minifi::Configuration::nifi_c2_agent_heartbeat_period, "1000"); + } + + void cleanup() override { + LogTestController::getInstance().reset(); + HTTPIntegrationBase::cleanup(); + } + + void runAssertions() override { + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(std::chrono::seconds(10), "Starting to reload Flow Controller with flow control name MiNiFi Flow, version")); + } +}; + +class VerifyFlowFetched : public HTTPIntegrationBase { + public: + using HTTPIntegrationBase::HTTPIntegrationBase; + + void testSetup() override { + LogTestController::getInstance().setInfo(); + LogTestController::getInstance().setDebug(); + LogTestController::getInstance().setDebug(); + LogTestController::getInstance().setDebug(); + } + + void configureC2() override { + configuration->set(minifi::Configuration::nifi_c2_agent_protocol_class, "RESTSender"); + configuration->set(minifi::Configuration::nifi_c2_enable, "true"); + configuration->set(minifi::Configuration::nifi_c2_agent_class, "test"); + configuration->set(minifi::Configuration::nifi_c2_agent_heartbeat_period, "1000"); + } + + void setFlowUrl(const std::string& url) { + configuration->set(minifi::Configuration::nifi_c2_flow_url, url); + } + + void cleanup() override { + LogTestController::getInstance().reset(); + HTTPIntegrationBase::cleanup(); + } + + void runAssertions() override { + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(std::chrono::seconds(10), "Successfully fetched valid flow configuration")); + } +}; + +class VerifyC2FailedUpdate : public VerifyC2Update { + public: + explicit VerifyC2FailedUpdate(std::chrono::milliseconds waitTime) + : VerifyC2Update(waitTime) { + } + + void testSetup() override { + LogTestController::getInstance().setInfo(); + LogTestController::getInstance().setDebug(); + minifi::utils::file::create_dir("content_repository"); + } + + void runAssertions() override { + REQUIRE(minifi::test::utils::verifyLogLinePresenceInPollTime(std::chrono::seconds(10), "Invalid configuration payload", "update failed")); + } + + void cleanup() override { + minifi::utils::file::delete_dir("content_repository", true); + VerifyC2Update::cleanup(); + } +}; + +} // namespace org::apache::nifi::minifi::test diff --git a/libminifi/test/libtest/integration/IntegrationBase.cpp b/libminifi/test/libtest/integration/IntegrationBase.cpp new file mode 100644 index 0000000000..75d3577067 --- /dev/null +++ b/libminifi/test/libtest/integration/IntegrationBase.cpp @@ -0,0 +1,156 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "IntegrationBase.h" + +#include + +#include "utils/net/DNS.h" +#include "utils/HTTPUtils.h" +#include "unit/ProvenanceTestHelper.h" +#include "utils/FifoExecutor.h" +#include "core/ConfigurationFactory.h" + +namespace org::apache::nifi::minifi::test { + +IntegrationBase::IntegrationBase(std::chrono::milliseconds waitTime) + : configuration(std::make_shared()), + wait_time_(waitTime) { +} + +void IntegrationBase::configureSecurity() { + if (!key_dir.empty()) { + configuration->set(minifi::Configure::nifi_security_client_certificate, (key_dir / "cn.crt.pem").string()); + configuration->set(minifi::Configure::nifi_security_client_private_key, (key_dir / "cn.ckey.pem").string()); + configuration->set(minifi::Configure::nifi_security_client_pass_phrase, (key_dir / "cn.pass").string()); + configuration->set(minifi::Configure::nifi_security_client_ca_certificate, (key_dir / "nifi-cert.pem").string()); + configuration->set(minifi::Configure::nifi_default_directory, key_dir.string()); + } +} + +void IntegrationBase::run(const std::optional& test_file_location, const std::optional& home_path) { + using namespace std::literals::chrono_literals; + testSetup(); + + std::shared_ptr test_repo = std::make_shared(); + std::shared_ptr test_flow_repo = std::make_shared(); + + if (test_file_location) { + configuration->set(minifi::Configure::nifi_flow_configuration_file, test_file_location->string()); + } + configuration->set(minifi::Configure::nifi_state_storage_local_class_name, "VolatileMapStateStorage"); + + configureC2(); + configureFullHeartbeat(); + + std::shared_ptr content_repo = std::make_shared(); + content_repo->initialize(configuration); + + std::atomic running = true; + minifi::utils::FifoExecutor assertion_runner; + std::future assertions_done; + while (running) { + running = false; // Stop running after this iteration, unless restart is explicitly requested + + bool should_encrypt_flow_config = (configuration->get(minifi::Configure::nifi_flow_configuration_encrypt) + | minifi::utils::andThen(minifi::utils::string::toBool)).value_or(false); + + std::shared_ptr filesystem; + if (home_path) { + filesystem = std::make_shared( + should_encrypt_flow_config, + minifi::utils::crypto::EncryptionProvider::create(*home_path)); + } else { + filesystem = std::make_shared(); + } + + std::optional sensitive_properties_encryptor = [&]() { + if (home_path) { + return minifi::utils::crypto::EncryptionProvider::createSensitivePropertiesEncryptor(*home_path); + } else { + auto encryption_key = minifi::utils::string::from_hex("e4bce4be67f417ed2530038626da57da7725ff8c0b519b692e4311e4d4fe8a28"); + return minifi::utils::crypto::EncryptionProvider{minifi::utils::crypto::XSalsa20Cipher{encryption_key}}; + } + }(); + + std::string nifi_configuration_class_name = "adaptiveconfiguration"; + configuration->get(minifi::Configure::nifi_configuration_class_name, nifi_configuration_class_name); + + std::shared_ptr flow_config = core::createFlowConfiguration( + core::ConfigurationContext{ + .flow_file_repo = test_repo, + .content_repo = content_repo, + .configuration = configuration, + .path = test_file_location, + .filesystem = filesystem, + .sensitive_properties_encryptor = sensitive_properties_encryptor + }, nifi_configuration_class_name); + + auto controller_service_provider = flow_config->getControllerServiceProvider(); + char state_dir_name_template[] = "/var/tmp/integrationstate.XXXXXX"; // NOLINT(cppcoreguidelines-avoid-c-arrays) + state_dir = minifi::utils::file::create_temp_directory(state_dir_name_template); + if (!configuration->get(minifi::Configure::nifi_state_storage_local_path)) { + configuration->set(minifi::Configure::nifi_state_storage_local_path, state_dir.string()); + } + core::ProcessContext::getOrCreateDefaultStateStorage(controller_service_provider.get(), configuration); + + std::shared_ptr pg(flow_config->getRoot()); + queryRootProcessGroup(pg); + + const auto request_restart = [&, this] { + ++restart_requested_count_; + running = true; + }; + + std::vector> repo_metric_sources{test_repo, test_flow_repo, content_repo}; + auto metrics_publisher_store = std::make_unique(configuration, repo_metric_sources, flow_config); + flowController_ = std::make_unique(test_repo, test_flow_repo, configuration, + std::move(flow_config), content_repo, std::move(metrics_publisher_store), filesystem, request_restart); + flowController_->load(); + updateProperties(*flowController_); + flowController_->start(); + + assertions_done = assertion_runner.enqueue([this] { runAssertions(); }); + std::future_status status = std::future_status::ready; + while (!running && (status = assertions_done.wait_for(10ms)) == std::future_status::timeout) { /* wait */ } + if (running && status != std::future_status::timeout) { + // cancel restart, because assertions have finished running + running = false; + } + + if (!running) { + // Only stop servers if we're shutting down + shutdownBeforeFlowController(); + } + flowController_->stop(); + } + + cleanup(); +} + +std::string parseUrl(std::string url) { +#ifdef WIN32 + if (url.find("localhost") != std::string::npos) { + std::string port, scheme, path; + minifi::utils::parse_http_components(url, port, scheme, path); + url = scheme + "://" + org::apache::nifi::minifi::utils::net::getMyHostName() + ":" + port + path; + } +#endif + return url; +} + +} // namespace org::apache::nifi::minifi::test diff --git a/libminifi/test/libtest/integration/IntegrationBase.h b/libminifi/test/libtest/integration/IntegrationBase.h new file mode 100644 index 0000000000..8c374159df --- /dev/null +++ b/libminifi/test/libtest/integration/IntegrationBase.h @@ -0,0 +1,121 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#define DEFAULT_WAITTIME_MSECS 10000 + +#include +#include +#include +#include +#include + +#include "core/ProcessGroup.h" +#include "FlowController.h" +#include "properties/Configure.h" + +namespace minifi = org::apache::nifi::minifi; +namespace core = minifi::core; +namespace utils = minifi::utils; + +namespace org::apache::nifi::minifi::test { + +class IntegrationBase { + public: + explicit IntegrationBase(std::chrono::milliseconds waitTime = std::chrono::milliseconds(DEFAULT_WAITTIME_MSECS)); + IntegrationBase(const IntegrationBase&) = delete; + IntegrationBase(IntegrationBase&& other) noexcept + :configuration{std::move(other.configuration)}, + flowController_{std::move(other.flowController_)}, + wait_time_{other.wait_time_}, + port{std::move(other.port)}, + scheme{std::move(other.scheme)}, + key_dir{std::move(other.key_dir)}, + state_dir{std::move(other.state_dir)}, + restart_requested_count_{other.restart_requested_count_.load()} + {} + IntegrationBase& operator=(const IntegrationBase&) = delete; + IntegrationBase& operator=(IntegrationBase&& other) noexcept { + if (&other == this) return *this; + configuration = std::move(other.configuration); + flowController_ = std::move(other.flowController_); + wait_time_ = other.wait_time_; + port = std::move(other.port); + scheme = std::move(other.scheme); + key_dir = std::move(other.key_dir); + state_dir = std::move(other.state_dir); + restart_requested_count_ = other.restart_requested_count_.load(); + return *this; + } + virtual ~IntegrationBase() = default; + + virtual void run(const std::optional& test_file_location = {}, const std::optional& home_path = {}); + + void setKeyDir(const std::filesystem::path& key_dir) { + this->key_dir = key_dir; + configureSecurity(); + } + + virtual void testSetup() = 0; + + virtual void shutdownBeforeFlowController() { + } + + const std::shared_ptr& getConfiguration() const { + return configuration; + } + + void setConfiguration(std::shared_ptr configuration) { + this->configuration = std::move(configuration); + } + + virtual void cleanup() { + if (!state_dir.empty()) { + minifi::utils::file::delete_dir(state_dir); + } + } + + virtual void runAssertions() = 0; + + protected: + virtual void configureC2() { + } + + virtual void queryRootProcessGroup(std::shared_ptr /*pg*/) { + } + + virtual void configureFullHeartbeat() { + } + + virtual void updateProperties(minifi::FlowController& /*fc*/) { + } + + void configureSecurity(); + std::shared_ptr configuration; + std::unique_ptr response_node_loader_; + std::unique_ptr flowController_; + std::chrono::milliseconds wait_time_; + std::string port, scheme; + std::filesystem::path key_dir; + std::filesystem::path state_dir; + std::atomic restart_requested_count_{0}; +}; + +std::string parseUrl(std::string url); + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/http-curl/tests/ServerAwareHandler.h b/libminifi/test/libtest/integration/ServerAwareHandler.h similarity index 97% rename from extensions/http-curl/tests/ServerAwareHandler.h rename to libminifi/test/libtest/integration/ServerAwareHandler.h index 3ca23d2610..797d3a7b40 100644 --- a/extensions/http-curl/tests/ServerAwareHandler.h +++ b/libminifi/test/libtest/integration/ServerAwareHandler.h @@ -20,6 +20,8 @@ #include #include +#include +#include #include "CivetServer.h" diff --git a/extensions/http-curl/tests/TestServer.h b/libminifi/test/libtest/integration/TestServer.h similarity index 92% rename from extensions/http-curl/tests/TestServer.h rename to libminifi/test/libtest/integration/TestServer.h index f690789f89..f864d4a972 100644 --- a/extensions/http-curl/tests/TestServer.h +++ b/libminifi/test/libtest/integration/TestServer.h @@ -23,7 +23,7 @@ #include "civetweb.h" #include "CivetLibrary.h" #include "CivetServer.h" -#include "HTTPUtils.h" +#include "utils/HTTPUtils.h" #include "ServerAwareHandler.h" /** @@ -31,9 +31,9 @@ * so they wouldn't get stuck (if a handler returns after shutdown is * initiated it might get stuck inside worker_thread_run > consume_socket) */ -class TestServer{ +class TestServer { public: - TestServer(std::string &port, std::string &rooturi, CivetHandler *handler, CivetCallbacks *callbacks, std::string& cert, std::string &ca_cert) { + TestServer(const std::string &port, const std::string &rooturi, CivetHandler *handler, CivetCallbacks *callbacks, const std::string& cert, const std::string &ca_cert) { if (!mg_check_feature(2)) { throw std::runtime_error("Error: Embedded example built with SSL support, " "but civetweb library build without.\n"); diff --git a/libminifi/test/libtest/unit/CMakeLists.txt b/libminifi/test/libtest/unit/CMakeLists.txt new file mode 100644 index 0000000000..dccb78a9ac --- /dev/null +++ b/libminifi/test/libtest/unit/CMakeLists.txt @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +file(GLOB TEST_BASE_SOURCES "*.cpp") +add_minifi_library(libminifi-unittest STATIC ${TEST_BASE_SOURCES}) +target_link_libraries(libminifi-unittest core-minifi Catch2::Catch2WithMain) +target_include_directories(libminifi-unittest BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include/") +if(WIN32) + target_include_directories(libminifi-unittest BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/opsys/win") +else() + target_include_directories(libminifi-unittest BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/opsys/posix") +endif() diff --git a/libminifi/test/Catch.h b/libminifi/test/libtest/unit/Catch.h similarity index 100% rename from libminifi/test/Catch.h rename to libminifi/test/libtest/unit/Catch.h diff --git a/libminifi/test/ConfigurationTestController.h b/libminifi/test/libtest/unit/ConfigurationTestController.h similarity index 98% rename from libminifi/test/ConfigurationTestController.h rename to libminifi/test/libtest/unit/ConfigurationTestController.h index 38ab6eaa60..d726cc2000 100644 --- a/libminifi/test/ConfigurationTestController.h +++ b/libminifi/test/libtest/unit/ConfigurationTestController.h @@ -20,7 +20,7 @@ #include -#include "TestBase.h" +#include "unit/TestBase.h" #include "core/FlowConfiguration.h" #include "core/RepositoryFactory.h" #include "core/yaml/YamlConfiguration.h" diff --git a/libminifi/test/unit/ContentRepositoryDependentTests.h b/libminifi/test/libtest/unit/ContentRepositoryDependentTests.h similarity index 95% rename from libminifi/test/unit/ContentRepositoryDependentTests.h rename to libminifi/test/libtest/unit/ContentRepositoryDependentTests.h index 80cbc655bd..24001c3a6d 100644 --- a/libminifi/test/unit/ContentRepositoryDependentTests.h +++ b/libminifi/test/libtest/unit/ContentRepositoryDependentTests.h @@ -25,9 +25,9 @@ #include "core/Processor.h" #include "core/ProcessSession.h" #include "core/Resource.h" -#include "../TestBase.h" -#include "../Catch.h" -#include "../DummyProcessor.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/DummyProcessor.h" #include "StreamPipe.h" #pragma once @@ -60,7 +60,7 @@ class Fixture { const core::Relationship Failure{"failure", "something has gone awry"}; explicit Fixture(std::shared_ptr content_repo) { - test_plan_ = test_controller_.createPlan(nullptr, std::nullopt, content_repo); + test_plan_ = test_controller_.createPlan(nullptr, std::nullopt, std::move(content_repo)); dummy_processor_ = test_plan_->addProcessor("DummyProcessor", "dummyProcessor"); context_ = [this] { test_plan_->runNextProcessor(); @@ -95,7 +95,7 @@ class Fixture { }; void testReadOnSmallerClonedFlowFiles(std::shared_ptr content_repo) { - Fixture fixture = Fixture(content_repo); + Fixture fixture = Fixture(std::move(content_repo)); core::ProcessSession& process_session = fixture.processSession(); const auto original_ff = process_session.create(); fixture.writeToFlowFile(original_ff, "foobar"); @@ -124,7 +124,7 @@ void testReadOnSmallerClonedFlowFiles(std::shared_ptr c } void testAppendToUnmanagedFlowFile(std::shared_ptr content_repo) { - Fixture fixture = Fixture(content_repo); + Fixture fixture = Fixture(std::move(content_repo)); core::ProcessSession& process_session = fixture.processSession(); const auto flow_file = process_session.create(); REQUIRE(flow_file); @@ -143,7 +143,7 @@ void testAppendToUnmanagedFlowFile(std::shared_ptr cont } void testAppendToManagedFlowFile(std::shared_ptr content_repo) { - Fixture fixture = Fixture(content_repo); + Fixture fixture = Fixture(std::move(content_repo)); core::ProcessSession& process_session = fixture.processSession(); const auto flow_file = process_session.create(); REQUIRE(flow_file); @@ -161,7 +161,7 @@ void testAppendToManagedFlowFile(std::shared_ptr conten } void testReadFromZeroLengthFlowFile(std::shared_ptr content_repo) { - Fixture fixture = Fixture(content_repo); + Fixture fixture = Fixture(std::move(content_repo)); core::ProcessSession& process_session = fixture.processSession(); const auto flow_file = process_session.create(); REQUIRE(flow_file); diff --git a/libminifi/test/DummyProcessor.cpp b/libminifi/test/libtest/unit/DummyProcessor.cpp similarity index 100% rename from libminifi/test/DummyProcessor.cpp rename to libminifi/test/libtest/unit/DummyProcessor.cpp diff --git a/libminifi/test/DummyProcessor.h b/libminifi/test/libtest/unit/DummyProcessor.h similarity index 100% rename from libminifi/test/DummyProcessor.h rename to libminifi/test/libtest/unit/DummyProcessor.h diff --git a/libminifi/test/EmptyFlow.h b/libminifi/test/libtest/unit/EmptyFlow.h similarity index 100% rename from libminifi/test/EmptyFlow.h rename to libminifi/test/libtest/unit/EmptyFlow.h diff --git a/libminifi/test/LogUtils.h b/libminifi/test/libtest/unit/LogUtils.h similarity index 100% rename from libminifi/test/LogUtils.h rename to libminifi/test/libtest/unit/LogUtils.h diff --git a/libminifi/test/unit/MockClasses.h b/libminifi/test/libtest/unit/MockClasses.h similarity index 93% rename from libminifi/test/unit/MockClasses.h rename to libminifi/test/libtest/unit/MockClasses.h index 20cff01dee..54af4c46a1 100644 --- a/libminifi/test/unit/MockClasses.h +++ b/libminifi/test/libtest/unit/MockClasses.h @@ -27,6 +27,7 @@ #include "core/PropertyDefinition.h" #include "core/PropertyDefinitionBuilder.h" #include "core/RelationshipDefinition.h" +#include "Catch.h" namespace minifi = org::apache::nifi::minifi; @@ -114,11 +115,11 @@ class MockProcessor : public minifi::core::Processor { std::shared_ptr service = context.getControllerService(linked_service); std::lock_guard lock(control_mutex); if (!disabled.load()) { - assert(true == context.isControllerServiceEnabled(linked_service)); - assert(nullptr != service); - assert("pushitrealgood" == std::static_pointer_cast(service)->doSomething()); + REQUIRE(context.isControllerServiceEnabled(linked_service)); + REQUIRE(nullptr != service); + REQUIRE("pushitrealgood" == std::static_pointer_cast(service)->doSomething()); } else { - assert(false == context.isControllerServiceEnabled(linked_service)); + REQUIRE_FALSE(context.isControllerServiceEnabled(linked_service)); } // verify we have access to the controller service // and verify that we can execute it. diff --git a/libminifi/test/unit/ProvenanceTestHelper.h b/libminifi/test/libtest/unit/ProvenanceTestHelper.h similarity index 100% rename from libminifi/test/unit/ProvenanceTestHelper.h rename to libminifi/test/libtest/unit/ProvenanceTestHelper.h diff --git a/libminifi/test/ReadFromFlowFileTestProcessor.cpp b/libminifi/test/libtest/unit/ReadFromFlowFileTestProcessor.cpp similarity index 100% rename from libminifi/test/ReadFromFlowFileTestProcessor.cpp rename to libminifi/test/libtest/unit/ReadFromFlowFileTestProcessor.cpp diff --git a/libminifi/test/ReadFromFlowFileTestProcessor.h b/libminifi/test/libtest/unit/ReadFromFlowFileTestProcessor.h similarity index 100% rename from libminifi/test/ReadFromFlowFileTestProcessor.h rename to libminifi/test/libtest/unit/ReadFromFlowFileTestProcessor.h diff --git a/libminifi/test/libtest/unit/SingleProcessorTestController.cpp b/libminifi/test/libtest/unit/SingleProcessorTestController.cpp new file mode 100644 index 0000000000..3df87c269e --- /dev/null +++ b/libminifi/test/libtest/unit/SingleProcessorTestController.cpp @@ -0,0 +1,104 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "SingleProcessorTestController.h" + +#include + +#include "FlowFileRecord.h" +#include "range/v3/algorithm/all_of.hpp" + +namespace org::apache::nifi::minifi::test { + +ProcessorTriggerResult SingleProcessorTestController::trigger() { + plan->runProcessor(processor_); + std::unordered_map>> result; + for (const auto& [relationship, connection]: outgoing_connections_) { + std::set> expired_flow_files; + std::vector> output_flow_files; + while (connection->isWorkAvailable()) { + auto output_flow_file = connection->poll(expired_flow_files); + assert(expired_flow_files.empty()); + if (!output_flow_file) continue; + output_flow_files.push_back(std::move(output_flow_file)); + } + result.insert_or_assign(relationship, std::move(output_flow_files)); + } + return result; +} + +ProcessorTriggerResult SingleProcessorTestController::trigger(InputFlowFileData&& input_flow_file_data) { + input_->put(createFlowFile(input_flow_file_data.content, std::move(input_flow_file_data.attributes))); + return trigger(); +} + +ProcessorTriggerResult SingleProcessorTestController::trigger(const std::string_view input_flow_file_content, std::unordered_map input_flow_file_attributes) { + return trigger({input_flow_file_content, std::move(input_flow_file_attributes)}); +} + +ProcessorTriggerResult SingleProcessorTestController::trigger(std::vector&& input_flow_file_datas) { + for (auto& input_flow_file_data : std::move(input_flow_file_datas)) { + input_->put(createFlowFile(input_flow_file_data.content, std::move(input_flow_file_data.attributes))); + } + return trigger(); +} + +bool SingleProcessorTestController::triggerUntil(const std::unordered_map& expected_quantities, + ProcessorTriggerResult& result, + const std::chrono::milliseconds max_duration, + const std::chrono::milliseconds wait_time) { + auto start_time = std::chrono::steady_clock::now(); + while (std::chrono::steady_clock::now() < start_time + max_duration) { + for (auto& [relationship, flow_files] : trigger()) { + result[relationship].insert(result[relationship].end(), flow_files.begin(), flow_files.end()); + } + if (ranges::all_of(expected_quantities, [&result](const auto& kv) { + const auto& [relationship, expected_quantity] = kv; + return result[relationship].size() >= expected_quantity; + })) { + return true; + } + std::this_thread::sleep_for(wait_time); + } + return false; +} + +core::Relationship SingleProcessorTestController::addDynamicRelationship(std::string name) { + auto relationship = core::Relationship{std::move(name), ""}; + outgoing_connections_.insert_or_assign(relationship, plan->addConnection(processor_, relationship, nullptr)); + return relationship; +} + +std::shared_ptr SingleProcessorTestController::createFlowFile(const std::string_view content, std::unordered_map attributes) { + const auto flow_file = std::make_shared(); + for (auto& attr : std::move(attributes)) { + flow_file->setAttribute(attr.first, std::move(attr.second)); + } + auto content_session = plan->getContentRepo()->createSession(); + auto claim = content_session->create(); + auto stream = content_session->write(claim); + stream->write(reinterpret_cast(content.data()), content.size()); + flow_file->setResourceClaim(claim); + flow_file->setSize(stream->size()); + flow_file->setOffset(0); + + stream->close(); + content_session->commit(); + return flow_file; +} + +} // namespace org::apache::nifi::minifi::test diff --git a/libminifi/test/libtest/unit/SingleProcessorTestController.h b/libminifi/test/libtest/unit/SingleProcessorTestController.h new file mode 100644 index 0000000000..994544fe47 --- /dev/null +++ b/libminifi/test/libtest/unit/SingleProcessorTestController.h @@ -0,0 +1,76 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once +#include +#include +#include +#include +#include +#include +#include "TestBase.h" +#include "core/Processor.h" + +using namespace std::literals::chrono_literals; + +namespace org::apache::nifi::minifi::test { +struct InputFlowFileData { + std::string_view content; + std::unordered_map attributes = {}; +}; + +using ProcessorTriggerResult = std::unordered_map>>; + +class SingleProcessorTestController : public TestController { + public: + explicit SingleProcessorTestController(const std::shared_ptr& processor) + : processor_{plan->addProcessor(processor, processor->getName())} + {} + + ProcessorTriggerResult trigger(); + ProcessorTriggerResult trigger(InputFlowFileData&& input_flow_file_data); + ProcessorTriggerResult trigger(const std::string_view input_flow_file_content, std::unordered_map input_flow_file_attributes = {}); + ProcessorTriggerResult trigger(std::vector&& input_flow_file_datas); + bool triggerUntil(const std::unordered_map& expected_quantities, + ProcessorTriggerResult& result, + const std::chrono::milliseconds max_duration, + const std::chrono::milliseconds wait_time = 50ms); + + core::Relationship addDynamicRelationship(std::string name); + + private: + std::shared_ptr createFlowFile(const std::string_view content, std::unordered_map attributes); + + public: + std::shared_ptr plan = createPlan(); + + protected: + core::Processor& getProcessor() const { return *processor_; } + + private: + std::shared_ptr processor_; + std::unordered_map outgoing_connections_{[this] { + std::unordered_map result; + for (const auto& relationship: processor_->getSupportedRelationships()) { + result.insert_or_assign(relationship, plan->addConnection(processor_, relationship, nullptr)); + } + return result; + }()}; + Connection* input_ = plan->addConnection(nullptr, core::Relationship{"success", "success"}, processor_); +}; + +} // namespace org::apache::nifi::minifi::test diff --git a/libminifi/test/unit/SiteToSiteHelper.h b/libminifi/test/libtest/unit/SiteToSiteHelper.h similarity index 100% rename from libminifi/test/unit/SiteToSiteHelper.h rename to libminifi/test/libtest/unit/SiteToSiteHelper.h diff --git a/libminifi/test/StatefulProcessor.cpp b/libminifi/test/libtest/unit/StatefulProcessor.cpp similarity index 100% rename from libminifi/test/StatefulProcessor.cpp rename to libminifi/test/libtest/unit/StatefulProcessor.cpp index cbb0193ee3..d3c786f8f3 100644 --- a/libminifi/test/StatefulProcessor.cpp +++ b/libminifi/test/libtest/unit/StatefulProcessor.cpp @@ -14,10 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include "StatefulProcessor.h" #include -#include "StatefulProcessor.h" #include "Exception.h" #include "core/Resource.h" #include "core/ProcessContext.h" diff --git a/libminifi/test/StatefulProcessor.h b/libminifi/test/libtest/unit/StatefulProcessor.h similarity index 100% rename from libminifi/test/StatefulProcessor.h rename to libminifi/test/libtest/unit/StatefulProcessor.h diff --git a/libminifi/test/TestBase.cpp b/libminifi/test/libtest/unit/TestBase.cpp similarity index 96% rename from libminifi/test/TestBase.cpp rename to libminifi/test/libtest/unit/TestBase.cpp index bc83785c6b..58ec2c7559 100644 --- a/libminifi/test/TestBase.cpp +++ b/libminifi/test/libtest/unit/TestBase.cpp @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "./TestBase.h" +#include "TestBase.h" #include #include @@ -30,9 +30,9 @@ #include "core/logging/LoggerConfiguration.h" #include "core/state/nodes/FlowInformation.h" #include "core/controller/StandardControllerServiceProvider.h" -#include "unit/ProvenanceTestHelper.h" +#include "ProvenanceTestHelper.h" #include "utils/ClassUtils.h" -#include "utils/IntegrationTestUtils.h" +#include "TestUtils.h" #include "utils/Id.h" #include "utils/StringUtils.h" #include "utils/span.h" @@ -71,7 +71,7 @@ void LogTestController::setLevel(std::string_view name, spdlog::level::level_enu } } -std::shared_ptr LogTestController::getLoggerByClassName(std::string_view class_name, const std::optional& id) { +std::shared_ptr LogTestController::getLoggerByClassName(std::string_view class_name, const std::optional& id) { return config ? config->getLogger(class_name, id) : logging::LoggerConfiguration::getConfiguration().getLogger(class_name, id); } @@ -238,8 +238,8 @@ TestPlan::~TestPlan() { controller_services_provider_->clearControllerServices(); } -std::shared_ptr TestPlan::addProcessor(const std::shared_ptr &processor, const std::string& /*name*/, const std::initializer_list& relationships, - bool linkToPrevious) { +std::shared_ptr TestPlan::addProcessor(const std::shared_ptr &processor, const std::string& /*name*/, + const std::initializer_list& relationships, bool linkToPrevious) { if (finalized) { return nullptr; } @@ -282,8 +282,10 @@ std::shared_ptr TestPlan::addProcessor(const std::share } std::shared_ptr node = std::make_shared(processor.get()); processor_nodes_.push_back(node); - std::shared_ptr contextBuilder = minifi::core::ClassLoader::getDefaultClassLoader().instantiate("ProcessContextBuilder", "ProcessContextBuilder"); - contextBuilder = contextBuilder->withContentRepository(content_repo_)->withFlowFileRepository(flow_repo_)->withProvider(controller_services_provider_.get())->withProvenanceRepository(prov_repo_)->withConfiguration(configuration_); + std::shared_ptr contextBuilder = + minifi::core::ClassLoader::getDefaultClassLoader().instantiate("ProcessContextBuilder", "ProcessContextBuilder"); + contextBuilder = contextBuilder->withContentRepository(content_repo_)->withFlowFileRepository(flow_repo_)->withProvider(controller_services_provider_.get()) + ->withProvenanceRepository(prov_repo_)->withConfiguration(configuration_); auto context = contextBuilder->build(node); processor_contexts_.push_back(context); processor_queue_.push_back(processor); @@ -317,7 +319,8 @@ std::shared_ptr TestPlan::addProcessor(const std::strin return addProcessor(processor_name, minifi::utils::IdGenerator::getIdGenerator()->generate(), name, relationships, linkToPrevious); } -minifi::Connection* TestPlan::addConnection(const std::shared_ptr& source_proc, const minifi::core::Relationship& source_relationship, const std::shared_ptr& destination_proc) { +minifi::Connection* TestPlan::addConnection(const std::shared_ptr& source_proc, const minifi::core::Relationship& source_relationship, + const std::shared_ptr& destination_proc) { std::stringstream connection_name; connection_name << (source_proc ? source_proc->getUUIDStr().c_str() : "none") @@ -481,7 +484,7 @@ void TestPlan::scheduleProcessor(const std::shared_ptr& } void TestPlan::scheduleProcessors() { - for(std::size_t target_location = 0; target_location < processor_queue_.size(); ++target_location) { + for (std::size_t target_location = 0; target_location < processor_queue_.size(); ++target_location) { std::shared_ptr processor = processor_queue_.at(target_location); std::shared_ptr context = processor_contexts_.at(target_location); scheduleProcessor(processor, context); @@ -550,7 +553,7 @@ bool TestPlan::runCurrentProcessor() { } bool TestPlan::runCurrentProcessorUntilFlowfileIsProduced(std::chrono::milliseconds wait_duration) { - using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; + using org::apache::nifi::minifi::test::utils::verifyEventHappenedInPollTime; const auto isFlowFileProduced = [&] { runCurrentProcessor(); const std::vector connections = getProcessorOutboundConnections(processor_queue_.at(location)); @@ -674,7 +677,7 @@ std::string TestPlan::getContent(const minifi::core::FlowFile& file) const { auto content_stream = content_repo_->read(*content_claim); auto output_stream = std::make_shared(); minifi::InputStreamPipe{*output_stream}(content_stream); - return utils::span_to(utils::as_span(output_stream->getBuffer()).subspan(file.getOffset(), file.getSize())); + return utils::span_to(minifi::utils::as_span(output_stream->getBuffer()).subspan(file.getOffset(), file.getSize())); } TestController::TestController() @@ -708,7 +711,8 @@ std::shared_ptr TestController::createPlan(PlanConfig config) { flow_version_, config.configuration, config.state_dir ? config.state_dir->string().c_str() : nullptr); } -std::shared_ptr TestController::createPlan(std::shared_ptr configuration, std::optional state_dir, std::shared_ptr content_repo) { +std::shared_ptr TestController::createPlan(std::shared_ptr configuration, std::optional state_dir, + std::shared_ptr content_repo) { return createPlan(PlanConfig{ .configuration = std::move(configuration), .state_dir = std::move(state_dir), diff --git a/libminifi/test/TestBase.h b/libminifi/test/libtest/unit/TestBase.h similarity index 100% rename from libminifi/test/TestBase.h rename to libminifi/test/libtest/unit/TestBase.h diff --git a/libminifi/test/libtest/unit/TestControllerWithFlow.cpp b/libminifi/test/libtest/unit/TestControllerWithFlow.cpp new file mode 100644 index 0000000000..10b68187e3 --- /dev/null +++ b/libminifi/test/libtest/unit/TestControllerWithFlow.cpp @@ -0,0 +1,83 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "TestControllerWithFlow.h" + +#include +#include + +#include "ProvenanceTestHelper.h" +#include "core/yaml/YamlConfiguration.h" +#include "core/repository/VolatileContentRepository.h" +#include "Catch.h" + +TestControllerWithFlow::TestControllerWithFlow(const char* yamlConfigContent, bool setup_flow) { + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + + home_ = createTempDirectory(); + + yaml_path_ = home_ / "config.yml"; + std::ofstream{yaml_path_} << yamlConfigContent; + + configuration_ = std::make_shared(); + configuration_->setHome(home_.string()); + configuration_->set(minifi::Configure::nifi_flow_configuration_file, yaml_path_.string()); + configuration_->set(minifi::Configure::nifi_c2_enable, "true"); + + if (setup_flow) { + setupFlow(); + } +} + +void TestControllerWithFlow::setupFlow() { + std::shared_ptr prov_repo = std::make_shared(); + std::shared_ptr ff_repo = std::make_shared(); + std::shared_ptr content_repo = std::make_shared(); + + REQUIRE(content_repo->initialize(configuration_)); + + auto flow = std::make_shared(core::ConfigurationContext{ + .flow_file_repo = ff_repo, + .content_repo = content_repo, + .configuration = configuration_, + .path = yaml_path_.string(), + .filesystem = std::make_shared(), + .sensitive_properties_encryptor = utils::crypto::EncryptionProvider{utils::crypto::XSalsa20Cipher{utils::crypto::XSalsa20Cipher::generateKey()}} + }); + auto root = flow->getRoot(); + root_ = root.get(); + std::vector> repo_metric_sources{prov_repo, ff_repo, content_repo}; + auto metrics_publisher_store = std::make_unique(configuration_, repo_metric_sources, flow); + metrics_publisher_store_ = metrics_publisher_store.get(); + controller_ = std::make_shared(prov_repo, ff_repo, configuration_, std::move(flow), content_repo, std::move(metrics_publisher_store)); + controller_->load(std::move(root)); +} + + +TestControllerWithFlow::~TestControllerWithFlow() { + if (controller_) { + controller_->stop(); + } + LogTestController::getInstance().reset(); +} diff --git a/extensions/http-curl/HTTPCurlLoader.cpp b/libminifi/test/libtest/unit/TestControllerWithFlow.h similarity index 55% rename from extensions/http-curl/HTTPCurlLoader.cpp rename to libminifi/test/libtest/unit/TestControllerWithFlow.h index c4043429c8..b545c89877 100644 --- a/extensions/http-curl/HTTPCurlLoader.cpp +++ b/libminifi/test/libtest/unit/TestControllerWithFlow.h @@ -1,4 +1,5 @@ /** + * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -15,25 +16,31 @@ * limitations under the License. */ -#ifdef WIN32 -#pragma comment(lib, "wldap32.lib" ) -#pragma comment(lib, "crypt32.lib" ) -#pragma comment(lib, "Ws2_32.lib") +#pragma once + + +#include +#include -#define CURL_STATICLIB -#include -#endif +#include "FlowController.h" +#include "TestBase.h" -#include "core/extension/Extension.h" +class TestControllerWithFlow: public TestController { + public: + explicit TestControllerWithFlow(const char* yamlConfigContent, bool setup_flow = true); -#include "client/HTTPClient.h" + void setupFlow(); -static bool init(const org::apache::nifi::minifi::core::extension::ExtensionConfig& /*config*/) { - return curl_global_init(CURL_GLOBAL_DEFAULT) == CURLE_OK; -} + void startFlow() { + controller_->start(); + } -static void deinit() { - curl_global_cleanup(); -} + ~TestControllerWithFlow(); -REGISTER_EXTENSION("HttpCurlExtension", init, deinit); + std::filesystem::path home_; + std::filesystem::path yaml_path_; + std::shared_ptr configuration_; + std::shared_ptr controller_; + core::ProcessGroup* root_{nullptr}; + minifi::state::MetricsPublisherStore* metrics_publisher_store_{nullptr}; +}; diff --git a/libminifi/test/Utils.h b/libminifi/test/libtest/unit/TestUtils.cpp similarity index 59% rename from libminifi/test/Utils.h rename to libminifi/test/libtest/unit/TestUtils.cpp index 6de35baa72..6c9e1342ff 100644 --- a/libminifi/test/Utils.h +++ b/libminifi/test/libtest/unit/TestUtils.cpp @@ -14,59 +14,123 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#pragma once - -#include -#include -#include -#include - -#include "rapidjson/document.h" -#include "asio.hpp" -#include "asio/ssl.hpp" -#include "net/Ssl.h" -#include "utils/IntegrationTestUtils.h" - -using namespace std::literals::chrono_literals; - -#undef GetObject // windows.h #defines GetObject = GetObjectA or GetObjectW, which conflicts with rapidjson -#include "Connection.h" -#include "FlowFileQueue.h" -#include "Catch.h" - -#define FIELD_ACCESSOR(field) \ - template \ - static auto get_##field(T&& instance) -> decltype((std::forward(instance).field)) { \ - return std::forward(instance).field; \ + +#include "TestUtils.h" + +#include + +#ifdef WIN32 +#include +#include +#endif + +#include "utils/gsl.h" + +#ifdef WIN32 +namespace { + +void setAclOnFileOrDirectory(std::string file_name, DWORD perms, ACCESS_MODE perm_options) { + PSECURITY_DESCRIPTOR security_descriptor = nullptr; + const auto security_descriptor_deleter = gsl::finally([&security_descriptor] { if (security_descriptor) { LocalFree((HLOCAL) security_descriptor); } }); + + PACL old_acl = nullptr; // GetNamedSecurityInfo will set this to a non-owning pointer to a field inside security_descriptor: no need to free it + if (GetNamedSecurityInfo(file_name.c_str(), SE_FILE_OBJECT, DACL_SECURITY_INFORMATION, NULL, NULL, &old_acl, NULL, &security_descriptor) != ERROR_SUCCESS) { + throw std::runtime_error("Could not get security info for file: " + file_name); + } + + char trustee_name[] = "Everyone"; + EXPLICIT_ACCESS explicit_access = { + .grfAccessPermissions = perms, + .grfAccessMode = perm_options, + .grfInheritance = CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE, + .Trustee = { .TrusteeForm = TRUSTEE_IS_NAME, .ptstrName = trustee_name } + }; + + PACL new_acl = nullptr; + const auto new_acl_deleter = gsl::finally([&new_acl] { if (new_acl) { LocalFree((HLOCAL) new_acl); } }); + + if (SetEntriesInAcl(1, &explicit_access, old_acl, &new_acl) != ERROR_SUCCESS) { + throw std::runtime_error("Could not create new ACL for file: " + file_name); } -#define METHOD_ACCESSOR(method) \ - template \ - static auto call_##method(T&& instance, Args&& ...args) -> decltype((std::forward(instance).method(std::forward(args)...))) { \ - return std::forward(instance).method(std::forward(args)...); \ + if (SetNamedSecurityInfo(file_name.data(), SE_FILE_OBJECT, DACL_SECURITY_INFORMATION, NULL, NULL, new_acl, NULL) != ERROR_SUCCESS) { + throw std::runtime_error("Could not set the new ACL for file: " + file_name); } +} + +} // namespace +#endif namespace org::apache::nifi::minifi::test::utils { -namespace internal { -struct JsonContext { - const JsonContext *parent{nullptr}; - std::string_view member; +std::filesystem::path putFileToDir(const std::filesystem::path& dir_path, const std::filesystem::path& file_name, const std::string& content) { + auto file_path = dir_path/file_name; + std::ofstream out_file(file_path, std::ios::binary | std::ios::out); + if (out_file.is_open()) { + out_file << content; + } + return file_path; +} - std::string path() const { - if (!parent) { - return "/"; - } - return minifi::utils::string::join_pack(parent->path(), member, "/"); +std::string getFileContent(const std::filesystem::path& file_name) { + std::ifstream file_handle(file_name, std::ios::binary | std::ios::in); + assert(file_handle.is_open()); + std::string file_content{ (std::istreambuf_iterator(file_handle)), (std::istreambuf_iterator()) }; + return file_content; +} + +void makeFileOrDirectoryNotWritable(const std::filesystem::path& file_name) { +#ifdef WIN32 + setAclOnFileOrDirectory(file_name.string(), FILE_GENERIC_WRITE, DENY_ACCESS); +#else + std::filesystem::permissions(file_name, std::filesystem::perms::owner_write, std::filesystem::perm_options::remove); +#endif +} + +void makeFileOrDirectoryWritable(const std::filesystem::path& file_name) { +#ifdef WIN32 + setAclOnFileOrDirectory(file_name.string(), FILE_GENERIC_WRITE, GRANT_ACCESS); +#else + std::filesystem::permissions(file_name, std::filesystem::perms::owner_write, std::filesystem::perm_options::add); +#endif +} + +minifi::utils::Identifier generateUUID() { + // TODO(hunyadi): Will make the Id generator manage lifetime using a unique_ptr and return a raw ptr on access + static std::shared_ptr id_generator = minifi::utils::IdGenerator::getIdGenerator(); + return id_generator->generate(); +} + +void ManualClock::advance(std::chrono::milliseconds elapsed_time) { + if (elapsed_time.count() < 0) { + throw std::logic_error("A steady clock can only be advanced forward"); + } + std::lock_guard lock(mtx_); + time_ += elapsed_time; + for (auto* cv : cvs_) { + cv->notify_all(); } -}; -} // namespace internal +} -#define REQUIRE_WARN(cond, msg) if (!(cond)) {WARN(msg); REQUIRE(cond);} +bool ManualClock::wait_until(std::condition_variable& cv, std::unique_lock& lck, std::chrono::milliseconds time, const std::function& pred) { + std::chrono::milliseconds now; + { + std::unique_lock lock(mtx_); + now = time_; + cvs_.insert(&cv); + } + cv.wait_for(lck, time - now, [&] { + now = timeSinceEpoch(); + return now >= time || pred(); + }); + { + std::unique_lock lock(mtx_); + cvs_.erase(&cv); + } + return pred(); +} -// carries out a loose match on objects, i.e. it doesn't matter if the -// actual object has extra fields than expected -void matchJSON(const internal::JsonContext& ctx, const rapidjson::Value& actual, const rapidjson::Value& expected, bool strict = false) { +void matchJSON(const internal::JsonContext& ctx, const rapidjson::Value& actual, const rapidjson::Value& expected, bool strict) { if (expected.IsObject()) { REQUIRE_WARN(actual.IsObject(), fmt::format("Expected object at {}", ctx.path())); for (const auto& expected_member : expected.GetObject()) { @@ -91,38 +155,19 @@ void matchJSON(const internal::JsonContext& ctx, const rapidjson::Value& actual, } } -void verifyJSON(const std::string& actual_str, const std::string& expected_str, bool strict = false) { - rapidjson::Document actual, expected; +void verifyJSON(const std::string& actual_str, const std::string& expected_str, bool strict) { + rapidjson::Document actual; + rapidjson::Document expected; REQUIRE_FALSE(actual.Parse(actual_str.c_str()).HasParseError()); REQUIRE_FALSE(expected.Parse(expected_str.c_str()).HasParseError()); matchJSON(internal::JsonContext{}, actual, expected, strict); } -template -class ExceptionSubStringMatcher : public Catch::Matchers::MatcherBase { - public: - explicit ExceptionSubStringMatcher(std::vector exception_substr) : - possible_exception_substrs_(std::move(exception_substr)) {} - - bool match(T const& script_exception) const override { - for (auto& possible_exception_substr : possible_exception_substrs_) { - if (std::string(script_exception.what()).find(possible_exception_substr) != std::string::npos) - return true; - } - return false; - } - - std::string describe() const override { return "Checks whether the exception message contains at least one of the provided exception substrings"; } - - private: - std::vector possible_exception_substrs_; -}; - bool countLogOccurrencesUntil(const std::string& pattern, const int occurrences, const std::chrono::milliseconds max_duration, - const std::chrono::milliseconds wait_time = 50ms) { + const std::chrono::milliseconds wait_time) { auto start_time = std::chrono::steady_clock::now(); while (std::chrono::steady_clock::now() < start_time + max_duration) { if (LogTestController::getInstance().countOccurrences(pattern) == occurrences) @@ -132,7 +177,7 @@ bool countLogOccurrencesUntil(const std::string& pattern, return false; } -std::error_code sendMessagesViaTCP(const std::vector& contents, const asio::ip::tcp::endpoint& remote_endpoint, const std::optional delimiter = std::nullopt) { +std::error_code sendMessagesViaTCP(const std::vector& contents, const asio::ip::tcp::endpoint& remote_endpoint, const std::optional delimiter) { asio::io_context io_context; asio::ip::tcp::socket socket(io_context); std::error_code err; @@ -147,7 +192,7 @@ std::error_code sendMessagesViaTCP(const std::vector& contents if (err) return err; } - return std::error_code(); + return {}; } std::error_code sendUdpDatagram(const asio::const_buffer content, const asio::ip::udp::endpoint& remote_endpoint) { @@ -177,25 +222,12 @@ bool isIPv6Disabled() { return error_code.value() == EADDRNOTAVAIL; } -struct ConnectionTestAccessor { - FIELD_ACCESSOR(queue_); -}; - -struct FlowFileQueueTestAccessor { - FIELD_ACCESSOR(min_size_); - FIELD_ACCESSOR(max_size_); - FIELD_ACCESSOR(target_size_); - FIELD_ACCESSOR(clock_); - FIELD_ACCESSOR(swapped_flow_files_); - FIELD_ACCESSOR(load_task_); - FIELD_ACCESSOR(queue_); -}; std::error_code sendMessagesViaSSL(const std::vector& contents, const asio::ip::tcp::endpoint& remote_endpoint, const std::filesystem::path& ca_cert_path, - const std::optional& ssl_data = std::nullopt, - asio::ssl::context::method method = asio::ssl::context::tls_client) { + const std::optional& ssl_data, + asio::ssl::context::method method) { asio::ssl::context ctx(method); ctx.load_verify_file(ca_cert_path.string()); if (ssl_data) { @@ -228,34 +260,7 @@ std::error_code sendMessagesViaSSL(const std::vector& contents return err; } } - return std::error_code(); -} - -#ifdef WIN32 -inline std::error_code hide_file(const std::filesystem::path& file_name) { - const bool success = SetFileAttributesA(file_name.string().c_str(), FILE_ATTRIBUTE_HIDDEN); - if (!success) { - // note: All possible documented error codes from GetLastError are in [0;15999] at the time of writing. - // The below casting is safe in [0;std::numeric_limits::max()], int max is guaranteed to be at least 32767 - return { static_cast(GetLastError()), std::system_category() }; - } return {}; } -#endif /* WIN32 */ - -template -concept NetworkingProcessor = std::derived_from - && requires(T x) { - {T::Port} -> std::convertible_to; - {x.getPort()} -> std::convertible_to; - }; // NOLINT(readability/braces) - -template -uint16_t scheduleProcessorOnRandomPort(const std::shared_ptr& test_plan, const std::shared_ptr& processor) { - REQUIRE(processor->setProperty(T::Port, "0")); - test_plan->scheduleProcessor(processor); - REQUIRE(minifi::utils::verifyEventHappenedInPollTime(250ms, [&processor] { return processor->getPort() != 0; }, 20ms)); - return processor->getPort(); -} } // namespace org::apache::nifi::minifi::test::utils diff --git a/libminifi/test/libtest/unit/TestUtils.h b/libminifi/test/libtest/unit/TestUtils.h new file mode 100644 index 0000000000..01cb8b404a --- /dev/null +++ b/libminifi/test/libtest/unit/TestUtils.h @@ -0,0 +1,223 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utils/file/FileUtils.h" +#include "utils/Environment.h" +#include "utils/Id.h" +#include "utils/TimeUtil.h" +#include "TestBase.h" + +#include "rapidjson/document.h" +#include "asio.hpp" +#include "asio/ssl.hpp" +#include "utils/net/Ssl.h" + +using namespace std::literals::chrono_literals; + +#undef GetObject // windows.h #defines GetObject = GetObjectA or GetObjectW, which conflicts with rapidjson +#include "Connection.h" +#include "utils/FlowFileQueue.h" +#include "Catch.h" + +#define FIELD_ACCESSOR(field) \ + template \ + static auto get_##field(T&& instance) -> decltype((std::forward(instance).field)) { \ + return std::forward(instance).field; \ + } + +#define METHOD_ACCESSOR(method) \ + template \ + static auto call_##method(T&& instance, Args&& ...args) -> decltype((std::forward(instance).method(std::forward(args)...))) { \ + return std::forward(instance).method(std::forward(args)...); \ + } + +namespace org::apache::nifi::minifi::test::utils { + +std::filesystem::path putFileToDir(const std::filesystem::path& dir_path, const std::filesystem::path& file_name, const std::string& content); +std::string getFileContent(const std::filesystem::path& file_name); + +void makeFileOrDirectoryNotWritable(const std::filesystem::path& file_name); +void makeFileOrDirectoryWritable(const std::filesystem::path& file_name); + +minifi::utils::Identifier generateUUID(); + +class ManualClock : public minifi::utils::timeutils::SteadyClock { + public: + [[nodiscard]] std::chrono::milliseconds timeSinceEpoch() const override { + std::lock_guard lock(mtx_); + return time_; + } + + [[nodiscard]] std::chrono::time_point now() const override { + return std::chrono::steady_clock::time_point{timeSinceEpoch()}; + } + + void advance(std::chrono::milliseconds elapsed_time); + bool wait_until(std::condition_variable& cv, std::unique_lock& lck, std::chrono::milliseconds time, const std::function& pred) override; + + private: + mutable std::mutex mtx_; + std::unordered_set cvs_; + std::chrono::milliseconds time_{0}; +}; + +template +bool verifyEventHappenedInPollTime( + const std::chrono::duration& wait_duration, + Fun&& check, + std::chrono::microseconds check_interval = std::chrono::milliseconds(100)) { + std::chrono::steady_clock::time_point wait_end = std::chrono::steady_clock::now() + wait_duration; + do { + if (std::forward(check)()) { + return true; + } + std::this_thread::sleep_for(check_interval); + } while (std::chrono::steady_clock::now() < wait_end); + return false; +} + +template +bool verifyLogLinePresenceInPollTime(const std::chrono::duration& wait_duration, String&&... patterns) { + auto check = [&patterns...] { + const std::string logs = LogTestController::getInstance().getLogs(); + return ((logs.find(patterns) != std::string::npos) && ...); + }; + return verifyEventHappenedInPollTime(wait_duration, check); +} + +template +bool verifyLogLineVariantPresenceInPollTime(const std::chrono::duration& wait_duration, String&&... patterns) { + auto check = [&patterns...] { + const std::string logs = LogTestController::getInstance().getLogs(); + return ((logs.find(patterns) != std::string::npos) || ...); + }; + return verifyEventHappenedInPollTime(wait_duration, check); +} + +namespace internal { +struct JsonContext { + const JsonContext *parent{nullptr}; + std::string_view member; + + std::string path() const { + if (!parent) { + return "/"; + } + return minifi::utils::string::join_pack(parent->path(), member, "/"); + } +}; +} // namespace internal + +#define REQUIRE_WARN(cond, msg) if (!(cond)) {WARN(msg); REQUIRE(cond);} + +// carries out a loose match on objects, i.e. it doesn't matter if the +// actual object has extra fields than expected +void matchJSON(const internal::JsonContext& ctx, const rapidjson::Value& actual, const rapidjson::Value& expected, bool strict = false); +void verifyJSON(const std::string& actual_str, const std::string& expected_str, bool strict = false); + +template +class ExceptionSubStringMatcher : public Catch::Matchers::MatcherBase { + public: + explicit ExceptionSubStringMatcher(std::vector exception_substr) : + possible_exception_substrs_(std::move(exception_substr)) {} + + bool match(T const& script_exception) const override { + for (auto& possible_exception_substr : possible_exception_substrs_) { + if (std::string(script_exception.what()).find(possible_exception_substr) != std::string::npos) + return true; + } + return false; + } + + std::string describe() const override { return "Checks whether the exception message contains at least one of the provided exception substrings"; } + + private: + std::vector possible_exception_substrs_; +}; + +bool countLogOccurrencesUntil(const std::string& pattern, + const int occurrences, + const std::chrono::milliseconds max_duration, + const std::chrono::milliseconds wait_time = 50ms); +std::error_code sendMessagesViaTCP(const std::vector& contents, const asio::ip::tcp::endpoint& remote_endpoint, const std::optional delimiter = std::nullopt); +std::error_code sendUdpDatagram(const asio::const_buffer content, const asio::ip::udp::endpoint& remote_endpoint); + +std::error_code sendUdpDatagram(const std::span content, const asio::ip::udp::endpoint& remote_endpoint); +std::error_code sendUdpDatagram(const std::string_view content, const asio::ip::udp::endpoint& remote_endpoint); + +bool isIPv6Disabled(); + +struct ConnectionTestAccessor { + FIELD_ACCESSOR(queue_); +}; + +struct FlowFileQueueTestAccessor { + FIELD_ACCESSOR(min_size_); + FIELD_ACCESSOR(max_size_); + FIELD_ACCESSOR(target_size_); + FIELD_ACCESSOR(clock_); + FIELD_ACCESSOR(swapped_flow_files_); + FIELD_ACCESSOR(load_task_); + FIELD_ACCESSOR(queue_); +}; + +std::error_code sendMessagesViaSSL(const std::vector& contents, + const asio::ip::tcp::endpoint& remote_endpoint, + const std::filesystem::path& ca_cert_path, + const std::optional& ssl_data = std::nullopt, + asio::ssl::context::method method = asio::ssl::context::tls_client); + +#ifdef WIN32 +inline std::error_code hide_file(const std::filesystem::path& file_name) { + const bool success = SetFileAttributesA(file_name.string().c_str(), FILE_ATTRIBUTE_HIDDEN); + if (!success) { + // note: All possible documented error codes from GetLastError are in [0;15999] at the time of writing. + // The below casting is safe in [0;std::numeric_limits::max()], int max is guaranteed to be at least 32767 + return { static_cast(GetLastError()), std::system_category() }; + } + return {}; +} +#endif /* WIN32 */ + +template +concept NetworkingProcessor = std::derived_from + && requires(T x) { + {T::Port} -> std::convertible_to; + {x.getPort()} -> std::convertible_to; + }; // NOLINT(readability/braces) + +template +uint16_t scheduleProcessorOnRandomPort(const std::shared_ptr& test_plan, const std::shared_ptr& processor) { + REQUIRE(processor->setProperty(T::Port, "0")); + test_plan->scheduleProcessor(processor); + REQUIRE(verifyEventHappenedInPollTime(250ms, [&processor] { return processor->getPort() != 0; }, 20ms)); + return processor->getPort(); +} + +} // namespace org::apache::nifi::minifi::test::utils diff --git a/libminifi/test/WriteToFlowFileTestProcessor.cpp b/libminifi/test/libtest/unit/WriteToFlowFileTestProcessor.cpp similarity index 100% rename from libminifi/test/WriteToFlowFileTestProcessor.cpp rename to libminifi/test/libtest/unit/WriteToFlowFileTestProcessor.cpp diff --git a/libminifi/test/WriteToFlowFileTestProcessor.h b/libminifi/test/libtest/unit/WriteToFlowFileTestProcessor.h similarity index 100% rename from libminifi/test/WriteToFlowFileTestProcessor.h rename to libminifi/test/libtest/unit/WriteToFlowFileTestProcessor.h diff --git a/libminifi/test/persistence-tests/CMakeLists.txt b/libminifi/test/persistence-tests/CMakeLists.txt index c2913bd6d6..c2b51a2afe 100644 --- a/libminifi/test/persistence-tests/CMakeLists.txt +++ b/libminifi/test/persistence-tests/CMakeLists.txt @@ -32,7 +32,7 @@ FOREACH(testfile ${PERSISTENCE_TESTS}) target_link_libraries(${testfilename} minifi-standard-processors) target_link_libraries(${testfilename} minifi-rocksdb-repos) createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) MATH(EXPR PERSISTENCE_TEST_COUNT "${PERSISTENCE_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) ENDFOREACH() diff --git a/libminifi/test/persistence-tests/PersistenceTests.cpp b/libminifi/test/persistence-tests/PersistenceTests.cpp index a1c4927094..50ab878b55 100644 --- a/libminifi/test/persistence-tests/PersistenceTests.cpp +++ b/libminifi/test/persistence-tests/PersistenceTests.cpp @@ -27,19 +27,19 @@ #include "FlowFileRecord.h" #include "FlowFileRepository.h" #include "properties/Configure.h" -#include "../unit/ProvenanceTestHelper.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/ProvenanceTestHelper.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "catch2/matchers/catch_matchers_string.hpp" #include "../../extensions/libarchive/MergeContent.h" #include "core/repository/VolatileFlowFileRepository.h" #include "../../extensions/rocksdb-repos/DatabaseContentRepository.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" using Connection = minifi::Connection; using MergeContent = minifi::processors::MergeContent; -using minifi::utils::verifyEventHappenedInPollTime; +using minifi::test::utils::verifyEventHappenedInPollTime; namespace { diff --git a/libminifi/test/schema-tests/CMakeLists.txt b/libminifi/test/schema-tests/CMakeLists.txt index a5befa0829..0a53ed1f10 100644 --- a/libminifi/test/schema-tests/CMakeLists.txt +++ b/libminifi/test/schema-tests/CMakeLists.txt @@ -25,7 +25,7 @@ FOREACH(testfile ${SCHEMA_TESTS}) get_filename_component(testfilename "${testfile}" NAME_WE) add_minifi_executable("${testfilename}" "${testfile}") createTests("${testfilename}") - target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} Catch2::Catch2WithMain) target_link_libraries(${testfilename} minifi-standard-processors) target_link_libraries(${testfilename} nlohmann_json_schema_validator) MATH(EXPR SCHEMA_TEST_COUNT "${SCHEMA_TEST_COUNT}+1") diff --git a/libminifi/test/schema-tests/SchemaTests.cpp b/libminifi/test/schema-tests/SchemaTests.cpp index 45a4a8ae5f..3a3f7dc58f 100644 --- a/libminifi/test/schema-tests/SchemaTests.cpp +++ b/libminifi/test/schema-tests/SchemaTests.cpp @@ -18,8 +18,8 @@ #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "../agent/JsonSchema.h" #include "nlohmann/json-schema.hpp" #include "utils/RegexUtils.h" diff --git a/libminifi/test/unit/AbstractProcessorTest.cpp b/libminifi/test/unit/AbstractProcessorTest.cpp index 29467fa81e..99506c63d5 100644 --- a/libminifi/test/unit/AbstractProcessorTest.cpp +++ b/libminifi/test/unit/AbstractProcessorTest.cpp @@ -17,7 +17,7 @@ */ #define EXTENSION_LIST "" // NOLINT(cppcoreguidelines-macro-usage) #include -#include "../Catch.h" +#include "unit/Catch.h" #include "core/AbstractProcessor.h" #include "core/PropertyDefinitionBuilder.h" diff --git a/libminifi/test/unit/ArrayUtilsTests.cpp b/libminifi/test/unit/ArrayUtilsTests.cpp index 31dd36460d..481e1d945f 100644 --- a/libminifi/test/unit/ArrayUtilsTests.cpp +++ b/libminifi/test/unit/ArrayUtilsTests.cpp @@ -15,7 +15,7 @@ * limitations under the License. */ -#include "../Catch.h" +#include "unit/Catch.h" #include "utils/ArrayUtils.h" namespace utils = org::apache::nifi::minifi::utils; diff --git a/libminifi/test/unit/BackTraceTests.cpp b/libminifi/test/unit/BackTraceTests.cpp index 87840b13ef..51eaa59d48 100644 --- a/libminifi/test/unit/BackTraceTests.cpp +++ b/libminifi/test/unit/BackTraceTests.cpp @@ -19,8 +19,8 @@ #include #include #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/BackTrace.h" #include "utils/Monitors.h" #include "utils/ThreadPool.h" diff --git a/libminifi/test/unit/CMakeLists.txt b/libminifi/test/unit/CMakeLists.txt new file mode 100644 index 0000000000..14752003b5 --- /dev/null +++ b/libminifi/test/unit/CMakeLists.txt @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set(NANOFI_TEST_DIR "${CMAKE_SOURCE_DIR}/nanofi/tests/") +GETSOURCEFILES(UNIT_TESTS "${TEST_DIR}/unit/") +GETSOURCEFILES(NANOFI_UNIT_TESTS "${NANOFI_TEST_DIR}") + +if (NOT WIN32) + list(REMOVE_ITEM UNIT_TESTS WindowsCertStoreLocationTests.cpp) +endif() + +SET(UNIT_TEST_COUNT 0) +FOREACH(testfile ${UNIT_TESTS}) + get_filename_component(testfilename "${testfile}" NAME_WE) + add_minifi_executable("${testfilename}" "${TEST_DIR}/unit/${testfile}") + target_compile_definitions("${testfilename}" PRIVATE TZ_DATA_DIR="${CMAKE_BINARY_DIR}/tzdata") + target_compile_definitions("${testfilename}" PRIVATE TEST_RESOURCES="${TEST_RESOURCES}") + createTests("${testfilename}") + target_link_libraries(${testfilename} Catch2::Catch2WithMain) + MATH(EXPR UNIT_TEST_COUNT "${UNIT_TEST_COUNT}+1") + add_test(NAME "${testfilename}" COMMAND "${testfilename}") +ENDFOREACH() +message("-- Finished building ${UNIT_TEST_COUNT} unit test file(s)...") + +if(NOT WIN32 AND ENABLE_NANOFI) + SET(UNIT_TEST_COUNT 0) + FOREACH(testfile ${NANOFI_UNIT_TESTS}) + get_filename_component(testfilename "${testfile}" NAME_WE) + add_minifi_executable("${testfilename}" "${NANOFI_TEST_DIR}/${testfile}") + target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/nanofi/include") + target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/") + target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/processors/") + appendIncludes("${testfilename}") + target_link_libraries(${testfilename} Catch2::Catch2WithMain Threads::Threads) + + target_wholearchive_library(${testfilename} nanofi) + + createTests(${testfilename}) + + target_link_libraries(${testfilename} minifi-standard-processors) + + MATH(EXPR UNIT_TEST_COUNT "${UNIT_TEST_COUNT}+1") + add_test(NAME "${testfilename}" COMMAND "${testfilename}") + ENDFOREACH() + message("-- Finished building ${UNIT_TEST_COUNT} NanoFi unit test file(s)...") +endif() diff --git a/libminifi/test/unit/CRCTests.cpp b/libminifi/test/unit/CRCTests.cpp index 3defb7ecac..6da6be1c4e 100644 --- a/libminifi/test/unit/CRCTests.cpp +++ b/libminifi/test/unit/CRCTests.cpp @@ -20,8 +20,8 @@ #include #include "io/CRCStream.h" #include "io/OutputStream.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/gsl.h" TEST_CASE("Test CRC1", "[testcrc1]") { diff --git a/libminifi/test/unit/CertificateUtilsTests.cpp b/libminifi/test/unit/CertificateUtilsTests.cpp index 8e76787eff..912eebb07a 100644 --- a/libminifi/test/unit/CertificateUtilsTests.cpp +++ b/libminifi/test/unit/CertificateUtilsTests.cpp @@ -15,7 +15,7 @@ * limitations under the License. */ -#include "../Catch.h" +#include "unit/Catch.h" #include "date/date.h" #include "openssl/core_names.h" #include "utils/file/FileUtils.h" diff --git a/libminifi/test/unit/ChecksumCalculatorTests.cpp b/libminifi/test/unit/ChecksumCalculatorTests.cpp index 5bd658d5e0..fabecddfe6 100644 --- a/libminifi/test/unit/ChecksumCalculatorTests.cpp +++ b/libminifi/test/unit/ChecksumCalculatorTests.cpp @@ -17,9 +17,9 @@ #include -#include "../TestBase.h" -#include "../Catch.h" -#include "utils/TestUtils.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/TestUtils.h" #include "utils/ChecksumCalculator.h" namespace { @@ -30,7 +30,7 @@ namespace { TEST_CASE("ChecksumCalculator can calculate the checksum, which is equal to sha256sum", "[ChecksumCalculator]") { TestController test_controller; auto test_dir = test_controller.createTempDirectory(); - auto file_location = utils::putFileToDir(test_dir, "simple.txt", "one line of text\n"); + auto file_location = minifi::test::utils::putFileToDir(test_dir, "simple.txt", "one line of text\n"); REQUIRE(std::string{utils::ChecksumCalculator::CHECKSUM_TYPE} == std::string{"SHA256"}); // the first size_t{} is required by Catch2; it can't use a constexpr expression directly @@ -44,7 +44,7 @@ TEST_CASE("ChecksumCalculator can calculate the checksum, which is equal to sha2 TEST_CASE("On Windows text files, the checksum calculated is also the same as sha256sum", "[ChecksumCalculator]") { TestController test_controller; auto test_dir = test_controller.createTempDirectory(); - auto file_location = utils::putFileToDir(test_dir, "simple.txt", "one line of text\r\n"); + auto file_location = minifi::test::utils::putFileToDir(test_dir, "simple.txt", "one line of text\r\n"); utils::ChecksumCalculator checksum_calculator; checksum_calculator.setFileLocation(file_location); @@ -54,7 +54,7 @@ TEST_CASE("On Windows text files, the checksum calculated is also the same as sh TEST_CASE("The checksum can be reset and recomputed", "[ChecksumCalculator]") { TestController test_controller; auto test_dir = test_controller.createTempDirectory(); - auto file_location = utils::putFileToDir(test_dir, "simple.txt", "one line of text\n"); + auto file_location = minifi::test::utils::putFileToDir(test_dir, "simple.txt", "one line of text\n"); utils::ChecksumCalculator checksum_calculator; checksum_calculator.setFileLocation(file_location); @@ -73,13 +73,13 @@ TEST_CASE("The checksum can be reset and recomputed", "[ChecksumCalculator]") { TEST_CASE("If the file location is updated, the checksum will be recomputed", "[ChecksumCalculator]") { TestController test_controller; auto test_dir = test_controller.createTempDirectory(); - auto file_location = utils::putFileToDir(test_dir, "simple.txt", "one line of text\n"); + auto file_location = minifi::test::utils::putFileToDir(test_dir, "simple.txt", "one line of text\n"); utils::ChecksumCalculator checksum_calculator; checksum_calculator.setFileLocation(file_location); REQUIRE(checksum_calculator.getChecksum() == CHECKSUM_FOR_ONE_LINE_OF_TEXT); - auto other_file_location = utils::putFileToDir(test_dir, "long.txt", "one line of text\nanother line of text\n"); + auto other_file_location = minifi::test::utils::putFileToDir(test_dir, "long.txt", "one line of text\nanother line of text\n"); checksum_calculator.setFileLocation(other_file_location); REQUIRE(checksum_calculator.getChecksum() == CHECKSUM_FOR_TWO_LINES_OF_TEXT); } @@ -89,7 +89,7 @@ TEST_CASE("Checksums can be computed for binary (eg. encrypted) files, too", "[C auto test_dir = test_controller.createTempDirectory(); std::string binary_data(size_t{256}, '\0'); std::iota(binary_data.begin(), binary_data.end(), 'x'); - auto file_location = utils::putFileToDir(test_dir, "simple.txt", binary_data); + auto file_location = minifi::test::utils::putFileToDir(test_dir, "simple.txt", binary_data); utils::ChecksumCalculator checksum_calculator; checksum_calculator.setFileLocation(file_location); @@ -99,11 +99,11 @@ TEST_CASE("Checksums can be computed for binary (eg. encrypted) files, too", "[C TEST_CASE("The agent identifier is excluded from the checksum", "[ChecksumCalculator]") { TestController test_controller; auto test_dir = test_controller.createTempDirectory(); - auto file_location_1 = utils::putFileToDir(test_dir, "agent_one.txt", + auto file_location_1 = minifi::test::utils::putFileToDir(test_dir, "agent_one.txt", "nifi.c2.agent.class=Test\n" "nifi.c2.agent.identifier=Test-111\n" "nifi.c2.agent.heartbeat.period=10 sec\n"); - auto file_location_2 = utils::putFileToDir(test_dir, "agent_two.txt", + auto file_location_2 = minifi::test::utils::putFileToDir(test_dir, "agent_two.txt", "nifi.c2.agent.class=Test\n" "nifi.c2.agent.identifier=Test-222\n" "nifi.c2.agent.heartbeat.period=10 sec\n"); diff --git a/libminifi/test/unit/ClassUtilsTests.cpp b/libminifi/test/unit/ClassUtilsTests.cpp index 95a9a79cf2..5b373439d7 100644 --- a/libminifi/test/unit/ClassUtilsTests.cpp +++ b/libminifi/test/unit/ClassUtilsTests.cpp @@ -19,8 +19,8 @@ #include #include #include "utils/ClassUtils.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" TEST_CASE("Test ShortNames", "[testcrc1]") { std::string className; diff --git a/libminifi/test/unit/CollectionUtilsTests.cpp b/libminifi/test/unit/CollectionUtilsTests.cpp index cf487cf601..8012025859 100644 --- a/libminifi/test/unit/CollectionUtilsTests.cpp +++ b/libminifi/test/unit/CollectionUtilsTests.cpp @@ -20,8 +20,8 @@ #include #include #include "utils/CollectionUtils.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" TEST_CASE("TestHaveCommonItem", "[haveCommonItem]") { auto verify = [](std::initializer_list a, std::initializer_list b, bool result) { diff --git a/libminifi/test/unit/ComponentManifestTests.cpp b/libminifi/test/unit/ComponentManifestTests.cpp index 907300750e..5ea244280a 100644 --- a/libminifi/test/unit/ComponentManifestTests.cpp +++ b/libminifi/test/unit/ComponentManifestTests.cpp @@ -17,8 +17,8 @@ */ #undef LOAD_EXTENSIONS -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/Core.h" #include "core/ConfigurableComponent.h" #include "core/controller/ControllerService.h" diff --git a/libminifi/test/unit/ConfigurationChecksumsTests.cpp b/libminifi/test/unit/ConfigurationChecksumsTests.cpp index 1984234418..6da67a66e6 100644 --- a/libminifi/test/unit/ConfigurationChecksumsTests.cpp +++ b/libminifi/test/unit/ConfigurationChecksumsTests.cpp @@ -15,9 +15,9 @@ * limitations under the License. */ -#include "../TestBase.h" -#include "../Catch.h" -#include "utils/TestUtils.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/TestUtils.h" #include "core/state/nodes/ConfigurationChecksums.h" #include "utils/ChecksumCalculator.h" @@ -36,7 +36,7 @@ TEST_CASE("If no checksum calculators are added, we get an empty node", "[Config TEST_CASE("If one checksum calculator is added, we get a node with one child", "[ConfigurationChecksums]") { TestController test_controller; auto test_dir = test_controller.createTempDirectory(); - auto file_location = utils::putFileToDir(test_dir, "simple.txt", "one line of text\n"); + auto file_location = minifi::test::utils::putFileToDir(test_dir, "simple.txt", "one line of text\n"); utils::ChecksumCalculator checksum_calculator; checksum_calculator.setFileLocation(file_location); @@ -58,8 +58,8 @@ TEST_CASE("If one checksum calculator is added, we get a node with one child", " TEST_CASE("If two checksum calculators are added, we get a node with two children", "[ConfigurationChecksums]") { TestController test_controller; auto test_dir = test_controller.createTempDirectory(); - auto file_location_1 = utils::putFileToDir(test_dir, "first.txt", "this is the first file\n"); - auto file_location_2 = utils::putFileToDir(test_dir, "second.txt", "this is the second file\n"); + auto file_location_1 = minifi::test::utils::putFileToDir(test_dir, "first.txt", "this is the first file\n"); + auto file_location_2 = minifi::test::utils::putFileToDir(test_dir, "second.txt", "this is the second file\n"); utils::ChecksumCalculator checksum_calculator_1; checksum_calculator_1.setFileLocation(file_location_1); diff --git a/libminifi/test/unit/ConnectionTests.cpp b/libminifi/test/unit/ConnectionTests.cpp index 9a82a893f0..343a9bd8e2 100644 --- a/libminifi/test/unit/ConnectionTests.cpp +++ b/libminifi/test/unit/ConnectionTests.cpp @@ -16,9 +16,9 @@ */ #include "Connection.h" -#include "../TestBase.h" -#include "../Catch.h" -#include "ProvenanceTestHelper.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/ProvenanceTestHelper.h" TEST_CASE("Connection::poll() works correctly", "[poll]") { const auto flow_repo = std::make_shared(); diff --git a/libminifi/test/unit/ControllerSocketMetricsPublisherTest.cpp b/libminifi/test/unit/ControllerSocketMetricsPublisherTest.cpp index 8db4e2f000..702583bbc3 100644 --- a/libminifi/test/unit/ControllerSocketMetricsPublisherTest.cpp +++ b/libminifi/test/unit/ControllerSocketMetricsPublisherTest.cpp @@ -14,8 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "c2/ControllerSocketMetricsPublisher.h" #include "core/state/nodes/MetricsBase.h" #include "core/state/nodes/QueueMetrics.h" diff --git a/libminifi/test/unit/CoreTests.cpp b/libminifi/test/unit/CoreTests.cpp index 32764521fc..8c3fd6ef68 100644 --- a/libminifi/test/unit/CoreTests.cpp +++ b/libminifi/test/unit/CoreTests.cpp @@ -15,7 +15,7 @@ * limitations under the License. */ -#include "../Catch.h" +#include "unit/Catch.h" #include "core/Core.h" namespace org::apache::nifi::minifi::test { diff --git a/libminifi/test/unit/CpuUsageTest.cpp b/libminifi/test/unit/CpuUsageTest.cpp index 2fa486df7e..6aca43b738 100644 --- a/libminifi/test/unit/CpuUsageTest.cpp +++ b/libminifi/test/unit/CpuUsageTest.cpp @@ -21,8 +21,8 @@ #include #include "utils/SystemCpuUsageTracker.h" #include "utils/ProcessCpuUsageTracker.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" using namespace std::literals::chrono_literals; using steady_clock = std::chrono::steady_clock; diff --git a/libminifi/test/unit/CronTests.cpp b/libminifi/test/unit/CronTests.cpp index ade38b86ea..6d16e6609a 100644 --- a/libminifi/test/unit/CronTests.cpp +++ b/libminifi/test/unit/CronTests.cpp @@ -16,7 +16,7 @@ */ #include -#include "../Catch.h" +#include "unit/Catch.h" #include "utils/Cron.h" #include "date/date.h" #include "date/tz.h" diff --git a/libminifi/test/unit/DecryptorTests.cpp b/libminifi/test/unit/DecryptorTests.cpp index ca95d191e8..68c613695c 100644 --- a/libminifi/test/unit/DecryptorTests.cpp +++ b/libminifi/test/unit/DecryptorTests.cpp @@ -16,8 +16,8 @@ */ #include "properties/Decryptor.h" -#include "../Catch.h" -#include "../TestBase.h" +#include "unit/Catch.h" +#include "unit/TestBase.h" #include "StringUtils.h" #include "properties/Configure.h" @@ -89,7 +89,8 @@ TEST_CASE("Decryptor can decrypt a configuration file", "[decryptSensitiveProper minifi::Decryptor decryptor{utils::crypto::EncryptionProvider{encryption_key}}; minifi::Configure configuration{decryptor}; - configuration.setHome("resources"); + std::filesystem::path resources_dir{TEST_RESOURCES}; + configuration.setHome(resources_dir); configuration.loadConfigureFile("encrypted.minifi.properties"); REQUIRE_FALSE(configuration.getConfiguredKeys().empty()); @@ -114,7 +115,8 @@ TEST_CASE("Decryptor can decrypt a configuration file", "[decryptSensitiveProper } TEST_CASE("Decryptor can be created from a bootstrap file", "[create]") { - const auto valid_decryptor = minifi::Decryptor::create("resources"); + std::filesystem::path resources_dir{TEST_RESOURCES}; + const auto valid_decryptor = minifi::Decryptor::create(resources_dir); REQUIRE(valid_decryptor); REQUIRE(valid_decryptor->decrypt("HvbPejGT3ur9/00gXQK/dJCYwaNqhopf||CiXKiNaljSN7VkLXP5zfJnb4+4UcKIG3ddwuVfSPpkRRfT4=") == "SpeakFriendAndEnter"); diff --git a/libminifi/test/unit/DiskSpaceWatchdogTests.cpp b/libminifi/test/unit/DiskSpaceWatchdogTests.cpp index 6feb771db9..7be7734086 100644 --- a/libminifi/test/unit/DiskSpaceWatchdogTests.cpp +++ b/libminifi/test/unit/DiskSpaceWatchdogTests.cpp @@ -18,8 +18,8 @@ #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "properties/Configure.h" #include "DiskSpaceWatchdog.h" diff --git a/libminifi/test/unit/DistinguishedNameTests.cpp b/libminifi/test/unit/DistinguishedNameTests.cpp index 65c5330835..40c1cd31f0 100644 --- a/libminifi/test/unit/DistinguishedNameTests.cpp +++ b/libminifi/test/unit/DistinguishedNameTests.cpp @@ -16,7 +16,7 @@ */ #include "utils/tls/DistinguishedName.h" -#include "../Catch.h" +#include "unit/Catch.h" namespace utils = org::apache::nifi::minifi::utils; diff --git a/libminifi/test/unit/DynamicPropertyTests.cpp b/libminifi/test/unit/DynamicPropertyTests.cpp index 2552c397d0..c603f90932 100644 --- a/libminifi/test/unit/DynamicPropertyTests.cpp +++ b/libminifi/test/unit/DynamicPropertyTests.cpp @@ -18,8 +18,8 @@ #include #include "core/ConfigurableComponent.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" namespace org::apache::nifi::minifi::core { diff --git a/libminifi/test/unit/EncryptionUtilsTests.cpp b/libminifi/test/unit/EncryptionUtilsTests.cpp index 505524c838..e4dcd0310c 100644 --- a/libminifi/test/unit/EncryptionUtilsTests.cpp +++ b/libminifi/test/unit/EncryptionUtilsTests.cpp @@ -17,8 +17,8 @@ #include "utils/crypto/EncryptionUtils.h" #include "utils/StringUtils.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" namespace utils = org::apache::nifi::minifi::utils; diff --git a/libminifi/test/unit/EnvironmentUtilsTests.cpp b/libminifi/test/unit/EnvironmentUtilsTests.cpp index aaaacad1bc..4441f59015 100644 --- a/libminifi/test/unit/EnvironmentUtilsTests.cpp +++ b/libminifi/test/unit/EnvironmentUtilsTests.cpp @@ -25,8 +25,8 @@ #include "utils/Environment.h" #include "utils/file/PathUtils.h" #include "utils/gsl.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" TEST_CASE("getenv already existing", "[getenv]") { auto res = utils::Environment::getEnvironmentVariable("PATH"); diff --git a/libminifi/test/unit/ExceptionTests.cpp b/libminifi/test/unit/ExceptionTests.cpp index 02c7925472..44046c2846 100644 --- a/libminifi/test/unit/ExceptionTests.cpp +++ b/libminifi/test/unit/ExceptionTests.cpp @@ -17,8 +17,8 @@ */ -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "Exception.h" TEST_CASE("Test general exception .what()", "[testi general what]") { diff --git a/libminifi/test/unit/ExpectedTest.cpp b/libminifi/test/unit/ExpectedTest.cpp index 5963fbeef2..6c491e4273 100644 --- a/libminifi/test/unit/ExpectedTest.cpp +++ b/libminifi/test/unit/ExpectedTest.cpp @@ -18,8 +18,8 @@ #define EXTENSION_LIST "" // NOLINT(cppcoreguidelines-macro-usage) #include #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/expected.h" #include "utils/gsl.h" diff --git a/libminifi/test/unit/ExtendedKeyUsageTests.cpp b/libminifi/test/unit/ExtendedKeyUsageTests.cpp index 0163a6d346..f2564dc5f5 100644 --- a/libminifi/test/unit/ExtendedKeyUsageTests.cpp +++ b/libminifi/test/unit/ExtendedKeyUsageTests.cpp @@ -14,13 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifdef OPENSSL_SUPPORT - #include #include "utils/gsl.h" #include "utils/tls/ExtendedKeyUsage.h" -#include "../Catch.h" +#include "unit/Catch.h" namespace utils = org::apache::nifi::minifi::utils; @@ -191,5 +189,3 @@ TEST_CASE("ExtendedKeyUsage ignores ASN.1 flags higher than 15", "[constructor]" REQUIRE(createExtendedKeyUsage({128}) == nullptr); REQUIRE(createExtendedKeyUsage({1, 2, 200}) == nullptr); } - -#endif // OPENSSL_SUPPORT diff --git a/libminifi/test/unit/ExtensionVerificationTests.cpp b/libminifi/test/unit/ExtensionVerificationTests.cpp index 9fdd06a5bd..22bc23cd55 100644 --- a/libminifi/test/unit/ExtensionVerificationTests.cpp +++ b/libminifi/test/unit/ExtensionVerificationTests.cpp @@ -14,16 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - #define CUSTOM_EXTENSION_INIT -#undef NDEBUG #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "agent/agent_version.h" #include "core/extension/Utils.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" using namespace std::literals; diff --git a/libminifi/test/unit/FileMutexTests.cpp b/libminifi/test/unit/FileMutexTests.cpp index c00ad40866..3d14c191cb 100644 --- a/libminifi/test/unit/FileMutexTests.cpp +++ b/libminifi/test/unit/FileMutexTests.cpp @@ -21,7 +21,7 @@ #include "utils/FileMutex.h" #include "utils/file/FileUtils.h" #include "utils/OsUtils.h" -#include "../TestBase.h" +#include "unit/TestBase.h" namespace minifi = org::apache::nifi::minifi; diff --git a/libminifi/test/unit/FilePatternTests.cpp b/libminifi/test/unit/FilePatternTests.cpp index 9032888bf4..ce04baa6fc 100644 --- a/libminifi/test/unit/FilePatternTests.cpp +++ b/libminifi/test/unit/FilePatternTests.cpp @@ -19,8 +19,8 @@ #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/file/FilePattern.h" #include "range/v3/view/map.hpp" #include "range/v3/view/join.hpp" diff --git a/libminifi/test/unit/FileStreamTests.cpp b/libminifi/test/unit/FileStreamTests.cpp index 314f2ba8ee..c514e86378 100644 --- a/libminifi/test/unit/FileStreamTests.cpp +++ b/libminifi/test/unit/FileStreamTests.cpp @@ -22,8 +22,8 @@ #include #include "io/FileStream.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/gsl.h" #include "utils/file/FileUtils.h" diff --git a/libminifi/test/unit/FileSystemRepositoryTests.cpp b/libminifi/test/unit/FileSystemRepositoryTests.cpp index 52c40a6dd0..5e7d49fb2f 100644 --- a/libminifi/test/unit/FileSystemRepositoryTests.cpp +++ b/libminifi/test/unit/FileSystemRepositoryTests.cpp @@ -24,12 +24,11 @@ #include "utils/gsl.h" #include "utils/OsUtils.h" -#include "utils/TestUtils.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestUtils.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/Literals.h" #include "core/repository/FileSystemRepository.h" -#include "utils/IntegrationTestUtils.h" #include "utils/file/FileUtils.h" using namespace std::literals::chrono_literals; @@ -63,7 +62,7 @@ TEST_CASE("Test Physical memory usage", "[testphysicalmemoryusage]") { stream->write(as_bytes(fragment)); } - using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; + using org::apache::nifi::minifi::test::utils::verifyEventHappenedInPollTime; CHECK(verifyEventHappenedInPollTime(5s, [&] { const auto end_memory = minifi::utils::OsUtils::getCurrentProcessPhysicalMemoryUsage(); REQUIRE(end_memory > 0); diff --git a/libminifi/test/unit/FileSystemTests.cpp b/libminifi/test/unit/FileSystemTests.cpp index d85ab906d4..06c63af7d4 100644 --- a/libminifi/test/unit/FileSystemTests.cpp +++ b/libminifi/test/unit/FileSystemTests.cpp @@ -20,8 +20,8 @@ #include #include #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/file/FileSystem.h" using utils::crypto::EncryptionProvider; diff --git a/libminifi/test/unit/FileTriggerTests.cpp b/libminifi/test/unit/FileTriggerTests.cpp index 86dfb1b2fc..c2132b42f0 100644 --- a/libminifi/test/unit/FileTriggerTests.cpp +++ b/libminifi/test/unit/FileTriggerTests.cpp @@ -20,8 +20,8 @@ #include #include "c2/triggers/FileUpdateTrigger.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/Processor.h" #include "core/ClassLoader.h" diff --git a/libminifi/test/unit/FileUtilsTests.cpp b/libminifi/test/unit/FileUtilsTests.cpp index 5e9af4f672..a2e9b9aa2f 100644 --- a/libminifi/test/unit/FileUtilsTests.cpp +++ b/libminifi/test/unit/FileUtilsTests.cpp @@ -23,8 +23,8 @@ #include #include #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/Core.h" #include "utils/file/FileUtils.h" #include "utils/gsl.h" diff --git a/libminifi/test/unit/FlatMapTests.cpp b/libminifi/test/unit/FlatMapTests.cpp index 6478210061..5272c7eff4 100644 --- a/libminifi/test/unit/FlatMapTests.cpp +++ b/libminifi/test/unit/FlatMapTests.cpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/FlatMap.h" diff --git a/libminifi/test/unit/FlowFileQueueSwapTests.cpp b/libminifi/test/unit/FlowFileQueueSwapTests.cpp index 80d0f4f745..4b53d661c8 100644 --- a/libminifi/test/unit/FlowFileQueueSwapTests.cpp +++ b/libminifi/test/unit/FlowFileQueueSwapTests.cpp @@ -14,16 +14,223 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#include +#include +#include +#include +#include +#include #include "Connection.h" - -#include "../TestBase.h" -#include "SwapTestController.h" +#include "unit/TestBase.h" +#include "unit/TestUtils.h" +#include "unit/Catch.h" +#include "unit/ProvenanceTestHelper.h" +#include "../utils/gsl.h" namespace org::apache::nifi::minifi::test { +using Timepoint = std::chrono::time_point; + +enum EventKind { + Store, Load +}; + +struct SwapEvent { + EventKind kind; + std::vector flow_files; + + void verifyTimes(std::initializer_list seconds) { + REQUIRE(flow_files.size() == seconds.size()); + size_t idx = 0; + for (auto& second : seconds) { + REQUIRE(flow_files[idx].to_be_processed_after == Timepoint{std::chrono::seconds{second}}); + ++idx; + } + } +}; + +class SwappingFlowFileTestRepo : public TestFlowRepository, public minifi::SwapManager { + public: + void store(std::vector> flow_files) override { + std::vector ids; + for (const auto& ff : flow_files) { + ids.push_back(minifi::SwappedFlowFile{ff->getUUID(), ff->getPenaltyExpiration()}); + minifi::io::BufferStream output; + std::static_pointer_cast(ff)->Serialize(output); + Put(ff->getUUIDStr().c_str(), reinterpret_cast(output.getBuffer().data()), output.size()); + } + swap_events_.push_back({Store, ids}); + } + + std::future>> load(std::vector flow_files) override { + swap_events_.push_back({Load, flow_files}); + LoadTask load_task; + auto future = load_task.promise.get_future(); + load_task.result.reserve(flow_files.size()); + for (const auto& ff_id : flow_files) { + std::string value; + Get(ff_id.id.to_string().c_str(), value); + minifi::utils::Identifier container_id; + auto ff = minifi::FlowFileRecord::DeSerialize(gsl::make_span(value).as_span(), content_repo_, container_id); + ff->setPenaltyExpiration(ff_id.to_be_processed_after); + load_task.result.push_back(std::move(ff)); + } + load_tasks_.push_back(std::move(load_task)); + return future; + } + + struct LoadTask { + std::promise>> promise; + std::vector> result; + + void complete() { + promise.set_value(result); + } + }; + + std::vector load_tasks_; + std::vector swap_events_; +}; + +using FlowFilePtr = std::shared_ptr; +using FlowFilePtrVec = std::vector; + +struct FlowFileComparator { + bool operator()(const FlowFilePtr& left, const FlowFilePtr& right) const { + return left->getPenaltyExpiration() < right->getPenaltyExpiration(); + } +}; + +struct VerifiedQueue { + void push(FlowFilePtr ff) { + size(); + impl.push(ff); + ref_.insert(std::lower_bound(ref_.begin(), ref_.end(), ff, FlowFileComparator{}), ff); + size(); + } + + FlowFilePtr poll() { + size(); + FlowFilePtr ff = impl.pop(); + REQUIRE(!ref_.empty()); + // the order when flow files have the same penalty is not fixed + REQUIRE(ff->getPenaltyExpiration() == ref_.front()->getPenaltyExpiration()); + ref_.erase(ref_.begin()); + size(); + return ff; + } + + void verify(std::initializer_list live, std::optional> inter, std::initializer_list swapped) const { + // check live ffs + auto live_copy = utils::FlowFileQueueTestAccessor::get_queue_(impl); + REQUIRE(live_copy.size() == live.size()); + for (auto sec : live) { + auto min = live_copy.popMin(); + REQUIRE(min->getPenaltyExpiration() == Timepoint{std::chrono::seconds{sec}}); + } + + // check inter ffs + if (!inter) { + REQUIRE_FALSE(utils::FlowFileQueueTestAccessor::get_load_task_(impl).has_value()); + } else { + auto& intermediate = utils::FlowFileQueueTestAccessor::get_load_task_(impl)->intermediate_items; + REQUIRE(intermediate.size() == inter->size()); + size_t idx = 0; + for (auto sec : inter.value()) { + REQUIRE(intermediate[idx]->getPenaltyExpiration() == Timepoint{std::chrono::seconds{sec}}); + ++idx; + } + } + + // check swapped ffs + auto swapped_copy = utils::FlowFileQueueTestAccessor::get_swapped_flow_files_(impl); + REQUIRE(swapped_copy.size() == swapped.size()); + for (auto sec : swapped) { + auto min = swapped_copy.popMin(); + REQUIRE(min.to_be_processed_after == Timepoint{std::chrono::seconds{sec}}); + } + } + + bool isWorkAvailable() const { + return impl.isWorkAvailable(); + } + + size_t size() const { + size_t result = impl.size(); + REQUIRE(result == ref_.size()); + return result; + } + + explicit VerifiedQueue(std::shared_ptr swap_manager) + : impl(std::move(swap_manager)) {} + + minifi::utils::FlowFileQueue impl; + FlowFilePtrVec ref_; +}; + +class SwapTestController : public TestController { + public: + SwapTestController() { + content_repo_ = std::make_shared(); + flow_repo_ = std::make_shared(); + flow_repo_->loadComponent(content_repo_); + clock_ = std::make_shared(); + minifi::utils::timeutils::setClock(clock_); + queue_ = std::make_shared(std::static_pointer_cast(flow_repo_)); + } + + void setLimits(size_t min_size, size_t target_size, size_t max_size) { + queue_->impl.setMinSize(min_size); + queue_->impl.setTargetSize(target_size); + queue_->impl.setMaxSize(max_size); + } + + struct SwapEventPattern { + EventKind kind; + std::initializer_list seconds; + }; + + void verifySwapEvents(std::vector events) { + REQUIRE(flow_repo_->swap_events_.size() == events.size()); + size_t idx = 0; + for (auto& pattern : events) { + REQUIRE(pattern.kind == flow_repo_->swap_events_[idx].kind); + flow_repo_->swap_events_[idx].verifyTimes(pattern.seconds); + } + } + + void clearSwapEvents() { + flow_repo_->swap_events_.clear(); + } + + void verifyQueue(std::initializer_list live, std::optional> inter, std::initializer_list swapped) { + queue_->verify(live, inter, swapped); + } + + void pushAll(std::initializer_list seconds) { + for (auto sec : seconds) { + auto ff = std::static_pointer_cast(std::make_shared()); + ff->setPenaltyExpiration(Timepoint{std::chrono::seconds{sec}}); + queue_->push(std::move(ff)); + } + } + + void popAll(std::initializer_list seconds, bool check_is_work_available = false) { + for (auto sec : seconds) { + if (check_is_work_available) { + REQUIRE(queue_->isWorkAvailable()); + } + auto ff = queue_->poll(); + REQUIRE(ff->getPenaltyExpiration() == Timepoint{std::chrono::seconds{sec}}); + } + } + + std::shared_ptr flow_repo_; + std::shared_ptr content_repo_; + std::shared_ptr queue_; + std::shared_ptr clock_; +}; + TEST_CASE("Setting swap threshold sets underlying queue limits", "[SwapTest1]") { const size_t target_size = 4; const size_t min_size = target_size / 2; diff --git a/libminifi/test/unit/FlowFileQueueTests.cpp b/libminifi/test/unit/FlowFileQueueTests.cpp index 0d0c1b2b9a..589575e76a 100644 --- a/libminifi/test/unit/FlowFileQueueTests.cpp +++ b/libminifi/test/unit/FlowFileQueueTests.cpp @@ -18,9 +18,9 @@ #include #include "FlowFileQueue.h" -#include "../TestBase.h" -#include "../Catch.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/TestUtils.h" namespace core = minifi::core; @@ -120,19 +120,19 @@ TEST_CASE("Penalized flow files are popped from the FlowFileQueue in the order t REQUIRE_FALSE(queue.isWorkAvailable()); - REQUIRE(utils::verifyEventHappenedInPollTime(std::chrono::seconds{1}, PenaltyHasExpired{flow_file_2}, std::chrono::milliseconds{10})); + REQUIRE(minifi::test::utils::verifyEventHappenedInPollTime(std::chrono::seconds{1}, PenaltyHasExpired{flow_file_2}, std::chrono::milliseconds{10})); REQUIRE(queue.isWorkAvailable()); REQUIRE(queue.pop() == flow_file_2); - REQUIRE(utils::verifyEventHappenedInPollTime(std::chrono::seconds{1}, PenaltyHasExpired{flow_file_4}, std::chrono::milliseconds{10})); + REQUIRE(minifi::test::utils::verifyEventHappenedInPollTime(std::chrono::seconds{1}, PenaltyHasExpired{flow_file_4}, std::chrono::milliseconds{10})); REQUIRE(queue.isWorkAvailable()); REQUIRE(queue.pop() == flow_file_4); - REQUIRE(utils::verifyEventHappenedInPollTime(std::chrono::seconds{1}, PenaltyHasExpired{flow_file_1}, std::chrono::milliseconds{10})); + REQUIRE(minifi::test::utils::verifyEventHappenedInPollTime(std::chrono::seconds{1}, PenaltyHasExpired{flow_file_1}, std::chrono::milliseconds{10})); REQUIRE(queue.isWorkAvailable()); REQUIRE(queue.pop() == flow_file_1); - REQUIRE(utils::verifyEventHappenedInPollTime(std::chrono::seconds{1}, PenaltyHasExpired{flow_file_3}, std::chrono::milliseconds{10})); + REQUIRE(minifi::test::utils::verifyEventHappenedInPollTime(std::chrono::seconds{1}, PenaltyHasExpired{flow_file_3}, std::chrono::milliseconds{10})); REQUIRE(queue.isWorkAvailable()); REQUIRE(queue.pop() == flow_file_3); @@ -154,7 +154,7 @@ TEST_CASE("If a penalized then a non-penalized flow file is added to the FlowFil } SECTION("Wait until the penalty expires, then pop") { - REQUIRE(utils::verifyEventHappenedInPollTime(std::chrono::seconds{1}, PenaltyHasExpired{penalized_flow_file}, std::chrono::milliseconds{10})); + REQUIRE(minifi::test::utils::verifyEventHappenedInPollTime(std::chrono::seconds{1}, PenaltyHasExpired{penalized_flow_file}, std::chrono::milliseconds{10})); REQUIRE(queue.isWorkAvailable()); REQUIRE(queue.pop() == flow_file); diff --git a/libminifi/test/unit/FlowFileSerializationTests.cpp b/libminifi/test/unit/FlowFileSerializationTests.cpp index 04c58bd190..45059642b1 100644 --- a/libminifi/test/unit/FlowFileSerializationTests.cpp +++ b/libminifi/test/unit/FlowFileSerializationTests.cpp @@ -22,8 +22,8 @@ #include "serialization/FlowFileV3Serializer.h" #include "serialization/PayloadSerializer.h" #include "core/FlowFile.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/gsl.h" #include "utils/span.h" #include "FlowFile.h" diff --git a/libminifi/test/unit/GeneralUtilsTest.cpp b/libminifi/test/unit/GeneralUtilsTest.cpp index f16191fa71..3adf57d829 100644 --- a/libminifi/test/unit/GeneralUtilsTest.cpp +++ b/libminifi/test/unit/GeneralUtilsTest.cpp @@ -20,8 +20,8 @@ #include #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/GeneralUtils.h" namespace org::apache::nifi::minifi::test { diff --git a/extensions/http-curl/tests/unit/HTTPStreamingCallbackTests.cpp b/libminifi/test/unit/HTTPStreamingCallbackTests.cpp similarity index 93% rename from extensions/http-curl/tests/unit/HTTPStreamingCallbackTests.cpp rename to libminifi/test/unit/HTTPStreamingCallbackTests.cpp index ece9e53ccd..ae8618b57f 100644 --- a/extensions/http-curl/tests/unit/HTTPStreamingCallbackTests.cpp +++ b/libminifi/test/unit/HTTPStreamingCallbackTests.cpp @@ -24,16 +24,16 @@ #include #include -#include "client/HTTPCallback.h" -#include "TestBase.h" -#include "Catch.h" +#include "http/HTTPCallback.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" -namespace org::apache::nifi::minifi::extensions::curl::testing { +namespace org::apache::nifi::minifi::test { class HttpStreamingCallbackTestsFixture { public: HttpStreamingCallbackTestsFixture() { - LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); } HttpStreamingCallbackTestsFixture(HttpStreamingCallbackTestsFixture&&) = delete; @@ -90,7 +90,7 @@ class HttpStreamingCallbackTestsFixture { } protected: - HttpStreamingCallback callback_; + http::HttpStreamingCallback callback_; std::mutex content_mutex_; std::vector content_; std::thread consumer_thread_; @@ -165,4 +165,4 @@ TEST_CASE_METHOD(HttpStreamingCallbackTestsFixture, "HttpStreamingCallback multi REQUIRE(input == content); } -} // namespace org::apache::nifi::minifi::extensions::curl::testing +} // namespace org::apache::nifi::minifi::test diff --git a/libminifi/test/unit/HTTPUtilTests.cpp b/libminifi/test/unit/HTTPUtilTests.cpp index cf8433e147..991c9b3713 100644 --- a/libminifi/test/unit/HTTPUtilTests.cpp +++ b/libminifi/test/unit/HTTPUtilTests.cpp @@ -17,12 +17,12 @@ */ #include -#include "../TestBase.h" -#include "../Catch.h" -#include "utils/BaseHTTPClient.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "http/BaseHTTPClient.h" TEST_CASE("The URL class can parse various URL strings", "[URL][parsing]") { - const auto canParseURL = [](const std::string& url_string) { return utils::URL{url_string}.toString() == url_string; }; + const auto canParseURL = [](const std::string& url_string) { return minifi::http::URL{url_string}.toString() == url_string; }; REQUIRE(canParseURL("http://nifi.io")); REQUIRE(canParseURL("http://nifi.io:777")); @@ -39,7 +39,7 @@ TEST_CASE("The URL class can parse various URL strings", "[URL][parsing]") { } TEST_CASE("The URL class will fail to parse invalid URL strings", "[URL][parsing]") { - const auto failToParseURL = [](const std::string& url_string) { return utils::URL{url_string}.toString() == "INVALID"; }; + const auto failToParseURL = [](const std::string& url_string) { return minifi::http::URL{url_string}.toString() == "INVALID"; }; REQUIRE(failToParseURL("mailto:santa.claus@north.pole.org")); REQUIRE(failToParseURL("http:nifi.io")); @@ -50,42 +50,42 @@ TEST_CASE("The URL class will fail to parse invalid URL strings", "[URL][parsing } TEST_CASE("The URL class can extract the port from URL strings", "[URL][port]") { - REQUIRE(utils::URL{"http://nifi.io"}.port() == 80); - REQUIRE(utils::URL{"http://nifi.io/nifi"}.port() == 80); - REQUIRE(utils::URL{"https://nifi.io"}.port() == 443); - REQUIRE(utils::URL{"https://nifi.io/nifi"}.port() == 443); - REQUIRE(utils::URL{"http://nifi.io:123"}.port() == 123); - REQUIRE(utils::URL{"http://nifi.io:123/nifi"}.port() == 123); - REQUIRE(utils::URL{"https://nifi.io:456"}.port() == 456); - REQUIRE(utils::URL{"https://nifi.io:456/nifi"}.port() == 456); + REQUIRE(minifi::http::URL{"http://nifi.io"}.port() == 80); + REQUIRE(minifi::http::URL{"http://nifi.io/nifi"}.port() == 80); + REQUIRE(minifi::http::URL{"https://nifi.io"}.port() == 443); + REQUIRE(minifi::http::URL{"https://nifi.io/nifi"}.port() == 443); + REQUIRE(minifi::http::URL{"http://nifi.io:123"}.port() == 123); + REQUIRE(minifi::http::URL{"http://nifi.io:123/nifi"}.port() == 123); + REQUIRE(minifi::http::URL{"https://nifi.io:456"}.port() == 456); + REQUIRE(minifi::http::URL{"https://nifi.io:456/nifi"}.port() == 456); } TEST_CASE("The URL class can extract the host", "[URL][host]") { - REQUIRE(utils::URL{"http://nifi.io"}.host() == "nifi.io"); - REQUIRE(utils::URL{"http://nifi.io:777"}.host() == "nifi.io"); - REQUIRE(utils::URL{"http://nifi.io/nifi"}.host() == "nifi.io"); - REQUIRE(utils::URL{"https://nifi.somewhere.far.away:321/nifi"}.host() == "nifi.somewhere.far.away"); - REQUIRE(utils::URL{"http://nifi.io?what_is_love"}.host() == "nifi.io"); - REQUIRE(utils::URL{"http://nifi.io:123?what_is_love"}.host() == "nifi.io"); - REQUIRE(utils::URL{"https://nifi.io/nifi_path?what_is_love"}.host() == "nifi.io"); - REQUIRE(utils::URL{"http://nifi.io:4321/nifi_path?what_is_love"}.host() == "nifi.io"); - REQUIRE(utils::URL{"http://nifi.io#anchors_aweigh"}.host() == "nifi.io"); - REQUIRE(utils::URL{"https://nifi.io:123#anchors_aweigh"}.host() == "nifi.io"); - REQUIRE(utils::URL{"http://nifi.io/nifi_path#anchors_aweigh"}.host() == "nifi.io"); - REQUIRE(utils::URL{"https://nifi.io:4321/nifi_path#anchors_aweigh"}.host() == "nifi.io"); + REQUIRE(minifi::http::URL{"http://nifi.io"}.host() == "nifi.io"); + REQUIRE(minifi::http::URL{"http://nifi.io:777"}.host() == "nifi.io"); + REQUIRE(minifi::http::URL{"http://nifi.io/nifi"}.host() == "nifi.io"); + REQUIRE(minifi::http::URL{"https://nifi.somewhere.far.away:321/nifi"}.host() == "nifi.somewhere.far.away"); + REQUIRE(minifi::http::URL{"http://nifi.io?what_is_love"}.host() == "nifi.io"); + REQUIRE(minifi::http::URL{"http://nifi.io:123?what_is_love"}.host() == "nifi.io"); + REQUIRE(minifi::http::URL{"https://nifi.io/nifi_path?what_is_love"}.host() == "nifi.io"); + REQUIRE(minifi::http::URL{"http://nifi.io:4321/nifi_path?what_is_love"}.host() == "nifi.io"); + REQUIRE(minifi::http::URL{"http://nifi.io#anchors_aweigh"}.host() == "nifi.io"); + REQUIRE(minifi::http::URL{"https://nifi.io:123#anchors_aweigh"}.host() == "nifi.io"); + REQUIRE(minifi::http::URL{"http://nifi.io/nifi_path#anchors_aweigh"}.host() == "nifi.io"); + REQUIRE(minifi::http::URL{"https://nifi.io:4321/nifi_path#anchors_aweigh"}.host() == "nifi.io"); } TEST_CASE("The URL class can extract the host and port", "[URL][hostPort]") { - REQUIRE(utils::URL{"http://nifi.io"}.hostPort() == "http://nifi.io"); - REQUIRE(utils::URL{"http://nifi.io:777"}.hostPort() == "http://nifi.io:777"); - REQUIRE(utils::URL{"http://nifi.io/nifi"}.hostPort() == "http://nifi.io"); - REQUIRE(utils::URL{"https://nifi.somewhere.far.away:321/nifi"}.hostPort() == "https://nifi.somewhere.far.away:321"); - REQUIRE(utils::URL{"http://nifi.io?what_is_love"}.hostPort() == "http://nifi.io"); - REQUIRE(utils::URL{"http://nifi.io:123?what_is_love"}.hostPort() == "http://nifi.io:123"); - REQUIRE(utils::URL{"https://nifi.io/nifi_path?what_is_love"}.hostPort() == "https://nifi.io"); - REQUIRE(utils::URL{"http://nifi.io:4321/nifi_path?what_is_love"}.hostPort() == "http://nifi.io:4321"); - REQUIRE(utils::URL{"http://nifi.io#anchors_aweigh"}.hostPort() == "http://nifi.io"); - REQUIRE(utils::URL{"https://nifi.io:123#anchors_aweigh"}.hostPort() == "https://nifi.io:123"); - REQUIRE(utils::URL{"http://nifi.io/nifi_path#anchors_aweigh"}.hostPort() == "http://nifi.io"); - REQUIRE(utils::URL{"https://nifi.io:4321/nifi_path#anchors_aweigh"}.hostPort() == "https://nifi.io:4321"); + REQUIRE(minifi::http::URL{"http://nifi.io"}.hostPort() == "http://nifi.io"); + REQUIRE(minifi::http::URL{"http://nifi.io:777"}.hostPort() == "http://nifi.io:777"); + REQUIRE(minifi::http::URL{"http://nifi.io/nifi"}.hostPort() == "http://nifi.io"); + REQUIRE(minifi::http::URL{"https://nifi.somewhere.far.away:321/nifi"}.hostPort() == "https://nifi.somewhere.far.away:321"); + REQUIRE(minifi::http::URL{"http://nifi.io?what_is_love"}.hostPort() == "http://nifi.io"); + REQUIRE(minifi::http::URL{"http://nifi.io:123?what_is_love"}.hostPort() == "http://nifi.io:123"); + REQUIRE(minifi::http::URL{"https://nifi.io/nifi_path?what_is_love"}.hostPort() == "https://nifi.io"); + REQUIRE(minifi::http::URL{"http://nifi.io:4321/nifi_path?what_is_love"}.hostPort() == "http://nifi.io:4321"); + REQUIRE(minifi::http::URL{"http://nifi.io#anchors_aweigh"}.hostPort() == "http://nifi.io"); + REQUIRE(minifi::http::URL{"https://nifi.io:123#anchors_aweigh"}.hostPort() == "https://nifi.io:123"); + REQUIRE(minifi::http::URL{"http://nifi.io/nifi_path#anchors_aweigh"}.hostPort() == "http://nifi.io"); + REQUIRE(minifi::http::URL{"https://nifi.io:4321/nifi_path#anchors_aweigh"}.hostPort() == "https://nifi.io:4321"); } diff --git a/libminifi/test/unit/IdTests.cpp b/libminifi/test/unit/IdTests.cpp index b2fa4a6623..4a9d3b7765 100644 --- a/libminifi/test/unit/IdTests.cpp +++ b/libminifi/test/unit/IdTests.cpp @@ -21,10 +21,10 @@ #include #include #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/Id.h" -#include "../Utils.h" +#include "unit/TestUtils.h" namespace org::apache::nifi::minifi::utils { struct IdentifierTestAccessor { diff --git a/libminifi/test/unit/IntervalSwitchTest.cpp b/libminifi/test/unit/IntervalSwitchTest.cpp index 16ffa91ee3..4d11e3b0cf 100644 --- a/libminifi/test/unit/IntervalSwitchTest.cpp +++ b/libminifi/test/unit/IntervalSwitchTest.cpp @@ -16,8 +16,8 @@ * limitations under the License. */ -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/IntervalSwitch.h" namespace utils = org::apache::nifi::minifi::utils; diff --git a/libminifi/test/unit/JsonFlowSerializerTests.cpp b/libminifi/test/unit/JsonFlowSerializerTests.cpp index dd6f72aa40..efdf9ea6d6 100644 --- a/libminifi/test/unit/JsonFlowSerializerTests.cpp +++ b/libminifi/test/unit/JsonFlowSerializerTests.cpp @@ -17,8 +17,8 @@ #include -#include "../Catch.h" -#include "../ConfigurationTestController.h" +#include "unit/Catch.h" +#include "unit/ConfigurationTestController.h" #include "catch2/generators/catch_generators.hpp" #include "core/flow/FlowSchema.h" #include "core/json/JsonFlowSerializer.h" @@ -198,7 +198,7 @@ constexpr std::string_view config_json_with_nifi_schema_part_1 = R"({ "type": "org.apache.nifi.minifi.processors.InvokeHTTP", "bundle": { "group": "org.apache.nifi.minifi", - "artifact": "minifi-http-curl", + "artifact": "minifi-standard-processors", "version": "1.23.06" }, "properties": { diff --git a/libminifi/test/unit/LineByLineInputOutputStreamCallbackTests.cpp b/libminifi/test/unit/LineByLineInputOutputStreamCallbackTests.cpp index 6d86fb56cd..29e6dfac43 100644 --- a/libminifi/test/unit/LineByLineInputOutputStreamCallbackTests.cpp +++ b/libminifi/test/unit/LineByLineInputOutputStreamCallbackTests.cpp @@ -16,8 +16,8 @@ */ #include "LineByLineInputOutputStreamCallback.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/logging/LoggerConfiguration.h" #include "io/BufferStream.h" #include "fmt/format.h" diff --git a/libminifi/test/unit/LogMetricsPublisherTests.cpp b/libminifi/test/unit/LogMetricsPublisherTests.cpp index 4d427ddb14..511113d378 100644 --- a/libminifi/test/unit/LogMetricsPublisherTests.cpp +++ b/libminifi/test/unit/LogMetricsPublisherTests.cpp @@ -18,12 +18,12 @@ #include #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/state/LogMetricsPublisher.h" #include "core/state/nodes/ResponseNodeLoader.h" #include "core/RepositoryFactory.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" #include "utils/file/FileUtils.h" using namespace std::literals::chrono_literals; @@ -70,7 +70,7 @@ TEST_CASE_METHOD(LogPublisherTestFixture, "Logging interval property is mandator } SECTION("Logging interval is set to 2 seconds") { configuration_->set(minifi::Configuration::nifi_metrics_publisher_log_metrics_logging_interval, "2s"); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; publisher_.initialize(configuration_, response_node_loader_); REQUIRE(verifyLogLinePresenceInPollTime(5s, "Metric logging interval is set to 2000ms")); } @@ -85,7 +85,7 @@ TEST_CASE_METHOD(LogPublisherTestFixture, "Verify empty metrics if no valid metr } publisher_.initialize(configuration_, response_node_loader_); publisher_.loadMetricNodes(); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; REQUIRE(verifyLogLinePresenceInPollTime(5s, "LogMetricsPublisher is configured without any valid metrics!")); } @@ -95,7 +95,7 @@ TEST_CASE_METHOD(LogPublisherTestFixture, "Verify multiple metric nodes in logs" configuration_->set(Configure::nifi_metrics_publisher_metrics, "RepositoryMetrics,DeviceInfoNode"); publisher_.initialize(configuration_, response_node_loader_); publisher_.loadMetricNodes(); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; std::string expected_log = R"([info] { "LogMetrics": { "RepositoryMetrics": { @@ -129,7 +129,7 @@ TEST_CASE_METHOD(LogPublisherTestFixture, "Verify reloading different metrics", configuration_->set(Configure::nifi_metrics_publisher_metrics, "RepositoryMetrics"); publisher_.initialize(configuration_, response_node_loader_); publisher_.loadMetricNodes(); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; std::string expected_log = R"([info] { "LogMetrics": { "RepositoryMetrics": { @@ -182,7 +182,7 @@ TEST_CASE_METHOD(LogPublisherTestFixture, "Verify generic and publisher specific } publisher_.initialize(configuration_, response_node_loader_); publisher_.loadMetricNodes(); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; std::string expected_log = R"([info] { "LogMetrics": { "RepositoryMetrics": { @@ -217,7 +217,7 @@ TEST_CASE_METHOD(LogPublisherTestFixture, "Verify changing log level property fo configuration_->set(Configure::nifi_metrics_publisher_metrics, "RepositoryMetrics"); publisher_.initialize(configuration_, response_node_loader_); publisher_.loadMetricNodes(); - using org::apache::nifi::minifi::utils::verifyLogLinePresenceInPollTime; + using org::apache::nifi::minifi::test::utils::verifyLogLinePresenceInPollTime; std::string expected_log = R"([debug] { "LogMetrics": { "RepositoryMetrics": { diff --git a/libminifi/test/unit/LoggerConfigurationTests.cpp b/libminifi/test/unit/LoggerConfigurationTests.cpp index 7244e61562..6583c52a1f 100644 --- a/libminifi/test/unit/LoggerConfigurationTests.cpp +++ b/libminifi/test/unit/LoggerConfigurationTests.cpp @@ -23,8 +23,8 @@ #include #include -#include "../Catch.h" -#include "../TestBase.h" +#include "unit/Catch.h" +#include "unit/TestBase.h" #include "core/logging/LoggerConfiguration.h" #include "spdlog/formatter.h" #include "spdlog/pattern_formatter.h" diff --git a/libminifi/test/unit/LoggerTests.cpp b/libminifi/test/unit/LoggerTests.cpp index d4f45fb5d6..573ad20296 100644 --- a/libminifi/test/unit/LoggerTests.cpp +++ b/libminifi/test/unit/LoggerTests.cpp @@ -21,12 +21,12 @@ #include #include #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/logging/LoggerConfiguration.h" #include "io/ZlibStream.h" #include "StreamPipe.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" #include "utils/span.h" #include "utils/net/AsioSocketUtils.h" @@ -220,7 +220,7 @@ class LoggerTestAccessor { } static bool waitForCompressionToHappen(logging::LoggerConfiguration& log_config) { auto sink = log_config.compression_manager_.getSink(); - return !sink || utils::verifyEventHappenedInPollTime(1s, [&]{ return sink->cached_logs_.itemCount() == 0; }); + return !sink || minifi::test::utils::verifyEventHappenedInPollTime(1s, [&]{ return sink->cached_logs_.itemCount() == 0; }); } static auto getCompressedLogs(logging::LoggerConfiguration& log_config) { return log_config.compression_manager_.getCompressedLogs(std::chrono::milliseconds{0}); @@ -265,7 +265,7 @@ TEST_CASE("Test Compression cache overflow is discarded intermittently", "[ttl10 for (size_t idx = 0; idx < 10000; ++idx) { logger->log_error("Hi there"); } - bool cache_shrunk = utils::verifyEventHappenedInPollTime(std::chrono::seconds{1}, [&] { + bool cache_shrunk = minifi::test::utils::verifyEventHappenedInPollTime(std::chrono::seconds{1}, [&] { return LoggerTestAccessor::getUncompressedSize(*log_config) <= 10_KiB; }); REQUIRE(cache_shrunk); diff --git a/libminifi/test/unit/MapUtilsTests.cpp b/libminifi/test/unit/MapUtilsTests.cpp index 06de6c5b94..e0d7cedbf7 100644 --- a/libminifi/test/unit/MapUtilsTests.cpp +++ b/libminifi/test/unit/MapUtilsTests.cpp @@ -21,8 +21,8 @@ #include #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/MapUtils.h" namespace MapUtils = org::apache::nifi::minifi::utils::MapUtils; diff --git a/libminifi/test/unit/MemoryUsageTest.cpp b/libminifi/test/unit/MemoryUsageTest.cpp index 54ce83bf28..82dbd34dc0 100644 --- a/libminifi/test/unit/MemoryUsageTest.cpp +++ b/libminifi/test/unit/MemoryUsageTest.cpp @@ -24,8 +24,8 @@ #include "utils/gsl.h" #include "utils/OsUtils.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" TEST_CASE("Test Physical memory usage", "[testphysicalmemoryusage]") { constexpr bool cout_enabled = true; diff --git a/libminifi/test/unit/MetricsPublisherStoreTests.cpp b/libminifi/test/unit/MetricsPublisherStoreTests.cpp index 77d620453a..38131b0af1 100644 --- a/libminifi/test/unit/MetricsPublisherStoreTests.cpp +++ b/libminifi/test/unit/MetricsPublisherStoreTests.cpp @@ -17,9 +17,9 @@ */ #include -#include "../TestBase.h" -#include "../Catch.h" -#include "ProvenanceTestHelper.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/ProvenanceTestHelper.h" #include "core/repository/VolatileContentRepository.h" #include "properties/Configuration.h" #include "core/Resource.h" diff --git a/libminifi/test/unit/MetricsTests.cpp b/libminifi/test/unit/MetricsTests.cpp index 5be43fb4fd..16e732c185 100644 --- a/libminifi/test/unit/MetricsTests.cpp +++ b/libminifi/test/unit/MetricsTests.cpp @@ -19,13 +19,13 @@ #include "../../include/core/state/nodes/QueueMetrics.h" #include "../../include/core/state/nodes/RepositoryMetrics.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/Processor.h" #include "core/ClassLoader.h" #include "repository/VolatileContentRepository.h" -#include "ProvenanceTestHelper.h" -#include "../DummyProcessor.h" +#include "unit/ProvenanceTestHelper.h" +#include "unit/DummyProcessor.h" #include "range/v3/algorithm/find_if.hpp" using namespace std::literals::chrono_literals; diff --git a/libminifi/test/unit/MinifiConcurrentQueueTests.cpp b/libminifi/test/unit/MinifiConcurrentQueueTests.cpp index 8d2bec8a44..8c48b98fa4 100644 --- a/libminifi/test/unit/MinifiConcurrentQueueTests.cpp +++ b/libminifi/test/unit/MinifiConcurrentQueueTests.cpp @@ -23,12 +23,12 @@ #include #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/gsl.h" #include "utils/MinifiConcurrentQueue.h" #include "utils/StringUtils.h" -#include "utils/IntegrationTestUtils.h" +#include "unit/TestUtils.h" namespace utils = org::apache::nifi::minifi::utils; @@ -293,7 +293,7 @@ TEST_CASE("TestConcurrentQueue: test untimed waiting consumers", "[ProducerConsu producer.join(); auto queue_is_empty = [&queue]() { return queue.empty(); }; - REQUIRE(utils::verifyEventHappenedInPollTime(std::chrono::seconds(1), queue_is_empty)); + REQUIRE(minifi::test::utils::verifyEventHappenedInPollTime(std::chrono::seconds(1), queue_is_empty)); queue.stop(); consumer.join(); @@ -311,7 +311,7 @@ TEST_CASE("TestConcurrentQueue: test the readding dequeue consumer", "[ProducerC std::thread producer { MinifiConcurrentQueueTestProducersConsumers::getSimpleProducerThread(queue) }; auto we_have_all_results = [&results_size]() { return results_size >= 3; }; - REQUIRE(utils::verifyEventHappenedInPollTime(std::chrono::seconds(1), we_have_all_results)); + REQUIRE(minifi::test::utils::verifyEventHappenedInPollTime(std::chrono::seconds(1), we_have_all_results)); queue.stop(); producer.join(); @@ -380,7 +380,7 @@ TEST_CASE("TestConcurrentQueues::highLoad", "[TestConcurrentQueuesHighLoad]") { relay.join(); auto queue_is_empty = [&cqueue]() { return cqueue.empty(); }; - REQUIRE(utils::verifyEventHappenedInPollTime(std::chrono::seconds(1), queue_is_empty)); + REQUIRE(minifi::test::utils::verifyEventHappenedInPollTime(std::chrono::seconds(1), queue_is_empty)); cqueue.stop(); consumer.join(); diff --git a/libminifi/test/unit/NetUtilsTest.cpp b/libminifi/test/unit/NetUtilsTest.cpp index 10b7b2cb03..341a8670d4 100644 --- a/libminifi/test/unit/NetUtilsTest.cpp +++ b/libminifi/test/unit/NetUtilsTest.cpp @@ -18,13 +18,13 @@ #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/net/DNS.h" #include "utils/net/AsioSocketUtils.h" #include "utils/StringUtils.h" #include "controllers/SSLContextService.h" -#include "../Utils.h" +#include "unit/TestUtils.h" namespace utils = org::apache::nifi::minifi::utils; namespace net = utils::net; diff --git a/libminifi/test/unit/NetworkInterfaceInfoTests.cpp b/libminifi/test/unit/NetworkInterfaceInfoTests.cpp index d8cb232377..6c42a4f33d 100644 --- a/libminifi/test/unit/NetworkInterfaceInfoTests.cpp +++ b/libminifi/test/unit/NetworkInterfaceInfoTests.cpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/NetworkInterfaceInfo.h" namespace utils = org::apache::nifi::minifi::utils; diff --git a/libminifi/test/unit/NetworkPrioritizerServiceTests.cpp b/libminifi/test/unit/NetworkPrioritizerServiceTests.cpp index e6f428e949..91004eb9e9 100644 --- a/libminifi/test/unit/NetworkPrioritizerServiceTests.cpp +++ b/libminifi/test/unit/NetworkPrioritizerServiceTests.cpp @@ -19,17 +19,17 @@ #include #include #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/controller/ControllerService.h" #include "controllers/NetworkPrioritizerService.h" -#include "utils/TestUtils.h" +#include "unit/TestUtils.h" namespace { std::shared_ptr createNetworkPrioritizerService( const std::string& name, - const std::shared_ptr& clock = std::make_shared()) { + const std::shared_ptr& clock = std::make_shared()) { return std::make_shared(name, utils::Identifier{}, clock); } @@ -61,7 +61,7 @@ TEST_CASE("TestPrioritizerOneInterfaceMaxPayload", "[test2]") { } TEST_CASE("TestPrioritizerOneInterfaceMaxThroughput", "[test3]") { - auto clock = std::make_shared(); + auto clock = std::make_shared(); auto controller = createNetworkPrioritizerService("TestService", clock); controller->initialize(); controller->setProperty(minifi::controllers::NetworkPrioritizerService::NetworkControllers, "eth0,eth1"); @@ -76,7 +76,7 @@ TEST_CASE("TestPrioritizerOneInterfaceMaxThroughput", "[test3]") { } TEST_CASE("TestPriorotizerMultipleInterfaces", "[test4]") { - auto clock = std::make_shared(); + auto clock = std::make_shared(); auto parent_controller = createNetworkPrioritizerService("TestService", clock); std::shared_ptr configuration = std::make_shared(); parent_controller->initialize(); diff --git a/libminifi/test/unit/OpenTelemetryLogDataModelTests.cpp b/libminifi/test/unit/OpenTelemetryLogDataModelTests.cpp index ccf7e4c176..1d5bd9f5b1 100644 --- a/libminifi/test/unit/OpenTelemetryLogDataModelTests.cpp +++ b/libminifi/test/unit/OpenTelemetryLogDataModelTests.cpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/OpenTelemetryLogDataModelUtils.h" namespace utils = org::apache::nifi::minifi::utils; diff --git a/libminifi/test/unit/OptionalTest.cpp b/libminifi/test/unit/OptionalTest.cpp index 044694d11b..222e5c797f 100644 --- a/libminifi/test/unit/OptionalTest.cpp +++ b/libminifi/test/unit/OptionalTest.cpp @@ -16,8 +16,8 @@ * limitations under the License. */ -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/OptionalUtils.h" namespace utils = org::apache::nifi::minifi::utils; diff --git a/libminifi/test/unit/OsUtilTests.cpp b/libminifi/test/unit/OsUtilTests.cpp index a7777d1770..651e1f4f25 100644 --- a/libminifi/test/unit/OsUtilTests.cpp +++ b/libminifi/test/unit/OsUtilTests.cpp @@ -18,8 +18,8 @@ #include "utils/OsUtils.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" namespace org::apache::nifi::minifi::test { diff --git a/libminifi/test/unit/PathUtilsTests.cpp b/libminifi/test/unit/PathUtilsTests.cpp index b2c894bdd9..9037664df1 100644 --- a/libminifi/test/unit/PathUtilsTests.cpp +++ b/libminifi/test/unit/PathUtilsTests.cpp @@ -16,7 +16,7 @@ */ #include -#include "../Catch.h" +#include "unit/Catch.h" #include "utils/file/PathUtils.h" namespace fs = org::apache::nifi::minifi::utils::file; diff --git a/libminifi/test/unit/PayloadParserTests.cpp b/libminifi/test/unit/PayloadParserTests.cpp index f335a32aed..bdb2f50469 100644 --- a/libminifi/test/unit/PayloadParserTests.cpp +++ b/libminifi/test/unit/PayloadParserTests.cpp @@ -20,8 +20,8 @@ #include #include "c2/C2Payload.h" #include "c2/PayloadParser.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" TEST_CASE("Test Valid Payload", "[tv1]") { std::string ident = "identifier"; diff --git a/libminifi/test/unit/ProcessContextTest.cpp b/libminifi/test/unit/ProcessContextTest.cpp index 9cc04c1832..8afce50b6f 100644 --- a/libminifi/test/unit/ProcessContextTest.cpp +++ b/libminifi/test/unit/ProcessContextTest.cpp @@ -15,7 +15,7 @@ * limitations under the License. */ #include -#include "../Catch.h" +#include "unit/Catch.h" #include "ProcessContext.h" #include "core/FlowFile.h" #include "utils/meta/detected.h" diff --git a/libminifi/test/unit/ProcessSessionTests.cpp b/libminifi/test/unit/ProcessSessionTests.cpp index 8a717f396a..008c914daf 100644 --- a/libminifi/test/unit/ProcessSessionTests.cpp +++ b/libminifi/test/unit/ProcessSessionTests.cpp @@ -21,13 +21,12 @@ #include "core/ProcessSession.h" #include "core/Resource.h" -#include "../TestBase.h" -#include "../Catch.h" -#include "ContentRepositoryDependentTests.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/ContentRepositoryDependentTests.h" #include "Processor.h" #include "core/repository/VolatileFlowFileRepository.h" -#include "IntegrationTestUtils.h" -#include "../Utils.h" +#include "unit/TestUtils.h" namespace { diff --git a/libminifi/test/unit/ProcessorConfigUtilsTests.cpp b/libminifi/test/unit/ProcessorConfigUtilsTests.cpp index 2a284356a9..38c28284bf 100644 --- a/libminifi/test/unit/ProcessorConfigUtilsTests.cpp +++ b/libminifi/test/unit/ProcessorConfigUtilsTests.cpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "PropertyDefinition.h" #include "core/Processor.h" #include "core/PropertyDefinitionBuilder.h" diff --git a/libminifi/test/unit/PropertyEncryptionUtilsTests.cpp b/libminifi/test/unit/PropertyEncryptionUtilsTests.cpp index 5c963f4764..93885b6727 100644 --- a/libminifi/test/unit/PropertyEncryptionUtilsTests.cpp +++ b/libminifi/test/unit/PropertyEncryptionUtilsTests.cpp @@ -17,7 +17,7 @@ #include -#include "../Catch.h" +#include "unit/Catch.h" #include "utils/crypto/EncryptionProvider.h" #include "utils/crypto/EncryptionUtils.h" #include "utils/crypto/property_encryption/PropertyEncryptionUtils.h" diff --git a/libminifi/test/unit/PropertyTests.cpp b/libminifi/test/unit/PropertyTests.cpp index 05f7238b2d..e45618d673 100644 --- a/libminifi/test/unit/PropertyTests.cpp +++ b/libminifi/test/unit/PropertyTests.cpp @@ -20,8 +20,8 @@ #include "core/Property.h" #include "utils/StringUtils.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" namespace { enum class ConversionTestTarget { MS, NS }; diff --git a/libminifi/test/unit/PropertyValidationTests.cpp b/libminifi/test/unit/PropertyValidationTests.cpp index 8e56101a90..933d925bfb 100644 --- a/libminifi/test/unit/PropertyValidationTests.cpp +++ b/libminifi/test/unit/PropertyValidationTests.cpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/ConfigurableComponent.h" #include "core/PropertyDefinitionBuilder.h" #include "utils/PropertyErrors.h" diff --git a/libminifi/test/unit/RegexUtilsTests.cpp b/libminifi/test/unit/RegexUtilsTests.cpp index d54a10ebe5..7b0195df73 100644 --- a/libminifi/test/unit/RegexUtilsTests.cpp +++ b/libminifi/test/unit/RegexUtilsTests.cpp @@ -20,8 +20,8 @@ #include #include "utils/RegexUtils.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "catch2/catch_all.hpp" #include "catch2/matchers/catch_matchers_string.hpp" diff --git a/libminifi/test/unit/RelationshipTests.cpp b/libminifi/test/unit/RelationshipTests.cpp index f983447545..a7544a296d 100644 --- a/libminifi/test/unit/RelationshipTests.cpp +++ b/libminifi/test/unit/RelationshipTests.cpp @@ -17,7 +17,7 @@ #include -#include "../Catch.h" +#include "unit/Catch.h" #include "core/Relationship.h" using org::apache::nifi::minifi::core::Relationship; diff --git a/libminifi/test/unit/ResourceQueueTests.cpp b/libminifi/test/unit/ResourceQueueTests.cpp index 8266d3b509..fc9906cdfd 100644 --- a/libminifi/test/unit/ResourceQueueTests.cpp +++ b/libminifi/test/unit/ResourceQueueTests.cpp @@ -20,8 +20,8 @@ #include #include "ResourceQueue.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "logging/LoggerConfiguration.h" using namespace std::literals::chrono_literals; diff --git a/libminifi/test/unit/ResponseNodeLoaderTests.cpp b/libminifi/test/unit/ResponseNodeLoaderTests.cpp index f0926523e1..a1d9703e13 100644 --- a/libminifi/test/unit/ResponseNodeLoaderTests.cpp +++ b/libminifi/test/unit/ResponseNodeLoaderTests.cpp @@ -18,13 +18,13 @@ #include #include #include -#include "../Catch.h" +#include "unit/Catch.h" #include "core/state/nodes/ResponseNodeLoader.h" -#include "../ReadFromFlowFileTestProcessor.h" -#include "../WriteToFlowFileTestProcessor.h" +#include "unit/ReadFromFlowFileTestProcessor.h" +#include "unit/WriteToFlowFileTestProcessor.h" #include "core/repository/VolatileContentRepository.h" #include "utils/Id.h" -#include "ProvenanceTestHelper.h" +#include "unit/ProvenanceTestHelper.h" namespace org::apache::nifi::minifi::test { diff --git a/libminifi/test/unit/ResponseNodeValueTests.cpp b/libminifi/test/unit/ResponseNodeValueTests.cpp index c6d92270e8..42818eb0ec 100644 --- a/libminifi/test/unit/ResponseNodeValueTests.cpp +++ b/libminifi/test/unit/ResponseNodeValueTests.cpp @@ -18,8 +18,8 @@ #include #include "../../include/core/state/Value.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" using ValueNode = org::apache::nifi::minifi::state::response::ValueNode; using Value = org::apache::nifi::minifi::state::response::Value; diff --git a/libminifi/test/unit/SchedulingAgentTests.cpp b/libminifi/test/unit/SchedulingAgentTests.cpp index 1dfa151cf7..872b364e76 100644 --- a/libminifi/test/unit/SchedulingAgentTests.cpp +++ b/libminifi/test/unit/SchedulingAgentTests.cpp @@ -17,10 +17,11 @@ #include -#include "../Catch.h" -#include "../TestBase.h" -#include "ProvenanceTestHelper.h" -#include "utils/TestUtils.h" +#include "unit/Catch.h" +#include "unit/TestBase.h" +#include "unit/ProvenanceTestHelper.h" +#include "unit/TestUtils.h" +#include "utils/TimeUtil.h" using namespace std::literals::chrono_literals; @@ -63,7 +64,7 @@ class SchedulingAgentTestFixture { count_proc_->setScheduledState(core::RUNNING); #ifdef WIN32 - utils::dateSetInstall(TZ_DATA_DIR); + minifi::utils::timeutils::dateSetInstall(TZ_DATA_DIR); date::set_install(TZ_DATA_DIR); #endif } diff --git a/libminifi/test/unit/SerializationTests.cpp b/libminifi/test/unit/SerializationTests.cpp index 8ba130f926..4affa33a7d 100644 --- a/libminifi/test/unit/SerializationTests.cpp +++ b/libminifi/test/unit/SerializationTests.cpp @@ -19,10 +19,9 @@ #include #include -#include "SiteToSiteHelper.h" -#include "../TestBase.h" -#include "../Catch.h" -#include "../unit/SiteToSiteHelper.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/SiteToSiteHelper.h" #define FMT_DEFAULT fmt_lower diff --git a/libminifi/test/unit/Site2SiteTests.cpp b/libminifi/test/unit/Site2SiteTests.cpp index c9ecb57ca9..dd20c2fbbc 100644 --- a/libminifi/test/unit/Site2SiteTests.cpp +++ b/libminifi/test/unit/Site2SiteTests.cpp @@ -25,9 +25,9 @@ #include "io/BaseStream.h" #include "sitetosite/Peer.h" #include "sitetosite/RawSocketProtocol.h" -#include "../TestBase.h" -#include "../Catch.h" -#include "../unit/SiteToSiteHelper.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/SiteToSiteHelper.h" #define FMT_DEFAULT fmt_lower diff --git a/libminifi/test/unit/SpanTests.cpp b/libminifi/test/unit/SpanTests.cpp index 4b648c8e49..faaa49f63a 100644 --- a/libminifi/test/unit/SpanTests.cpp +++ b/libminifi/test/unit/SpanTests.cpp @@ -18,8 +18,8 @@ #include #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/span.h" namespace utils = org::apache::nifi::minifi::utils; diff --git a/libminifi/test/unit/StagingQueueTests.cpp b/libminifi/test/unit/StagingQueueTests.cpp index 8debcfc605..fc07bdd7c4 100644 --- a/libminifi/test/unit/StagingQueueTests.cpp +++ b/libminifi/test/unit/StagingQueueTests.cpp @@ -19,8 +19,8 @@ #include #include "utils/StringUtils.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/StagingQueue.h" using org::apache::nifi::minifi::utils::StagingQueue; diff --git a/libminifi/test/unit/StreamTests.cpp b/libminifi/test/unit/StreamTests.cpp index f68872cba2..dd959b8cd6 100644 --- a/libminifi/test/unit/StreamTests.cpp +++ b/libminifi/test/unit/StreamTests.cpp @@ -23,8 +23,8 @@ #include #include #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "io/BaseStream.h" #include "io/StreamSlice.h" #include "utils/span.h" diff --git a/libminifi/test/unit/StringUtilsTests.cpp b/libminifi/test/unit/StringUtilsTests.cpp index b029d82785..405d6825b9 100644 --- a/libminifi/test/unit/StringUtilsTests.cpp +++ b/libminifi/test/unit/StringUtilsTests.cpp @@ -22,8 +22,8 @@ #include #include #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/Core.h" #include "utils/StringUtils.h" #include "utils/Environment.h" diff --git a/libminifi/test/unit/SwapTestController.h b/libminifi/test/unit/SwapTestController.h deleted file mode 100644 index c092601eee..0000000000 --- a/libminifi/test/unit/SwapTestController.h +++ /dev/null @@ -1,238 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include -#include - -#include "../TestBase.h" -#include "TestUtils.h" -#include "../Utils.h" -#include "../Catch.h" -#include "../unit/ProvenanceTestHelper.h" -#include "../utils/gsl.h" - -namespace org::apache::nifi::minifi::test { - -using Timepoint = std::chrono::time_point; - -enum EventKind { - Store, Load -}; - -struct SwapEvent { - EventKind kind; - std::vector flow_files; - - void verifyTimes(std::initializer_list seconds) { - REQUIRE(flow_files.size() == seconds.size()); - size_t idx = 0; - for (auto& second : seconds) { - REQUIRE(flow_files[idx].to_be_processed_after == Timepoint{std::chrono::seconds{second}}); - ++idx; - } - } -}; - -class SwappingFlowFileTestRepo : public TestFlowRepository, public minifi::SwapManager { - public: - void store(std::vector> flow_files) override { - std::vector ids; - for (const auto& ff : flow_files) { - ids.push_back(minifi::SwappedFlowFile{ff->getUUID(), ff->getPenaltyExpiration()}); - minifi::io::BufferStream output; - std::static_pointer_cast(ff)->Serialize(output); - Put(ff->getUUIDStr().c_str(), reinterpret_cast(output.getBuffer().data()), output.size()); - } - swap_events_.push_back({Store, ids}); - } - - std::future>> load(std::vector flow_files) override { - swap_events_.push_back({Load, flow_files}); - LoadTask load_task; - auto future = load_task.promise.get_future(); - load_task.result.reserve(flow_files.size()); - for (const auto& ff_id : flow_files) { - std::string value; - Get(ff_id.id.to_string().c_str(), value); - minifi::utils::Identifier container_id; - auto ff = minifi::FlowFileRecord::DeSerialize(gsl::make_span(value).as_span(), content_repo_, container_id); - ff->setPenaltyExpiration(ff_id.to_be_processed_after); - load_task.result.push_back(std::move(ff)); - } - load_tasks_.push_back(std::move(load_task)); - return future; - } - - struct LoadTask { - std::promise>> promise; - std::vector> result; - - void complete() { - promise.set_value(result); - } - }; - - std::vector load_tasks_; - std::vector swap_events_; -}; - -using FlowFilePtr = std::shared_ptr; -using FlowFilePtrVec = std::vector; - -struct FlowFileComparator { - bool operator()(const FlowFilePtr& left, const FlowFilePtr& right) const { - return left->getPenaltyExpiration() < right->getPenaltyExpiration(); - } -}; - -struct VerifiedQueue { - void push(FlowFilePtr ff) { - size(); - impl.push(ff); - ref_.insert(std::lower_bound(ref_.begin(), ref_.end(), ff, FlowFileComparator{}), ff); - size(); - } - - FlowFilePtr poll() { - size(); - FlowFilePtr ff = impl.pop(); - REQUIRE(!ref_.empty()); - // the order when flow files have the same penalty is not fixed - REQUIRE(ff->getPenaltyExpiration() == ref_.front()->getPenaltyExpiration()); - ref_.erase(ref_.begin()); - size(); - return ff; - } - - void verify(std::initializer_list live, std::optional> inter, std::initializer_list swapped) const { - // check live ffs - auto live_copy = utils::FlowFileQueueTestAccessor::get_queue_(impl); - REQUIRE(live_copy.size() == live.size()); - for (auto sec : live) { - auto min = live_copy.popMin(); - REQUIRE(min->getPenaltyExpiration() == Timepoint{std::chrono::seconds{sec}}); - } - - // check inter ffs - if (!inter) { - REQUIRE_FALSE(utils::FlowFileQueueTestAccessor::get_load_task_(impl).has_value()); - } else { - auto& intermediate = utils::FlowFileQueueTestAccessor::get_load_task_(impl)->intermediate_items; - REQUIRE(intermediate.size() == inter->size()); - size_t idx = 0; - for (auto sec : inter.value()) { - REQUIRE(intermediate[idx]->getPenaltyExpiration() == Timepoint{std::chrono::seconds{sec}}); - ++idx; - } - } - - // check swapped ffs - auto swapped_copy = utils::FlowFileQueueTestAccessor::get_swapped_flow_files_(impl); - REQUIRE(swapped_copy.size() == swapped.size()); - for (auto sec : swapped) { - auto min = swapped_copy.popMin(); - REQUIRE(min.to_be_processed_after == Timepoint{std::chrono::seconds{sec}}); - } - } - - bool isWorkAvailable() const { - return impl.isWorkAvailable(); - } - - size_t size() const { - size_t result = impl.size(); - REQUIRE(result == ref_.size()); - return result; - } - - explicit VerifiedQueue(std::shared_ptr swap_manager) - : impl(std::move(swap_manager)) {} - - minifi::utils::FlowFileQueue impl; - FlowFilePtrVec ref_; -}; - -class SwapTestController : public TestController { - public: - SwapTestController() { - content_repo_ = std::make_shared(); - flow_repo_ = std::make_shared(); - flow_repo_->loadComponent(content_repo_); - clock_ = std::make_shared(); - minifi::utils::timeutils::setClock(clock_); - queue_ = std::make_shared(std::static_pointer_cast(flow_repo_)); - } - - void setLimits(size_t min_size, size_t target_size, size_t max_size) { - queue_->impl.setMinSize(min_size); - queue_->impl.setTargetSize(target_size); - queue_->impl.setMaxSize(max_size); - } - - struct SwapEventPattern { - EventKind kind; - std::initializer_list seconds; - }; - - void verifySwapEvents(std::vector events) { - REQUIRE(flow_repo_->swap_events_.size() == events.size()); - size_t idx = 0; - for (auto& pattern : events) { - REQUIRE(pattern.kind == flow_repo_->swap_events_[idx].kind); - flow_repo_->swap_events_[idx].verifyTimes(pattern.seconds); - } - } - - void clearSwapEvents() { - flow_repo_->swap_events_.clear(); - } - - void verifyQueue(std::initializer_list live, std::optional> inter, std::initializer_list swapped) { - queue_->verify(live, inter, swapped); - } - - void pushAll(std::initializer_list seconds) { - for (auto sec : seconds) { - auto ff = std::static_pointer_cast(std::make_shared()); - ff->setPenaltyExpiration(Timepoint{std::chrono::seconds{sec}}); - queue_->push(std::move(ff)); - } - } - - void popAll(std::initializer_list seconds, bool check_is_work_available = false) { - for (auto sec : seconds) { - if (check_is_work_available) { - REQUIRE(queue_->isWorkAvailable()); - } - auto ff = queue_->poll(); - REQUIRE(ff->getPenaltyExpiration() == Timepoint{std::chrono::seconds{sec}}); - } - } - - std::shared_ptr flow_repo_; - std::shared_ptr content_repo_; - std::shared_ptr queue_; - std::shared_ptr clock_; -}; - -} // namespace org::apache::nifi::minifi::test diff --git a/libminifi/test/unit/ThreadPoolTests.cpp b/libminifi/test/unit/ThreadPoolTests.cpp index cbf0547b5a..914d107e1e 100644 --- a/libminifi/test/unit/ThreadPoolTests.cpp +++ b/libminifi/test/unit/ThreadPoolTests.cpp @@ -19,8 +19,8 @@ #include #include #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "utils/ThreadPool.h" using namespace std::literals::chrono_literals; diff --git a/libminifi/test/unit/TimeUtilTests.cpp b/libminifi/test/unit/TimeUtilTests.cpp index cd7c6f3449..e67a0c00bf 100644 --- a/libminifi/test/unit/TimeUtilTests.cpp +++ b/libminifi/test/unit/TimeUtilTests.cpp @@ -16,8 +16,8 @@ */ #include "utils/TimeUtil.h" -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" using namespace std::literals::chrono_literals; diff --git a/libminifi/test/unit/TypeListTests.cpp b/libminifi/test/unit/TypeListTests.cpp index 463a7a26cf..568e4b56ac 100644 --- a/libminifi/test/unit/TypeListTests.cpp +++ b/libminifi/test/unit/TypeListTests.cpp @@ -15,7 +15,7 @@ * limitations under the License. */ -#include "../Catch.h" +#include "unit/Catch.h" #include "core/Core.h" #include "utils/meta/type_list.h" diff --git a/libminifi/test/unit/UpdatePolicyTests.cpp b/libminifi/test/unit/UpdatePolicyTests.cpp index 84ccd23f7c..4a45cf2a34 100644 --- a/libminifi/test/unit/UpdatePolicyTests.cpp +++ b/libminifi/test/unit/UpdatePolicyTests.cpp @@ -19,8 +19,8 @@ #include #include #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/Processor.h" #include "../../controller/Controller.h" #include "core/controller/ControllerService.h" diff --git a/libminifi/test/unit/ValueParserTests.cpp b/libminifi/test/unit/ValueParserTests.cpp index cade20ba2c..e6bf57b491 100644 --- a/libminifi/test/unit/ValueParserTests.cpp +++ b/libminifi/test/unit/ValueParserTests.cpp @@ -18,7 +18,7 @@ #include #include "utils/ValueParser.h" -#include "../Catch.h" +#include "unit/Catch.h" namespace utils = org::apache::nifi::minifi::utils; diff --git a/libminifi/test/unit/WindowsCertStoreLocationTests.cpp b/libminifi/test/unit/WindowsCertStoreLocationTests.cpp index 87dd3a2d1e..e3622c618d 100644 --- a/libminifi/test/unit/WindowsCertStoreLocationTests.cpp +++ b/libminifi/test/unit/WindowsCertStoreLocationTests.cpp @@ -18,8 +18,8 @@ #include #include -#include "TestUtils.h" -#include "../Catch.h" +#include "unit/TestUtils.h" +#include "unit/Catch.h" #include "utils/tls/WindowsCertStoreLocation.h" #include "range/v3/algorithm/contains.hpp" diff --git a/libminifi/test/unit/YamlFlowSerializerTests.cpp b/libminifi/test/unit/YamlFlowSerializerTests.cpp index 14d8062477..f5e765a1b9 100644 --- a/libminifi/test/unit/YamlFlowSerializerTests.cpp +++ b/libminifi/test/unit/YamlFlowSerializerTests.cpp @@ -17,8 +17,8 @@ #include -#include "../Catch.h" -#include "../ConfigurationTestController.h" +#include "unit/Catch.h" +#include "unit/ConfigurationTestController.h" #include "catch2/generators/catch_generators.hpp" #include "core/flow/FlowSchema.h" #include "core/yaml/YamlFlowSerializer.h" diff --git a/libminifi/test/unit/ZlibStreamTests.cpp b/libminifi/test/unit/ZlibStreamTests.cpp index 296f633bd9..99fe64bbe7 100644 --- a/libminifi/test/unit/ZlibStreamTests.cpp +++ b/libminifi/test/unit/ZlibStreamTests.cpp @@ -19,8 +19,8 @@ #include #include #include -#include "../TestBase.h" -#include "../Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "io/ZlibStream.h" #include "utils/gsl.h" #include "utils/StringUtils.h" diff --git a/minifi_main/CMakeLists.txt b/minifi_main/CMakeLists.txt index 62eb234f6e..25fa0fd2d9 100644 --- a/minifi_main/CMakeLists.txt +++ b/minifi_main/CMakeLists.txt @@ -65,7 +65,7 @@ endif() get_property(extensions GLOBAL PROPERTY EXTENSION-OPTIONS) include(ArgParse) -target_link_libraries(minifiexe spdlog libsodium gsl-lite argparse ${LIBMINIFI}) +target_link_libraries(minifiexe spdlog::spdlog libsodium gsl-lite argparse ${LIBMINIFI}) set_target_properties(minifiexe PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") set_target_properties(minifiexe PROPERTIES OUTPUT_NAME minifi) diff --git a/minifi_main/tests/CMakeLists.txt b/minifi_main/tests/CMakeLists.txt index 2549c349b0..12afbf7b72 100644 --- a/minifi_main/tests/CMakeLists.txt +++ b/minifi_main/tests/CMakeLists.txt @@ -22,10 +22,8 @@ SET(MINIFI_MAIN_UNIT_TEST_COUNT 0) FOREACH(TEST_FILE ${MINIFI_MAIN_UNIT_TESTS}) get_filename_component(TEST_TARGET "${TEST_FILE}" NAME_WE) add_minifi_executable("${TEST_TARGET}" "${TEST_FILE}" "../TableFormatter.cpp") - target_include_directories(${TEST_TARGET} PRIVATE BEFORE - "${CMAKE_SOURCE_DIR}/libminifi/test" - "${CMAKE_SOURCE_DIR}/minifi_main") - target_link_libraries(${TEST_TARGET} core-minifi Catch2WithMain) + target_include_directories(${TEST_TARGET} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/minifi_main") + target_link_libraries(${TEST_TARGET} core-minifi Catch2::Catch2WithMain) createTests("${TEST_TARGET}") add_test(NAME ${TEST_TARGET} COMMAND "${TEST_TARGET}" WORKING_DIRECTORY "${TEST_DIR}") MATH(EXPR MINIFI_MAIN_UNIT_TEST_COUNT "${MINIFI_MAIN_UNIT_TEST_COUNT}+1") diff --git a/minifi_main/tests/TableFormatterTests.cpp b/minifi_main/tests/TableFormatterTests.cpp index bfd8121ced..8f5e2aa407 100644 --- a/minifi_main/tests/TableFormatterTests.cpp +++ b/minifi_main/tests/TableFormatterTests.cpp @@ -15,7 +15,7 @@ * limitations under the License. */ -#include "Catch.h" +#include "unit/Catch.h" #include "TableFormatter.h" namespace minifi = org::apache::nifi::minifi; diff --git a/nanofi/CMakeLists.txt b/nanofi/CMakeLists.txt index d8fe2c99a7..a5f6cf59a1 100644 --- a/nanofi/CMakeLists.txt +++ b/nanofi/CMakeLists.txt @@ -42,9 +42,7 @@ set_cpp_version() add_minifi_library(nanofi STATIC ${NANOFI_SOURCES}) -target_link_libraries(nanofi spdlog ut core-minifi minifi-standard-processors) +target_link_libraries(nanofi spdlog::spdlog ut core-minifi minifi-standard-processors) -if (ENABLE_CURL) - add_subdirectory(examples) - add_subdirectory(ecu) -endif() +add_subdirectory(examples) +add_subdirectory(ecu) diff --git a/nanofi/ecu/CMakeLists.txt b/nanofi/ecu/CMakeLists.txt index c627091d05..cc6f5ca826 100644 --- a/nanofi/ecu/CMakeLists.txt +++ b/nanofi/ecu/CMakeLists.txt @@ -22,13 +22,10 @@ cmake_minimum_required(VERSION 3.24) if (NOT WIN32) add_minifi_executable(log_aggregator log_aggregator.c) target_link_libraries(log_aggregator nanofi Threads::Threads) - target_link_libraries(log_aggregator minifi-http-curl) add_minifi_executable(tailfile_chunk tailfile_chunk.c) target_link_libraries(tailfile_chunk nanofi Threads::Threads) - target_link_libraries(tailfile_chunk minifi-http-curl) add_minifi_executable(tailfile_delimited tailfile_delimited.c) target_link_libraries(tailfile_delimited nanofi Threads::Threads) - target_link_libraries(tailfile_delimited minifi-http-curl) endif() diff --git a/nanofi/examples/CMakeLists.txt b/nanofi/examples/CMakeLists.txt index 28c7595631..e81558c102 100644 --- a/nanofi/examples/CMakeLists.txt +++ b/nanofi/examples/CMakeLists.txt @@ -27,7 +27,6 @@ if (NOT WIN32) add_minifi_executable(terminate_handler terminate_handler.c) - target_link_libraries(generate_flow minifi-http-curl) target_link_libraries(generate_flow nanofi Threads::Threads) target_link_libraries(terminate_handler nanofi Threads::Threads) @@ -38,16 +37,13 @@ if (NOT WIN32) add_minifi_executable(transmit_flow transmit_flow.c) - target_link_libraries(transmit_flow minifi-http-curl) target_link_libraries(transmit_flow nanofi Threads::Threads) add_minifi_executable(transmit_payload transmit_payload.c) - target_link_libraries(transmit_payload minifi-http-curl) target_link_libraries(transmit_payload nanofi Threads::Threads) add_minifi_executable(monitor_directory monitor_directory.c) - target_link_libraries(monitor_directory minifi-http-curl) target_link_libraries(monitor_directory nanofi Threads::Threads) endif() diff --git a/nanofi/examples/hash_file.c b/nanofi/examples/hash_file.c index e155fedbca..3c253977ef 100644 --- a/nanofi/examples/hash_file.c +++ b/nanofi/examples/hash_file.c @@ -14,9 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -#ifdef OPENSSL_SUPPORT - #include "api/nanofi.h" #include #include @@ -116,5 +113,3 @@ int main(int argc, char **argv) { return 0; } - -#endif // OPENSSL_SUPPORT diff --git a/nanofi/tests/CAPITests.cpp b/nanofi/tests/CAPITests.cpp index 490f01b0f4..b68c0e6d90 100644 --- a/nanofi/tests/CAPITests.cpp +++ b/nanofi/tests/CAPITests.cpp @@ -20,8 +20,8 @@ #include #include #include "utils/file/FileUtils.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "api/nanofi.h" const std::string test_file_content = "C API raNdOMcaSe test d4t4 th1s is!"; diff --git a/nanofi/tests/CLogAggregatorTests.cpp b/nanofi/tests/CLogAggregatorTests.cpp index f491e682f3..c0e2ba626d 100644 --- a/nanofi/tests/CLogAggregatorTests.cpp +++ b/nanofi/tests/CLogAggregatorTests.cpp @@ -17,7 +17,7 @@ */ #ifndef _WIN32 -#include "Catch.h" +#include "unit/Catch.h" #include #include diff --git a/nanofi/tests/CSite2SiteTests.cpp b/nanofi/tests/CSite2SiteTests.cpp index 10633f17c3..f2e6874794 100644 --- a/nanofi/tests/CSite2SiteTests.cpp +++ b/nanofi/tests/CSite2SiteTests.cpp @@ -30,8 +30,8 @@ #include #include "io/BaseStream.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "unit/SiteToSiteHelper.h" #include "sitetosite/CPeer.h" #include "sitetosite/CRawSocketProtocol.h" diff --git a/nanofi/tests/CTailFileChunkTests.cpp b/nanofi/tests/CTailFileChunkTests.cpp index 7baa030c1b..d595ed7650 100644 --- a/nanofi/tests/CTailFileChunkTests.cpp +++ b/nanofi/tests/CTailFileChunkTests.cpp @@ -16,7 +16,7 @@ * limitations under the License. */ #ifndef _WIN32 -#include "Catch.h" +#include "unit/Catch.h" #include #include diff --git a/nanofi/tests/CTailFileDelimitedTests.cpp b/nanofi/tests/CTailFileDelimitedTests.cpp index f6e58b5fbd..31a5f7e452 100644 --- a/nanofi/tests/CTailFileDelimitedTests.cpp +++ b/nanofi/tests/CTailFileDelimitedTests.cpp @@ -16,7 +16,7 @@ * limitations under the License. */ -#include "Catch.h" +#include "unit/Catch.h" #include "CTestsBase.h" diff --git a/nanofi/tests/CTestsBase.h b/nanofi/tests/CTestsBase.h index 997398b291..02bc553086 100644 --- a/nanofi/tests/CTestsBase.h +++ b/nanofi/tests/CTestsBase.h @@ -29,8 +29,8 @@ #include "core/file_utils.h" #include "api/ecu.h" #include "api/nanofi.h" -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" class FileManager { public: diff --git a/nanofi/tests/CUUIDTests.cpp b/nanofi/tests/CUUIDTests.cpp index c40516a6c8..29a51173a3 100644 --- a/nanofi/tests/CUUIDTests.cpp +++ b/nanofi/tests/CUUIDTests.cpp @@ -19,8 +19,8 @@ #include #include #include -#include "TestBase.h" -#include "Catch.h" +#include "unit/TestBase.h" +#include "unit/Catch.h" #include "core/cuuid.h" bool verify_uuid(const char * uuid) { diff --git a/rheldistro.sh b/rheldistro.sh index 7414e3a203..aba5ce6474 100644 --- a/rheldistro.sh +++ b/rheldistro.sh @@ -80,9 +80,8 @@ build_deps(){ wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm sudo yum -y install epel-release-latest-7.noarch.rpm - COMMAND="sudo yum install cmake3 libuuid libuuid-devel" + COMMAND="sudo yum install cmake3 libuuid libuuid-devel perl bzip2-devel" INSTALLED=() - INSTALLED+=("bzip2-devel") for option in "${OPTIONS[@]}" ; do option_value="${!option}" if [ "$option_value" = "${TRUE}" ]; then @@ -117,8 +116,6 @@ build_deps(){ INSTALLED+=("xz-devel") elif [ "$FOUND_VALUE" = "libssh2" ]; then INSTALLED+=("libssh2-devel") - elif [ "$FOUND_VALUE" = "opensslbuild" ]; then - INSTALLED+=("perl") fi fi done diff --git a/suse.sh b/suse.sh index c1ea7b081a..ab24d39778 100644 --- a/suse.sh +++ b/suse.sh @@ -66,9 +66,8 @@ bootstrap_compiler() { } build_deps(){ # Install epel-release so that cmake3 will be available for installation - COMMAND="sudo zypper in -y libuuid1 libuuid-devel" + COMMAND="sudo zypper in -y libuuid1 libuuid-devel perl libbz2-devel" INSTALLED=() - INSTALLED+=("libbz2-devel") for option in "${OPTIONS[@]}" ; do option_value="${!option}" if [ "$option_value" = "${TRUE}" ]; then @@ -103,8 +102,6 @@ build_deps(){ INSTALLED+=("xz-devel") elif [ "$FOUND_VALUE" = "libssh2" ]; then INSTALLED+=("libssh2-devel") - elif [ "$FOUND_VALUE" = "opensslbuild" ]; then - INSTALLED+=("perl") fi fi done diff --git a/test_package/.gitignore b/test_package/.gitignore new file mode 100644 index 0000000000..8a787a020e --- /dev/null +++ b/test_package/.gitignore @@ -0,0 +1,8 @@ +cmake-build-release* +CMakeUserPresets.json +conan.lock +conanbuildinfo.txt +conanbuildinfo.cmake +conaninfo.txt +graph_info.json +build diff --git a/test_package/CMakeLists.txt b/test_package/CMakeLists.txt new file mode 100644 index 0000000000..590dc160a9 --- /dev/null +++ b/test_package/CMakeLists.txt @@ -0,0 +1,69 @@ +cmake_minimum_required(VERSION 3.15) +project(test-minifi-cpp CXX) + +find_package(minifi-cpp CONFIG REQUIRED) +find_package(gsl-lite CONFIG REQUIRED) +find_package(fmt CONFIG REQUIRED) +find_package(spdlog CONFIG REQUIRED) +find_package(Catch2 CONFIG REQUIRED) + +add_executable(abstract_processor_test src/AbstractProcessorTest.cpp) + +target_link_libraries(abstract_processor_test +minifi-cpp::minifi-cpp +gsl::gsl-lite +fmt::fmt +spdlog::spdlog +Catch2::Catch2WithMain +) + +list(APPEND gsl-lite_DEFINITIONS_RELEASE gsl_CONFIG_DEFAULTS_VERSION=1) + +target_compile_definitions(gsl::gsl-lite INTERFACE ${gsl-lite_DEFINITIONS_RELEASE}) + +target_include_directories(abstract_processor_test PUBLIC + ${fmt_INCLUDE_DIRS} + ${minifi-cpp_INCLUDE_DIRS} + ${minifi-cpp_INCLUDE_DIRS}/libminifi/include + ${minifi-cpp_INCLUDE_DIRS}/libminifi/test/libtest + ${minifi-cpp_INCLUDE_DIRS}/_deps/magic_enum-src/include + ${minifi-cpp_INCLUDE_DIRS}/_deps/date_src-src/include + ${minifi-cpp_INCLUDE_DIRS}/_deps/catch2-src/src + ${minifi-cpp_INCLUDE_DIRS}/_deps/catch2-build/generated-includes + ${minifi-cpp_INCLUDE_DIRS}/_deps/spdlog-src/include + ${minifi-cpp_INCLUDE_DIRS}/thirdparty/rapidjson-48fbd8cd202ca54031fe799db2ad44ffa8e77c13/include + ${minifi-cpp_INCLUDE_DIRS}/extensions/standard-processors/processors +) + +add_test(NAME abstract_processor_test COMMAND abstract_processor_test) + +# add_executable(minifi_generate_flow_file_test src/MinifiGenerateFlowFileTest.cpp) + +# target_link_libraries(minifi_generate_flow_file_test +# minifi-cpp::minifi-cpp +# gsl::gsl-lite +# fmt::fmt +# spdlog::spdlog +# Catch2::Catch2WithMain +# ) + +# list(APPEND gsl-lite_DEFINITIONS_RELEASE gsl_CONFIG_DEFAULTS_VERSION=1) + +# target_compile_definitions(gsl::gsl-lite INTERFACE ${gsl-lite_DEFINITIONS_RELEASE}) + +# target_include_directories(minifi_generate_flow_file_test PUBLIC +# ${fmt_INCLUDE_DIRS} +# ${minifi-cpp_INCLUDE_DIRS} +# ${minifi-cpp_INCLUDE_DIRS}/libminifi/include +# ${minifi-cpp_INCLUDE_DIRS}/libminifi/test/libtest +# ${minifi-cpp_INCLUDE_DIRS}/_deps/magic_enum-src/include +# ${minifi-cpp_INCLUDE_DIRS}/_deps/date_src-src/include +# ${minifi-cpp_INCLUDE_DIRS}/_deps/catch2-src/src +# ${minifi-cpp_INCLUDE_DIRS}/_deps/catch2-build/generated-includes +# ${minifi-cpp_INCLUDE_DIRS}/_deps/spdlog-src/include +# ${minifi-cpp_INCLUDE_DIRS}/thirdparty/rapidjson-48fbd8cd202ca54031fe799db2ad44ffa8e77c13/include +# ${minifi-cpp_INCLUDE_DIRS}/extensions/standard-processors/processors +# ) + +# add_test(NAME minifi_generate_flow_file_test COMMAND minifi_generate_flow_file_test) + diff --git a/test_package/conanfile.py b/test_package/conanfile.py new file mode 100644 index 0000000000..f6f113c940 --- /dev/null +++ b/test_package/conanfile.py @@ -0,0 +1,32 @@ +import os + +from conan import ConanFile +from conan.tools.build import can_run +from conan.tools.cmake import CMake, cmake_layout + +class MiNiFiCppTestConan(ConanFile): + settings = "os", "compiler", "build_type", "arch" + generators = "CMakeDeps", "CMakeToolchain", "VirtualRunEnv" + test_type = "explicit" + + def requirements(self): + self.requires(self.tested_reference_str) + self.requires("gsl-lite/0.41.0") + self.requires("fmt/10.2.1") + self.requires("spdlog/1.14.0") + self.requires("catch2/3.5.4") + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def layout(self): + cmake_layout(self) + + def test(self): + if can_run(self): + cmd = os.path.join(self.cpp.build.bindirs[0], "abstract_processor_test") + self.run(cmd, env="conanrun") + # cmd = os.path.join(self.cpp.build.bindirs[0], "minifi_generate_flow_file_test") + # self.run(cmd, env="conanrun") diff --git a/test_package/src/AbstractProcessorTest.cpp b/test_package/src/AbstractProcessorTest.cpp new file mode 100644 index 0000000000..99506c63d5 --- /dev/null +++ b/test_package/src/AbstractProcessorTest.cpp @@ -0,0 +1,114 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#define EXTENSION_LIST "" // NOLINT(cppcoreguidelines-macro-usage) +#include +#include "unit/Catch.h" +#include "core/AbstractProcessor.h" +#include "core/PropertyDefinitionBuilder.h" + +namespace org::apache::nifi::minifi::test { + +struct AbstractProcessorTestCase1 : core::AbstractProcessor { + static constexpr auto SupportsDynamicProperties = true; + static constexpr auto SupportsDynamicRelationships = true; + static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_FORBIDDEN; + static constexpr auto IsSingleThreaded = true; + static constexpr auto Property1 = core::PropertyDefinitionBuilder<>::createProperty("Property 1") + .withDefaultValue("foo") + .supportsExpressionLanguage(true) + .isRequired(true) + .build(); + static constexpr auto Property2 = core::PropertyDefinitionBuilder<>::createProperty("Property 2") + .withDefaultValue("bar") + .supportsExpressionLanguage(false) + .isRequired(false) + .build(); + static constexpr auto Properties = std::array{Property1, Property2}; + static constexpr core::RelationshipDefinition Rel1{"rel1", "rel1 description"}; + static constexpr auto Relationships = std::array{Rel1}; + AbstractProcessorTestCase1() :core::AbstractProcessor{"TestCase1"} {} + void onSchedule(core::ProcessContext&, core::ProcessSessionFactory&) override {} + void onTrigger(core::ProcessContext&, core::ProcessSession&) override {} +}; + +TEST_CASE("AbstractProcessor case1", "[processor][abstractprocessor][case1]") { + AbstractProcessorTestCase1 processor; + processor.initialize(); + REQUIRE(processor.getName() == "TestCase1"); + REQUIRE(processor.supportsDynamicProperties() == true); + REQUIRE(processor.supportsDynamicRelationships() == true); + REQUIRE(processor.getInputRequirement() == core::annotation::Input::INPUT_FORBIDDEN); + REQUIRE(processor.isSingleThreaded()); + const auto properties = processor.getProperties(); + REQUIRE(properties.size() == 2); + REQUIRE(properties.contains("Property 1")); + REQUIRE(properties.at("Property 1").supportsExpressionLanguage()); + REQUIRE(properties.at("Property 1").getRequired()); + REQUIRE(properties.contains("Property 2")); + REQUIRE(!properties.at("Property 2").supportsExpressionLanguage()); + REQUIRE(!properties.at("Property 2").getRequired()); + REQUIRE(processor.getSupportedRelationships().size() == 1); + const auto relationships = processor.getSupportedRelationships(); + REQUIRE(relationships[0].getName() == "rel1"); +} + +struct AbstractProcessorTestCase2 : core::AbstractProcessor { + static constexpr bool SupportsDynamicProperties = false; + static constexpr bool SupportsDynamicRelationships = false; + static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_REQUIRED; + static constexpr bool IsSingleThreaded = false; + static constexpr auto Property1 = core::PropertyDefinitionBuilder<>::createProperty("prop 1") + .withDefaultValue("foo") + .supportsExpressionLanguage(true) + .isRequired(true) + .build(); + static constexpr auto Property2 = core::PropertyDefinitionBuilder<>::createProperty("prop 2") + .withDefaultValue("bar") + .supportsExpressionLanguage(false) + .isRequired(false) + .build(); + static constexpr auto Properties = std::array{Property1, Property2}; + static constexpr core::RelationshipDefinition Rel1{"Relationship1", "rel1 description"}; + static constexpr auto Relationships = std::array{Rel1}; + AbstractProcessorTestCase2() :core::AbstractProcessor{"TestCase2"} {} + void onSchedule(core::ProcessContext&, core::ProcessSessionFactory&) override {} + void onTrigger(core::ProcessContext&, core::ProcessSession&) override {} +}; + +TEST_CASE("AbstractProcessor case2", "[processor][abstractprocessor][case2]") { + AbstractProcessorTestCase2 processor; + processor.initialize(); + REQUIRE(processor.getName() == "TestCase2"); + REQUIRE(!processor.supportsDynamicProperties()); + REQUIRE(!processor.supportsDynamicRelationships()); + REQUIRE(processor.getInputRequirement() == core::annotation::Input::INPUT_REQUIRED); + REQUIRE(!processor.isSingleThreaded()); + const auto properties = processor.getProperties(); + REQUIRE(properties.size() == 2); + REQUIRE(properties.contains("prop 1")); + REQUIRE(properties.at("prop 1").supportsExpressionLanguage()); + REQUIRE(properties.at("prop 1").getRequired()); + REQUIRE(properties.contains("prop 2")); + REQUIRE(!properties.at("prop 2").supportsExpressionLanguage()); + REQUIRE(!properties.at("prop 2").getRequired()); + REQUIRE(processor.getSupportedRelationships().size() == 1); + const auto relationships = processor.getSupportedRelationships(); + REQUIRE(relationships[0].getName() == "Relationship1"); +} + +} // namespace org::apache::nifi::minifi::test diff --git a/test_package/src/MinifiGenerateFlowFileTest.cpp b/test_package/src/MinifiGenerateFlowFileTest.cpp new file mode 100644 index 0000000000..336a80ce4f --- /dev/null +++ b/test_package/src/MinifiGenerateFlowFileTest.cpp @@ -0,0 +1,157 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +#include "unit/TestBase.h" +#include "unit/SingleProcessorTestController.h" +#include "unit/Catch.h" +#include "GenerateFlowFile.h" + +using minifi::processors::GenerateFlowFile; + +TEST_CASE("GenerateFlowFileWithBinaryData") { + std::optional is_unique; + + SECTION("Not unique") { + is_unique = false; + } + + SECTION("Unique") { + is_unique = true; + } + + std::shared_ptr generate_flow_file = std::make_shared("GenerateFlowFile"); + minifi::test::SingleProcessorTestController test_controller{generate_flow_file}; + LogTestController::getInstance().setWarn(); + + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::FileSize, "10"); + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::BatchSize, "2"); + + // This property will be ignored if binary files are used + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::CustomText, "Current time: ${now()}"); + + REQUIRE(is_unique.has_value()); + std::string result; + if (is_unique.has_value()) { + result = is_unique.value() ? "true" : "false"; + } + else { + result = "nullopt"; + } + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::UniqueFlowFiles, result); + + auto first_batch = test_controller.trigger(); + REQUIRE(first_batch.at(GenerateFlowFile::Success).size() == 2); + auto first_batch_0 = test_controller.plan->getContent(first_batch.at(GenerateFlowFile::Success)[0]); + auto first_batch_1 = test_controller.plan->getContent(first_batch.at(GenerateFlowFile::Success)[1]); + CHECK(first_batch_0.size() == 10); + + if (*is_unique) { + CHECK(first_batch_0 != first_batch_1); + } else { + CHECK(first_batch_0 == first_batch_1); + } + + CHECK(LogTestController::getInstance().contains("Custom Text property is set but not used. For Custom Text to be used, Data Format needs to be Text, and Unique FlowFiles needs to be false.")); +} + +TEST_CASE("GenerateFlowFileTestEmpty") { + std::shared_ptr generate_flow_file = std::make_shared("GenerateFlowFile"); + minifi::test::SingleProcessorTestController test_controller{generate_flow_file}; + + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::FileSize, "0"); + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::UniqueFlowFiles, "false"); + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::DataFormat, "Text"); + + auto result = test_controller.trigger(); + REQUIRE(result.at(GenerateFlowFile::Success).size() == 1); + auto result_0 = test_controller.plan->getContent(result.at(GenerateFlowFile::Success)[0]); + CHECK(result_0.empty()); +} + +TEST_CASE("GenerateFlowFileCustomTextTest") { + std::shared_ptr generate_flow_file = std::make_shared("GenerateFlowFile"); + minifi::test::SingleProcessorTestController test_controller{generate_flow_file}; + + constexpr auto uuid_string_length = 36; + + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::CustomText, "${UUID()}"); + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::UniqueFlowFiles, "false"); + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::DataFormat, "Text"); + + auto result = test_controller.trigger(); + REQUIRE(result.at(GenerateFlowFile::Success).size() == 1); + auto result_0 = test_controller.plan->getContent(result.at(GenerateFlowFile::Success)[0]); + CHECK(result_0.length() == uuid_string_length); +} + +TEST_CASE("GenerateFlowFileCustomTextEmptyTest") { + std::shared_ptr generate_flow_file = std::make_shared("GenerateFlowFile"); + minifi::test::SingleProcessorTestController test_controller{generate_flow_file}; + + constexpr int32_t file_size = 10; + + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::FileSize, std::to_string(file_size)); + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::UniqueFlowFiles, "false"); + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::DataFormat, "Text"); + SECTION("Empty custom data") { + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::CustomText, ""); + } + + SECTION("No custom data") { + } + + auto result = test_controller.trigger(); + REQUIRE(result.at(GenerateFlowFile::Success).size() == 1); + auto result_0 = test_controller.plan->getContent(result.at(GenerateFlowFile::Success)[0]); + CHECK(result_0.length() == file_size); +} + +TEST_CASE("GenerateFlowFile reevaluating CustomText") { + std::shared_ptr generate_flow_file = std::make_shared("GenerateFlowFile"); + minifi::test::SingleProcessorTestController test_controller{generate_flow_file}; + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::DataFormat, "Text"); + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::UniqueFlowFiles, "false"); + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::CustomText, "${nextInt()}"); + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::BatchSize, "2"); + + for (auto i = 0; i < 100; ++i) { + auto batch = test_controller.trigger(); + auto batch_0 = test_controller.plan->getContent(batch.at(GenerateFlowFile::Success)[0]); + auto batch_1 = test_controller.plan->getContent(batch.at(GenerateFlowFile::Success)[1]); + CHECK(batch_0 == batch_1); + CHECK(batch_0 == std::to_string(i)); + } +} + +TEST_CASE("GenerateFlowFile CustomText evaluates to empty string") { + std::shared_ptr generate_flow_file = std::make_shared("GenerateFlowFile"); + minifi::test::SingleProcessorTestController test_controller{generate_flow_file}; + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::DataFormat, "Text"); + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::UniqueFlowFiles, "false"); + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::CustomText, "${invalid_variable}"); + test_controller.plan->setProperty(generate_flow_file, GenerateFlowFile::BatchSize, "2"); + + auto batch = test_controller.trigger(); + auto batch_0 = test_controller.plan->getContent(batch.at(GenerateFlowFile::Success)[0]); + auto batch_1 = test_controller.plan->getContent(batch.at(GenerateFlowFile::Success)[1]); + CHECK(batch_0 == batch_1); + CHECK(batch_0.empty()); +} diff --git a/win_build_vs.bat b/win_build_vs.bat index 020cbdafa1..0cf8ea35e8 100644 --- a/win_build_vs.bat +++ b/win_build_vs.bat @@ -120,7 +120,7 @@ if [%generator%] EQU ["Ninja"] ( ) echo on cmake -G %generator% %build_platform_cmd% -DMINIFI_INCLUDE_VC_REDIST_MERGE_MODULES=%installer_merge_modules% -DTEST_CUSTOM_WEL_PROVIDER=%test_custom_wel_provider% -DENABLE_SQL=%enable_sql% -DMINIFI_USE_REAL_ODBC_TEST_DRIVER=%real_odbc% ^ - -DCMAKE_BUILD_TYPE_INIT=%cmake_build_type% -DCMAKE_BUILD_TYPE=%cmake_build_type% -DWIN32=WIN32 -DENABLE_LIBRDKAFKA=%enable_kafka% -DENABLE_JNI=%enable_jni% -DMINIFI_OPENSSL=ON ^ + -DCMAKE_BUILD_TYPE_INIT=%cmake_build_type% -DCMAKE_BUILD_TYPE=%cmake_build_type% -DWIN32=WIN32 -DENABLE_LIBRDKAFKA=%enable_kafka% -DENABLE_JNI=%enable_jni% ^ -DENABLE_COAP=%enable_coap% -DENABLE_AWS=%enable_aws% -DENABLE_PDH=%enable_pdh% -DENABLE_AZURE=%enable_azure% -DENABLE_SFTP=%enable_sftp% -DENABLE_SPLUNK=%enable_splunk% -DENABLE_GCP=%enable_gcp% ^ -DENABLE_NANOFI=%enable_nanofi% -DENABLE_OPENCV=%enable_opencv% -DENABLE_PROMETHEUS=%enable_prometheus% -DENABLE_ELASTICSEARCH=%enable_elastic% -DUSE_SHARED_LIBS=OFF -DENABLE_CONTROLLER=OFF ^ -DENABLE_BUSTACHE=%enable_bustache% -DENABLE_ENCRYPT_CONFIG=%enable_encrypt_config% -DENABLE_LUA_SCRIPTING=%enable_lua_scripting% -DENABLE_SMB=%enable_smb% ^