From 03f1b8f51db71a52bc9ee1b364bf688186606af9 Mon Sep 17 00:00:00 2001 From: Marc Parisi Date: Mon, 9 Oct 2017 18:42:24 -0400 Subject: [PATCH] MINIFICPP-60: Add initial implementation of Site to Site changes. MINIFICPP-60: Update interfaces to allow for seamless extension MINIFICPP-60: Update functions to accomodate changed signatures MINIFICPP-60: Update per pull request comments. Fix issues with most tests failing in parallel MINIFICPP-275: Update CompressContent to use signature MINIFICPP-60: Correct issue with HTTP where we were using the remote port from the JSON response --- README.md | 7 +- cmake/BuildTests.cmake | 1 - extensions/http-curl/CMakeLists.txt | 12 +- ...{HttpCurlLoader.cpp => HTTPCurlLoader.cpp} | 3 +- .../{HttpCurlLoader.h => HTTPCurlLoader.h} | 30 +- extensions/http-curl/client/HTTPCallback.h | 188 +++ .../client/{HttpClient.cpp => HTTPClient.cpp} | 84 +- extensions/http-curl/client/HTTPClient.h | 61 +- extensions/http-curl/client/HTTPStream.cpp | 127 ++ extensions/http-curl/client/HTTPStream.h | 172 +++ .../http-curl/processors/InvokeHTTP.cpp | 8 +- extensions/http-curl/processors/InvokeHTTP.h | 10 +- extensions/http-curl/protocols/RESTProtocol.h | 3 +- extensions/http-curl/protocols/RESTReceiver.h | 4 +- extensions/http-curl/protocols/RESTSender.cpp | 60 +- extensions/http-curl/protocols/RESTSender.h | 11 +- .../http-curl/sitetosite/HTTPProtocol.cpp | 310 ++++ .../http-curl/sitetosite/HTTPProtocol.h | 197 +++ .../http-curl/sitetosite/HTTPTransaction.h | 70 + extensions/http-curl/sitetosite/PeersEntity.h | 112 ++ extensions/libarchive/ArchiveLoader.h | 10 +- extensions/libarchive/BinFiles.cpp | 2 +- extensions/libarchive/BinFiles.h | 2 +- extensions/libarchive/CompressContent.cpp | 5 +- extensions/libarchive/CompressContent.h | 4 +- libminifi/CMakeLists.txt | 2 +- libminifi/include/RemoteProcessorGroupPort.h | 36 +- libminifi/include/SchedulingAgent.h | 2 +- libminifi/include/core/Processor.h | 12 +- .../SiteToSiteProvenanceReportingTask.h | 9 +- .../include/core/yaml/YamlConfiguration.h | 4 +- libminifi/include/io/BaseStream.h | 12 +- libminifi/include/io/CRCStream.h | 32 +- libminifi/include/io/DataStream.h | 1 + libminifi/include/io/NonConvertingStream.h | 200 +++ libminifi/include/processors/GetTCP.h | 4 +- libminifi/include/properties/Configure.h | 1 + .../{Site2SitePeer.h => sitetosite/Peer.h} | 210 ++- .../include/sitetosite/RawSocketProtocol.h | 211 +++ .../SiteToSite.h} | 555 +++----- .../include/sitetosite/SiteToSiteClient.h | 330 +++++ .../include/sitetosite/SiteToSiteFactory.h | 89 ++ ...yteInputCallBack.h => ByteArrayCallback.h} | 74 +- libminifi/include/utils/HTTPClient.h | 89 +- libminifi/include/utils/StringUtils.h | 11 + libminifi/src/Configure.cpp | 1 + libminifi/src/RemoteProcessorGroupPort.cpp | 141 +- libminifi/src/SchedulingAgent.cpp | 6 +- libminifi/src/Site2SiteClientProtocol.cpp | 1261 ----------------- libminifi/src/core/Processor.cpp | 2 +- .../SiteToSiteProvenanceReportingTask.cpp | 39 +- libminifi/src/core/yaml/YamlConfiguration.cpp | 6 +- libminifi/src/io/DataStream.cpp | 2 + libminifi/src/io/FileStream.cpp | 4 +- libminifi/src/io/NonConvertingStream.cpp | 190 +++ libminifi/src/io/Serializable.cpp | 1 - libminifi/src/processors/GetTCP.cpp | 4 +- .../Peer.cpp} | 9 +- .../src/sitetosite/RawSocketProtocol.cpp | 629 ++++++++ libminifi/src/sitetosite/SiteToSiteClient.cpp | 773 ++++++++++ libminifi/src/utils/ByteArrayCallback.cpp | 155 ++ libminifi/test/TestBase.cpp | 5 +- libminifi/test/TestBase.h | 19 +- .../test/curl-tests/C2NullConfiguration.cpp | 2 - .../test/curl-tests/C2VerifyServeResults.cpp | 2 - libminifi/test/curl-tests/CMakeLists.txt | 7 +- .../ControllerServiceIntegrationTests.cpp | 5 - .../test/curl-tests/HTTPSiteToSiteTests.cpp | 262 ++++ .../curl-tests/sitetositehttp/CivetStream.h | 138 ++ .../curl-tests/sitetositehttp/HTTPHandlers.h | 322 +++++ libminifi/test/integration/IntegrationBase.h | 13 +- .../integration/ProvenanceReportingTest.cpp | 2 +- .../resources/C2VerifyHeartbeatAndStop.yml | 73 + .../test/resources/C2VerifyServeResults.yml | 73 + .../test/resources/TestHTTPGetSecure.yml | 2 +- libminifi/test/resources/TestHTTPPost.yml | 4 +- .../resources/TestHTTPPostChunkedEncoding.yml | 4 +- .../test/resources/TestHTTPSiteToSite.yml | 91 ++ .../test/resources/TestSite2SiteRest.yml | 4 +- .../resources/TestSite2SiteRestSecure.yml | 2 +- libminifi/test/resources/ThreadPoolAdjust.yml | 97 ++ libminifi/test/unit/GetTCPTests.cpp | 12 +- libminifi/test/unit/LoggerTests.cpp | 1 - libminifi/test/unit/ProcessorTests.cpp | 15 +- libminifi/test/unit/SerializationTests.cpp | 2 - libminifi/test/unit/Site2SiteTests.cpp | 35 +- 86 files changed, 5733 insertions(+), 2059 deletions(-) rename extensions/http-curl/{HttpCurlLoader.cpp => HTTPCurlLoader.cpp} (97%) rename extensions/http-curl/{HttpCurlLoader.h => HTTPCurlLoader.h} (68%) create mode 100644 extensions/http-curl/client/HTTPCallback.h rename extensions/http-curl/client/{HttpClient.cpp => HTTPClient.cpp} (78%) create mode 100644 extensions/http-curl/client/HTTPStream.cpp create mode 100644 extensions/http-curl/client/HTTPStream.h create mode 100644 extensions/http-curl/sitetosite/HTTPProtocol.cpp create mode 100644 extensions/http-curl/sitetosite/HTTPProtocol.h create mode 100644 extensions/http-curl/sitetosite/HTTPTransaction.h create mode 100644 extensions/http-curl/sitetosite/PeersEntity.h create mode 100644 libminifi/include/io/NonConvertingStream.h rename libminifi/include/{Site2SitePeer.h => sitetosite/Peer.h} (54%) create mode 100644 libminifi/include/sitetosite/RawSocketProtocol.h rename libminifi/include/{Site2SiteClientProtocol.h => sitetosite/SiteToSite.h} (50%) create mode 100644 libminifi/include/sitetosite/SiteToSiteClient.h create mode 100644 libminifi/include/sitetosite/SiteToSiteFactory.h rename libminifi/include/utils/{ByteInputCallBack.h => ByteArrayCallback.h} (55%) delete mode 100644 libminifi/src/Site2SiteClientProtocol.cpp create mode 100644 libminifi/src/io/NonConvertingStream.cpp rename libminifi/src/{Site2SitePeer.cpp => sitetosite/Peer.cpp} (91%) create mode 100644 libminifi/src/sitetosite/RawSocketProtocol.cpp create mode 100644 libminifi/src/sitetosite/SiteToSiteClient.cpp create mode 100644 libminifi/src/utils/ByteArrayCallback.cpp create mode 100644 libminifi/test/curl-tests/HTTPSiteToSiteTests.cpp create mode 100644 libminifi/test/curl-tests/sitetositehttp/CivetStream.h create mode 100644 libminifi/test/curl-tests/sitetositehttp/HTTPHandlers.h create mode 100644 libminifi/test/resources/C2VerifyHeartbeatAndStop.yml create mode 100644 libminifi/test/resources/C2VerifyServeResults.yml create mode 100644 libminifi/test/resources/TestHTTPSiteToSite.yml create mode 100644 libminifi/test/resources/ThreadPoolAdjust.yml diff --git a/README.md b/README.md index 413333bbe4..cc833e0cf3 100644 --- a/README.md +++ b/README.md @@ -307,7 +307,7 @@ Additionally, users can utilize the MiNiFi Toolkit Converter (version 0.0.1 - sc max concurrent tasks: 1 Properties: -### Site2Site Security Configuration +### SiteToSite Security Configuration in minifi.properties @@ -327,6 +327,11 @@ Additionally, users can utilize the MiNiFi Toolkit Converter (version 0.0.1 - sc if you do not want to enable client certificate base authorization nifi.security.need.ClientAuth=false +### HTTP SiteToSite Configuration +To enable HTTPSiteToSite you must set the following flag to true + + nifi.remote.input.http.enabled=true + ### Command and Control Configuration For more more insight into the API used within the C2 agent, please visit: https://cwiki.apache.org/confluence/display/MINIFI/C2+Design+Proposal diff --git a/cmake/BuildTests.cmake b/cmake/BuildTests.cmake index 30542cabb4..99e980b2bc 100644 --- a/cmake/BuildTests.cmake +++ b/cmake/BuildTests.cmake @@ -86,7 +86,6 @@ FOREACH(testfile ${INTEGRATION_TESTS}) ENDFOREACH() message("-- Finished building ${INT_TEST_COUNT} integration test file(s)...") - get_property(extensions GLOBAL PROPERTY EXTENSION-TESTS) foreach(EXTENSION ${extensions}) message("Adding ${EXTENSION} ? ") diff --git a/extensions/http-curl/CMakeLists.txt b/extensions/http-curl/CMakeLists.txt index d851aff31c..cd9f3bacd2 100644 --- a/extensions/http-curl/CMakeLists.txt +++ b/extensions/http-curl/CMakeLists.txt @@ -21,11 +21,12 @@ find_package(CURL REQUIRED) set(CMAKE_EXE_LINKER_FLAGS "-Wl,--export-all-symbols") set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--export-symbols") +set(CMAKE_CXX_VISIBILITY_PRESET default) include_directories(../../libminifi/include ../../libminifi/include/core/yaml ../../libminifi/include/core ../../thirdparty/spdlog-20170710/include ../../thirdparty/concurrentqueue ../../thirdparty/yaml-cpp-yaml-cpp-0.5.3/include ../../thirdparty/civetweb-1.9.1/include ../../thirdparty/jsoncpp/include ../../thirdparty/) -include_directories(protocols client processors) +include_directories(protocols client processors sitetosite) -file(GLOB SOURCES "*.cpp" "protocols/*.cpp" "client/*.cpp" "processors/*.cpp") +file(GLOB SOURCES "*.cpp" "protocols/*.cpp" "client/*.cpp" "processors/*.cpp" "sitetosite/*.cpp") add_library(minifi-http-curl STATIC ${SOURCES}) set_property(TARGET minifi-http-curl PROPERTY POSITION_INDEPENDENT_CODE ON) @@ -43,8 +44,8 @@ endif(CURL_FOUND) # Include UUID find_package(UUID REQUIRED) -target_link_libraries(minifi-http-curl ${LIBMINIFI} ${UUID_LIBRARIES} ${JSONCPP_LIB}) -add_dependencies(minifi-http-curl jsoncpp_project) +#set(LINK_FLAGS ${LINK_FLAGS} "-Wl,-whole-archive") +# Include OpenSSL find_package(OpenSSL REQUIRED) include_directories(${OPENSSL_INCLUDE_DIR}) target_link_libraries(minifi-http-curl ${CMAKE_DL_LIBS} ) @@ -66,5 +67,4 @@ else () endif () SET (HTTP-CURL minifi-http-curl PARENT_SCOPE) - -register_extension(minifi-http-curl) +register_extension(minifi-http-curl) \ No newline at end of file diff --git a/extensions/http-curl/HttpCurlLoader.cpp b/extensions/http-curl/HTTPCurlLoader.cpp similarity index 97% rename from extensions/http-curl/HttpCurlLoader.cpp rename to extensions/http-curl/HTTPCurlLoader.cpp index b1f3eb53f0..010888a44c 100644 --- a/extensions/http-curl/HttpCurlLoader.cpp +++ b/extensions/http-curl/HTTPCurlLoader.cpp @@ -15,12 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "HttpCurlLoader.h" +#include "HTTPCurlLoader.h" #include "core/FlowConfiguration.h" bool HttpCurlObjectFactory::added = core::FlowConfiguration::add_static_func("createHttpCurlFactory"); - extern "C" { void *createHttpCurlFactory(void) { diff --git a/extensions/http-curl/HttpCurlLoader.h b/extensions/http-curl/HTTPCurlLoader.h similarity index 68% rename from extensions/http-curl/HttpCurlLoader.h rename to extensions/http-curl/HTTPCurlLoader.h index cb9a30de50..797aebf189 100644 --- a/extensions/http-curl/HttpCurlLoader.h +++ b/extensions/http-curl/HTTPCurlLoader.h @@ -15,8 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef EXTENSIONS_HTTPCURLLOADER_H -#define EXTENSIONS_HTTPCURLLOADER_H +#ifndef EXTENSIONS_HTTPCURLLOADER_H_ +#define EXTENSIONS_HTTPCURLLOADER_H_ #include "protocols/RESTProtocol.h" #include "protocols/RESTSender.h" @@ -24,6 +24,8 @@ #include "processors/InvokeHTTP.h" #include "client/HTTPClient.h" #include "core/ClassLoader.h" +#include "sitetosite/HTTPProtocol.h" +#include "utils/StringUtils.h" class __attribute__((visibility("default"))) HttpCurlObjectFactory : public core::ObjectFactory { public: @@ -35,36 +37,40 @@ class __attribute__((visibility("default"))) HttpCurlObjectFactory : public core * Gets the name of the object. * @return class name of processor */ - virtual std::string getName() { + virtual std::string getName() override{ return "HttpCurlObjectFactory"; } - virtual std::string getClassName() { + virtual std::string getClassName() override{ return "HttpCurlObjectFactory"; } /** * Gets the class name for the object * @return class name for the processor. */ - virtual std::vector getClassNames() { + virtual std::vector getClassNames() override{ std::vector class_names; class_names.push_back("RESTProtocol"); + class_names.push_back("HttpProtocol"); class_names.push_back("RESTReceiver"); class_names.push_back("RESTSender"); class_names.push_back("InvokeHTTP"); class_names.push_back("HTTPClient"); + class_names.push_back("HttpSiteToSiteClient"); return class_names; } - virtual std::unique_ptr assign(const std::string &class_name) { - if (class_name == "RESTReceiver") { + virtual std::unique_ptr assign(const std::string &class_name) override{ + if (utils::StringUtils::equalsIgnoreCase(class_name, "RESTReceiver")) { return std::unique_ptr(new core::DefautObjectFactory()); - } else if (class_name == "RESTSender") { + } else if (utils::StringUtils::equalsIgnoreCase(class_name, "RESTSender")) { return std::unique_ptr(new core::DefautObjectFactory()); - } else if (class_name == "InvokeHTTP") { + } else if (utils::StringUtils::equalsIgnoreCase(class_name, "InvokeHTTP")) { return std::unique_ptr(new core::DefautObjectFactory()); - } else if (class_name == "HTTPClient") { + } else if (utils::StringUtils::equalsIgnoreCase(class_name, "HTTPClient")) { return std::unique_ptr(new core::DefautObjectFactory()); + } else if (utils::StringUtils::equalsIgnoreCase(class_name, "HttpProtocol") || utils::StringUtils::equalsIgnoreCase(class_name, "HttpSiteToSiteClient")) { + return std::unique_ptr(new core::DefautObjectFactory()); } else { return nullptr; } @@ -72,11 +78,9 @@ class __attribute__((visibility("default"))) HttpCurlObjectFactory : public core static bool added; - - }; extern "C" { void *createHttpCurlFactory(void); } -#endif /* EXTENSIONS_HTTPCURLLOADER_H */ +#endif /* EXTENSIONS_HTTPCURLLOADER_H_ */ diff --git a/extensions/http-curl/client/HTTPCallback.h b/extensions/http-curl/client/HTTPCallback.h new file mode 100644 index 0000000000..aeca2a9c60 --- /dev/null +++ b/extensions/http-curl/client/HTTPCallback.h @@ -0,0 +1,188 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef EXTENSIONS_HTTP_CURL_CLIENT_HTTPCALLBACK_H_ +#define EXTENSIONS_HTTP_CURL_CLIENT_HTTPCALLBACK_H_ + +#include "concurrentqueue.h" +#include +#include +#include +#include + +#include "utils/ByteArrayCallback.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace utils { + +/** + * will stream as items are processed. + */ +class HttpStreamingCallback : public ByteInputCallBack { + public: + HttpStreamingCallback() + : ptr(nullptr), + is_alive_(true) { + previous_pos_ = 0; + rolling_count_ = 0; + } + + virtual ~HttpStreamingCallback() { + + } + + void close() { + is_alive_ = false; + cv.notify_all(); + } + + virtual void seek(size_t pos) { + if ((pos - previous_pos_) >= current_vec_.size() || current_vec_.size() == 0) + load_buffer(); + } + + virtual int64_t process(std::shared_ptr stream) { + + std::vector vec; + + if (stream->getSize() > 0) { + vec.resize(stream->getSize()); + + stream->readData(reinterpret_cast(vec.data()), stream->getSize()); + } + + size_t added_size = vec.size(); + + byte_arrays_.enqueue(std::move(vec)); + + cv.notify_all(); + + return added_size; + + } + + virtual int64_t process(uint8_t *vector, size_t size) { + + std::vector vec; + + if (size > 0) { + vec.resize(size); + + memcpy(vec.data(), vector, size); + + size_t added_size = vec.size(); + + byte_arrays_.enqueue(std::move(vec)); + + cv.notify_all(); + + return added_size; + } else { + return 0; + } + + } + + virtual void write(std::string content) { + std::vector vec; + vec.assign(content.begin(), content.end()); + byte_arrays_.enqueue(vec); + } + + virtual char *getBuffer(size_t pos) { + + // if there is no space remaining in our current buffer, + // we should load the next. If none exists after that we have no more buffer + std::lock_guard lock(mutex_); + + if ((pos - previous_pos_) >= current_vec_.size() || current_vec_.size() == 0) + load_buffer(); + + if (ptr == nullptr) + return nullptr; + + size_t absolute_position = pos - previous_pos_; + + current_pos_ = pos; + for (int i = 0; i < current_vec_.size(); i++) { + } + + return ptr + absolute_position; + } + + virtual const size_t getRemaining(size_t pos) { + return current_vec_.size(); + } + + virtual const size_t getBufferSize() { + std::lock_guard lock(mutex_); + + if (ptr == nullptr || current_pos_ >= rolling_count_) { + load_buffer(); + } + return rolling_count_; + } + + private: + + inline void load_buffer() { + std::unique_lock lock(mutex_); + cv.wait(lock, [&] {return byte_arrays_.size_approx() > 0 || is_alive_==false;}); + if (!is_alive_ && byte_arrays_.size_approx() == 0) { + lock.unlock(); + return; + } + try { + if (byte_arrays_.try_dequeue(current_vec_)) { + ptr = ¤t_vec_[0]; + previous_pos_.store(rolling_count_.load()); + current_pos_ = 0; + rolling_count_ += current_vec_.size(); + } else { + ptr = nullptr; + } + lock.unlock(); + } catch (...) { + lock.unlock(); + } + } + + std::atomic is_alive_; + std::atomic rolling_count_; + std::condition_variable_any cv; + std::atomic previous_pos_; + std::atomic current_pos_; + + std::recursive_mutex mutex_; + + moodycamel::ConcurrentQueue> byte_arrays_; + + char *ptr; + + std::vector current_vec_; +}; + +} /* namespace utils */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif /* EXTENSIONS_HTTP_CURL_CLIENT_HTTPCALLBACK_H_ */ diff --git a/extensions/http-curl/client/HttpClient.cpp b/extensions/http-curl/client/HTTPClient.cpp similarity index 78% rename from extensions/http-curl/client/HttpClient.cpp rename to extensions/http-curl/client/HTTPClient.cpp index f2bc0e1ff5..2d5d1e377c 100644 --- a/extensions/http-curl/client/HttpClient.cpp +++ b/extensions/http-curl/client/HTTPClient.cpp @@ -17,6 +17,7 @@ */ #include "HTTPClient.h" #include +#include #include #include #include @@ -37,8 +38,10 @@ HTTPClient::HTTPClient(const std::string &url, const std::shared_ptr::getLogger()), connect_timeout_(0), read_timeout_(0), + callback(nullptr), content_type(nullptr), + read_callback_(INT_MAX), headers_(nullptr), http_code(0), - header_response_(1), + header_response_(-1), res(CURLE_OK) { HTTPClientInitializer *initializer = HTTPClientInitializer::getInstance(); http_session_ = curl_easy_init(); @@ -63,6 +68,7 @@ HTTPClient::HTTPClient(std::string name, uuid_t uuid) HTTPClient::HTTPClient() : core::Connectable("HTTPClient", 0), ssl_context_service_(nullptr), + callback(nullptr), url_(), logger_(logging::LoggerFactory::getLogger()), connect_timeout_(0), @@ -70,17 +76,26 @@ HTTPClient::HTTPClient() content_type(nullptr), headers_(nullptr), http_code(0), - header_response_(1), + read_callback_(INT_MAX), + header_response_(-1), res(CURLE_OK) { HTTPClientInitializer *initializer = HTTPClientInitializer::getInstance(); http_session_ = curl_easy_init(); } HTTPClient::~HTTPClient() { + forceClose(); +} + +void HTTPClient::forceClose(){ if (nullptr != headers_) { curl_slist_free_all(headers_); + headers_ = nullptr; + } + if (http_session_ != nullptr){ + curl_easy_cleanup(http_session_); + http_session_ = nullptr; } - curl_easy_cleanup(http_session_); } void HTTPClient::setVerbose() { @@ -108,18 +123,23 @@ void HTTPClient::setDisablePeerVerification() { void HTTPClient::setConnectionTimeout(int64_t timeout) { connect_timeout_ = timeout; + curl_easy_setopt(http_session_,CURLOPT_NOSIGNAL,1); } void HTTPClient::setReadTimeout(int64_t timeout) { read_timeout_ = timeout; } +void HTTPClient::setReadCallback(HTTPReadCallback *callbackObj) { + callback = callbackObj; + curl_easy_setopt(http_session_, CURLOPT_WRITEFUNCTION, &utils::HTTPRequestResponse::recieve_write); + curl_easy_setopt(http_session_, CURLOPT_WRITEDATA, static_cast(callbackObj)); +} + void HTTPClient::setUploadCallback(HTTPUploadCallback *callbackObj) { logger_->log_info("Setting callback"); if (method_ == "put" || method_ == "PUT") { curl_easy_setopt(http_session_, CURLOPT_INFILESIZE_LARGE, (curl_off_t ) callbackObj->ptr->getBufferSize()); - } else { - curl_easy_setopt(http_session_, CURLOPT_POSTFIELDSIZE, (curl_off_t ) callbackObj->ptr->getBufferSize()); } curl_easy_setopt(http_session_, CURLOPT_READFUNCTION, &utils::HTTPRequestResponse::send_write); curl_easy_setopt(http_session_, CURLOPT_READDATA, static_cast(callbackObj)); @@ -159,6 +179,12 @@ void HTTPClient::appendHeader(const std::string &new_header) { headers_ = curl_slist_append(headers_, new_header.c_str()); } +void HTTPClient::appendHeader(const std::string &key, const std::string &value) { + std::stringstream new_header; + new_header << key << ": " << value; + headers_ = curl_slist_append(headers_, new_header.str().c_str()); +} + void HTTPClient::setUseChunkedEncoding() { headers_ = curl_slist_append(headers_, "Transfer-Encoding: chunked"); } @@ -175,12 +201,14 @@ bool HTTPClient::submit() { curl_easy_setopt(http_session_, CURLOPT_URL, url_.c_str()); logger_->log_info("Submitting to %s", url_); - curl_easy_setopt(http_session_, CURLOPT_WRITEFUNCTION, &utils::HTTPRequestResponse::recieve_write); - curl_easy_setopt(http_session_, CURLOPT_WRITEDATA, static_cast(&content_)); - + if (callback == nullptr) { + content_.ptr = &read_callback_; + curl_easy_setopt(http_session_, CURLOPT_WRITEFUNCTION, &utils::HTTPRequestResponse::recieve_write); + curl_easy_setopt(http_session_, CURLOPT_WRITEDATA, static_cast(&content_)); + } curl_easy_setopt(http_session_, CURLOPT_HEADERFUNCTION, &utils::HTTPHeaderResponse::receive_headers); curl_easy_setopt(http_session_, CURLOPT_HEADERDATA, static_cast(&header_response_)); - + curl_easy_setopt(http_session_, CURLOPT_TCP_KEEPALIVE, 0L); res = curl_easy_perform(http_session_); curl_easy_getinfo(http_session_, CURLINFO_RESPONSE_CODE, &http_code); curl_easy_getinfo(http_session_, CURLINFO_CONTENT_TYPE, &content_type); @@ -188,6 +216,38 @@ bool HTTPClient::submit() { logger_->log_error("curl_easy_perform() failed %s\n", curl_easy_strerror(res)); return false; } + + logger_->log_info("Finished with %s", url_); + std::string key = ""; + for (auto header_line : header_response_.header_tokens_) { + int i = 0; + for (i = 0; i < header_line.length(); i++) { + if (header_line.at(i) == ':') { + break; + } + } + if (i >= header_line.length()) { + if (key.empty()) + header_response_.append("HEADER", header_line); + else + header_response_.append(key, header_line); + } else { + key = header_line.substr(0, i); + int length = header_line.length() - (i + 2); + if (length <= 0) { + continue; + } + std::string value = header_line.substr(i + 2, length); + int end_find = value.size() - 1; + for (; end_find > 0; end_find--) { + if (value.at(end_find) > 32) { + break; + } + } + value = value.substr(0, end_find + 1); + header_response_.append(key, value); + } + } return true; } @@ -204,7 +264,9 @@ const char *HTTPClient::getContentType() { } const std::vector &HTTPClient::getResponseBody() { - return content_.data; + if (response_body_.size() == 0) + response_body_ = std::move(read_callback_.to_string()); + return response_body_; } void HTTPClient::set_request_method(const std::string method) { diff --git a/extensions/http-curl/client/HTTPClient.h b/extensions/http-curl/client/HTTPClient.h index 609a052498..99200d3693 100644 --- a/extensions/http-curl/client/HTTPClient.h +++ b/extensions/http-curl/client/HTTPClient.h @@ -1,5 +1,4 @@ /** - * HTTPUtils class declaration * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -28,8 +27,9 @@ #include #include #include + +#include "utils/ByteArrayCallback.h" #include "controllers/SSLContextService.h" -#include "utils/ByteInputCallBack.h" #include "core/logging/Logger.h" #include "core/logging/LoggerConfiguration.h" #include "properties/Configure.h" @@ -78,53 +78,63 @@ class HTTPClient : public BaseHTTPClient, public core::Connectable { ~HTTPClient(); - void setVerbose(); + virtual void setVerbose() override ; + + void forceClose(); - void initialize(const std::string &method, const std::string url = "", const std::shared_ptr ssl_context_service = nullptr); + virtual void initialize(const std::string &method, const std::string url = "", const std::shared_ptr ssl_context_service = nullptr) override; - void setConnectionTimeout(int64_t timeout); + virtual void setConnectionTimeout(int64_t timeout) override; - void setReadTimeout(int64_t timeout); + virtual void setReadTimeout(int64_t timeout) override; - void setUploadCallback(HTTPUploadCallback *callbackObj); + virtual void setUploadCallback(HTTPUploadCallback *callbackObj) override; + + virtual void setReadCallback(HTTPReadCallback *callbackObj) ; struct curl_slist *build_header_list(std::string regex, const std::map &attributes); - void setContentType(std::string content_type); + virtual void setContentType(std::string content_type) override; - std::string escape(std::string string_to_escape); + virtual std::string escape(std::string string_to_escape) override; - void setPostFields(std::string input); + virtual void setPostFields(std::string input) override; void setHeaders(struct curl_slist *list); - void appendHeader(const std::string &new_header); + virtual void appendHeader(const std::string &new_header) override; + + void appendHeader(const std::string &key, const std::string &value); - bool submit(); + bool submit() override; CURLcode getResponseResult(); - int64_t &getResponseCode(); + int64_t &getResponseCode() override; - const char *getContentType(); + const char *getContentType() override; - const std::vector &getResponseBody(); + const std::vector &getResponseBody() override; - void set_request_method(const std::string method); + void set_request_method(const std::string method) override; - void setUseChunkedEncoding(); + void setUseChunkedEncoding() override; - void setDisablePeerVerification(); + void setDisablePeerVerification() override; - const std::vector &getHeaders() { + const std::vector &getHeaders() override{ return header_response_.header_tokens_; } + virtual const std::map &getParsedHeaders() override{ + return header_response_.header_mapping_; + } + /** * Determines if we are connected and operating */ - virtual bool isRunning() { + virtual bool isRunning() override{ return true; } @@ -135,7 +145,7 @@ class HTTPClient : public BaseHTTPClient, public core::Connectable { void waitForWork(uint64_t timeoutMs) { } - virtual void yield() { + virtual void yield() override{ } @@ -143,13 +153,13 @@ class HTTPClient : public BaseHTTPClient, public core::Connectable { * Determines if work is available by this connectable * @return boolean if work is available. */ - virtual bool isWorkAvailable() { + virtual bool isWorkAvailable() override{ return true; } protected: - inline bool matches(const std::string &value, const std::string &sregex); + inline bool matches(const std::string &value, const std::string &sregex) override; static CURLcode configure_ssl_context(CURL *curl, void *ctx, void *param) { minifi::controllers::SSLContextService *ssl_context_service = static_cast(param); @@ -161,9 +171,12 @@ class HTTPClient : public BaseHTTPClient, public core::Connectable { void configure_secure_connection(CURL *http_session); + HTTPReadCallback *callback; + bool isSecure(const std::string &url); struct curl_slist *headers_; - utils::HTTPRequestResponse content_; + HTTPReadCallback content_; + ByteOutputCallback read_callback_; utils::HTTPHeaderResponse header_response_; CURLcode res; int64_t http_code; diff --git a/extensions/http-curl/client/HTTPStream.cpp b/extensions/http-curl/client/HTTPStream.cpp new file mode 100644 index 0000000000..8a5a4b912f --- /dev/null +++ b/extensions/http-curl/client/HTTPStream.cpp @@ -0,0 +1,127 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "HTTPStream.h" + +#include +#include +#include +#include + +#include "HTTPCallback.h" +#include "io/validation.h" +#include "io/NonConvertingStream.h" +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace io { + +HttpStream::HttpStream(std::shared_ptr client) + : logger_(logging::LoggerFactory::getLogger()), + http_client_(client), + written(0), + http_read_callback_(16384), + started_(false) { + // submit early on +} + +void HttpStream::closeStream() { + http_callback_.close(); + http_read_callback_.close(); +} + +void HttpStream::seek(uint64_t offset) { + // seek is an unnecessary part of this implementatino + throw std::exception(); +} + +int HttpStream::writeData(std::vector &buf, int buflen) { + if (buf.capacity() < buflen) { + return -1; + } + return writeData(reinterpret_cast(&buf[0]), buflen); +} + +// data stream overrides + +int HttpStream::writeData(uint8_t *value, int size) { + if (!IsNullOrEmpty(value)) { + if (!started_) { + std::lock_guard lock(mutex_); + if (!started_) { + callback_.ptr = &http_callback_; + callback_.pos = 0; + http_client_->setUploadCallback(&callback_); + http_client_future_ = std::async(submit_client, http_client_); + started_ = true; + } + } + http_callback_.process(value,size); + return size; + } else { + return -1; + } +} + +template +inline std::vector HttpStream::readBuffer(const T& t) { + std::vector buf; + buf.resize(sizeof t); + readData(reinterpret_cast(&buf[0]), sizeof(t)); + return buf; +} + +int HttpStream::readData(std::vector &buf, int buflen) { + if (buf.capacity() < buflen) { + buf.resize(buflen); + } + int ret = readData(reinterpret_cast(&buf[0]), buflen); + + if (ret < buflen) { + buf.resize(ret); + } + return ret; +} + +int HttpStream::readData(uint8_t *buf, int buflen) { + if (!IsNullOrEmpty(buf)) { + if (!started_) { + std::lock_guard lock(mutex_); + if (!started_) { + read_callback_.ptr = &http_read_callback_; + read_callback_.pos = 0; + http_client_->setReadCallback(&read_callback_); + http_client_future_ = std::async(submit_read_client, http_client_, &http_read_callback_); + started_ = true; + } + } + + return http_read_callback_.readFully((char*) buf, buflen); + + } else { + return -1; + } +} + +} /* namespace io */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + diff --git a/extensions/http-curl/client/HTTPStream.h b/extensions/http-curl/client/HTTPStream.h new file mode 100644 index 0000000000..de7c7d8a67 --- /dev/null +++ b/extensions/http-curl/client/HTTPStream.h @@ -0,0 +1,172 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef EXTENSIONS_HTTP_CURL_CLIENT_HTTPSTREAM_H_ +#define EXTENSIONS_HTTP_CURL_CLIENT_HTTPSTREAM_H_ + +#include +#include +#include +#include +#include + +#include "HTTPCallback.h" +#include "io/BaseStream.h" +#include "HTTPClient.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace io { + +class HttpStream : public io::BaseStream { + public: + /** + * File Stream constructor that accepts an fstream shared pointer. + * It must already be initialized for read and write. + */ + explicit HttpStream(std::shared_ptr http_client_); + + virtual ~HttpStream() { + closeStream(); + http_client_future_.get(); + } + + virtual void closeStream() override; + + const std::shared_ptr &getClient() { + http_client_future_.get(); + return http_client_; + } + + void forceClose(){ + closeStream(); + http_client_->forceClose(); + } + /** + * Skip to the specified offset. + * @param offset offset to which we will skip + */ + virtual void seek(uint64_t offset) override; + + virtual const uint64_t getSize() const override{ + return written; + } + + // data stream extensions + /** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ + virtual int readData(std::vector &buf, int buflen) override; + /** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ + virtual int readData(uint8_t *buf, int buflen) override; + + /** + * Write value to the stream using std::vector + * @param buf incoming buffer + * @param buflen buffer to write + * + */ + virtual int writeData(std::vector &buf, int buflen); + + /** + * writes value to stream + * @param value value to write + * @param size size of value + */ + virtual int writeData(uint8_t *value, int size) override; + + /** + * Returns the underlying buffer + * @return vector's array + **/ + const uint8_t *getBuffer() const { + throw std::runtime_error("Stream does not support this operation"); + } + + static bool submit_client(std::shared_ptr client) { + bool submit_status = client->submit(); + + return submit_status; + } + + static bool submit_read_client( std::shared_ptr client, utils::ByteOutputCallback *callback){ + bool submit_status = client->submit(); + callback->close(); + + return submit_status; + } + + inline bool isFinished() { + if (http_client_future_.wait_for(std::chrono::seconds(0)) == std::future_status::ready && (http_read_callback_.getSize() == 0 && http_read_callback_.waitingOps())) { + // http_read_callback_.close(); + return true; + } + else{ + return false; + } + } + + protected: + + /** + * Creates a vector and returns the vector using the provided + * type name. + * @param t incoming object + * @returns vector. + */ + template + std::vector readBuffer(const T&); + + void reset(); + + std::vector array; + + std::shared_ptr http_client_; + std::future http_client_future_; + + size_t written; + + std::mutex mutex_; + std::atomic started_; + + utils::HttpStreamingCallback http_callback_; + + utils::HTTPUploadCallback callback_; + + utils::ByteOutputCallback http_read_callback_; + + utils::HTTPReadCallback read_callback_; + + private: + + std::shared_ptr logger_; +}; +} /* namespace io */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif /* EXTENSIONS_HTTP_CURL_CLIENT_HTTPSTREAM_H_ */ diff --git a/extensions/http-curl/processors/InvokeHTTP.cpp b/extensions/http-curl/processors/InvokeHTTP.cpp index 8e16122743..71e8cda34d 100644 --- a/extensions/http-curl/processors/InvokeHTTP.cpp +++ b/extensions/http-curl/processors/InvokeHTTP.cpp @@ -33,6 +33,7 @@ #include #include +#include "utils/ByteArrayCallback.h" #include "core/FlowFile.h" #include "core/logging/Logger.h" #include "core/ProcessContext.h" @@ -41,7 +42,6 @@ #include "io/StreamFactory.h" #include "ResourceClaim.h" #include "utils/StringUtils.h" -#include "utils/ByteInputCallBack.h" namespace org { namespace apache { @@ -140,7 +140,7 @@ void InvokeHTTP::initialize() { setSupportedRelationships(relationships); } -void InvokeHTTP::onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory) { +void InvokeHTTP::onSchedule(const std::shared_ptr &context, const std::shared_ptr &sessionFactory) { if (!context->getProperty(Method.getName(), method_)) { logger_->log_info("%s attribute is missing, so default value of %s will be used", Method.getName().c_str(), Method.getValue().c_str()); return; @@ -240,7 +240,7 @@ bool InvokeHTTP::emitFlowFile(const std::string &method) { return ("POST" == method || "PUT" == method || "PATCH" == method); } -void InvokeHTTP::onTrigger(core::ProcessContext *context, core::ProcessSession *session) { +void InvokeHTTP::onTrigger(const std::shared_ptr &context, const std::shared_ptr &session) { logger_->log_info("onTrigger InvokeHTTP with %s to %s", method_, url_); std::shared_ptr flowFile = std::static_pointer_cast(session->get()); @@ -352,7 +352,7 @@ void InvokeHTTP::onTrigger(core::ProcessContext *context, core::ProcessSession * } } -void InvokeHTTP::route(std::shared_ptr &request, std::shared_ptr &response, core::ProcessSession *session, core::ProcessContext *context, bool isSuccess, +void InvokeHTTP::route(std::shared_ptr &request, std::shared_ptr &response, const std::shared_ptr &session, const std::shared_ptr &context, bool isSuccess, int statusCode) { // check if we should yield the processor if (!isSuccess && request == nullptr) { diff --git a/extensions/http-curl/processors/InvokeHTTP.h b/extensions/http-curl/processors/InvokeHTTP.h index 2a00cef087..1aa3faafef 100644 --- a/extensions/http-curl/processors/InvokeHTTP.h +++ b/extensions/http-curl/processors/InvokeHTTP.h @@ -23,6 +23,7 @@ #include #include +#include "utils/ByteArrayCallback.h" #include "FlowFileRecord.h" #include "core/Processor.h" #include "core/ProcessSession.h" @@ -30,7 +31,6 @@ #include "core/Property.h" #include "core/Resource.h" #include "controllers/SSLContextService.h" -#include "utils/ByteInputCallBack.h" #include "core/logging/LoggerConfiguration.h" #include "utils/Id.h" #include "../client/HTTPClient.h" @@ -104,9 +104,9 @@ class InvokeHTTP : public core::Processor { static core::Relationship RelNoRetry; static core::Relationship RelFailure; - virtual void onTrigger(core::ProcessContext *context, core::ProcessSession *session); - virtual void initialize(); - virtual void onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory); + virtual void onTrigger(const std::shared_ptr &context, const std::shared_ptr &session) override; + virtual void initialize() override; + virtual void onSchedule(const std::shared_ptr &context, const std::shared_ptr &sessionFactory) override; /** * Provides a reference to the URL. */ @@ -131,7 +131,7 @@ class InvokeHTTP : public core::Processor { * @param isSuccess success code or not * @param statuscode http response code. */ - void route(std::shared_ptr &request, std::shared_ptr &response, core::ProcessSession *session, core::ProcessContext *context, bool isSuccess, int statusCode); + void route(std::shared_ptr &request, std::shared_ptr &response, const std::shared_ptr &session, const std::shared_ptr &context, bool isSuccess, int statusCode); /** * Determine if we should emit a new flowfile based on our activity * @param method method type diff --git a/extensions/http-curl/protocols/RESTProtocol.h b/extensions/http-curl/protocols/RESTProtocol.h index ed62d9ce50..4767e77a0d 100644 --- a/extensions/http-curl/protocols/RESTProtocol.h +++ b/extensions/http-curl/protocols/RESTProtocol.h @@ -22,11 +22,12 @@ #include "json/writer.h" #include #include + +#include "utils/ByteArrayCallback.h" #include "CivetServer.h" #include "c2/C2Protocol.h" #include "c2/HeartBeatReporter.h" #include "controllers/SSLContextService.h" -#include "utils/ByteInputCallBack.h" #include "utils/HTTPClient.h" namespace org { diff --git a/extensions/http-curl/protocols/RESTReceiver.h b/extensions/http-curl/protocols/RESTReceiver.h index f4e4a3c908..77cfc902a9 100644 --- a/extensions/http-curl/protocols/RESTReceiver.h +++ b/extensions/http-curl/protocols/RESTReceiver.h @@ -51,8 +51,8 @@ class RESTReceiver : public RESTProtocol, public HeartBeatReporter { public: RESTReceiver(std::string name, uuid_t uuid = nullptr); - void initialize(const std::shared_ptr &controller, const std::shared_ptr &configure); - virtual int16_t heartbeat(const C2Payload &heartbeat); + virtual void initialize(const std::shared_ptr &controller, const std::shared_ptr &configure) override; + virtual int16_t heartbeat(const C2Payload &heartbeat) override; protected: diff --git a/extensions/http-curl/protocols/RESTSender.cpp b/extensions/http-curl/protocols/RESTSender.cpp index e67661c343..9959814876 100644 --- a/extensions/http-curl/protocols/RESTSender.cpp +++ b/extensions/http-curl/protocols/RESTSender.cpp @@ -98,42 +98,38 @@ void RESTSender::update(const std::shared_ptr &configure) { } const C2Payload RESTSender::sendPayload(const std::string url, const Direction direction, const C2Payload &payload, const std::string outputConfig) { - if (!url.empty()) { - utils::HTTPClient client(url, ssl_context_service_); - client.setConnectionTimeout(2); - - std::unique_ptr input = nullptr; - std::unique_ptr callback = nullptr; - if (direction == Direction::TRANSMIT) { - input = std::unique_ptr(new utils::ByteInputCallBack()); - callback = std::unique_ptr(new utils::HTTPUploadCallback); - input->write(outputConfig); - callback->ptr = input.get(); - callback->pos = 0; - client.set_request_method("POST"); - client.setUploadCallback(callback.get()); - } else { - // we do not need to set the uplaod callback - // since we are not uploading anything on a get - client.set_request_method("GET"); - } - client.setContentType("application/json"); - bool isOkay = client.submit(); - int64_t respCode = client.getResponseCode(); + utils::HTTPClient client(url, ssl_context_service_); + client.setConnectionTimeout(2); + + std::unique_ptr input = nullptr; + std::unique_ptr callback = nullptr; + if (direction == Direction::TRANSMIT) { + input = std::unique_ptr(new utils::ByteInputCallBack()); + callback = std::unique_ptr(new utils::HTTPUploadCallback); + input->write(outputConfig); + callback->ptr = input.get(); + callback->pos = 0; + client.set_request_method("POST"); + client.setUploadCallback(callback.get()); + } else { + // we do not need to set the uplaod callback + // since we are not uploading anything on a get + client.set_request_method("GET"); + } + client.setContentType("application/json"); + bool isOkay = client.submit(); + int64_t respCode = client.getResponseCode(); - if (isOkay && respCode) { - if (payload.isRaw()) { - C2Payload response_payload(payload.getOperation(), state::UpdateState::READ_COMPLETE, true, true); + if (isOkay && respCode) { + if (payload.isRaw()) { + C2Payload response_payload(payload.getOperation(), state::UpdateState::READ_COMPLETE, true, true); - response_payload.setRawData(client.getResponseBody()); - return std::move(response_payload); - } - return parseJsonResponse(payload, client.getResponseBody()); - } else { - return C2Payload(payload.getOperation(), state::UpdateState::READ_ERROR, true); + response_payload.setRawData(client.getResponseBody()); + return std::move(response_payload); } + return parseJsonResponse(payload, client.getResponseBody()); } else { - return C2Payload(payload.getOperation(), state::UpdateState::READ_COMPLETE, true); + return C2Payload(payload.getOperation(), state::UpdateState::READ_ERROR, true); } } diff --git a/extensions/http-curl/protocols/RESTSender.h b/extensions/http-curl/protocols/RESTSender.h index 09a1a4dc95..e4c1e5ea31 100644 --- a/extensions/http-curl/protocols/RESTSender.h +++ b/extensions/http-curl/protocols/RESTSender.h @@ -22,12 +22,13 @@ #include "json/writer.h" #include #include + +#include "utils/ByteArrayCallback.h" #include "CivetServer.h" #include "c2/C2Protocol.h" #include "RESTProtocol.h" #include "c2/HeartBeatReporter.h" #include "controllers/SSLContextService.h" -#include "utils/ByteInputCallBack.h" #include "../client/HTTPClient.h" namespace org { @@ -49,13 +50,13 @@ class RESTSender : public RESTProtocol, public C2Protocol { explicit RESTSender(std::string name, uuid_t uuid = nullptr); - virtual C2Payload consumePayload(const std::string &url, const C2Payload &payload, Direction direction, bool async); + virtual C2Payload consumePayload(const std::string &url, const C2Payload &payload, Direction direction, bool async) override; - virtual C2Payload consumePayload(const C2Payload &payload, Direction direction, bool async); + virtual C2Payload consumePayload(const C2Payload &payload, Direction direction, bool async) override; - virtual void update(const std::shared_ptr &configure); + virtual void update(const std::shared_ptr &configure) override; - virtual void initialize(const std::shared_ptr &controller, const std::shared_ptr &configure); + virtual void initialize(const std::shared_ptr &controller, const std::shared_ptr &configure) override; protected: diff --git a/extensions/http-curl/sitetosite/HTTPProtocol.cpp b/extensions/http-curl/sitetosite/HTTPProtocol.cpp new file mode 100644 index 0000000000..940d3e35ba --- /dev/null +++ b/extensions/http-curl/sitetosite/HTTPProtocol.cpp @@ -0,0 +1,310 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "HTTPProtocol.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "PeersEntity.h" +#include "io/CRCStream.h" +#include "sitetosite/Peer.h" +#include "io/validation.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace sitetosite { + +std::shared_ptr HttpSiteToSiteClient::id_generator_ = utils::IdGenerator::getIdGenerator(); + +const std::string HttpSiteToSiteClient::parseTransactionId(const std::string &uri) { + int i = 0; + for (i = uri.length() - 1; i >= 0; i--) { + if (uri.at(i) == '/') + break; + } + return uri.substr(i + 1, uri.length() - (i + 1)); +} + +std::shared_ptr HttpSiteToSiteClient::createTransaction(std::string &transactionID, TransferDirection direction) { + std::string dir_str = direction == SEND ? "input-ports" : "output-ports"; + std::stringstream uri; + uri << getBaseURI() << "data-transfer/" << dir_str << "/" << getPortId() << "/transactions"; + auto client = create_http_client(uri.str(), "POST"); + + client->appendHeader(PROTOCOL_VERSION_HEADER, "1"); + + client->setConnectionTimeout(5); + + client->setContentType("application/json"); + client->appendHeader("Accept: application/json"); + client->setUseChunkedEncoding(); + client->setPostFields(""); + client->submit(); + peer_->setStream(nullptr); + if (client->getResponseCode() == 201) { + // parse the headers + auto headers = client->getParsedHeaders(); + auto intent_name = headers["x-location-uri-intent"]; + if (intent_name == "transaction-url") { + auto url = headers["Location"]; + + if (IsNullOrEmpty(&url)) { + logger_->log_debug("Location is empty"); + } else { + + org::apache::nifi::minifi::io::CRCStream crcstream(peer_.get()); + auto transaction = std::make_shared(direction, crcstream); + transaction->initialize(this, url); + auto transactionId = parseTransactionId(url); + if (IsNullOrEmpty(transactionId)) + return nullptr; + transaction->setTransactionId(transactionId); + std::shared_ptr client; + if (transaction->getDirection() == SEND) { + client = openConnectionForSending(transaction); + } else { + client = openConnectionForReceive(transaction); + transaction->setDataAvailable(true); + // a 201 tells us that data is available. A 200 would mean that nothing is available. + } + + client->appendHeader(PROTOCOL_VERSION_HEADER, "1"); + peer_->setStream(std::unique_ptr(new io::HttpStream(client))); + transactionID = transaction->getUUIDStr(); + logger_->log_debug("Created transaction id -%s-", transactionID); + known_transactions_[transaction->getUUIDStr()] = transaction; + return transaction; + } + } else { + logger_->log_debug("Could not create transaction, intent is %s", intent_name); + } + } else { + logger_->log_debug("Could not create transaction, received %d", client->getResponseCode()); + } + return nullptr; +} + +int HttpSiteToSiteClient::readResponse(const std::shared_ptr &transaction, RespondCode &code, std::string &message) { + if (current_code == FINISH_TRANSACTION) { + + if (transaction->getDirection() == SEND) { + auto stream = dynamic_cast(peer_->getStream()); + stream->closeStream(); + auto client = stream->getClient(); + if (client->getResponseCode() == 202) { + code = CONFIRM_TRANSACTION; + message = std::string(client->getResponseBody().data(), client->getResponseBody().size()); + } else { + code = UNRECOGNIZED_RESPONSE_CODE; + } + return 1; + } else { + return 1; + } + } else if (transaction->getDirection() == RECEIVE) { + if (transaction->getState() == TRANSACTION_STARTED || transaction->getState() == DATA_EXCHANGED) { + + if (current_code == CONFIRM_TRANSACTION && transaction->getState() == DATA_EXCHANGED) { + auto stream = dynamic_cast(peer_->getStream()); + if (!stream->isFinished()) { + logger_->log_info("confirm read for %s, but not finished ", transaction->getUUIDStr()); + } + + closeTransaction(transaction->getUUIDStr()); + code = CONFIRM_TRANSACTION; + } else { + auto stream = dynamic_cast(peer_->getStream()); + if (stream->isFinished()) { + code = FINISH_TRANSACTION; + current_code = FINISH_TRANSACTION; + } else { + code = CONTINUE_TRANSACTION; + } + } + } else if (transaction->getState() == TRANSACTION_CONFIRMED) { + closeTransaction(transaction->getUUIDStr()); + code = CONFIRM_TRANSACTION; + } else { + + } + return 1; + } else if (transaction->getState() == TRANSACTION_CONFIRMED) { + closeTransaction(transaction->getUUIDStr()); + code = TRANSACTION_FINISHED; + + return 1; + } + return SiteToSiteClient::readResponse(transaction, code, message); + +} +// write respond +int HttpSiteToSiteClient::writeResponse(const std::shared_ptr &transaction, RespondCode code, std::string message) { + current_code = code; + if (code == CONFIRM_TRANSACTION || code == FINISH_TRANSACTION) { + return 1; + + } else if (code == CONTINUE_TRANSACTION) { + logger_->log_info("Continuing HTTP transaction"); + return 1; + } + return SiteToSiteClient::writeResponse(transaction, code, message); +} + +bool HttpSiteToSiteClient::getPeerList(std::vector &peers) { + std::stringstream uri; + uri << getBaseURI() << "site-to-site/peers"; + + auto client = create_http_client(uri.str(), "GET"); + + client->appendHeader(PROTOCOL_VERSION_HEADER, "1"); + + client->submit(); + + if (client->getResponseCode() == 200) { + if (sitetosite::PeersEntity::parse(logger_, std::string(client->getResponseBody().data(), client->getResponseBody().size()), port_id_, peers)) { + return true; + } + } + return false; +} + +std::shared_ptr HttpSiteToSiteClient::openConnectionForSending(const std::shared_ptr &transaction) { + std::stringstream uri; + uri << transaction->getTransactionUrl() << "/flow-files"; + std::shared_ptr client = std::move(create_http_client(uri.str(), "POST")); + client->setContentType("application/octet-stream"); + client->appendHeader("Accept", "text/plain"); + client->setUseChunkedEncoding(); + return client; +} + +std::shared_ptr HttpSiteToSiteClient::openConnectionForReceive(const std::shared_ptr &transaction) { + std::stringstream uri; + uri << transaction->getTransactionUrl() << "/flow-files"; + std::shared_ptr client = std::move(create_http_client(uri.str(), "GET")); + return client; +} + +//! Transfer string for the process session +bool HttpSiteToSiteClient::transmitPayload(const std::shared_ptr &context, const std::shared_ptr &session, const std::string &payload, + std::map attributes) { + return false; +} + +void HttpSiteToSiteClient::tearDown() { + + if (peer_state_ >= ESTABLISHED) { + logger_->log_info("Site2Site Protocol tearDown"); + } + known_transactions_.clear(); + peer_->Close(); + peer_state_ = IDLE; + +} + +void HttpSiteToSiteClient::closeTransaction(const std::string &transactionID) { + std::shared_ptr transaction = NULL; + + std::map >::iterator it = this->known_transactions_.find(transactionID); + + if (it == known_transactions_.end()) { + return; + } else { + transaction = it->second; + if (transaction->closed_) { + return; + } + } + + std::string append_str; + logger_->log_info("Site2Site close transaction %s", transaction->getUUIDStr().c_str()); + + int code = UNRECOGNIZED_RESPONSE_CODE; + if (transaction->getState() == TRANSACTION_CONFIRMED) { + code = CONFIRM_TRANSACTION; + } else if (transaction->getDirection() == RECEIVE && current_code == CONFIRM_TRANSACTION) { + if (transaction->_bytes > 0) + code = CONFIRM_TRANSACTION; + else + code = CANCEL_TRANSACTION; + + } else if (transaction->current_transfers_ == 0 && !transaction->isDataAvailable()) { + code = CANCEL_TRANSACTION; + } + + std::stringstream uri; + std::string dir_str = transaction->getDirection() == SEND ? "input-ports" : "output-ports"; + + uri << getBaseURI() << "data-transfer/" << dir_str << "/" << getPortId() << "/transactions/" << transactionID << "?responseCode=" << code; + + if (transaction->getDirection() == RECEIVE && current_code == CONFIRM_TRANSACTION && transaction->_bytes > 0) { + uri << "&checksum=" << transaction->getCRC(); + } + auto client = create_http_client(uri.str(), "DELETE"); + + client->appendHeader(PROTOCOL_VERSION_HEADER, "1"); + + client->setConnectionTimeout(5); + + client->appendHeader("Accept", "application/json"); + + client->submit(); + + logger_->log_debug("Received %d response code from delete", client->getResponseCode()); + + transaction->closed_ = true; + + transaction->current_transfers_--; +} + +void HttpSiteToSiteClient::deleteTransaction(std::string transactionID) { + std::shared_ptr transaction = NULL; + + std::map >::iterator it = this->known_transactions_.find(transactionID); + + if (it == known_transactions_.end()) { + return; + } else { + transaction = it->second; + } + + std::string append_str; + logger_->log_info("Site2Site delete transaction %s", transaction->getUUIDStr().c_str()); + + closeTransaction(transactionID); + + known_transactions_.erase(transactionID); + +} + +} /* namespace sitetosite */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/extensions/http-curl/sitetosite/HTTPProtocol.h b/extensions/http-curl/sitetosite/HTTPProtocol.h new file mode 100644 index 0000000000..bbef3c9c80 --- /dev/null +++ b/extensions/http-curl/sitetosite/HTTPProtocol.h @@ -0,0 +1,197 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __SITE2SITE_CLIENT_PROTOCOL_H__ +#define __SITE2SITE_CLIENT_PROTOCOL_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "HTTPTransaction.h" +#include "sitetosite/SiteToSite.h" +#include "sitetosite/SiteToSiteClient.h" +#include "core/Property.h" +#include "properties/Configure.h" +#include "FlowFileRecord.h" +#include "core/logging/LoggerConfiguration.h" +#include "core/ProcessContext.h" +#include "core/ProcessSession.h" +#include "io/CRCStream.h" +#include "sitetosite/Peer.h" +#include "utils/Id.h" +#include "../client/HTTPClient.h" + + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace sitetosite { + +/** + * Site2Site Peer + */ +typedef struct Site2SitePeerStatus { + std::string host_; + int port_;bool isSecure_; +} Site2SitePeerStatus; + +// HttpSiteToSiteClient Class +class HttpSiteToSiteClient : public sitetosite::SiteToSiteClient { + + static constexpr char const* PROTOCOL_VERSION_HEADER = "x-nifi-site-to-site-protocol-version"; + public: + + /*! + * Create a new http protocol + */ + HttpSiteToSiteClient(std::string name, uuid_t uuid = 0) + : SiteToSiteClient(), + current_code(UNRECOGNIZED_RESPONSE_CODE), + logger_(logging::LoggerFactory::getLogger()) { + peer_state_ = READY; + } + + /*! + * Create a new http protocol + */ + HttpSiteToSiteClient(std::unique_ptr peer) + : SiteToSiteClient(), + current_code(UNRECOGNIZED_RESPONSE_CODE), + logger_(logging::LoggerFactory::getLogger()) { + peer_ = std::move(peer); + peer_state_ = READY; + } + // Destructor + virtual ~HttpSiteToSiteClient() { + + } + + void setPeer(std::unique_ptr peer) override { + peer_ = std::move(peer); + } + + bool getPeerList(std::vector &peers) override; + + /** + * Establish the protocol connection. Not needed for the HTTP connection, so we simply + * return true. + */ + virtual bool establish() override { + return true; + } + + virtual std::shared_ptr createTransaction(std::string &transactionID, TransferDirection direction) override; + + // Transfer flow files for the process session + //virtual bool transferFlowFiles(const std::shared_ptr &context, const std::shared_ptr &session); + //! Transfer string for the process session + virtual bool transmitPayload(const std::shared_ptr &context, const std::shared_ptr &session, const std::string &payload, + std::map attributes) override; + // deleteTransaction + virtual void deleteTransaction(std::string transactionID) override; + + protected: + + /** + * Closes the transaction + * @param transactionID transaction id reference. + */ + void closeTransaction(const std::string &transactionID); + + virtual int readResponse(const std::shared_ptr &transaction, RespondCode &code, std::string &message) override; + // write respond + virtual int writeResponse(const std::shared_ptr &transaction, RespondCode code, std::string message) override; + + /** + * Bootstrapping is not really required for the HTTP Site To Site so we will set the peer state and return true. + */ + virtual bool bootstrap() override { + peer_state_ = READY; + return true; + } + + /** + * Creates a connection for sending, returning an HTTP client that is structured and configured + * to receive flow files. + * @param transaction transaction against which we are performing our sends + * @return HTTP Client shared pointer. + */ + std::shared_ptr openConnectionForSending(const std::shared_ptr &transaction); + + /** + * Creates a connection for receiving, returning an HTTP client that is structured and configured + * to receive flow files. + * @param transaction transaction against which we are performing our reads + * @return HTTP Client shared pointer. + */ + std::shared_ptr openConnectionForReceive(const std::shared_ptr &transaction); + + const std::string getBaseURI() { + std::string uri = "http://"; + uri.append(peer_->getHostName()); + uri.append(":"); + uri.append(std::to_string(peer_->getPort())); + uri.append("/nifi-api/"); + return uri; + } + + virtual void tearDown() override; + + const std::string parseTransactionId(const std::string &uri); + + std::unique_ptr create_http_client(const std::string &uri, const std::string &method = "POST", bool setPropertyHeaders = false) { + std::unique_ptr http_client_ = std::unique_ptr(new minifi::utils::HTTPClient(uri, nullptr)); + http_client_->initialize(method, uri, nullptr); + if (setPropertyHeaders) { + if (_currentVersion >= 5) { + // batch count, size, and duratin don't appear to be set through the interfaces. + } + } + return http_client_; + } + + private: + + std::shared_ptr logger_; + + RespondCode current_code; + // Prevent default copy constructor and assignment operation + // Only support pass by reference or pointer + HttpSiteToSiteClient(const HttpSiteToSiteClient &parent); + HttpSiteToSiteClient &operator=(const HttpSiteToSiteClient &parent); + static std::shared_ptr id_generator_; +}; + +} /* namespace sitetosite */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ +#endif diff --git a/extensions/http-curl/sitetosite/HTTPTransaction.h b/extensions/http-curl/sitetosite/HTTPTransaction.h new file mode 100644 index 0000000000..6a0dbc3be6 --- /dev/null +++ b/extensions/http-curl/sitetosite/HTTPTransaction.h @@ -0,0 +1,70 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef EXTENSIONS_HTTP_CURL_SITETOSITE_HTTPTRANSACTION_H_ +#define EXTENSIONS_HTTP_CURL_SITETOSITE_HTTPTRANSACTION_H_ + +#include "sitetosite/SiteToSite.h" +#include "io/CRCStream.h" +#include "sitetosite/SiteToSiteClient.h" +#include "sitetosite/Peer.h" +#include "HTTPStream.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace sitetosite { + +/** + * Purpose: HTTP Transaction is an implementation that exposes the site to site client. + * Includes the transaction URL. + */ +class HttpTransaction : public sitetosite::Transaction { + public: + explicit HttpTransaction(sitetosite::TransferDirection direction, org::apache::nifi::minifi::io::CRCStream &stream) + : Transaction(direction, stream), + client_ref_(nullptr) { + + } + + ~HttpTransaction(){ + auto stream = dynamic_cast< org::apache::nifi::minifi::io::HttpStream*>( dynamic_cast(crcStream.getstream())->getStream() ); + stream->forceClose(); + } + + void initialize(sitetosite::SiteToSiteClient *client, const std::string &url) { + client_ref_ = client; + transaction_url_ = url; + } + + + const std::string &getTransactionUrl(){ + return transaction_url_; + } + protected: + sitetosite::SiteToSiteClient *client_ref_; + std::string transaction_url_; +}; + +} /* namespace sitetosite */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif /* EXTENSIONS_HTTP_CURL_SITETOSITE_HTTPTRANSACTION_H_ */ diff --git a/extensions/http-curl/sitetosite/PeersEntity.h b/extensions/http-curl/sitetosite/PeersEntity.h new file mode 100644 index 0000000000..54be488316 --- /dev/null +++ b/extensions/http-curl/sitetosite/PeersEntity.h @@ -0,0 +1,112 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef EXTENSIONS_HTTP_CURL_SITETOSITE_PEERSENTITY_H_ +#define EXTENSIONS_HTTP_CURL_SITETOSITE_PEERSENTITY_H_ + +#include "json/json.h" +#include "sitetosite/Peer.h" +#include "utils/StringUtils.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace sitetosite { + +/** + * Represents a peer. Contains the parser for the Peer JSON. + */ +class PeersEntity { + public: + + static bool parse(const std::shared_ptr &logger, const std::string &entity, uuid_t id, std::vector &peer_statuses) { + + Json::Reader reader; + Json::Value root; + try { + if (reader.parse(entity, root)) { + if (root.isMember("peers") && root["peers"].size() > 0) { + for (const auto &peer : root["peers"]) { + + std::string hostname; + int port = 0, flowFileCount = 0; + bool secure = false; + + if (peer.isMember("hostname") && peer.isMember("port")) { + hostname = peer["hostname"].asString(); + port = peer["port"].asInt(); + } + if (peer.isMember("secure")) { + // do not assume that secure parses incorrectly that we can continue without security + try { + secure = peer["secure"].asBool(); + } catch (...) { + logger->log_debug("Could not properly parse secure, so we're going to try it as a string"); + std::string secureStr = peer["secure"].asString(); + if (utils::StringUtils::equalsIgnoreCase(secureStr,"true")) + secure = true; + else if (utils::StringUtils::equalsIgnoreCase(secureStr,"false")) + secure = false; + else{ + logger->log_error("could not parse secure string %s",secureStr); + throw std::exception(); + } + + } + } + if (peer.isMember("flowFileCount")) { + // we can assume that if flowFileCount cannot parse as an integer that we CAN continue. + try { + flowFileCount = peer["flowFileCount"].asInt(); + } catch (...) { + logger->log_debug("Could not properly parse flowFileCount, so we're going to continue without it"); + } + } + // host name and port are required. + if (!IsNullOrEmpty(hostname) && port > 0) { + sitetosite::PeerStatus status(std::make_shared(id, hostname, port, secure), flowFileCount, true); + peer_statuses.push_back(std::move(status)); + } else { + logger->log_debug("hostname empty or port is zero. hostname: %s, port: %d", hostname, port); + } + } + } else { + logger->log_debug("Peers is either not a member or is empty. String to analyze: %s", entity); + } + } + return true; + } catch (Json::RuntimeError &er) { + logger->log_debug("JSON runtime error occurred. Message: %s", er.what()); + return false; + } catch (...) { + logger->log_debug("General exception occurred"); + return false; + } + + } + +} +; + +} /* namespace sitetosite */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif /* EXTENSIONS_HTTP_CURL_SITETOSITE_PEERSENTITY_H_ */ diff --git a/extensions/libarchive/ArchiveLoader.h b/extensions/libarchive/ArchiveLoader.h index 54f0a74652..2b7c9347a4 100644 --- a/extensions/libarchive/ArchiveLoader.h +++ b/extensions/libarchive/ArchiveLoader.h @@ -55,15 +55,13 @@ class __attribute__((visibility("default"))) ArchiveFactory : public core::Objec } virtual std::unique_ptr assign(const std::string &class_name) { - std::string name = class_name; - std::transform(name.begin(), name.end(), name.begin(), ::tolower); - if (name == "mergecontent") { + if (utils::StringUtils::equalsIgnoreCase(class_name, "MergeContent")) { return std::unique_ptr(new core::DefautObjectFactory()); - } else if (name == "compresscontent") { + } else if (utils::StringUtils::equalsIgnoreCase(class_name, "CompressContent")) { return std::unique_ptr(new core::DefautObjectFactory()); - } else if (name == "focusarchiveentry") { + } else if (utils::StringUtils::equalsIgnoreCase(class_name,"FocusArchiveEntry")) { return std::unique_ptr(new core::DefautObjectFactory()); - } else if (name == "unfocusarchiveentry") { + } else if (utils::StringUtils::equalsIgnoreCase(class_name,"UnfocusArchiveEntry")) { return std::unique_ptr(new core::DefautObjectFactory()); } else { return nullptr; diff --git a/extensions/libarchive/BinFiles.cpp b/extensions/libarchive/BinFiles.cpp index 41df9fcdb1..34afcf0b94 100644 --- a/extensions/libarchive/BinFiles.cpp +++ b/extensions/libarchive/BinFiles.cpp @@ -234,7 +234,7 @@ bool BinManager::offer(const std::string &group, std::shared_ptr return true; } -void BinFiles::onTrigger(std::shared_ptr context, std::shared_ptr session) { +void BinFiles::onTrigger(const std::shared_ptr &context, const std::shared_ptr &session) { std::shared_ptr flow = std::static_pointer_cast < FlowFileRecord > (session->get()); if (flow != nullptr) { diff --git a/extensions/libarchive/BinFiles.h b/extensions/libarchive/BinFiles.h index 240fb09a9d..db8a6b8789 100644 --- a/extensions/libarchive/BinFiles.h +++ b/extensions/libarchive/BinFiles.h @@ -270,7 +270,7 @@ class BinFiles : public core::Processor { virtual void onTrigger(core::ProcessContext *context, core::ProcessSession *session) { } // OnTrigger method, implemented by NiFi BinFiles - virtual void onTrigger(std::shared_ptr context, std::shared_ptr session); + virtual void onTrigger(const std::shared_ptr &context, const std::shared_ptr &session); // Initialize, over write by NiFi BinFiles virtual void initialize(void); diff --git a/extensions/libarchive/CompressContent.cpp b/extensions/libarchive/CompressContent.cpp index 4f250e9a49..fdd4d5745c 100644 --- a/extensions/libarchive/CompressContent.cpp +++ b/extensions/libarchive/CompressContent.cpp @@ -87,7 +87,7 @@ void CompressContent::onSchedule(core::ProcessContext *context, core::ProcessSes fileExtension_[COMPRESSION_FORMAT_XZ_LZMA2] = ".xz"; } -void CompressContent::onTrigger(std::shared_ptr context, std::shared_ptr session) { +void CompressContent::onTrigger(const std::shared_ptr &context, const std::shared_ptr &session) { std::shared_ptr flowFile = session->get(); if (!flowFile) { @@ -153,8 +153,7 @@ void CompressContent::onTrigger(std::shared_ptr context, s } else { session->removeAttribute(processFlowFile, FlowAttributeKey(MIME_TYPE)); if (updateFileName_) { - if (fileName.size() >= fileExtension.size() && - fileName.compare(fileName.size() - fileExtension.size(), fileExtension.size(), fileExtension) == 0) { + if (fileName.size() >= fileExtension.size() && fileName.compare(fileName.size() - fileExtension.size(), fileExtension.size(), fileExtension) == 0) { fileName = fileName.substr(0, fileName.size() - fileExtension.size()); session->putAttribute(processFlowFile, FlowAttributeKey(FILENAME), fileName); } diff --git a/extensions/libarchive/CompressContent.h b/extensions/libarchive/CompressContent.h index b34aee7588..6041668321 100644 --- a/extensions/libarchive/CompressContent.h +++ b/extensions/libarchive/CompressContent.h @@ -146,7 +146,7 @@ class CompressContent: public core::Processor { class WriteCallback: public OutputStreamCallback { public: WriteCallback(std::string &compress_mode, int64_t compress_level, std::string &compress_format, - std::shared_ptr &flow, std::shared_ptr &session) : + std::shared_ptr &flow, const std::shared_ptr &session) : compress_mode_(compress_mode), compress_level_(compress_level), compress_format_(compress_format), flow_(flow), session_(session), logger_(logging::LoggerFactory::getLogger()), @@ -354,7 +354,7 @@ class CompressContent: public core::Processor { virtual void onTrigger(core::ProcessContext *context, core::ProcessSession *session) { } // OnTrigger method, implemented by NiFi CompressContent - virtual void onTrigger(std::shared_ptr context, std::shared_ptr session); + virtual void onTrigger(const std::shared_ptr &context, const std::shared_ptr &session); // Initialize, over write by NiFi CompressContent virtual void initialize(void); diff --git a/libminifi/CMakeLists.txt b/libminifi/CMakeLists.txt index 6eea752ea4..4f7156ac18 100644 --- a/libminifi/CMakeLists.txt +++ b/libminifi/CMakeLists.txt @@ -58,7 +58,7 @@ include_directories(../thirdparty/jsoncpp/include) include_directories(../thirdparty/concurrentqueue/) include_directories(include) -file(GLOB SOURCES "src/core/logging/*.cpp" "src/core/state/*.cpp" "src/c2/protocols/*.cpp" "src/c2/*.cpp" "src/io/*.cpp" "src/io/tls/*.cpp" "src/core/controller/*.cpp" "src/controllers/*.cpp" "src/core/*.cpp" "src/core/repository/*.cpp" "src/core/yaml/*.cpp" "src/core/reporting/*.cpp" "src/provenance/*.cpp" "src/utils/*.cpp" "src/*.cpp") +file(GLOB SOURCES "src/sitetosite/*.cpp" "src/core/logging/*.cpp" "src/core/state/*.cpp" "src/c2/protocols/*.cpp" "src/c2/*.cpp" "src/io/*.cpp" "src/io/tls/*.cpp" "src/core/controller/*.cpp" "src/controllers/*.cpp" "src/core/*.cpp" "src/core/repository/*.cpp" "src/core/yaml/*.cpp" "src/core/reporting/*.cpp" "src/provenance/*.cpp" "src/utils/*.cpp" "src/*.cpp") file(GLOB PROCESSOR_SOURCES "src/processors/*.cpp" ) diff --git a/libminifi/include/RemoteProcessorGroupPort.h b/libminifi/include/RemoteProcessorGroupPort.h index cefce454b0..e89d443d3e 100644 --- a/libminifi/include/RemoteProcessorGroupPort.h +++ b/libminifi/include/RemoteProcessorGroupPort.h @@ -28,7 +28,7 @@ #include "FlowFileRecord.h" #include "core/Processor.h" #include "core/ProcessSession.h" -#include "Site2SiteClientProtocol.h" +#include "sitetosite/SiteToSiteClient.h" #include "io/StreamFactory.h" #include "controllers/SSLContextService.h" #include "core/logging/LoggerConfiguration.h" @@ -47,7 +47,7 @@ class RemoteProcessorGroupPort : public core::Processor { RemoteProcessorGroupPort(const std::shared_ptr &stream_factory, std::string name, std::string url, const std::shared_ptr &configure, uuid_t uuid = nullptr) : core::Processor(name, uuid), configure_(configure), - direction_(SEND), + direction_(sitetosite::SEND), transmitting_(false), timeout_(0), logger_(logging::LoggerFactory::getLogger()), @@ -59,7 +59,7 @@ class RemoteProcessorGroupPort : public core::Processor { } site2site_port_ = -1; site2site_secure_ = false; - site2site_peer_index_ = -1; + peer_index_ = -1; // REST API port and host port_ = -1; utils::parse_url(url_, host_, port_, protocol_); @@ -79,15 +79,16 @@ class RemoteProcessorGroupPort : public core::Processor { // Supported Relationships static core::Relationship relation; public: - void onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory); + virtual void onSchedule(const std::shared_ptr &context, const std::shared_ptr &sessionFactory); // OnTrigger method, implemented by NiFi RemoteProcessorGroupPort - virtual void onTrigger(core::ProcessContext *context, core::ProcessSession *session); + virtual void onTrigger(const std::shared_ptr &context, const std::shared_ptr &session); + // Initialize, over write by NiFi RemoteProcessorGroupPort virtual void initialize(void); // Set Direction - void setDirection(TransferDirection direction) { + void setDirection(sitetosite::TransferDirection direction) { direction_ = direction; - if (direction_ == RECEIVE) + if (direction_ == sitetosite::RECEIVE) this->setTriggerWhenEmpty(true); } // Set Timeout @@ -120,16 +121,16 @@ class RemoteProcessorGroupPort : public core::Processor { protected: std::shared_ptr stream_factory_; - std::unique_ptr getNextProtocol(bool create); - void returnProtocol(std::unique_ptr protocol); + std::unique_ptr getNextProtocol(bool create); + void returnProtocol(std::unique_ptr protocol); - moodycamel::ConcurrentQueue> available_protocols_; + moodycamel::ConcurrentQueue> available_protocols_; std::shared_ptr configure_; // Logger std::shared_ptr logger_; // Transaction Direction - TransferDirection direction_; + sitetosite::TransferDirection direction_; // Transmitting bool transmitting_; // timeout @@ -142,12 +143,17 @@ class RemoteProcessorGroupPort : public core::Processor { int port_; std::string protocol_; std::string url_; + bool http_enabled_; + + sitetosite::CLIENT_TYPE client_type_; + // Remote Site2Site Info - int site2site_port_;bool site2site_secure_; - std::vector site2site_peer_status_list_; - std::atomic site2site_peer_index_; - std::mutex site2site_peer_mutex_; + int site2site_port_; + bool site2site_secure_; + std::vector peers_; + std::atomic peer_index_; + std::mutex peer_mutex_; std::string rest_user_name_; std::string rest_password_; diff --git a/libminifi/include/SchedulingAgent.h b/libminifi/include/SchedulingAgent.h index 3fc72c698a..a4f2adeac4 100644 --- a/libminifi/include/SchedulingAgent.h +++ b/libminifi/include/SchedulingAgent.h @@ -71,7 +71,7 @@ class SchedulingAgent { } // onTrigger, return whether the yield is need - bool onTrigger(std::shared_ptr processor, std::shared_ptr processContext, std::shared_ptr sessionFactory); + bool onTrigger(const std::shared_ptr &processor, const std::shared_ptr &processContext, const std::shared_ptr &sessionFactory); // Whether agent has work to do bool hasWorkToDo(std::shared_ptr processor); // Whether the outgoing need to be backpressure diff --git a/libminifi/include/core/Processor.h b/libminifi/include/core/Processor.h index 955a04441b..b083e75334 100644 --- a/libminifi/include/core/Processor.h +++ b/libminifi/include/core/Processor.h @@ -42,9 +42,7 @@ #include "ProcessSession.h" #include "ProcessSessionFactory.h" #include "Scheduling.h" - #include -#include "Site2SiteClientProtocol.h" namespace org { namespace apache { @@ -206,7 +204,7 @@ class __attribute__((visibility("default"))) Processor : public Connectable, pub std::shared_ptr getNextIncomingConnection(); // On Trigger - void onTrigger(std::shared_ptr context, std::shared_ptr sessionFactory); + void onTrigger(const std::shared_ptr &context, const std::shared_ptr &sessionFactory); void onTrigger(ProcessContext *context, ProcessSessionFactory *sessionFactory); @@ -217,15 +215,17 @@ class __attribute__((visibility("default"))) Processor : public Connectable, pub public: // OnTrigger method, implemented by NiFi Processor Designer - virtual void onTrigger(std::shared_ptr context, std::shared_ptr session) { + virtual void onTrigger(const std::shared_ptr &context, const std::shared_ptr &session) { onTrigger(context.get(), session.get()); } - virtual void onTrigger(ProcessContext *context, ProcessSession *session) = 0; + virtual void onTrigger(ProcessContext *context, ProcessSession *session){ + + } // Initialize, overridden by NiFi Process Designer virtual void initialize() { } // Scheduled event hook, overridden by NiFi Process Designer - virtual void onSchedule(std::shared_ptr context, std::shared_ptr sessionFactory) { + virtual void onSchedule(const std::shared_ptr &context, const std::shared_ptr &sessionFactory) { onSchedule(context.get(), sessionFactory.get()); } virtual void onSchedule(ProcessContext *context, ProcessSessionFactory *sessionFactory) { diff --git a/libminifi/include/core/reporting/SiteToSiteProvenanceReportingTask.h b/libminifi/include/core/reporting/SiteToSiteProvenanceReportingTask.h index 0e4ba81699..545d5e532a 100644 --- a/libminifi/include/core/reporting/SiteToSiteProvenanceReportingTask.h +++ b/libminifi/include/core/reporting/SiteToSiteProvenanceReportingTask.h @@ -27,7 +27,6 @@ #include "core/Processor.h" #include "core/ProcessSession.h" #include "RemoteProcessorGroupPort.h" -#include "Site2SiteClientProtocol.h" #include "io/StreamFactory.h" #include "core/logging/LoggerConfiguration.h" @@ -61,11 +60,13 @@ class SiteToSiteProvenanceReportingTask : public minifi::RemoteProcessorGroupPor public: //! Get provenance json report - void getJsonReport(core::ProcessContext *context, core::ProcessSession *session, std::vector> &records, std::string &report); + void getJsonReport(const std::shared_ptr &context, const std::shared_ptr &session, std::vector> &records, std::string &report); - void onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory); + + void onSchedule(const std::shared_ptr &context, const std::shared_ptr &sessionFactory); //! OnTrigger method, implemented by NiFi SiteToSiteProvenanceReportingTask - virtual void onTrigger(core::ProcessContext *context, core::ProcessSession *session); + void onTrigger(const std::shared_ptr &context, const std::shared_ptr &session); + //! Initialize, over write by NiFi SiteToSiteProvenanceReportingTask virtual void initialize(void); //! Set Port UUID diff --git a/libminifi/include/core/yaml/YamlConfiguration.h b/libminifi/include/core/yaml/YamlConfiguration.h index 0f247e76cd..d595183ec1 100644 --- a/libminifi/include/core/yaml/YamlConfiguration.h +++ b/libminifi/include/core/yaml/YamlConfiguration.h @@ -22,7 +22,7 @@ #include "yaml-cpp/yaml.h" #include "processors/LoadProcessors.h" #include "core/FlowConfiguration.h" -#include "Site2SiteClientProtocol.h" +#include "sitetosite/SiteToSite.h" #include #include "io/validation.h" #include "io/StreamFactory.h" @@ -170,7 +170,7 @@ class YamlConfiguration : public FlowConfiguration { * @param parent the parent ProcessGroup for the port * @param direction the TransferDirection of the port */ - void parsePortYaml(YAML::Node *portNode, core::ProcessGroup *parent, TransferDirection direction); + void parsePortYaml(YAML::Node *portNode, core::ProcessGroup *parent, sitetosite::TransferDirection direction); /** * Parses the root level YAML node for the flow configuration and diff --git a/libminifi/include/io/BaseStream.h b/libminifi/include/io/BaseStream.h index 3410be6763..fa6fe1b9df 100644 --- a/libminifi/include/io/BaseStream.h +++ b/libminifi/include/io/BaseStream.h @@ -58,7 +58,7 @@ class BaseStream : public DataStream, public Serializable { **/ virtual int write(uint32_t base_value, bool is_little_endian = EndiannessCheck::IS_LITTLE); - int writeData(uint8_t *value, int size); + virtual int writeData(uint8_t *value, int size); virtual void seek(uint64_t offset) { if (composable_stream_ != this) { @@ -171,6 +171,16 @@ class BaseStream : public DataStream, public Serializable { **/ virtual int read(uint64_t &value, bool is_little_endian = EndiannessCheck::IS_LITTLE); + virtual const uint64_t getSize() const { + if (composable_stream_ == this){ + return buffer.size(); + } + else{ + return composable_stream_->getSize(); + } + + } + /** * read UTF from stream * @param str reference string diff --git a/libminifi/include/io/CRCStream.h b/libminifi/include/io/CRCStream.h index 70cb89a33b..70313329fe 100644 --- a/libminifi/include/io/CRCStream.h +++ b/libminifi/include/io/CRCStream.h @@ -48,6 +48,14 @@ class CRCStream : public BaseStream { } + T *getstream(){ + return child_stream_; + } + + void disableEncoding(){ + disable_encoding_ = true; + } + /** * Reads data and places it into buf * @param buf buffer in which we extract data @@ -132,6 +140,7 @@ class CRCStream : public BaseStream { return crc_; } + void reset(); protected: @@ -151,19 +160,19 @@ class CRCStream : public BaseStream { uint64_t crc_; T *child_stream_; + bool disable_encoding_; }; template CRCStream::CRCStream(T *other) - : child_stream_(other) { + : child_stream_(other), disable_encoding_(false) { crc_ = crc32(0L, Z_NULL, 0); } template CRCStream::CRCStream(CRCStream &&move) - : crc_(std::move(move.crc_)), + : crc_(std::move(move.crc_)), disable_encoding_(false), child_stream_(std::move(move.child_stream_)) { - } template @@ -177,7 +186,7 @@ int CRCStream::readData(std::vector &buf, int buflen) { template int CRCStream::readData(uint8_t *buf, int buflen) { int ret = child_stream_->read(buf, buflen); - crc_ = crc32(crc_, buf, buflen); + crc_ = crc32(crc_, buf, ret); return ret; } @@ -209,6 +218,8 @@ void CRCStream::updateCRC(uint8_t *buffer, uint32_t length) { template int CRCStream::write(uint64_t base_value, bool is_little_endian) { + if (disable_encoding_) + is_little_endian=false; const uint64_t value = is_little_endian == 1 ? htonll_r(base_value) : base_value; uint8_t bytes[sizeof value]; std::copy(static_cast(static_cast(&value)), static_cast(static_cast(&value)) + sizeof value, bytes); @@ -217,6 +228,8 @@ int CRCStream::write(uint64_t base_value, bool is_little_endian) { template int CRCStream::write(uint32_t base_value, bool is_little_endian) { + if (disable_encoding_) + is_little_endian=false; const uint32_t value = is_little_endian ? htonl(base_value) : base_value; uint8_t bytes[sizeof value]; std::copy(static_cast(static_cast(&value)), static_cast(static_cast(&value)) + sizeof value, bytes); @@ -225,6 +238,8 @@ int CRCStream::write(uint32_t base_value, bool is_little_endian) { template int CRCStream::write(uint16_t base_value, bool is_little_endian) { + if (disable_encoding_) + is_little_endian=false; const uint16_t value = is_little_endian == 1 ? htons(base_value) : base_value; uint8_t bytes[sizeof value]; std::copy(static_cast(static_cast(&value)), static_cast(static_cast(&value)) + sizeof value, bytes); @@ -233,7 +248,8 @@ int CRCStream::write(uint16_t base_value, bool is_little_endian) { template int CRCStream::read(uint64_t &value, bool is_little_endian) { - + if (disable_encoding_) + is_little_endian=false; auto buf = readBuffer(value); if (is_little_endian) { @@ -248,7 +264,8 @@ int CRCStream::read(uint64_t &value, bool is_little_endian) { template int CRCStream::read(uint32_t &value, bool is_little_endian) { - + if (disable_encoding_) + is_little_endian=false; auto buf = readBuffer(value); if (is_little_endian) { @@ -263,7 +280,8 @@ int CRCStream::read(uint32_t &value, bool is_little_endian) { template int CRCStream::read(uint16_t &value, bool is_little_endian) { - + if (disable_encoding_) + is_little_endian=false; auto buf = readBuffer(value); if (is_little_endian) { diff --git a/libminifi/include/io/DataStream.h b/libminifi/include/io/DataStream.h index e0fa4befa0..be7ebe5bf5 100644 --- a/libminifi/include/io/DataStream.h +++ b/libminifi/include/io/DataStream.h @@ -19,6 +19,7 @@ #ifndef LIBMINIFI_INCLUDE_IO_DATASTREAM_H_ #define LIBMINIFI_INCLUDE_IO_DATASTREAM_H_ +#include #include #include #include "EndianCheck.h" diff --git a/libminifi/include/io/NonConvertingStream.h b/libminifi/include/io/NonConvertingStream.h new file mode 100644 index 0000000000..3196617cce --- /dev/null +++ b/libminifi/include/io/NonConvertingStream.h @@ -0,0 +1,200 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBMINIFI_INCLUDE_IO_NonConvertingStream_H_ +#define LIBMINIFI_INCLUDE_IO_NonConvertingStream_H_ +#include +#include +#include "EndianCheck.h" +#include "BaseStream.h" +#include "Serializable.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace io { + +/** + * Base Stream. Not intended to be thread safe as it is not intended to be shared + * + * Extensions may be thread safe and thus shareable, but that is up to the implementation. + */ +class NonConvertingStream : public BaseStream { + + public: + NonConvertingStream() + : composable_stream_(this) { + } + + NonConvertingStream(DataStream *other) + : composable_stream_(other) { + } + + virtual ~NonConvertingStream() { + + } + /** + * write 4 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ + virtual int write(uint32_t base_value, bool is_little_endian = false); + + int writeData(uint8_t *value, int size); + + virtual void seek(uint32_t offset) { + if (composable_stream_ != this) { + composable_stream_->seek(offset); + } else { + DataStream::seek(offset); + } + } + + /** + * write 2 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ + virtual int write(uint16_t base_value, bool is_little_endian = false); + + /** + * write valueto stream + * @param value non encoded value + * @param len length of value + * @param strema output stream + * @return resulting write size + **/ + virtual int write(uint8_t *value, int len); + + /** + * write 8 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ + virtual int write(uint64_t base_value, bool is_little_endian = false); + + /** + * write bool to stream + * @param value non encoded value + * @return resulting write size + **/ + virtual int write(bool value); + + /** + * write UTF string to stream + * @param str string to write + * @return resulting write size + **/ + virtual int writeUTF(std::string str, bool widen = false); + + /** + * reads a byte from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ + virtual int read(uint8_t &value); + + /** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ + virtual int readData(std::vector &buf, int buflen); + /** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ + virtual int readData(uint8_t *buf, int buflen); + + /** + * reads two bytes from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ + virtual int read(uint16_t &base_value, bool is_little_endian = false); + + /** + * reads a byte from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ + virtual int read(char &value); + + /** + * reads a byte array from the stream + * @param value reference in which will set the result + * @param len length to read + * @param stream stream from which we will read + * @return resulting read size + **/ + virtual int read(uint8_t *value, int len); + + /** + * reads four bytes from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ + virtual int read(uint32_t &value, bool is_little_endian = false); + + /** + * reads eight byte from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ + virtual int read(uint64_t &value, bool is_little_endian = false); + + virtual const uint64_t getSize() const { + if (composable_stream_ == this){ + return buffer.size(); + } + else{ + return composable_stream_->getSize(); + } + + } + + /** + * read UTF from stream + * @param str reference string + * @param stream stream from which we will read + * @return resulting read size + **/ + virtual int readUTF(std::string &str, bool widen = false); + protected: + DataStream *composable_stream_; +}; + +} /* namespace io */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ +#endif /* LIBMINIFI_INCLUDE_IO_NonConvertingStream_H_ */ diff --git a/libminifi/include/processors/GetTCP.h b/libminifi/include/processors/GetTCP.h index f649286b28..5ee263f762 100644 --- a/libminifi/include/processors/GetTCP.h +++ b/libminifi/include/processors/GetTCP.h @@ -215,7 +215,7 @@ class GetTCP : public core::Processor, public state::metrics::MetricsSource { * @param sessionFactory process session factory that is used when creating * ProcessSession objects. */ - virtual void onSchedule(std::shared_ptr processContext, std::shared_ptr sessionFactory); + virtual void onSchedule(const std::shared_ptr &processContext, const std::shared_ptr &sessionFactory); void onSchedule(core::ProcessContext *processContext, core::ProcessSessionFactory *sessionFactory) { throw std::exception(); @@ -225,7 +225,7 @@ class GetTCP : public core::Processor, public state::metrics::MetricsSource { * @param context processor context * @param session processor session reference. */ - virtual void onTrigger(std::shared_ptr context, std::shared_ptr session); + virtual void onTrigger(const std::shared_ptr &context, const std::shared_ptr &session); virtual void onTrigger(core::ProcessContext *context, core::ProcessSession *session) { throw std::exception(); diff --git a/libminifi/include/properties/Configure.h b/libminifi/include/properties/Configure.h index 0d7bb55a44..6911ae23d1 100644 --- a/libminifi/include/properties/Configure.h +++ b/libminifi/include/properties/Configure.h @@ -56,6 +56,7 @@ class Configure : public Properties { static const char *nifi_flowfile_repository_directory_default; static const char *nifi_flowfile_repository_enable; static const char *nifi_remote_input_secure; + static const char *nifi_remote_input_http; static const char *nifi_security_need_ClientAuth; // site2site security config static const char *nifi_security_client_certificate; diff --git a/libminifi/include/Site2SitePeer.h b/libminifi/include/sitetosite/Peer.h similarity index 54% rename from libminifi/include/Site2SitePeer.h rename to libminifi/include/sitetosite/Peer.h index 59f5cfe960..4617a7a1ac 100644 --- a/libminifi/include/Site2SitePeer.h +++ b/libminifi/include/sitetosite/Peer.h @@ -1,6 +1,4 @@ /** - * @file Site2SitePeer.h - * Site2SitePeer class declaration for site to site peer * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -17,8 +15,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef __SITE2SITE_PEER_H__ -#define __SITE2SITE_PEER_H__ +#ifndef LIBMINIFI_INCLUDE_SITETOSITE_PEER_H_ +#define LIBMINIFI_INCLUDE_SITETOSITE_PEER_H_ #include #include @@ -26,6 +24,8 @@ #include #include #include +#include +#include #include #include #include @@ -41,71 +41,166 @@ namespace org { namespace apache { namespace nifi { namespace minifi { +namespace sitetosite { + +class Peer { + public: + explicit Peer(uuid_t port_id, const std::string &host, uint16_t port, bool secure = false) + : host_(host), + port_(port), + secure_(secure) { + uuid_copy(port_id_, port_id); + } + + explicit Peer(const std::string &host, uint16_t port, bool secure = false) + : host_(host), + port_(port), + secure_(secure) { + } + + explicit Peer(const Peer &other) + : host_(other.host_), + port_(other.port_), + secure_(other.secure_) { + uuid_copy(port_id_, other.port_id_); + } + + explicit Peer(const Peer &&other) + : host_(std::move(other.host_)), + port_(std::move(other.port_)), + secure_(std::move(other.secure_)) { + uuid_copy(port_id_, other.port_id_); + } + + uint16_t getPort() const { + return port_; + } + + const std::string &getHost() const { + return host_; + } + + bool isSecure() const { + return secure_; + } + + void getPortId(uuid_t other) const { + uuid_copy(other, port_id_); + } + + protected: + uint16_t port_; + + std::string host_; + + uuid_t port_id_; + // secore comms + + bool secure_; +}; + +class PeerStatus { + public: + PeerStatus(const std::shared_ptr &peer, uint32_t flow_file_count, bool query_for_peers) + : peer_(peer), + flow_file_count_(flow_file_count), + query_for_peers_(query_for_peers) { + + } + PeerStatus(const PeerStatus &&other) + : peer_(std::move(other.peer_)), + flow_file_count_(std::move(other.flow_file_count_)), + query_for_peers_(std::move(other.query_for_peers_)) { + + } + const std::shared_ptr &getPeer() const { + return peer_; + } + + uint32_t getFlowFileCount() { + return flow_file_count_; + } + + bool getQueryForPeers() { + return query_for_peers_; + } + protected: + std::shared_ptr peer_; + uint32_t flow_file_count_; + bool query_for_peers_; +}; static const char MAGIC_BYTES[] = { 'N', 'i', 'F', 'i' }; // Site2SitePeer Class -class Site2SitePeer : public org::apache::nifi::minifi::io::BaseStream { +class SiteToSitePeer : public org::apache::nifi::minifi::io::BaseStream { public: - Site2SitePeer() + SiteToSitePeer() : stream_(nullptr), host_(""), port_(-1), - logger_(logging::LoggerFactory::getLogger()) { + logger_(logging::LoggerFactory::getLogger()) { } /* * Create a new site2site peer */ - explicit Site2SitePeer(std::unique_ptr injected_socket, const std::string host_, uint16_t port_) - : host_(host_), - port_(port_), - stream_(injected_socket.release()), - logger_(logging::LoggerFactory::getLogger()) { - _yieldExpiration = 0; - _timeOut = 30000; // 30 seconds - _url = "nifi://" + host_ + ":" + std::to_string(port_); + explicit SiteToSitePeer(std::unique_ptr injected_socket, const std::string host, uint16_t port) + : SiteToSitePeer(host, port) { + stream_ = std::move(injected_socket); + } + + explicit SiteToSitePeer(const std::string &host, uint16_t port) + : host_(host), + port_(port), + stream_(nullptr), + yield_expiration_(0), + timeout_(30000), + logger_(logging::LoggerFactory::getLogger()) { + url_ = "nifi://" + host_ + ":" + std::to_string(port_); + yield_expiration_ = 0; + timeout_ = 30000; // 30 seconds } - explicit Site2SitePeer(Site2SitePeer &&ss) + explicit SiteToSitePeer(SiteToSitePeer &&ss) : stream_(ss.stream_.release()), host_(std::move(ss.host_)), port_(std::move(ss.port_)), logger_(std::move(ss.logger_)) { - _yieldExpiration.store(ss._yieldExpiration); - _timeOut.store(ss._timeOut); - _url = std::move(ss._url); + yield_expiration_.store(ss.yield_expiration_); + timeout_.store(ss.timeout_); + url_ = std::move(ss.url_); } // Destructor - ~Site2SitePeer() { + ~SiteToSitePeer() { Close(); } // Set Processor yield period in MilliSecond void setYieldPeriodMsec(uint64_t period) { - _yieldPeriodMsec = period; + yield_period_msec_ = period; } // get URL std::string getURL() { - return _url; + return url_; } // Get Processor yield period in MilliSecond uint64_t getYieldPeriodMsec(void) { - return (_yieldPeriodMsec); + return (yield_period_msec_); } // Yield based on the yield period void yield() { - _yieldExpiration = (getTimeMillis() + _yieldPeriodMsec); + yield_expiration_ = (getTimeMillis() + yield_period_msec_); } // setHostName void setHostName(std::string host_) { this->host_ = host_; - _url = "nifi://" + host_ + ":" + std::to_string(port_); + url_ = "nifi://" + host_ + ":" + std::to_string(port_); } // setPort void setPort(uint16_t port_) { this->port_ = port_; - _url = "nifi://" + host_ + ":" + std::to_string(port_); + url_ = "nifi://" + host_ + ":" + std::to_string(port_); } // getHostName std::string getHostName() { @@ -117,36 +212,36 @@ class Site2SitePeer : public org::apache::nifi::minifi::io::BaseStream { } // Yield based on the input time void yield(uint64_t time) { - _yieldExpiration = (getTimeMillis() + time); + yield_expiration_ = (getTimeMillis() + time); } // whether need be to yield bool isYield() { - if (_yieldExpiration > 0) - return (_yieldExpiration >= getTimeMillis()); + if (yield_expiration_ > 0) + return (yield_expiration_ >= getTimeMillis()); else return false; } // clear yield expiration void clearYield() { - _yieldExpiration = 0; + yield_expiration_ = 0; } // Yield based on the yield period void yield(std::string portId) { std::lock_guard lock(mutex_); - uint64_t yieldExpiration = (getTimeMillis() + _yieldPeriodMsec); - _yieldExpirationPortIdMap[portId] = yieldExpiration; + uint64_t yieldExpiration = (getTimeMillis() + yield_period_msec_); + yield_expiration_PortIdMap[portId] = yieldExpiration; } // Yield based on the input time void yield(std::string portId, uint64_t time) { std::lock_guard lock(mutex_); uint64_t yieldExpiration = (getTimeMillis() + time); - _yieldExpirationPortIdMap[portId] = yieldExpiration; + yield_expiration_PortIdMap[portId] = yieldExpiration; } // whether need be to yield bool isYield(std::string portId) { std::lock_guard lock(mutex_); - std::map::iterator it = this->_yieldExpirationPortIdMap.find(portId); - if (it != _yieldExpirationPortIdMap.end()) { + std::map::iterator it = this->yield_expiration_PortIdMap.find(portId); + if (it != yield_expiration_PortIdMap.end()) { uint64_t yieldExpiration = it->second; return (yieldExpiration >= getTimeMillis()); } else { @@ -156,19 +251,28 @@ class Site2SitePeer : public org::apache::nifi::minifi::io::BaseStream { // clear yield expiration void clearYield(std::string portId) { std::lock_guard lock(mutex_); - std::map::iterator it = this->_yieldExpirationPortIdMap.find(portId); - if (it != _yieldExpirationPortIdMap.end()) { - _yieldExpirationPortIdMap.erase(portId); + std::map::iterator it = this->yield_expiration_PortIdMap.find(portId); + if (it != yield_expiration_PortIdMap.end()) { + yield_expiration_PortIdMap.erase(portId); } } // setTimeOut void setTimeOut(uint64_t time) { - _timeOut = time; + timeout_ = time; } // getTimeOut uint64_t getTimeOut() { - return _timeOut; + return timeout_; } + + void setStream(std::unique_ptr stream) { + stream_ = std::move(stream); + } + + org::apache::nifi::minifi::io::DataStream *getStream() { + return stream_.get(); + } + int write(uint8_t value) { return Serializable::write(value, stream_.get()); } @@ -225,19 +329,19 @@ class Site2SitePeer : public org::apache::nifi::minifi::io::BaseStream { /** * Move assignment operator. */ - Site2SitePeer& operator=(Site2SitePeer&& other) { + SiteToSitePeer& operator=(SiteToSitePeer&& other) { stream_ = std::unique_ptr(other.stream_.release()); host_ = std::move(other.host_); port_ = std::move(other.port_); - _yieldExpiration = 0; - _timeOut = 30000; // 30 seconds - _url = "nifi://" + host_ + ":" + std::to_string(port_); + yield_expiration_ = 0; + timeout_ = 30000; // 30 seconds + url_ = "nifi://" + host_ + ":" + std::to_string(port_); return *this; } - Site2SitePeer(const Site2SitePeer &parent) = delete; - Site2SitePeer &operator=(const Site2SitePeer &parent) = delete; + SiteToSitePeer(const SiteToSitePeer &parent) = delete; + SiteToSitePeer &operator=(const SiteToSitePeer &parent) = delete; protected: @@ -251,25 +355,27 @@ class Site2SitePeer : public org::apache::nifi::minifi::io::BaseStream { // Mutex for protection std::mutex mutex_; // URL - std::string _url; + std::string url_; // socket timeout; - std::atomic _timeOut; + std::atomic timeout_; // Logger std::shared_ptr logger_; // Yield Period in Milliseconds - std::atomic _yieldPeriodMsec; + std::atomic yield_period_msec_; // Yield Expiration - std::atomic _yieldExpiration; + std::atomic yield_expiration_; // Yield Expiration per destination PortID - std::map _yieldExpirationPortIdMap; + std::map yield_expiration_PortIdMap; // OpenSSL connection state // Prevent default copy constructor and assignment operation // Only support pass by reference or pointer }; +} /* namespace sitetosite */ } /* namespace minifi */ } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ -#endif + +#endif /* LIBMINIFI_INCLUDE_SITETOSITE_PEER_H_ */ diff --git a/libminifi/include/sitetosite/RawSocketProtocol.h b/libminifi/include/sitetosite/RawSocketProtocol.h new file mode 100644 index 0000000000..b54b3801c3 --- /dev/null +++ b/libminifi/include/sitetosite/RawSocketProtocol.h @@ -0,0 +1,211 @@ +/** + * @file RawSiteToSiteClient.h + * RawSiteToSiteClient class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __SITE2SITE_CLIENT_PROTOCOL_H__ +#define __SITE2SITE_CLIENT_PROTOCOL_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "SiteToSite.h" +#include "SiteToSiteClient.h" +#include "core/Property.h" +#include "properties/Configure.h" +#include "FlowFileRecord.h" +#include "core/logging/LoggerConfiguration.h" +#include "core/ProcessContext.h" +#include "core/ProcessSession.h" +#include "io/CRCStream.h" +#include "Peer.h" +#include "utils/Id.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace sitetosite { + +/** + * Site2Site Peer + */ +typedef struct Site2SitePeerStatus { + std::string host_; + int port_;bool isSecure_; +} Site2SitePeerStatus; + +// RawSiteToSiteClient Class +class RawSiteToSiteClient : public sitetosite::SiteToSiteClient { + public: + // Constructor + /*! + * Create a new control protocol + */ + RawSiteToSiteClient(std::unique_ptr peer) + : logger_(logging::LoggerFactory::getLogger()) { + peer_ = std::move(peer); + _batchSize = 0; + _batchCount = 0; + _batchDuration = 0; + _batchSendNanos = 5000000000; // 5 seconds + _timeOut = 30000; // 30 seconds + _supportedVersion[0] = 5; + _supportedVersion[1] = 4; + _supportedVersion[2] = 3; + _supportedVersion[3] = 2; + _supportedVersion[4] = 1; + _currentVersion = _supportedVersion[0]; + _currentVersionIndex = 0; + _supportedCodecVersion[0] = 1; + _currentCodecVersion = _supportedCodecVersion[0]; + _currentCodecVersionIndex = 0; + } + // Destructor + virtual ~RawSiteToSiteClient() { + tearDown(); + } + + public: + // setBatchSize + void setBatchSize(uint64_t size) { + _batchSize = size; + } + // setBatchCount + void setBatchCount(uint64_t count) { + _batchCount = count; + } + // setBatchDuration + void setBatchDuration(uint64_t duration) { + _batchDuration = duration; + } + // setTimeOut + void setTimeOut(uint64_t time) { + _timeOut = time; + if (peer_) + peer_->setTimeOut(time); + + } + + void setPeer(std::unique_ptr peer) { + peer_ = std::move(peer); + } + /** + * Provides a reference to the time out + * @returns timeout + */ + const uint64_t getTimeOut() const { + return _timeOut; + } + + // getResourceName + std::string getResourceName() { + return "SocketFlowFileProtocol"; + } + // getCodecResourceName + std::string getCodecResourceName() { + return "StandardFlowFileCodec"; + } + + // get peerList + virtual bool getPeerList(std::vector &peer); + // negotiateCodec + virtual bool negotiateCodec(); + // initiateResourceNegotiation + virtual bool initiateResourceNegotiation(); + // initiateCodecResourceNegotiation + virtual bool initiateCodecResourceNegotiation(); + // tearDown + virtual void tearDown(); + // write Request Type + virtual int writeRequestType(RequestType type); + // read Request Type + virtual int readRequestType(RequestType &type); + // read Respond + virtual int readRespond(const std::shared_ptr &transaction, RespondCode &code, std::string &message); + // write respond + virtual int writeRespond(const std::shared_ptr &transaction, RespondCode code, std::string message); + // getRespondCodeContext + virtual RespondCodeContext *getRespondCodeContext(RespondCode code) { + for (unsigned int i = 0; i < sizeof(respondCodeContext) / sizeof(RespondCodeContext); i++) { + if (respondCodeContext[i].code == code) { + return &respondCodeContext[i]; + } + } + return NULL; + } + + // Creation of a new transaction, return the transaction ID if success, + // Return NULL when any error occurs + virtual std::shared_ptr createTransaction(std::string &transactionID, TransferDirection direction); + + //! Transfer string for the process session + virtual bool transmitPayload(const std::shared_ptr &context, const std::shared_ptr &session, const std::string &payload, + std::map attributes); + + // bootstrap the protocol to the ready for transaction state by going through the state machine + virtual bool bootstrap(); + protected: + + // establish + virtual bool establish(); + // handShake + virtual bool handShake(); + + private: + + // Mutex for protection + std::mutex mutex_; + // Logger + std::shared_ptr logger_; + // Batch Count + std::atomic _batchCount; + // Batch Size + std::atomic _batchSize; + // Batch Duration in msec + std::atomic _batchDuration; + // Timeout in msec + std::atomic _timeOut; + + // commsIdentifier + std::string _commsIdentifier; + + // Prevent default copy constructor and assignment operation + // Only support pass by reference or pointer + RawSiteToSiteClient(const RawSiteToSiteClient &parent); + RawSiteToSiteClient &operator=(const RawSiteToSiteClient &parent); + static std::shared_ptr id_generator_; +}; + +} /* namespace sitetosite */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ +#endif diff --git a/libminifi/include/Site2SiteClientProtocol.h b/libminifi/include/sitetosite/SiteToSite.h similarity index 50% rename from libminifi/include/Site2SiteClientProtocol.h rename to libminifi/include/sitetosite/SiteToSite.h index df974fb54f..953c5b1878 100644 --- a/libminifi/include/Site2SiteClientProtocol.h +++ b/libminifi/include/sitetosite/SiteToSite.h @@ -1,6 +1,4 @@ /** - * @file Site2SiteClientProtocol.h - * Site2SiteClientProtocol class declaration * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -17,38 +15,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef __SITE2SITE_CLIENT_PROTOCOL_H__ -#define __SITE2SITE_CLIENT_PROTOCOL_H__ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#ifndef LIBMINIFI_INCLUDE_CORE_SITETOSITE_SITETOSITE_H_ +#define LIBMINIFI_INCLUDE_CORE_SITETOSITE_SITETOSITE_H_ +#include "controllers/SSLContextService.h" +#include "Peer.h" #include "core/Property.h" #include "properties/Configure.h" -#include "Site2SitePeer.h" -#include "FlowFileRecord.h" -#include "core/logging/LoggerConfiguration.h" -#include "core/ProcessContext.h" -#include "core/ProcessSession.h" #include "io/CRCStream.h" +#include "io/StreamFactory.h" #include "utils/Id.h" namespace org { namespace apache { namespace nifi { namespace minifi { +namespace sitetosite { // Resource Negotiated Status Code #define RESOURCE_OK 20 @@ -57,6 +39,94 @@ namespace minifi { // ! Max attributes #define MAX_NUM_ATTRIBUTES 25000 +// Respond Code Sequence Pattern +static const uint8_t CODE_SEQUENCE_VALUE_1 = (uint8_t) 'R'; +static const uint8_t CODE_SEQUENCE_VALUE_2 = (uint8_t) 'C'; + +/** + * Enumeration of Properties that can be used for the Site-to-Site Socket + * Protocol. + */ +typedef enum { + /** + * Boolean value indicating whether or not the contents of a FlowFile should + * be GZipped when transferred. + */ + GZIP, + /** + * The unique identifier of the port to communicate with + */ + PORT_IDENTIFIER, + /** + * Indicates the number of milliseconds after the request was made that the + * client will wait for a response. If no response has been received by the + * time this value expires, the server can move on without attempting to + * service the request because the client will have already disconnected. + */ + REQUEST_EXPIRATION_MILLIS, + /** + * The preferred number of FlowFiles that the server should send to the + * client when pulling data. This property was introduced in version 5 of + * the protocol. + */ + BATCH_COUNT, + /** + * The preferred number of bytes that the server should send to the client + * when pulling data. This property was introduced in version 5 of the + * protocol. + */ + BATCH_SIZE, + /** + * The preferred amount of time that the server should send data to the + * client when pulling data. This property was introduced in version 5 of + * the protocol. Value is in milliseconds. + */ + BATCH_DURATION, + MAX_HANDSHAKE_PROPERTY +} HandshakeProperty; + +// HandShakeProperty Str +static const char *HandShakePropertyStr[MAX_HANDSHAKE_PROPERTY] = { +/** + * Boolean value indicating whether or not the contents of a FlowFile should + * be GZipped when transferred. + */ +"GZIP", +/** + * The unique identifier of the port to communicate with + */ +"PORT_IDENTIFIER", +/** + * Indicates the number of milliseconds after the request was made that the + * client will wait for a response. If no response has been received by the + * time this value expires, the server can move on without attempting to + * service the request because the client will have already disconnected. + */ +"REQUEST_EXPIRATION_MILLIS", +/** + * The preferred number of FlowFiles that the server should send to the + * client when pulling data. This property was introduced in version 5 of + * the protocol. + */ +"BATCH_COUNT", +/** + * The preferred number of bytes that the server should send to the client + * when pulling data. This property was introduced in version 5 of the + * protocol. + */ +"BATCH_SIZE", +/** + * The preferred amount of time that the server should send data to the + * client when pulling data. This property was introduced in version 5 of + * the protocol. Value is in milliseconds. + */ +"BATCH_DURATION" }; + +typedef enum { + RAW, + HTTP +} CLIENT_TYPE; + /** * An enumeration for specifying the direction in which data should be * transferred between a client and a remote NiFi instance. @@ -115,6 +185,11 @@ typedef enum { * * The Transaction has been canceled. * */ TRANSACTION_CANCELED, + + /** + * * Transaction has been successfully closed. + * */ + TRANSACTION_CLOSED, /** * * The Transaction ended in an error. * */ @@ -182,121 +257,47 @@ static RespondCodeContext respondCodeContext[] = { { RESERVED, "Reserved for Fut "Port's Destination is Full", false }, { UNAUTHORIZED, "User Not Authorized", true }, { ABORT, "Abort", true }, { UNRECOGNIZED_RESPONSE_CODE, "Unrecognized Response Code", false }, { END_OF_STREAM, "End of Stream", false } }; -// Respond Code Sequence Pattern -static const uint8_t CODE_SEQUENCE_VALUE_1 = (uint8_t) 'R'; -static const uint8_t CODE_SEQUENCE_VALUE_2 = (uint8_t) 'C'; - -/** - * Enumeration of Properties that can be used for the Site-to-Site Socket - * Protocol. - */ -typedef enum { - /** - * Boolean value indicating whether or not the contents of a FlowFile should - * be GZipped when transferred. - */ - GZIP, - /** - * The unique identifier of the port to communicate with - */ - PORT_IDENTIFIER, - /** - * Indicates the number of milliseconds after the request was made that the - * client will wait for a response. If no response has been received by the - * time this value expires, the server can move on without attempting to - * service the request because the client will have already disconnected. - */ - REQUEST_EXPIRATION_MILLIS, - /** - * The preferred number of FlowFiles that the server should send to the - * client when pulling data. This property was introduced in version 5 of - * the protocol. - */ - BATCH_COUNT, - /** - * The preferred number of bytes that the server should send to the client - * when pulling data. This property was introduced in version 5 of the - * protocol. - */ - BATCH_SIZE, - /** - * The preferred amount of time that the server should send data to the - * client when pulling data. This property was introduced in version 5 of - * the protocol. Value is in milliseconds. - */ - BATCH_DURATION, - MAX_HANDSHAKE_PROPERTY -} HandshakeProperty; - -// HandShakeProperty Str -static const char *HandShakePropertyStr[MAX_HANDSHAKE_PROPERTY] = { -/** - * Boolean value indicating whether or not the contents of a FlowFile should - * be GZipped when transferred. - */ -"GZIP", -/** - * The unique identifier of the port to communicate with - */ -"PORT_IDENTIFIER", -/** - * Indicates the number of milliseconds after the request was made that the - * client will wait for a response. If no response has been received by the - * time this value expires, the server can move on without attempting to - * service the request because the client will have already disconnected. - */ -"REQUEST_EXPIRATION_MILLIS", -/** - * The preferred number of FlowFiles that the server should send to the - * client when pulling data. This property was introduced in version 5 of - * the protocol. - */ -"BATCH_COUNT", -/** - * The preferred number of bytes that the server should send to the client - * when pulling data. This property was introduced in version 5 of the - * protocol. - */ -"BATCH_SIZE", -/** - * The preferred amount of time that the server should send data to the - * client when pulling data. This property was introduced in version 5 of - * the protocol. Value is in milliseconds. - */ -"BATCH_DURATION" }; - -class Site2SiteClientProtocol; - // Transaction Class class Transaction { - friend class Site2SiteClientProtocol; public: // Constructor /*! * Create a new transaction */ - explicit Transaction(TransferDirection direction, org::apache::nifi::minifi::io::CRCStream &stream) - : crcStream(std::move(stream)) { + explicit Transaction(TransferDirection direction, org::apache::nifi::minifi::io::CRCStream &stream) + : crcStream(std::move(stream)), closed_(false) { _state = TRANSACTION_STARTED; _direction = direction; _dataAvailable = false; - _transfers = 0; + current_transfers_ = 0; + total_transfers_ = 0; _bytes = 0; char uuidStr[37]; // Generate the global UUID for the transaction - id_generator_->generate(_uuid); - uuid_unparse_lower(_uuid, uuidStr); - _uuidStr = uuidStr; + id_generator_->generate(uuid_); + uuid_unparse_lower(uuid_, uuidStr); + uuid_str_ = uuidStr; } // Destructor virtual ~Transaction() { } // getUUIDStr std::string getUUIDStr() { - return _uuidStr; + return uuid_str_; } + + void setTransactionId(const std::string str) { + setUUIDStr(str); + } + + void setUUIDStr(const std::string &str) { + uuid_str_ = str; + uuid_parse(str.c_str(), uuid_); + } + + // getState TransactionState getState() { return _state; @@ -322,312 +323,94 @@ class Transaction { crcStream.updateCRC(buffer, length); } - org::apache::nifi::minifi::io::CRCStream &getStream() { + org::apache::nifi::minifi::io::CRCStream &getStream() { return crcStream; } Transaction(const Transaction &parent) = delete; Transaction &operator=(const Transaction &parent) = delete; - protected: + // Number of current transfers + int current_transfers_; + // number of total seen transfers + int total_transfers_; - private: + // Number of content bytes + uint64_t _bytes; - org::apache::nifi::minifi::io::CRCStream crcStream; // Transaction State TransactionState _state; - // Transaction Direction - TransferDirection _direction; + + bool closed_; + // Whether received data is available bool _dataAvailable; + + protected: + + org::apache::nifi::minifi::io::CRCStream crcStream; + + private: + + // Transaction Direction + TransferDirection _direction; + // A global unique identifier - uuid_t _uuid; + uuid_t uuid_; // UUID string - std::string _uuidStr; - // Number of transfer - int _transfers; - // Number of content bytes - uint64_t _bytes; + std::string uuid_str_; + static std::shared_ptr id_generator_; }; -/** - * Represents a piece of data that is to be sent to or that was received from a - * NiFi instance. - */ -class DataPacket { +class SiteToSiteClientConfiguration { public: - DataPacket(Site2SiteClientProtocol *protocol, Transaction *transaction, std::map attributes, std::string &payload) - : payload_(payload) { - _protocol = protocol; - _size = 0; - _transaction = transaction; - _attributes = attributes; + SiteToSiteClientConfiguration(std::shared_ptr stream_factory, const std::shared_ptr &peer, CLIENT_TYPE type = RAW) + : stream_factory_(stream_factory), + peer_(peer), + ssl_service_(nullptr) { + client_type_ = type; } - std::map _attributes; - uint64_t _size; - Site2SiteClientProtocol *_protocol; - Transaction *_transaction; - std::string & payload_; -}; + SiteToSiteClientConfiguration(const SiteToSiteClientConfiguration &other) = delete; -/** - * Site2Site Peer - */ -typedef struct Site2SitePeerStatus { - std::string host_; - int port_;bool isSecure_; -} Site2SitePeerStatus; - -// Site2SiteClientProtocol Class -class Site2SiteClientProtocol { - public: - // Constructor - /*! - * Create a new control protocol - */ - Site2SiteClientProtocol(std::unique_ptr peer) - : logger_(logging::LoggerFactory::getLogger()) { - peer_ = std::move(peer); - _batchSize = 0; - _batchCount = 0; - _batchDuration = 0; - _batchSendNanos = 5000000000; // 5 seconds - _timeOut = 30000; // 30 seconds - _peerState = IDLE; - _supportedVersion[0] = 5; - _supportedVersion[1] = 4; - _supportedVersion[2] = 3; - _supportedVersion[3] = 2; - _supportedVersion[4] = 1; - _currentVersion = _supportedVersion[0]; - _currentVersionIndex = 0; - _supportedCodecVersion[0] = 1; - _currentCodecVersion = _supportedCodecVersion[0]; - _currentCodecVersionIndex = 0; - } - // Destructor - virtual ~Site2SiteClientProtocol() { - tearDown(); + CLIENT_TYPE getClientType() const { + return client_type_; } - public: - // setBatchSize - void setBatchSize(uint64_t size) { - _batchSize = size; - } - // setBatchCount - void setBatchCount(uint64_t count) { - _batchCount = count; + const std::shared_ptr &getPeer() const { + return peer_; } - // setBatchDuration - void setBatchDuration(uint64_t duration) { - _batchDuration = duration; - } - // setTimeOut - void setTimeOut(uint64_t time) { - _timeOut = time; - if (peer_) - peer_->setTimeOut(time); + void setSecurityContext(const std::shared_ptr &ssl_service) { + ssl_service_ = ssl_service; } - void setPeer(std::unique_ptr peer) { - peer_ = std::move(peer); - } - /** - * Provides a reference to the time out - * @returns timeout - */ - const uint64_t getTimeOut() const { - return _timeOut; + const std::shared_ptr &getSecurityContext() const { + return ssl_service_; } - /** - * Provides a reference to the port identifier - * @returns port identifier - */ - const std::string getPortId() const { - return _portIdStr; - } - // setPortId - void setPortId(uuid_t id) { - uuid_copy(_portId, id); - char idStr[37]; - uuid_unparse_lower(id, idStr); - _portIdStr = idStr; - } - // getResourceName - std::string getResourceName() { - return "SocketFlowFileProtocol"; - } - // getCodecResourceName - std::string getCodecResourceName() { - return "StandardFlowFileCodec"; - } - // bootstrap the protocol to the ready for transaction state by going through the state machine - bool bootstrap(); - // get peerList - bool getPeerList(std::vector &peer); - // establish - bool establish(); - // handShake - bool handShake(); - // negotiateCodec - bool negotiateCodec(); - // initiateResourceNegotiation - bool initiateResourceNegotiation(); - // initiateCodecResourceNegotiation - bool initiateCodecResourceNegotiation(); - // tearDown - void tearDown(); - // write Request Type - int writeRequestType(RequestType type); - // read Request Type - int readRequestType(RequestType &type); - // read Respond - int readRespond(RespondCode &code, std::string &message); - // write respond - int writeRespond(RespondCode code, std::string message); - // getRespondCodeContext - RespondCodeContext *getRespondCodeContext(RespondCode code) { - for (unsigned int i = 0; i < sizeof(respondCodeContext) / sizeof(RespondCodeContext); i++) { - if (respondCodeContext[i].code == code) { - return &respondCodeContext[i]; - } - } - return NULL; + const std::shared_ptr &getStreamFactory() const { + return stream_factory_; } - // Creation of a new transaction, return the transaction ID if success, - // Return NULL when any error occurs - Transaction *createTransaction(std::string &transactionID, TransferDirection direction); - // Receive the data packet from the transaction - // Return false when any error occurs - bool receive(std::string transactionID, DataPacket *packet, bool &eof); - // Send the data packet from the transaction - // Return false when any error occurs - int16_t send(std::string transactionID, DataPacket *packet, std::shared_ptr flowFile, core::ProcessSession *session); - // Confirm the data that was sent or received by comparing CRC32's of the data sent and the data received. - bool confirm(std::string transactionID); - // Cancel the transaction - void cancel(std::string transactionID); - // Complete the transaction - bool complete(std::string transactionID); - // Error the transaction - void error(std::string transactionID); - // Receive flow files for the process session - void receiveFlowFiles(core::ProcessContext *context, core::ProcessSession *session); - // Transfer flow files for the process session - void transferFlowFiles(core::ProcessContext *context, core::ProcessSession *session); - //! Transfer string for the process session - void transferString(core::ProcessContext *context, core::ProcessSession *session, std::string &payload, std::map attributes); - // deleteTransaction - void deleteTransaction(std::string transactionID); - // Nest Callback Class for write stream - class WriteCallback : public OutputStreamCallback { - public: - WriteCallback(DataPacket *packet) - : _packet(packet) { - } - DataPacket *_packet; - //void process(std::ofstream *stream) { - int64_t process(std::shared_ptr stream) { - uint8_t buffer[8192]; - int len = _packet->_size; - while (len > 0) { - int size = std::min(len, (int) sizeof(buffer)); - int ret = _packet->_transaction->getStream().readData(buffer, size); - if (ret != size) { - _packet->_protocol->logger_->log_error("Site2Site Receive Flow Size %d Failed %d", size, ret); - return -1; - } - stream->write(buffer, size); - len -= size; - } - return len; - } - }; - // Nest Callback Class for read stream - class ReadCallback : public InputStreamCallback { - public: - ReadCallback(DataPacket *packet) - : _packet(packet) { - } - DataPacket *_packet; - int64_t process(std::shared_ptr stream) { - _packet->_size = 0; - uint8_t buffer[8192] = { 0 }; - int readSize; - size_t size = 0; - do { - readSize = stream->read(buffer, 8192); - - if (readSize == 0) { - break; - } - if (readSize < 0) { - return -1; - } - int ret = _packet->_transaction->getStream().writeData(buffer, readSize); - if (ret != readSize) { - _packet->_protocol->logger_->log_error("Site2Site Send Flow Size %d Failed %d", readSize, ret); - return -1; - } - size += readSize; - } while (size < stream->getSize()); - _packet->_size = size; - return size; - } - }; - protected: - private: + std::shared_ptr stream_factory_; - // Mutex for protection - std::mutex mutex_; - // Logger - std::shared_ptr logger_; - // Batch Count - std::atomic _batchCount; - // Batch Size - std::atomic _batchSize; - // Batch Duration in msec - std::atomic _batchDuration; - // Timeout in msec - std::atomic _timeOut; - // Peer Connection - std::unique_ptr peer_; - // portId - uuid_t _portId; - // portIDStr - std::string _portIdStr; - // BATCH_SEND_NANOS - uint64_t _batchSendNanos; - // Peer State - PeerState _peerState; - uint32_t _supportedVersion[5]; - uint32_t _currentVersion; - int _currentVersionIndex; - uint32_t _supportedCodecVersion[1]; - uint32_t _currentCodecVersion; - int _currentCodecVersionIndex; - // commsIdentifier - std::string _commsIdentifier; - // transaction map - std::map _transactionMap; - - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - Site2SiteClientProtocol(const Site2SiteClientProtocol &parent); - Site2SiteClientProtocol &operator=(const Site2SiteClientProtocol &parent); - static std::shared_ptr id_generator_; + std::shared_ptr peer_; + + CLIENT_TYPE client_type_; + + // secore comms + + std::shared_ptr ssl_service_; }; +} /* namespace sitetosite */ } /* namespace minifi */ } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ -#endif + +#endif /* LIBMINIFI_INCLUDE_CORE_SITETOSITE_SITETOSITE_H_ */ diff --git a/libminifi/include/sitetosite/SiteToSiteClient.h b/libminifi/include/sitetosite/SiteToSiteClient.h new file mode 100644 index 0000000000..1b13f2e9fd --- /dev/null +++ b/libminifi/include/sitetosite/SiteToSiteClient.h @@ -0,0 +1,330 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBMINIFI_INCLUDE_CORE_SITETOSITE_SITETOSITECLIENT_H_ +#define LIBMINIFI_INCLUDE_CORE_SITETOSITE_SITETOSITECLIENT_H_ + +#include "Peer.h" +#include "SiteToSite.h" +#include "core/ProcessSession.h" +#include "core/ProcessContext.h" +#include "core/Connectable.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace sitetosite { + +/** + * Represents a piece of data that is to be sent to or that was received from a + * NiFi instance. + */ +class DataPacket { + public: + DataPacket(const std::shared_ptr &logger, const std::shared_ptr &transaction, std::map attributes, const std::string &payload) + : payload_(payload), + logger_reference_(logger) { + _size = 0; + transaction_ = transaction; + _attributes = attributes; + } + std::map _attributes; + uint64_t _size; + std::shared_ptr logger_reference_; + std::shared_ptr transaction_; + const std::string & payload_; + +}; + +class SiteToSiteClient : public core::Connectable { + + public: + + SiteToSiteClient() + : core::Connectable("SitetoSiteClient", 0), + peer_state_(IDLE), + _batchSendNanos(5000000000), + logger_(logging::LoggerFactory::getLogger()) { + _supportedVersion[0] = 5; + _supportedVersion[1] = 4; + _supportedVersion[2] = 3; + _supportedVersion[3] = 2; + _supportedVersion[4] = 1; + _currentVersion = _supportedVersion[0]; + _currentVersionIndex = 0; + _supportedCodecVersion[0] = 1; + _currentCodecVersion = _supportedCodecVersion[0]; + _currentCodecVersionIndex = 0; + } + + virtual ~SiteToSiteClient() { + + } + + /** + * Creates a transaction using the transaction ID and the direction + * @param transactionID transaction identifier + * @param direction direction of transfer + */ + virtual std::shared_ptr createTransaction(std::string &transactionID, TransferDirection direction) = 0; + + /** + * Transfers flow files + * @param direction transfer direction + * @param context process context + * @param session process session + * @returns true if the process succeeded, failure OR exception thrown otherwise + */ + virtual bool transfer(TransferDirection direction, const std::shared_ptr &context, const std::shared_ptr &session) { + if (__builtin_expect(direction == SEND, 1)) { + return transferFlowFiles(context, session); + } else { + return receiveFlowFiles(context, session); + } + } + + /** + * Transfers flow files to server + * @param context process context + * @param session process session + * @returns true if the process succeeded, failure OR exception thrown otherwise + */ + virtual bool transferFlowFiles(const std::shared_ptr &context, const std::shared_ptr &session); + + /** + * Receive flow files from server + * @param context process context + * @param session process session + * @returns true if the process succeeded, failure OR exception thrown otherwise + */ + + // Confirm the data that was sent or received by comparing CRC32's of the data sent and the data received. + // Receive flow files for the process session + bool receiveFlowFiles(const std::shared_ptr &context, const std::shared_ptr &session); + + // Receive the data packet from the transaction + // Return false when any error occurs + bool receive(std::string transactionID, DataPacket *packet, bool &eof); + /** + * Transfers raw data and attributes to server + * @param context process context + * @param session process session + * @param payload data to transmit + * @param attributes + * @returns true if the process succeeded, failure OR exception thrown otherwise + */ + virtual bool transmitPayload(const std::shared_ptr &context, const std::shared_ptr &session, const std::string &payload, + std::map attributes) = 0; + + void setPortId(uuid_t id) { + uuid_copy(port_id_, id); + char idStr[37]; + uuid_unparse_lower(id, idStr); + port_id_str_ = idStr; + } + + /** + * Sets the base peer for this interface. + */ + virtual void setPeer(std::unique_ptr peer) { + peer_ = std::move(peer); + } + + /** + * Provides a reference to the port identifier + * @returns port identifier + */ + const std::string getPortId() const { + return port_id_str_; + } + + /** + * Obtains the peer list and places them into the provided vector + * @param peers peer vector. + * @return true if successful, false otherwise + */ + virtual bool getPeerList(std::vector &peers) = 0; + + /** + * Establishes the interface. + * @return true if successful, false otherwise + */ + virtual bool establish() = 0; + + const std::shared_ptr &getLogger() { + return logger_; + } + + virtual void yield() { + + } + + /** + * Determines if we are connected and operating + */ + virtual bool isRunning() { + return running_; + } + + /** + * Determines if work is available by this connectable + * @return boolean if work is available. + */ + virtual bool isWorkAvailable() { + return true; + } + + virtual bool bootstrap() { + return true; + } + + // Return -1 when any error occurs + virtual int16_t send(std::string transactionID, DataPacket *packet, const std::shared_ptr &flowFile, const std::shared_ptr &session); + + protected: + + // Cancel the transaction + virtual void cancel(std::string transactionID); + // Complete the transaction + virtual bool complete(std::string transactionID); + // Error the transaction + virtual void error(std::string transactionID); + + virtual bool confirm(std::string transactionID); + // deleteTransaction + virtual void deleteTransaction(std::string transactionID); + + virtual void tearDown(); + + // write Request Type + virtual int writeRequestType(RequestType type); + // read Request Type + virtual int readRequestType(RequestType &type); + // read Respond + virtual int readResponse(const std::shared_ptr &transaction, RespondCode &code, std::string &message); + // write respond + virtual int writeResponse(const std::shared_ptr &transaction, RespondCode code, std::string message); + // getRespondCodeContext + virtual RespondCodeContext *getRespondCodeContext(RespondCode code) { + for (unsigned int i = 0; i < sizeof(respondCodeContext) / sizeof(RespondCodeContext); i++) { + if (respondCodeContext[i].code == code) { + return &respondCodeContext[i]; + } + } + return NULL; + } + + // Peer State + PeerState peer_state_; + + std::shared_ptr logger_; + + // portIDStr + std::string port_id_str_; + + // portId + uuid_t port_id_; + + // Peer Connection + std::unique_ptr peer_; + + std::atomic running_; + + // transaction map + std::map> known_transactions_; + + // BATCH_SEND_NANOS + uint64_t _batchSendNanos; + + /*** + * versioning + */ + uint32_t _supportedVersion[5]; + uint32_t _currentVersion; + int _currentVersionIndex; + uint32_t _supportedCodecVersion[1]; + uint32_t _currentCodecVersion; + int _currentCodecVersionIndex; + +}; + +// Nest Callback Class for write stream +class WriteCallback : public OutputStreamCallback { + public: + WriteCallback(DataPacket *packet) + : _packet(packet) { + } + DataPacket *_packet; + //void process(std::ofstream *stream) { + int64_t process(std::shared_ptr stream) { + uint8_t buffer[8192]; + int len = _packet->_size; + while (len > 0) { + int size = std::min(len, (int) sizeof(buffer)); + int ret = _packet->transaction_->getStream().readData(buffer, size); + if (ret != size) { + _packet->logger_reference_->log_error("Site2Site Receive Flow Size %d Failed %d", size, ret); + return -1; + } + stream->write(buffer, size); + len -= size; + } + return len; + } +}; +// Nest Callback Class for read stream +class ReadCallback : public InputStreamCallback { + public: + ReadCallback(DataPacket *packet) + : _packet(packet) { + } + DataPacket *_packet; + int64_t process(std::shared_ptr stream) { + _packet->_size = 0; + uint8_t buffer[8192] = { 0 }; + int readSize; + size_t size = 0; + do { + readSize = stream->read(buffer, 8192); + + if (readSize == 0) { + break; + } + if (readSize < 0) { + return -1; + } + int ret = _packet->transaction_->getStream().writeData(buffer, readSize); + if (ret != readSize) { + _packet->logger_reference_->log_error("Site2Site Send Flow Size %d Failed %d", readSize, ret); + return -1; + } + size += readSize; + } while (size < stream->getSize()); + _packet->_size = size; + return size; + } +}; + +} /* namespace sitetosite */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif /* LIBMINIFI_INCLUDE_CORE_SITETOSITE_SITETOSITECLIENT_H_ */ diff --git a/libminifi/include/sitetosite/SiteToSiteFactory.h b/libminifi/include/sitetosite/SiteToSiteFactory.h new file mode 100644 index 0000000000..dda8337da0 --- /dev/null +++ b/libminifi/include/sitetosite/SiteToSiteFactory.h @@ -0,0 +1,89 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBMINIFI_INCLUDE_SITETOSITE_SITETOSITEFACTORY_H_ +#define LIBMINIFI_INCLUDE_SITETOSITE_SITETOSITEFACTORY_H_ + +#include "RawSocketProtocol.h" +#include "SiteToSite.h" +#include +#include "Peer.h" +#include "SiteToSiteClient.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace sitetosite { + +/** + * Create a streaming peer from the provided client configuration + * @param client_configuration client configuration reference + * @returns SiteToSitePeer + */ +static std::unique_ptr createStreamingPeer(const SiteToSiteClientConfiguration &client_configuration) { + std::unique_ptr str = std::unique_ptr( + client_configuration.getStreamFactory()->createSocket(client_configuration.getPeer()->getHost(), client_configuration.getPeer()->getPort())); + auto peer = std::unique_ptr(new SiteToSitePeer(std::move(str), client_configuration.getPeer()->getHost(), client_configuration.getPeer()->getPort())); + return std::move(peer); +} + +/** + * Creates a raw socket client. + * RawSiteToSiteClient will be instantiated and returned through a unique ptr. + */ +static std::unique_ptr createRawSocket(const SiteToSiteClientConfiguration &client_configuration) { + uuid_t uuid; + client_configuration.getPeer()->getPortId(uuid); + auto ptr = std::unique_ptr(new RawSiteToSiteClient(createStreamingPeer(client_configuration))); + ptr->setPortId(uuid); + return std::move(ptr); +} + +/** + * Returns a client based on the client configuratin's client type. + * Currently only HTTP and RAW are supported. + * @param client_configuration client configuration reference + * @returns site to site client or nullptr. + */ +static std::unique_ptr createClient(const SiteToSiteClientConfiguration &client_configuration) { + uuid_t uuid; + client_configuration.getPeer()->getPortId(uuid); + switch (client_configuration.getClientType()) { + case RAW: + return createRawSocket(client_configuration); + case HTTP: + auto http_protocol = core::ClassLoader::getDefaultClassLoader().instantiateRaw("HttpProtocol", "HttpProtocol"); + if (nullptr != http_protocol) { + auto ptr = std::unique_ptr(static_cast(http_protocol)); + auto peer = std::unique_ptr(new SiteToSitePeer(client_configuration.getPeer()->getHost(), client_configuration.getPeer()->getPort())); + ptr->setPortId(uuid); + ptr->setPeer(std::move(peer)); + return std::move(ptr); + } + return nullptr; + }; + return nullptr; +} + +} /* namespace sitetosite */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif /* LIBMINIFI_INCLUDE_SITETOSITE_SITETOSITEFACTORY_H_ */ diff --git a/libminifi/include/utils/ByteInputCallBack.h b/libminifi/include/utils/ByteArrayCallback.h similarity index 55% rename from libminifi/include/utils/ByteInputCallBack.h rename to libminifi/include/utils/ByteArrayCallback.h index 676d67b99f..354a58655e 100644 --- a/libminifi/include/utils/ByteInputCallBack.h +++ b/libminifi/include/utils/ByteArrayCallback.h @@ -14,8 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef LIBMINIFI_INCLUDE_UTILS_BYTEINPUTCALLBACK_H_ -#define LIBMINIFI_INCLUDE_UTILS_BYTEINPUTCALLBACK_H_ +#ifndef LIBMINIFI_INCLUDE_UTILS_BYTEARRAYCALLBACK_H_ +#define LIBMINIFI_INCLUDE_UTILS_BYTEARRAYCALLBACK_H_ #include #include @@ -40,7 +40,7 @@ class ByteInputCallBack : public InputStreamCallback { } - int64_t process(std::shared_ptr stream) { + virtual int64_t process(std::shared_ptr stream) { stream->seek(0); @@ -56,18 +56,26 @@ class ByteInputCallBack : public InputStreamCallback { } - void write(std::string content) { + virtual void seek(size_t pos) { + + } + + virtual void write(std::string content) { //vec.resize(content.length()); //std::copy(content.begin(), content.end(), std::back_inserter(vec)); vec.assign(content.begin(), content.end()); ptr = &vec[0]; } - char *getBuffer() { - return ptr; + virtual char *getBuffer(size_t pos) { + return ptr + pos; } - const size_t getBufferSize() { + virtual const size_t getRemaining(size_t pos) { + return getBufferSize() - pos; + } + + virtual const size_t getBufferSize() { return vec.size(); } @@ -76,10 +84,60 @@ class ByteInputCallBack : public InputStreamCallback { std::vector vec; }; +/** + * General vector based uint8_t callback. + */ +class ByteOutputCallback : public OutputStreamCallback { + public: + ByteOutputCallback(size_t max_hold) + : ptr(nullptr), + max_size_(max_hold) { + current_str_pos = 0; + size_ = 0; + is_alive_ = true; + } + + virtual ~ByteOutputCallback() { + + } + + virtual int64_t process(std::shared_ptr stream); + + const std::vector to_string(); + + void close(); + + size_t getSize(); + + bool waitingOps(); + + virtual void write(char *data, size_t size); + + size_t readFully(char *buffer, size_t size); + + private: + + inline size_t read_current_str(char *buffer, size_t size); + + inline void preload_next_str(); + + std::atomic is_alive_; + size_t max_size_; + std::condition_variable_any spinner_; + std::recursive_mutex vector_lock_; + std::atomic size_; + char *ptr; + + size_t current_str_pos; + std::string current_str; + std::queue vec; +} +; + } /* namespace utils */ } /* namespace minifi */ } /* namespace nifi */ } /* namespace apache */ } /* namespace org */ -#endif /* LIBMINIFI_INCLUDE_UTILS_BYTEINPUTCALLBACK_H_ */ +#endif /* LIBMINIFI_INCLUDE_UTILS_BYTEARRAYCALLBACK_H_ */ diff --git a/libminifi/include/utils/HTTPClient.h b/libminifi/include/utils/HTTPClient.h index eb68b9da05..b1062efcdb 100644 --- a/libminifi/include/utils/HTTPClient.h +++ b/libminifi/include/utils/HTTPClient.h @@ -17,7 +17,7 @@ */ #ifndef LIBMINIFI_INCLUDE_UTILS_BaseHTTPClient_H_ #define LIBMINIFI_INCLUDE_UTILS_BaseHTTPClient_H_ -#include "ByteInputCallBack.h" +#include "ByteArrayCallback.h" #include "controllers/SSLContextService.h" namespace org { namespace apache { @@ -26,8 +26,25 @@ namespace minifi { namespace utils { struct HTTPUploadCallback { + std::mutex mutex; ByteInputCallBack *ptr; size_t pos; + + size_t getPos() { + std::lock_guard lock(mutex); + return pos; + } +}; + +struct HTTPReadCallback { + std::mutex mutex; + ByteOutputCallback *ptr; + size_t pos; + + size_t getPos() { + std::lock_guard lock(mutex); + return pos; + } }; struct HTTPHeaderResponse { @@ -38,13 +55,18 @@ struct HTTPHeaderResponse { } void append(const std::string &header) { - if (header_tokens_.size() <= max_tokens_) { + if (max_tokens_ == -1 || header_tokens_.size() <= max_tokens_) { header_tokens_.push_back(header); } } + void append(const std::string &key, const std::string &value) { + header_mapping_[key].append(value); + } + int max_tokens_; std::vector header_tokens_; + std::map header_mapping_; static size_t receive_headers(void *buffer, size_t size, size_t nmemb, void *userp) { HTTPHeaderResponse *pHeaders = (HTTPHeaderResponse *) (userp); @@ -62,14 +84,36 @@ struct HTTPHeaderResponse { /** * HTTP Response object */ -struct HTTPRequestResponse { +class HTTPRequestResponse { + std::vector data; + std::condition_variable space_available_; + std::mutex data_mutex_; + + size_t max_queue; + + public: + + const std::vector &getData() { + return data; + } + HTTPRequestResponse(const HTTPRequestResponse &other) + : max_queue(other.max_queue) { + + } + + HTTPRequestResponse(size_t max) + : max_queue(max) { + + } /** * Receive HTTP Response. */ static size_t recieve_write(char * data, size_t size, size_t nmemb, void * p) { - return static_cast(p)->write_content(data, size, nmemb); + HTTPReadCallback *callback = static_cast(p); + callback->ptr->write(data, (size * nmemb)); + return (size * nmemb); } /** @@ -83,16 +127,21 @@ struct HTTPRequestResponse { static size_t send_write(char * data, size_t size, size_t nmemb, void * p) { if (p != 0) { HTTPUploadCallback *callback = (HTTPUploadCallback*) p; - if (callback->pos <= callback->ptr->getBufferSize()) { - char *ptr = callback->ptr->getBuffer(); - int len = callback->ptr->getBufferSize() - callback->pos; - if (len <= 0) { + size_t buffer_size = callback->ptr->getBufferSize(); + if (callback->getPos() <= buffer_size) { + int len = buffer_size - callback->pos; + if (len <= 0) + return 0; + char *ptr = callback->ptr->getBuffer(callback->getPos()); + + if (ptr == nullptr) { return 0; } if (len > size * nmemb) len = size * nmemb; - memcpy(data, callback->ptr->getBuffer() + callback->pos, len); + memcpy(data, ptr, len); callback->pos += len; + callback->ptr->seek(callback->getPos()); return len; } } else { @@ -102,7 +151,21 @@ struct HTTPRequestResponse { return 0; } + int read_data(uint8_t *buf, size_t size) { + size_t size_to_read = size; + if (size_to_read > data.size()) { + size_to_read = data.size(); + } + memcpy(buf, data.data(), size_to_read); + return size_to_read; + } + size_t write_content(char* ptr, size_t size, size_t nmemb) { + + if (data.size() + (size * nmemb) > max_queue) { + std::unique_lock lock(data_mutex_); + space_available_.wait(lock, [&] {return data.size() + (size*nmemb) < max_queue;}); + } data.insert(data.end(), ptr, ptr + size * nmemb); return size * nmemb; } @@ -181,12 +244,17 @@ class BaseHTTPClient { } + virtual const std::map &getParsedHeaders() { + return header_mapping_; + } + protected: int64_t response_code; std::vector response_body_; std::vector headers_; + std::map header_mapping_; - virtual inline bool matches(const std::string &value, const std::string &sregex){ + virtual inline bool matches(const std::string &value, const std::string &sregex) { return false; } @@ -197,7 +265,6 @@ static std::string get_token(utils::BaseHTTPClient *client, std::string username if (nullptr == client) { return ""; } - utils::HTTPRequestResponse content; std::string token; client->setContentType("application/x-www-form-urlencoded"); diff --git a/libminifi/include/utils/StringUtils.h b/libminifi/include/utils/StringUtils.h index 3f8fbeaecd..f50ec91b47 100644 --- a/libminifi/include/utils/StringUtils.h +++ b/libminifi/include/utils/StringUtils.h @@ -81,6 +81,17 @@ class StringUtils { return s; } + /** + * Compares strings by lower casing them. + */ + static inline bool equalsIgnoreCase(const std::string &left, const std::string right) { + if (left.length() == right.length()) { + return std::equal(right.begin(), right.end(), left.begin(), [](unsigned char lc, unsigned char rc) {return std::tolower(lc) == std::tolower(rc);}); + } else { + return false; + } + } + static std::vector split(const std::string &str, const std::string &delimiter) { std::vector result; int last = 0; diff --git a/libminifi/src/Configure.cpp b/libminifi/src/Configure.cpp index db9840e27b..a8ea78f98f 100644 --- a/libminifi/src/Configure.cpp +++ b/libminifi/src/Configure.cpp @@ -46,6 +46,7 @@ const char *Configure::nifi_flowfile_repository_max_storage_time = "nifi.flowfil const char *Configure::nifi_flowfile_repository_directory_default = "nifi.flowfile.repository.directory.default"; const char *Configure::nifi_dbcontent_repository_directory_default = "nifi.database.content.repository.directory.default"; const char *Configure::nifi_remote_input_secure = "nifi.remote.input.secure"; +const char *Configure::nifi_remote_input_http = "nifi.remote.input.http.enabled"; const char *Configure::nifi_security_need_ClientAuth = "nifi.security.need.ClientAuth"; const char *Configure::nifi_security_client_certificate = "nifi.security.client.certificate"; const char *Configure::nifi_security_client_private_key = "nifi.security.client.private.key"; diff --git a/libminifi/src/RemoteProcessorGroupPort.cpp b/libminifi/src/RemoteProcessorGroupPort.cpp index 89b78db3d8..8604c6e2c7 100644 --- a/libminifi/src/RemoteProcessorGroupPort.cpp +++ b/libminifi/src/RemoteProcessorGroupPort.cpp @@ -33,6 +33,9 @@ #include #include #include + +#include "sitetosite/Peer.h" +#include "sitetosite/SiteToSiteFactory.h" #include "json/json.h" #include "json/writer.h" @@ -42,7 +45,6 @@ #include "core/ProcessorNode.h" #include "core/Property.h" #include "core/Relationship.h" -#include "Site2SitePeer.h" #include "utils/HTTPClient.h" namespace org { @@ -59,50 +61,45 @@ core::Property RemoteProcessorGroupPort::port("Port", "Remote Port", ""); core::Property RemoteProcessorGroupPort::portUUID("Port UUID", "Specifies remote NiFi Port UUID.", ""); core::Relationship RemoteProcessorGroupPort::relation; -std::unique_ptr RemoteProcessorGroupPort::getNextProtocol(bool create = true) { - std::unique_ptr nextProtocol = nullptr; +std::unique_ptr RemoteProcessorGroupPort::getNextProtocol(bool create = true) { + std::unique_ptr nextProtocol = nullptr; if (!available_protocols_.try_dequeue(nextProtocol)) { if (create) { // create if (url_.empty()) { - nextProtocol = std::unique_ptr(new Site2SiteClientProtocol(nullptr)); - nextProtocol->setPortId(protocol_uuid_); - std::unique_ptr str = std::unique_ptr(stream_factory_->createSocket(host_, port_)); - std::unique_ptr peer_ = std::unique_ptr(new Site2SitePeer(std::move(str), host_, port_)); - nextProtocol->setPeer(std::move(peer_)); - } else if (site2site_peer_index_ >= 0) { - nextProtocol = std::unique_ptr(new Site2SiteClientProtocol(nullptr)); - minifi::Site2SitePeerStatus peer; - nextProtocol->setPortId(protocol_uuid_); - { - std::lock_guard lock(site2site_peer_mutex_); - peer = site2site_peer_status_list_[this->site2site_peer_index_]; - site2site_peer_index_++; - if (site2site_peer_index_ >= site2site_peer_status_list_.size()) { - site2site_peer_index_ = 0; - } + sitetosite::SiteToSiteClientConfiguration config(stream_factory_, std::make_shared(protocol_uuid_, host_, port_, ssl_service != nullptr), client_type_); + nextProtocol = std::move(sitetosite::createClient(config)); + } else if (peer_index_ >= 0) { + std::lock_guard lock(peer_mutex_); + logger_->log_info("Creating client from peer %d", peer_index_.load()); + sitetosite::SiteToSiteClientConfiguration config(stream_factory_, peers_[this->peer_index_].getPeer(), client_type_); + + peer_index_++; + if (peer_index_ >= peers_.size()) { + peer_index_ = 0; } - logger_->log_info("creating new protocol with %s and %d", peer.host_, peer.port_); - std::unique_ptr str = std::unique_ptr(stream_factory_->createSocket(peer.host_, peer.port_)); - std::unique_ptr peer_ = std::unique_ptr(new Site2SitePeer(std::move(str), peer.host_, peer.port_)); - nextProtocol->setPeer(std::move(peer_)); + + nextProtocol = std::move(sitetosite::createClient(config)); } else { logger_->log_info("Refreshing the peer list since there are none configured."); refreshPeerList(); } } } + logger_->log_info("Obtained protocol from available_protocols_"); return std::move(nextProtocol); } -void RemoteProcessorGroupPort::returnProtocol(std::unique_ptr return_protocol) { - int count = site2site_peer_status_list_.size(); +void RemoteProcessorGroupPort::returnProtocol(std::unique_ptr return_protocol) { + int count = peers_.size(); if (max_concurrent_tasks_ > count) count = max_concurrent_tasks_; if (available_protocols_.size_approx() >= count) { + logger_->log_info("not enqueueing protocol %s", getUUIDStr()); // let the memory be freed return; } + logger_->log_info("enqueueing protocol %s, have a total of %lu", getUUIDStr(), available_protocols_.size_approx()); available_protocols_.enqueue(std::move(return_protocol)); } @@ -118,35 +115,43 @@ void RemoteProcessorGroupPort::initialize() { std::set relationships; relationships.insert(relation); setSupportedRelationships(relationships); - std::lock_guard lock(site2site_peer_mutex_); + + client_type_ = sitetosite::RAW; + std::string http_enabled_str; + if (configure_->get(Configure::nifi_remote_input_http, http_enabled_str)) { + if (utils::StringUtils::StringToBool(http_enabled_str, http_enabled_)) { + client_type_ = sitetosite::HTTP; + } + } + + std::lock_guard lock(peer_mutex_); if (!url_.empty()) { refreshPeerList(); - if (site2site_peer_status_list_.size() > 0) - site2site_peer_index_ = 0; + if (peers_.size() > 0) + peer_index_ = 0; } // populate the site2site protocol for load balancing between them - if (site2site_peer_status_list_.size() > 0) { - int count = site2site_peer_status_list_.size(); + if (peers_.size() > 0) { + int count = peers_.size(); if (max_concurrent_tasks_ > count) count = max_concurrent_tasks_; for (int i = 0; i < count; i++) { - std::unique_ptr nextProtocol = nullptr; - nextProtocol = std::unique_ptr(new Site2SiteClientProtocol(nullptr)); - nextProtocol->setPortId(protocol_uuid_); - minifi::Site2SitePeerStatus peer = site2site_peer_status_list_[this->site2site_peer_index_]; - site2site_peer_index_++; - if (site2site_peer_index_ >= site2site_peer_status_list_.size()) { - site2site_peer_index_ = 0; + std::unique_ptr nextProtocol = nullptr; + sitetosite::SiteToSiteClientConfiguration config(stream_factory_, peers_[this->peer_index_].getPeer(), client_type_); + peer_index_++; + if (peer_index_ >= peers_.size()) { + peer_index_ = 0; } - std::unique_ptr str = std::unique_ptr(stream_factory_->createSocket(peer.host_, peer.port_)); - std::unique_ptr peer_ = std::unique_ptr(new Site2SitePeer(std::move(str), peer.host_, peer.port_)); - nextProtocol->setPeer(std::move(peer_)); + logger_->log_info("Creating client"); + nextProtocol = std::move(sitetosite::createClient(config)); + logger_->log_info("Created client, moving into available protocols"); returnProtocol(std::move(nextProtocol)); } } + logger_->log_info("Finished initialization"); } -void RemoteProcessorGroupPort::onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory) { +void RemoteProcessorGroupPort::onSchedule(const std::shared_ptr &context, const std::shared_ptr &sessionFactory) { std::string value; if (context->getProperty(portUUID.getName(), value)) { uuid_parse(value.c_str(), protocol_uuid_); @@ -161,13 +166,15 @@ void RemoteProcessorGroupPort::onSchedule(core::ProcessContext *context, core::P } } -void RemoteProcessorGroupPort::onTrigger(core::ProcessContext *context, core::ProcessSession *session) { +void RemoteProcessorGroupPort::onTrigger(const std::shared_ptr &context, const std::shared_ptr &session) { + logger_->log_info("On trigger %s", getUUIDStr()); if (!transmitting_) { return; } std::string value; + logger_->log_info("On trigger %s", getUUIDStr()); if (url_.empty()) { if (context->getProperty(hostName.getName(), value) && !value.empty()) { host_ = value; @@ -183,29 +190,20 @@ void RemoteProcessorGroupPort::onTrigger(core::ProcessContext *context, core::Pr uuid_parse(value.c_str(), protocol_uuid_); } - std::unique_ptr protocol_ = nullptr; + std::unique_ptr protocol_ = nullptr; try { + logger_->log_info("get protocol in on trigger"); protocol_ = getNextProtocol(); if (!protocol_) { - logger_->log_info("no protocol"); - context->yield(); - return; - } - logger_->log_info("got protocol"); - if (!protocol_->bootstrap()) { - // bootstrap the client protocol if needed + logger_->log_info("no protocol, yielding"); context->yield(); - std::shared_ptr processor = std::static_pointer_cast(context->getProcessorNode()->getProcessor()); - logger_->log_error("Site2Site bootstrap failed yield period %d peer ", processor->getYieldPeriodMsec()); - return; } - if (direction_ == RECEIVE) { - protocol_->receiveFlowFiles(context, session); - } else { - protocol_->transferFlowFiles(context, session); + if (!protocol_->transfer(direction_, context, session)) { + logger_->log_info("protocol transmission failed, yielding"); + context->yield(); } returnProtocol(std::move(protocol_)); @@ -250,9 +248,9 @@ void RemoteProcessorGroupPort::refreshRemoteSite2SiteInfo() { auto client_ptr = core::ClassLoader::getDefaultClassLoader().instantiateRaw("HTTPClient", "HTTPClient"); if (nullptr == client_ptr) { - logger_->log_error("Could not locate HTTPClient. You do not have cURL support!"); - return; - } + logger_->log_error("Could not locate HTTPClient. You do not have cURL support!"); + return; + } client = std::unique_ptr(dynamic_cast(client_ptr)); client->initialize("GET", fullUrl.c_str(), ssl_service); @@ -273,8 +271,10 @@ void RemoteProcessorGroupPort::refreshRemoteSite2SiteInfo() { Json::Value controllerValue = value["controller"]; if (!controllerValue.empty()) { Json::Value port = controllerValue["remoteSiteListeningPort"]; - if (!port.empty()) + if (client_type_ == sitetosite::CLIENT_TYPE::RAW && !port.empty()) this->site2site_port_ = port.asInt(); + else + this->site2site_port_ = port_; Json::Value secure = controllerValue["siteToSiteSecure"]; if (!secure.empty()) this->site2site_secure_ = secure.asBool(); @@ -294,18 +294,17 @@ void RemoteProcessorGroupPort::refreshPeerList() { if (site2site_port_ == -1) return; - this->site2site_peer_status_list_.clear(); + this->peers_.clear(); + + std::unique_ptr protocol; + sitetosite::SiteToSiteClientConfiguration config(stream_factory_, std::make_shared(protocol_uuid_, host_, site2site_port_, ssl_service != nullptr), client_type_); + protocol = std::move(sitetosite::createClient(config)); - std::unique_ptr protocol; - protocol = std::unique_ptr(new Site2SiteClientProtocol(nullptr)); - protocol->setPortId(protocol_uuid_); - std::unique_ptr str = std::unique_ptr(stream_factory_->createSocket(host_, site2site_port_)); - std::unique_ptr peer_ = std::unique_ptr(new Site2SitePeer(std::move(str), host_, site2site_port_)); - protocol->setPeer(std::move(peer_)); - protocol->getPeerList(site2site_peer_status_list_); + protocol->getPeerList(peers_); - if (site2site_peer_status_list_.size() > 0) - site2site_peer_index_ = 0; + logger_->log_info("Have %d peers", peers_.size()); + if (peers_.size() > 0) + peer_index_ = 0; } } /* namespace minifi */ diff --git a/libminifi/src/SchedulingAgent.cpp b/libminifi/src/SchedulingAgent.cpp index 4a227b5084..2451a08a13 100644 --- a/libminifi/src/SchedulingAgent.cpp +++ b/libminifi/src/SchedulingAgent.cpp @@ -75,7 +75,8 @@ bool SchedulingAgent::hasTooMuchOutGoing(std::shared_ptr proces return processor->flowFilesOutGoingFull(); } -bool SchedulingAgent::onTrigger(std::shared_ptr processor, std::shared_ptr processContext, std::shared_ptr sessionFactory) { +bool SchedulingAgent::onTrigger(const std::shared_ptr &processor, const std::shared_ptr &processContext, + const std::shared_ptr &sessionFactory) { if (processor->isYield()) { logger_->log_debug("Not running %s since it must yield", processor->getName()); return false; @@ -89,14 +90,13 @@ bool SchedulingAgent::onTrigger(std::shared_ptr processor, std: return true; } if (hasTooMuchOutGoing(processor)) { - logger_->log_debug("backpressure applied because too much outgoing"); + logger_->log_debug("backpressure applied because too much outgoing for %s", processor->getUUIDStr()); // need to apply backpressure return true; } processor->incrementActiveTasks(); try { - logger_->log_debug("Triggering %s", processor->getName()); processor->onTrigger(processContext, sessionFactory); processor->decrementActiveTask(); } catch (Exception &exception) { diff --git a/libminifi/src/Site2SiteClientProtocol.cpp b/libminifi/src/Site2SiteClientProtocol.cpp deleted file mode 100644 index 8b8b64620d..0000000000 --- a/libminifi/src/Site2SiteClientProtocol.cpp +++ /dev/null @@ -1,1261 +0,0 @@ -/** - * @file Site2SiteProtocol.cpp - * Site2SiteProtocol class implementation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "io/CRCStream.h" -#include "Site2SitePeer.h" -#include "Site2SiteClientProtocol.h" - -namespace org { -namespace apache { -namespace nifi { -namespace minifi { - -std::shared_ptr Site2SiteClientProtocol::id_generator_ = utils::IdGenerator::getIdGenerator(); -std::shared_ptr Transaction::id_generator_ = utils::IdGenerator::getIdGenerator(); - -bool Site2SiteClientProtocol::establish() { - if (_peerState != IDLE) { - logger_->log_error("Site2Site peer state is not idle while try to establish"); - return false; - } - - bool ret = peer_->Open(); - - if (!ret) { - logger_->log_error("Site2Site peer socket open failed"); - return false; - } - - // Negotiate the version - ret = initiateResourceNegotiation(); - - if (!ret) { - logger_->log_error("Site2Site Protocol Version Negotiation failed"); - return false; - } - - logger_->log_info("Site2Site socket established"); - _peerState = ESTABLISHED; - - return true; -} - -bool Site2SiteClientProtocol::initiateResourceNegotiation() { - // Negotiate the version - if (_peerState != IDLE) { - logger_->log_error("Site2Site peer state is not idle while initiateResourceNegotiation"); - return false; - } - - logger_->log_info("Negotiate protocol version with destination port %s current version %d", _portIdStr.c_str(), _currentVersion); - - int ret = peer_->writeUTF(this->getResourceName()); - - logger_->log_info("result of writing resource name is %i", ret); - if (ret <= 0) { - logger_->log_debug("result of writing resource name is %i", ret); - // tearDown(); - return false; - } - - ret = peer_->write(_currentVersion); - - if (ret <= 0) { - logger_->log_info("result of writing version is %i", ret); - return false; - } - - uint8_t statusCode; - ret = peer_->read(statusCode); - - if (ret <= 0) { - logger_->log_info("result of writing version status code %i", ret); - return false; - } - logger_->log_info("status code is %i", statusCode); - switch (statusCode) { - case RESOURCE_OK: - logger_->log_info("Site2Site Protocol Negotiate protocol version OK"); - return true; - case DIFFERENT_RESOURCE_VERSION: - uint32_t serverVersion; - ret = peer_->read(serverVersion); - if (ret <= 0) { - return false; - } - logger_->log_info("Site2Site Server Response asked for a different protocol version %d", serverVersion); - for (unsigned int i = (_currentVersionIndex + 1); i < sizeof(_supportedVersion) / sizeof(uint32_t); i++) { - if (serverVersion >= _supportedVersion[i]) { - _currentVersion = _supportedVersion[i]; - _currentVersionIndex = i; - return initiateResourceNegotiation(); - } - } - ret = -1; - return false; - case NEGOTIATED_ABORT: - logger_->log_info("Site2Site Negotiate protocol response ABORT"); - ret = -1; - return false; - default: - logger_->log_info("Negotiate protocol response unknown code %d", statusCode); - return true; - } - - return true; -} - -bool Site2SiteClientProtocol::initiateCodecResourceNegotiation() { - // Negotiate the version - if (_peerState != HANDSHAKED) { - logger_->log_error("Site2Site peer state is not handshaked while initiateCodecResourceNegotiation"); - return false; - } - - logger_->log_info("Negotiate Codec version with destination port %s current version %d", _portIdStr.c_str(), _currentCodecVersion); - - int ret = peer_->writeUTF(this->getCodecResourceName()); - - if (ret <= 0) { - logger_->log_debug("result of getCodecResourceName is %i", ret); - return false; - } - - ret = peer_->write(_currentCodecVersion); - - if (ret <= 0) { - logger_->log_debug("result of _currentCodecVersion is %i", ret); - return false; - } - - uint8_t statusCode; - ret = peer_->read(statusCode); - - if (ret <= 0) { - return false; - } - - switch (statusCode) { - case RESOURCE_OK: - logger_->log_info("Site2Site Codec Negotiate version OK"); - return true; - case DIFFERENT_RESOURCE_VERSION: - uint32_t serverVersion; - ret = peer_->read(serverVersion); - if (ret <= 0) { - return false; - } - logger_->log_info("Site2Site Server Response asked for a different codec version %d", serverVersion); - for (unsigned int i = (_currentCodecVersionIndex + 1); i < sizeof(_supportedCodecVersion) / sizeof(uint32_t); i++) { - if (serverVersion >= _supportedCodecVersion[i]) { - _currentCodecVersion = _supportedCodecVersion[i]; - _currentCodecVersionIndex = i; - return initiateCodecResourceNegotiation(); - } - } - ret = -1; - return false; - case NEGOTIATED_ABORT: - logger_->log_info("Site2Site Codec Negotiate response ABORT"); - ret = -1; - return false; - default: - logger_->log_info("Negotiate Codec response unknown code %d", statusCode); - return true; - } - - return true; -} - -bool Site2SiteClientProtocol::handShake() { - if (_peerState != ESTABLISHED) { - logger_->log_error("Site2Site peer state is not established while handshake"); - return false; - } - logger_->log_info("Site2Site Protocol Perform hand shake with destination port %s", _portIdStr.c_str()); - uuid_t uuid; - // Generate the global UUID for the com identify - id_generator_->generate(uuid); - char uuidStr[37]; - uuid_unparse_lower(uuid, uuidStr); - _commsIdentifier = uuidStr; - - int ret = peer_->writeUTF(_commsIdentifier); - - if (ret <= 0) { - return false; - } - - std::map properties; - properties[HandShakePropertyStr[GZIP]] = "false"; - properties[HandShakePropertyStr[PORT_IDENTIFIER]] = _portIdStr; - properties[HandShakePropertyStr[REQUEST_EXPIRATION_MILLIS]] = std::to_string(this->_timeOut); - if (this->_currentVersion >= 5) { - if (this->_batchCount > 0) - properties[HandShakePropertyStr[BATCH_COUNT]] = std::to_string(this->_batchCount); - if (this->_batchSize > 0) - properties[HandShakePropertyStr[BATCH_SIZE]] = std::to_string(this->_batchSize); - if (this->_batchDuration > 0) - properties[HandShakePropertyStr[BATCH_DURATION]] = std::to_string(this->_batchDuration); - } - - if (_currentVersion >= 3) { - ret = peer_->writeUTF(peer_->getURL()); - if (ret <= 0) { - return false; - } - } - - uint32_t size = properties.size(); - ret = peer_->write(size); - if (ret <= 0) { - return false; - } - - std::map::iterator it; - for (it = properties.begin(); it != properties.end(); it++) { - ret = peer_->writeUTF(it->first); - if (ret <= 0) { - return false; - } - ret = peer_->writeUTF(it->second); - if (ret <= 0) { - return false; - } - logger_->log_info("Site2Site Protocol Send handshake properties %s %s", it->first.c_str(), it->second.c_str()); - } - - RespondCode code; - std::string message; - - ret = this->readRespond(code, message); - - if (ret <= 0) { - return false; - } - - switch (code) { - case PROPERTIES_OK: - logger_->log_info("Site2Site HandShake Completed"); - _peerState = HANDSHAKED; - return true; - case PORT_NOT_IN_VALID_STATE: - case UNKNOWN_PORT: - case PORTS_DESTINATION_FULL: - logger_->log_error("Site2Site HandShake Failed because destination port, %s, is either invalid or full", _portIdStr); - ret = -1; - return false; - default: - logger_->log_info("HandShake Failed because of unknown respond code %d", code); - ret = -1; - return false; - } - - return false; -} - -void Site2SiteClientProtocol::tearDown() { - if (_peerState >= ESTABLISHED) { - logger_->log_info("Site2Site Protocol tearDown"); - // need to write shutdown request - writeRequestType(SHUTDOWN); - } - - std::map::iterator it; - for (it = _transactionMap.begin(); it != _transactionMap.end(); it++) { - delete it->second; - } - _transactionMap.clear(); - peer_->Close(); - _peerState = IDLE; -} - -bool Site2SiteClientProtocol::getPeerList(std::vector &peer) { - if (establish() && handShake()) { - int status = this->writeRequestType(REQUEST_PEER_LIST); - - if (status <= 0) { - tearDown(); - return false; - } - - uint32_t number; - status = peer_->read(number); - - if (status <= 0) { - tearDown(); - return false; - } - - for (int i = 0; i < number; i++) { - std::string host; - status = peer_->readUTF(host); - if (status <= 0) { - tearDown(); - return false; - } - uint32_t port; - status = peer_->read(port); - if (status <= 0) { - tearDown(); - return false; - } - uint8_t secure; - status = peer_->read(secure); - if (status <= 0) { - tearDown(); - return false; - } - uint32_t count; - status = peer_->read(count); - if (status <= 0) { - tearDown(); - return false; - } - minifi::Site2SitePeerStatus status; - status.host_ = host; - status.isSecure_ = secure; - status.port_ = port; - peer.push_back(status); - logger_->log_info("Site2Site Peer host %s, port %d, Secure %d", host, port, secure); - } - - tearDown(); - return true; - } else { - tearDown(); - return false; - } -} - -int Site2SiteClientProtocol::writeRequestType(RequestType type) { - if (type >= MAX_REQUEST_TYPE) - return -1; - - return peer_->writeUTF(RequestTypeStr[type]); -} - -int Site2SiteClientProtocol::readRequestType(RequestType &type) { - std::string requestTypeStr; - - int ret = peer_->readUTF(requestTypeStr); - - if (ret <= 0) - return ret; - - for (int i = NEGOTIATE_FLOWFILE_CODEC; i <= SHUTDOWN; i++) { - if (RequestTypeStr[i] == requestTypeStr) { - type = (RequestType) i; - return ret; - } - } - - return -1; -} - -int Site2SiteClientProtocol::readRespond(RespondCode &code, std::string &message) { - uint8_t firstByte; - - int ret = peer_->read(firstByte); - - if (ret <= 0 || firstByte != CODE_SEQUENCE_VALUE_1) - return -1; - - uint8_t secondByte; - - ret = peer_->read(secondByte); - - if (ret <= 0 || secondByte != CODE_SEQUENCE_VALUE_2) - return -1; - - uint8_t thirdByte; - - ret = peer_->read(thirdByte); - - if (ret <= 0) - return ret; - - code = (RespondCode) thirdByte; - - RespondCodeContext *resCode = this->getRespondCodeContext(code); - - if (resCode == NULL) { - // Not a valid respond code - return -1; - } - if (resCode->hasDescription) { - ret = peer_->readUTF(message); - if (ret <= 0) - return -1; - } - return 3 + message.size(); -} - -int Site2SiteClientProtocol::writeRespond(RespondCode code, std::string message) { - RespondCodeContext *resCode = this->getRespondCodeContext(code); - - if (resCode == NULL) { - // Not a valid respond code - return -1; - } - - uint8_t codeSeq[3]; - codeSeq[0] = CODE_SEQUENCE_VALUE_1; - codeSeq[1] = CODE_SEQUENCE_VALUE_2; - codeSeq[2] = (uint8_t) code; - - int ret = peer_->write(codeSeq, 3); - - if (ret != 3) - return -1; - - if (resCode->hasDescription) { - ret = peer_->writeUTF(message); - if (ret > 0) { - return (3 + ret); - } else { - return ret; - } - } else { - return 3; - } -} - -bool Site2SiteClientProtocol::negotiateCodec() { - if (_peerState != HANDSHAKED) { - logger_->log_error("Site2Site peer state is not handshaked while negotiate codec"); - return false; - } - - logger_->log_info("Site2Site Protocol Negotiate Codec with destination port %s", _portIdStr.c_str()); - - int status = this->writeRequestType(NEGOTIATE_FLOWFILE_CODEC); - - if (status <= 0) { - return false; - } - - // Negotiate the codec version - bool ret = initiateCodecResourceNegotiation(); - - if (!ret) { - logger_->log_error("Site2Site Codec Version Negotiation failed"); - return false; - } - - logger_->log_info("Site2Site Codec Completed and move to READY state for data transfer"); - _peerState = READY; - - return true; -} - -bool Site2SiteClientProtocol::bootstrap() { - if (_peerState == READY) - return true; - - tearDown(); - - if (establish() && handShake() && negotiateCodec()) { - logger_->log_info("Site2Site Ready For data transaction"); - return true; - } else { - peer_->yield(); - tearDown(); - return false; - } -} - -Transaction* Site2SiteClientProtocol::createTransaction(std::string &transactionID, TransferDirection direction) { - int ret; - bool dataAvailable; - Transaction *transaction = NULL; - - if (_peerState != READY) { - bootstrap(); - } - - if (_peerState != READY) { - return NULL; - } - - if (direction == RECEIVE) { - ret = writeRequestType(RECEIVE_FLOWFILES); - - if (ret <= 0) { - return NULL; - } - - RespondCode code; - std::string message; - - ret = readRespond(code, message); - - if (ret <= 0) { - return NULL; - } - - org::apache::nifi::minifi::io::CRCStream crcstream(peer_.get()); - switch (code) { - case MORE_DATA: - dataAvailable = true; - logger_->log_info("Site2Site peer indicates that data is available"); - transaction = new Transaction(direction, crcstream); - _transactionMap[transaction->getUUIDStr()] = transaction; - transactionID = transaction->getUUIDStr(); - transaction->setDataAvailable(dataAvailable); - logger_->log_info("Site2Site create transaction %s", transaction->getUUIDStr().c_str()); - return transaction; - case NO_MORE_DATA: - dataAvailable = false; - logger_->log_info("Site2Site peer indicates that no data is available"); - transaction = new Transaction(direction, crcstream); - _transactionMap[transaction->getUUIDStr()] = transaction; - transactionID = transaction->getUUIDStr(); - transaction->setDataAvailable(dataAvailable); - logger_->log_info("Site2Site create transaction %s", transaction->getUUIDStr().c_str()); - return transaction; - default: - logger_->log_info("Site2Site got unexpected response %d when asking for data", code); - return NULL; - } - } else { - ret = writeRequestType(SEND_FLOWFILES); - - if (ret <= 0) { - return NULL; - } else { - org::apache::nifi::minifi::io::CRCStream crcstream(peer_.get()); - transaction = new Transaction(direction, crcstream); - _transactionMap[transaction->getUUIDStr()] = transaction; - transactionID = transaction->getUUIDStr(); - logger_->log_info("Site2Site create transaction %s", transaction->getUUIDStr().c_str()); - return transaction; - } - } -} - -bool Site2SiteClientProtocol::receive(std::string transactionID, DataPacket *packet, bool &eof) { - int ret; - Transaction *transaction = NULL; - - if (_peerState != READY) { - bootstrap(); - } - - if (_peerState != READY) { - return false; - } - - std::map::iterator it = this->_transactionMap.find(transactionID); - - if (it == _transactionMap.end()) { - return false; - } else { - transaction = it->second; - } - - if (transaction->getState() != TRANSACTION_STARTED && transaction->getState() != DATA_EXCHANGED) { - logger_->log_info("Site2Site transaction %s is not at started or exchanged state", transactionID.c_str()); - return false; - } - - if (transaction->getDirection() != RECEIVE) { - logger_->log_info("Site2Site transaction %s direction is wrong", transactionID.c_str()); - return false; - } - - if (!transaction->isDataAvailable()) { - eof = true; - return true; - } - - if (transaction->_transfers > 0) { - // if we already has transfer before, check to see whether another one is available - RespondCode code; - std::string message; - - ret = readRespond(code, message); - - if (ret <= 0) { - return false; - } - if (code == CONTINUE_TRANSACTION) { - logger_->log_info("Site2Site transaction %s peer indicate continue transaction", transactionID.c_str()); - transaction->_dataAvailable = true; - } else if (code == FINISH_TRANSACTION) { - logger_->log_info("Site2Site transaction %s peer indicate finish transaction", transactionID.c_str()); - transaction->_dataAvailable = false; - } else { - logger_->log_info("Site2Site transaction %s peer indicate wrong respond code %d", transactionID.c_str(), code); - return false; - } - } - - if (!transaction->isDataAvailable()) { - eof = true; - return true; - } - - // start to read the packet - uint32_t numAttributes; - ret = transaction->getStream().read(numAttributes); - if (ret <= 0 || numAttributes > MAX_NUM_ATTRIBUTES) { - return false; - } - - // read the attributes - for (unsigned int i = 0; i < numAttributes; i++) { - std::string key; - std::string value; - ret = transaction->getStream().readUTF(key, true); - if (ret <= 0) { - return false; - } - ret = transaction->getStream().readUTF(value, true); - if (ret <= 0) { - return false; - } - packet->_attributes[key] = value; - logger_->log_info("Site2Site transaction %s receives attribute key %s value %s", transactionID.c_str(), key.c_str(), value.c_str()); - } - - uint64_t len; - ret = transaction->getStream().read(len); - if (ret <= 0) { - return false; - } - - packet->_size = len; - transaction->_transfers++; - transaction->_state = DATA_EXCHANGED; - transaction->_bytes += len; - logger_->log_info("Site2Site transaction %s receives flow record %d, total length %d", transactionID.c_str(), transaction->_transfers, transaction->_bytes); - - return true; -} - -int16_t Site2SiteClientProtocol::send(std::string transactionID, DataPacket *packet, std::shared_ptr flowFile, core::ProcessSession *session) { - int ret; - Transaction *transaction = NULL; - - if (flowFile && !flowFile->getResourceClaim()->exists()) { - logger_->log_info("Claim %s does not exist for FlowFile %s", flowFile->getResourceClaim()->getContentFullPath(), flowFile->getUUIDStr()); - return -2; - } - - if (_peerState != READY) { - bootstrap(); - } - - if (_peerState != READY) { - return -1; - } - - std::map::iterator it = this->_transactionMap.find(transactionID); - - if (it == _transactionMap.end()) { - return -1; - } else { - transaction = it->second; - } - - if (transaction->getState() != TRANSACTION_STARTED && transaction->getState() != DATA_EXCHANGED) { - logger_->log_info("Site2Site transaction %s is not at started or exchanged state", transactionID.c_str()); - return -1; - } - - if (transaction->getDirection() != SEND) { - logger_->log_info("Site2Site transaction %s direction is wrong", transactionID.c_str()); - return -1; - } - - if (transaction->_transfers > 0) { - ret = writeRespond(CONTINUE_TRANSACTION, "CONTINUE_TRANSACTION"); - if (ret <= 0) { - return -1; - } - } - - // start to read the packet - uint32_t numAttributes = packet->_attributes.size(); - ret = transaction->getStream().write(numAttributes); - if (ret != 4) { - return -1; - } - - std::map::iterator itAttribute; - for (itAttribute = packet->_attributes.begin(); itAttribute != packet->_attributes.end(); itAttribute++) { - ret = transaction->getStream().writeUTF(itAttribute->first, true); - - if (ret <= 0) { - return -1; - } - ret = transaction->getStream().writeUTF(itAttribute->second, true); - if (ret <= 0) { - return -1; - } - logger_->log_info("Site2Site transaction %s send attribute key %s value %s", transactionID.c_str(), itAttribute->first.c_str(), itAttribute->second.c_str()); - } - - uint64_t len = 0; - if (flowFile) { - len = flowFile->getSize(); - ret = transaction->getStream().write(len); - if (ret != 8) { - logger_->log_info("ret != 8"); - return -1; - } - if (flowFile->getSize() > 0) { - Site2SiteClientProtocol::ReadCallback callback(packet); - session->read(flowFile, &callback); - if (flowFile->getSize() != packet->_size) { - logger_->log_info("MisMatched sizes %d %d", flowFile->getSize(), packet->_size); - return -2; - } - } - if (packet->payload_.length() == 0 && len == 0) { - if (flowFile->getResourceClaim() == nullptr) - logger_->log_debug("no claim"); - else - logger_->log_debug("Flowfile empty %s", flowFile->getResourceClaim()->getContentFullPath()); - } - } else if (packet->payload_.length() > 0) { - len = packet->payload_.length(); - - ret = transaction->getStream().write(len); - if (ret != 8) { - return -1; - } - - ret = transaction->getStream().writeData(reinterpret_cast(const_cast(packet->payload_.c_str())), len); - if (ret != len) { - logger_->log_info("ret != len"); - return -1; - } - packet->_size += len; - } - - transaction->_transfers++; - transaction->_state = DATA_EXCHANGED; - transaction->_bytes += len; - logger_->log_info("Site2Site transaction %s send flow record %d, total length %d", transactionID.c_str(), transaction->_transfers, transaction->_bytes); - - return 0; -} - -void Site2SiteClientProtocol::receiveFlowFiles(core::ProcessContext *context, core::ProcessSession *session) { - uint64_t bytes = 0; - int transfers = 0; - Transaction *transaction = NULL; - - if (_peerState != READY) { - bootstrap(); - } - - if (_peerState != READY) { - context->yield(); - tearDown(); - throw Exception(SITE2SITE_EXCEPTION, "Can not establish handshake with peer"); - } - - // Create the transaction - std::string transactionID; - transaction = createTransaction(transactionID, RECEIVE); - - if (transaction == NULL) { - context->yield(); - tearDown(); - throw Exception(SITE2SITE_EXCEPTION, "Can not create transaction"); - } - - try { - while (true) { - std::map empty; - uint64_t startTime = getTimeMillis(); - std::string payload; - DataPacket packet(this, transaction, empty, payload); - bool eof = false; - - if (!receive(transactionID, &packet, eof)) { - throw Exception(SITE2SITE_EXCEPTION, "Receive Failed"); - } - if (eof) { - // transaction done - break; - } - std::shared_ptr flowFile = std::static_pointer_cast(session->create()); - - if (!flowFile) { - throw Exception(SITE2SITE_EXCEPTION, "Flow File Creation Failed"); - } - std::map::iterator it; - std::string sourceIdentifier; - for (it = packet._attributes.begin(); it != packet._attributes.end(); it++) { - if (it->first == FlowAttributeKey(UUID)) - sourceIdentifier = it->second; - flowFile->addAttribute(it->first, it->second); - } - - if (packet._size > 0) { - Site2SiteClientProtocol::WriteCallback callback(&packet); - session->write(flowFile, &callback); - if (flowFile->getSize() != packet._size) { - throw Exception(SITE2SITE_EXCEPTION, "Receive Size Not Right"); - } - } - core::Relationship relation; // undefined relationship - uint64_t endTime = getTimeMillis(); - std::string transitUri = peer_->getURL() + "/" + sourceIdentifier; - std::string details = "urn:nifi:" + sourceIdentifier + "Remote Host=" + peer_->getHostName(); - session->getProvenanceReporter()->receive(flowFile, transitUri, sourceIdentifier, details, endTime - startTime); - session->transfer(flowFile, relation); - // receive the transfer for the flow record - bytes += packet._size; - transfers++; - } // while true - - if (!confirm(transactionID)) { - throw Exception(SITE2SITE_EXCEPTION, "Confirm Transaction Failed"); - } - if (!complete(transactionID)) { - throw Exception(SITE2SITE_EXCEPTION, "Complete Transaction Failed"); - } - logger_->log_info("Site2Site transaction %s successfully receive flow record %d, content bytes %d", transactionID.c_str(), transfers, bytes); - // we yield the receive if we did not get anything - if (transfers == 0) - context->yield(); - } catch (std::exception &exception) { - if (transaction) - deleteTransaction(transactionID); - context->yield(); - tearDown(); - logger_->log_debug("Caught Exception %s", exception.what()); - throw; - } catch (...) { - if (transaction) - deleteTransaction(transactionID); - context->yield(); - tearDown(); - logger_->log_debug("Caught Exception during Site2SiteClientProtocol::receiveFlowFiles"); - throw; - } - - deleteTransaction(transactionID); - - return; -} - -bool Site2SiteClientProtocol::confirm(std::string transactionID) { - int ret; - Transaction *transaction = NULL; - - if (_peerState != READY) { - bootstrap(); - } - - if (_peerState != READY) { - return false; - } - - std::map::iterator it = this->_transactionMap.find(transactionID); - - if (it == _transactionMap.end()) { - return false; - } else { - transaction = it->second; - } - - if (transaction->getState() == TRANSACTION_STARTED && !transaction->isDataAvailable() && transaction->getDirection() == RECEIVE) { - transaction->_state = TRANSACTION_CONFIRMED; - return true; - } - - if (transaction->getState() != DATA_EXCHANGED) - return false; - - if (transaction->getDirection() == RECEIVE) { - if (transaction->isDataAvailable()) - return false; - // we received a FINISH_TRANSACTION indicator. Send back a CONFIRM_TRANSACTION message - // to peer so that we can verify that the connection is still open. This is a two-phase commit, - // which helps to prevent the chances of data duplication. Without doing this, we may commit the - // session and then when we send the response back to the peer, the peer may have timed out and may not - // be listening. As a result, it will re-send the data. By doing this two-phase commit, we narrow the - // Critical Section involved in this transaction so that rather than the Critical Section being the - // time window involved in the entire transaction, it is reduced to a simple round-trip conversation. - int64_t crcValue = transaction->getCRC(); - std::string crc = std::to_string(crcValue); - logger_->log_info("Site2Site Send confirm with CRC %d to transaction %s", transaction->getCRC(), transactionID.c_str()); - ret = writeRespond(CONFIRM_TRANSACTION, crc); - if (ret <= 0) - return false; - RespondCode code; - std::string message; - readRespond(code, message); - if (ret <= 0) - return false; - - if (code == CONFIRM_TRANSACTION) { - logger_->log_info("Site2Site transaction %s peer confirm transaction", transactionID.c_str()); - transaction->_state = TRANSACTION_CONFIRMED; - return true; - } else if (code == BAD_CHECKSUM) { - logger_->log_info("Site2Site transaction %s peer indicate bad checksum", transactionID.c_str()); - /* - transaction->_state = TRANSACTION_CONFIRMED; - return true; */ - return false; - } else { - logger_->log_info("Site2Site transaction %s peer unknown respond code %d", transactionID.c_str(), code); - return false; - } - } else { - logger_->log_info("Site2Site Send FINISH TRANSACTION for transaction %s", transactionID.c_str()); - ret = writeRespond(FINISH_TRANSACTION, "FINISH_TRANSACTION"); - if (ret <= 0) - return false; - RespondCode code; - std::string message; - readRespond(code, message); - if (ret <= 0) - return false; - - // we've sent a FINISH_TRANSACTION. Now we'll wait for the peer to send a 'Confirm Transaction' response - if (code == CONFIRM_TRANSACTION) { - logger_->log_info("Site2Site transaction %s peer confirm transaction with CRC %s", transactionID.c_str(), message.c_str()); - if (this->_currentVersion > 3) { - int64_t crcValue = transaction->getCRC(); - std::string crc = std::to_string(crcValue); - if (message == crc) { - logger_->log_info("Site2Site transaction %s CRC matched", transactionID.c_str()); - ret = writeRespond(CONFIRM_TRANSACTION, "CONFIRM_TRANSACTION"); - if (ret <= 0) - return false; - transaction->_state = TRANSACTION_CONFIRMED; - return true; - } else { - logger_->log_info("Site2Site transaction %s CRC not matched %s", transactionID.c_str(), crc.c_str()); - ret = writeRespond(BAD_CHECKSUM, "BAD_CHECKSUM"); - return false; - } - } - ret = writeRespond(CONFIRM_TRANSACTION, "CONFIRM_TRANSACTION"); - if (ret <= 0) - return false; - transaction->_state = TRANSACTION_CONFIRMED; - return true; - } else { - logger_->log_info("Site2Site transaction %s peer unknown respond code %d", transactionID.c_str(), code); - return false; - } - return false; - } -} - -void Site2SiteClientProtocol::cancel(std::string transactionID) { - Transaction *transaction = NULL; - - if (_peerState != READY) { - return; - } - - std::map::iterator it = this->_transactionMap.find(transactionID); - - if (it == _transactionMap.end()) { - return; - } else { - transaction = it->second; - } - - if (transaction->getState() == TRANSACTION_CANCELED || transaction->getState() == TRANSACTION_COMPLETED || transaction->getState() == TRANSACTION_ERROR) { - return; - } - - this->writeRespond(CANCEL_TRANSACTION, "Cancel"); - transaction->_state = TRANSACTION_CANCELED; - - tearDown(); - return; -} - -void Site2SiteClientProtocol::deleteTransaction(std::string transactionID) { - Transaction *transaction = NULL; - - std::map::iterator it = this->_transactionMap.find(transactionID); - - if (it == _transactionMap.end()) { - return; - } else { - transaction = it->second; - } - - logger_->log_info("Site2Site delete transaction %s", transaction->getUUIDStr().c_str()); - delete transaction; - _transactionMap.erase(transactionID); -} - -void Site2SiteClientProtocol::error(std::string transactionID) { - Transaction *transaction = NULL; - - std::map::iterator it = this->_transactionMap.find(transactionID); - - if (it == _transactionMap.end()) { - return; - } else { - transaction = it->second; - } - - transaction->_state = TRANSACTION_ERROR; - tearDown(); - return; -} - -// Complete the transaction -bool Site2SiteClientProtocol::complete(std::string transactionID) { - int ret; - Transaction *transaction = NULL; - - if (_peerState != READY) { - bootstrap(); - } - - if (_peerState != READY) { - return false; - } - - std::map::iterator it = this->_transactionMap.find(transactionID); - - if (it == _transactionMap.end()) { - return false; - } else { - transaction = it->second; - } - - if (transaction->getState() != TRANSACTION_CONFIRMED) { - return false; - } - - if (transaction->getDirection() == RECEIVE) { - if (transaction->_transfers == 0) { - transaction->_state = TRANSACTION_COMPLETED; - return true; - } else { - logger_->log_info("Site2Site transaction %s send finished", transactionID.c_str()); - ret = this->writeRespond(TRANSACTION_FINISHED, "Finished"); - if (ret <= 0) { - return false; - } else { - transaction->_state = TRANSACTION_COMPLETED; - return true; - } - } - } else { - RespondCode code; - std::string message; - int ret; - - ret = readRespond(code, message); - - if (ret <= 0) - return false; - - if (code == TRANSACTION_FINISHED) { - logger_->log_info("Site2Site transaction %s peer finished transaction", transactionID.c_str()); - transaction->_state = TRANSACTION_COMPLETED; - return true; - } else { - logger_->log_info("Site2Site transaction %s peer unknown respond code %d", transactionID.c_str(), code); - return false; - } - } -} - -void Site2SiteClientProtocol::transferFlowFiles(core::ProcessContext *context, core::ProcessSession *session) { - std::shared_ptr flow = std::static_pointer_cast(session->get()); - - Transaction *transaction = NULL; - - if (!flow) { - return; - } - - if (_peerState != READY) { - bootstrap(); - } - - if (_peerState != READY) { - context->yield(); - tearDown(); - throw Exception(SITE2SITE_EXCEPTION, "Can not establish handshake with peer"); - } - - // Create the transaction - std::string transactionID; - transaction = createTransaction(transactionID, SEND); - - if (transaction == NULL) { - context->yield(); - tearDown(); - throw Exception(SITE2SITE_EXCEPTION, "Can not create transaction"); - } - - bool continueTransaction = true; - uint64_t startSendingNanos = getTimeNano(); - - try { - while (continueTransaction) { - uint64_t startTime = getTimeMillis(); - std::string payload; - DataPacket packet(this, transaction, flow->getAttributes(), payload); - - int16_t resp = send(transactionID, &packet, flow, session); - if (resp == -1) { - throw Exception(SITE2SITE_EXCEPTION, "Send Failed"); - } - - logger_->log_info("Site2Site transaction %s send flow record %s", transactionID.c_str(), flow->getUUIDStr().c_str()); - if (resp == 0) { - uint64_t endTime = getTimeMillis(); - std::string transitUri = peer_->getURL() + "/" + flow->getUUIDStr(); - std::string details = "urn:nifi:" + flow->getUUIDStr() + "Remote Host=" + peer_->getHostName(); - session->getProvenanceReporter()->send(flow, transitUri, details, endTime - startTime, false); - } - session->remove(flow); - - uint64_t transferNanos = getTimeNano() - startSendingNanos; - if (transferNanos > _batchSendNanos) - break; - - flow = std::static_pointer_cast(session->get()); - - if (!flow) { - continueTransaction = false; - } - } // while true - - if (!confirm(transactionID)) { - std::stringstream ss; - ss << "Confirm Failed for " << transactionID; - throw Exception(SITE2SITE_EXCEPTION, ss.str().c_str()); - } - if (!complete(transactionID)) { - std::stringstream ss; - ss << "Complete Failed for " << transactionID; - throw Exception(SITE2SITE_EXCEPTION, ss.str().c_str()); - } - logger_->log_info("Site2Site transaction %s successfully send flow record %d, content bytes %d", transactionID.c_str(), transaction->_transfers, transaction->_bytes); - } catch (std::exception &exception) { - if (transaction) - deleteTransaction(transactionID); - context->yield(); - tearDown(); - logger_->log_debug("Caught Exception %s", exception.what()); - throw; - } catch (...) { - if (transaction) - deleteTransaction(transactionID); - context->yield(); - tearDown(); - logger_->log_debug("Caught Exception during Site2SiteClientProtocol::transferFlowFiles"); - throw; - } - - deleteTransaction(transactionID); - - return; -} - -void Site2SiteClientProtocol::transferString(core::ProcessContext *context, core::ProcessSession *session, std::string &payload, std::map attributes) { - Transaction *transaction = NULL; - - if (payload.length() <= 0) - return; - - if (_peerState != READY) { - bootstrap(); - } - - if (_peerState != READY) { - context->yield(); - tearDown(); - throw Exception(SITE2SITE_EXCEPTION, "Can not establish handshake with peer"); - } - - // Create the transaction - std::string transactionID; - transaction = createTransaction(transactionID, SEND); - - if (transaction == NULL) { - context->yield(); - tearDown(); - throw Exception(SITE2SITE_EXCEPTION, "Can not create transaction"); - } - - try { - DataPacket packet(this, transaction, attributes, payload); - - int16_t resp = send(transactionID, &packet, nullptr, session); - if (resp == -1) { - throw Exception(SITE2SITE_EXCEPTION, "Send Failed"); - } - logger_->log_info("Site2Site transaction %s send bytes length %d", transactionID.c_str(), payload.length()); - - if (!confirm(transactionID)) { - throw Exception(SITE2SITE_EXCEPTION, "Confirm Failed"); - } - if (!complete(transactionID)) { - throw Exception(SITE2SITE_EXCEPTION, "Complete Failed"); - } - logger_->log_info("Site2Site transaction %s successfully send flow record %d, content bytes %d", transactionID.c_str(), transaction->_transfers, transaction->_bytes); - } catch (std::exception &exception) { - if (transaction) - deleteTransaction(transactionID); - context->yield(); - tearDown(); - logger_->log_debug("Caught Exception %s", exception.what()); - throw; - } catch (...) { - if (transaction) - deleteTransaction(transactionID); - context->yield(); - tearDown(); - logger_->log_debug("Caught Exception during Site2SiteClientProtocol::transferBytes"); - throw; - } - - deleteTransaction(transactionID); - - return; -} - -} /* namespace minifi */ -} /* namespace nifi */ -} /* namespace apache */ -} /* namespace org */ diff --git a/libminifi/src/core/Processor.cpp b/libminifi/src/core/Processor.cpp index d35f283f31..2bceac506c 100644 --- a/libminifi/src/core/Processor.cpp +++ b/libminifi/src/core/Processor.cpp @@ -229,7 +229,7 @@ void Processor::onTrigger(ProcessContext *context, ProcessSessionFactory *sessio } } -void Processor::onTrigger(std::shared_ptr context, std::shared_ptr sessionFactory) { +void Processor::onTrigger(const std::shared_ptr &context, const std::shared_ptr &sessionFactory) { auto session = sessionFactory->createSession(); try { diff --git a/libminifi/src/core/reporting/SiteToSiteProvenanceReportingTask.cpp b/libminifi/src/core/reporting/SiteToSiteProvenanceReportingTask.cpp index c556701dc4..542d026da4 100644 --- a/libminifi/src/core/reporting/SiteToSiteProvenanceReportingTask.cpp +++ b/libminifi/src/core/reporting/SiteToSiteProvenanceReportingTask.cpp @@ -53,8 +53,8 @@ void SiteToSiteProvenanceReportingTask::initialize() { RemoteProcessorGroupPort::initialize(); } -void SiteToSiteProvenanceReportingTask::getJsonReport(core::ProcessContext *context, core::ProcessSession *session, std::vector> &records, - std::string &report) { +void SiteToSiteProvenanceReportingTask::getJsonReport(const std::shared_ptr &context, const std::shared_ptr &session, + std::vector> &records, std::string &report) { Json::Value array; for (auto sercomp : records) { std::shared_ptr record = std::dynamic_pointer_cast(sercomp); @@ -103,37 +103,18 @@ void SiteToSiteProvenanceReportingTask::getJsonReport(core::ProcessContext *cont report = writer.write(array); } -void SiteToSiteProvenanceReportingTask::onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory) { +void SiteToSiteProvenanceReportingTask::onSchedule(const std::shared_ptr &context, const std::shared_ptr &sessionFactory) { } -void SiteToSiteProvenanceReportingTask::onTrigger(core::ProcessContext *context, core::ProcessSession *session) { - std::unique_ptr protocol_ = getNextProtocol(true); - - if (!protocol_) { - context->yield(); - return; - } - +void SiteToSiteProvenanceReportingTask::onTrigger(const std::shared_ptr &context, const std::shared_ptr &session) { logger_->log_debug("SiteToSiteProvenanceReportingTask -- onTrigger"); - - if (!protocol_->bootstrap()) { - // bootstrap the client protocol if needeed - context->yield(); - std::shared_ptr processor = std::static_pointer_cast(context->getProcessorNode()->getProcessor()); - logger_->log_error("Site2Site bootstrap failed yield period %d peer ", processor->getYieldPeriodMsec()); - returnProtocol(std::move(protocol_)); - return; - } - std::vector> records; - logger_->log_debug("batch size %d records", batch_size_); size_t deserialized = batch_size_; std::shared_ptr repo = context->getProvenanceRepository(); std::function()> constructor = []() {return std::make_shared();}; if (!repo->DeSerialize(records, deserialized, constructor) && deserialized == 0) { logger_->log_debug("Not sending because deserialized is %d", deserialized); - returnProtocol(std::move(protocol_)); return; } @@ -143,13 +124,21 @@ void SiteToSiteProvenanceReportingTask::onTrigger(core::ProcessContext *context, std::string jsonStr; this->getJsonReport(context, session, records, jsonStr); if (jsonStr.length() <= 0) { - returnProtocol(std::move(protocol_)); + return; + } + + auto protocol_ = getNextProtocol(true); + + if (!protocol_) { + context->yield(); return; } try { std::map attributes; - protocol_->transferString(context, session, jsonStr, attributes); + if (!protocol_->transmitPayload(context, session, jsonStr, attributes)) { + context->yield(); + } } catch (...) { // if transfer bytes failed, return instead of purge the provenance records return; diff --git a/libminifi/src/core/yaml/YamlConfiguration.cpp b/libminifi/src/core/yaml/YamlConfiguration.cpp index 370da210b1..aab22f7063 100644 --- a/libminifi/src/core/yaml/YamlConfiguration.cpp +++ b/libminifi/src/core/yaml/YamlConfiguration.cpp @@ -296,7 +296,7 @@ void YamlConfiguration::parseRemoteProcessGroupYaml(YAML::Node *rpgNode, core::P YAML::Node currPort = portIter->as(); - this->parsePortYaml(&currPort, group, SEND); + this->parsePortYaml(&currPort, group, sitetosite::SEND); } // for node } YAML::Node outputPorts = currRpgNode["Output Ports"].as(); @@ -306,7 +306,7 @@ void YamlConfiguration::parseRemoteProcessGroupYaml(YAML::Node *rpgNode, core::P YAML::Node currPort = portIter->as(); - this->parsePortYaml(&currPort, group, RECEIVE); + this->parsePortYaml(&currPort, group, sitetosite::RECEIVE); } // for node } } @@ -567,7 +567,7 @@ void YamlConfiguration::parseConnectionYaml(YAML::Node *connectionsNode, core::P } } -void YamlConfiguration::parsePortYaml(YAML::Node *portNode, core::ProcessGroup *parent, TransferDirection direction) { +void YamlConfiguration::parsePortYaml(YAML::Node *portNode, core::ProcessGroup *parent, sitetosite::TransferDirection direction) { uuid_t uuid; std::shared_ptr processor = NULL; std::shared_ptr port = NULL; diff --git a/libminifi/src/io/DataStream.cpp b/libminifi/src/io/DataStream.cpp index 92c7cda12a..14ff5f078a 100644 --- a/libminifi/src/io/DataStream.cpp +++ b/libminifi/src/io/DataStream.cpp @@ -32,6 +32,8 @@ namespace minifi { namespace io { int DataStream::writeData(uint8_t *value, int size) { + if (value == nullptr) + return 0; std::copy(value, value + size, std::back_inserter(buffer)); return size; } diff --git a/libminifi/src/io/FileStream.cpp b/libminifi/src/io/FileStream.cpp index 93a38df062..2b605eb51a 100644 --- a/libminifi/src/io/FileStream.cpp +++ b/libminifi/src/io/FileStream.cpp @@ -136,14 +136,14 @@ int FileStream::readData(uint8_t *buf, int buflen) { file_stream_->read(reinterpret_cast(buf), buflen); if ((file_stream_->rdstate() & (file_stream_->eofbit | file_stream_->failbit)) != 0) { file_stream_->clear(); - size_t prev_offset = offset_; file_stream_->seekg(0, file_stream_->end); file_stream_->seekp(0, file_stream_->end); int len = file_stream_->tellg(); + size_t ret = len - offset_; offset_ = len; length_ = len; logger_->log_info("%s eof bit, ended at %d", path_, offset_); - return offset_-prev_offset; + return ret; } else { offset_ += buflen; file_stream_->seekp(offset_); diff --git a/libminifi/src/io/NonConvertingStream.cpp b/libminifi/src/io/NonConvertingStream.cpp new file mode 100644 index 0000000000..f40bede04e --- /dev/null +++ b/libminifi/src/io/NonConvertingStream.cpp @@ -0,0 +1,190 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "io/NonConvertingStream.h" +#include +#include +#include "io/Serializable.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace io { +/** + * write 4 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ +int NonConvertingStream::write(uint32_t base_value, bool is_little_endian) { + return Serializable::write(base_value, reinterpret_cast(composable_stream_), is_little_endian); +} + +int NonConvertingStream::writeData(uint8_t *value, int size) { + if (composable_stream_ == this) { + return DataStream::writeData(value, size); + } else { + return composable_stream_->writeData(value, size); + } +} + +/** + * write 2 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ +int NonConvertingStream::write(uint16_t base_value, bool is_little_endian) { + return Serializable::write(base_value, reinterpret_cast(composable_stream_), is_little_endian); +} + +/** + * write valueto stream + * @param value non encoded value + * @param len length of value + * @param strema output stream + * @return resulting write size + **/ +int NonConvertingStream::write(uint8_t *value, int len) { + return Serializable::write(value, len, reinterpret_cast(composable_stream_)); +} + +/** + * write 8 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ +int NonConvertingStream::write(uint64_t base_value, bool is_little_endian) { + return Serializable::write(base_value, reinterpret_cast(composable_stream_), is_little_endian); +} + +/** + * write bool to stream + * @param value non encoded value + * @return resulting write size + **/ +int NonConvertingStream::write(bool value) { + uint8_t v = value; + return Serializable::write(v); +} + +/** + * write UTF string to stream + * @param str string to write + * @return resulting write size + **/ +int NonConvertingStream::writeUTF(std::string str, bool widen) { + return Serializable::writeUTF(str, reinterpret_cast(composable_stream_), widen); +} + +/** + * reads a byte from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ +int NonConvertingStream::read(uint8_t &value) { + return Serializable::read(value, reinterpret_cast(composable_stream_)); +} + +/** + * reads two bytes from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ +int NonConvertingStream::read(uint16_t &base_value, bool is_little_endian) { + return Serializable::read(base_value, reinterpret_cast(composable_stream_)); +} + +/** + * reads a byte from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ +int NonConvertingStream::read(char &value) { + return Serializable::read(value, reinterpret_cast(composable_stream_)); +} + +/** + * reads a byte array from the stream + * @param value reference in which will set the result + * @param len length to read + * @param stream stream from which we will read + * @return resulting read size + **/ +int NonConvertingStream::read(uint8_t *value, int len) { + return Serializable::read(value, len, reinterpret_cast(composable_stream_)); +} + +/** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ +int NonConvertingStream::readData(std::vector &buf, int buflen) { + return Serializable::read(&buf[0], buflen, reinterpret_cast(composable_stream_)); +} +/** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ +int NonConvertingStream::readData(uint8_t *buf, int buflen) { + return Serializable::read(buf, buflen, reinterpret_cast(composable_stream_)); +} + +/** + * reads four bytes from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ +int NonConvertingStream::read(uint32_t &value, bool is_little_endian) { + return Serializable::read(value, reinterpret_cast(composable_stream_), is_little_endian); +} + +/** + * reads eight byte from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ +int NonConvertingStream::read(uint64_t &value, bool is_little_endian) { + return Serializable::read(value, reinterpret_cast(composable_stream_), is_little_endian); +} + +/** + * read UTF from stream + * @param str reference string + * @param stream stream from which we will read + * @return resulting read size + **/ +int NonConvertingStream::readUTF(std::string &str, bool widen) { + return Serializable::readUTF(str, reinterpret_cast(composable_stream_), widen); +} +} /* namespace io */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/io/Serializable.cpp b/libminifi/src/io/Serializable.cpp index 5e57c80848..342d2ba535 100644 --- a/libminifi/src/io/Serializable.cpp +++ b/libminifi/src/io/Serializable.cpp @@ -126,7 +126,6 @@ int Serializable::readUTF(std::string &str, DataStream *stream, bool widen) { uint16_t shortLength = 0; ret = read(shortLength, stream); utflen = shortLength; - if (ret <= 0) return ret; } else { diff --git a/libminifi/src/processors/GetTCP.cpp b/libminifi/src/processors/GetTCP.cpp index bcf3d58f9c..0db082f214 100644 --- a/libminifi/src/processors/GetTCP.cpp +++ b/libminifi/src/processors/GetTCP.cpp @@ -98,7 +98,7 @@ void GetTCP::initialize() { setSupportedRelationships(relationships); } -void GetTCP::onSchedule(std::shared_ptr context, std::shared_ptr sessionFactory) { +void GetTCP::onSchedule(const std::shared_ptr &context, const std::shared_ptr &sessionFactory) { std::string value; stay_connected_ = true; if (context->getProperty(EndpointList.getName(), value)) { @@ -222,7 +222,7 @@ void GetTCP::notifyStop() { socket_ring_buffer_.try_dequeue(socket_ptr); } } -void GetTCP::onTrigger(std::shared_ptr context, std::shared_ptr session) { +void GetTCP::onTrigger(const std::shared_ptr &context, const std::shared_ptr &session) { // Perform directory list metrics_->iterations_++; std::lock_guard lock(mutex_); diff --git a/libminifi/src/Site2SitePeer.cpp b/libminifi/src/sitetosite/Peer.cpp similarity index 91% rename from libminifi/src/Site2SitePeer.cpp rename to libminifi/src/sitetosite/Peer.cpp index 7c46564d1b..e8678d76c8 100644 --- a/libminifi/src/Site2SitePeer.cpp +++ b/libminifi/src/sitetosite/Peer.cpp @@ -17,7 +17,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "Site2SitePeer.h" #include #include #include @@ -26,6 +25,8 @@ #include #include #include + +#include "sitetosite/Peer.h" #include "io/ClientSocket.h" #include "io/validation.h" #include "FlowController.h" @@ -34,8 +35,9 @@ namespace org { namespace apache { namespace nifi { namespace minifi { +namespace sitetosite { -bool Site2SitePeer::Open() { +bool SiteToSitePeer::Open() { if (IsNullOrEmpty(host_)) return false; @@ -51,11 +53,12 @@ bool Site2SitePeer::Open() { return true; } -void Site2SitePeer::Close() { +void SiteToSitePeer::Close() { if (stream_ != nullptr) stream_->closeStream(); } +} /* namespace sitetosite */ } /* namespace minifi */ } /* namespace nifi */ } /* namespace apache */ diff --git a/libminifi/src/sitetosite/RawSocketProtocol.cpp b/libminifi/src/sitetosite/RawSocketProtocol.cpp new file mode 100644 index 0000000000..e162a36879 --- /dev/null +++ b/libminifi/src/sitetosite/RawSocketProtocol.cpp @@ -0,0 +1,629 @@ +/** + * Site2SiteProtocol class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sitetosite/RawSocketProtocol.h" +#include "io/CRCStream.h" +#include "sitetosite/Peer.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace sitetosite { + +std::shared_ptr RawSiteToSiteClient::id_generator_ = utils::IdGenerator::getIdGenerator(); +std::shared_ptr Transaction::id_generator_ = utils::IdGenerator::getIdGenerator(); + +bool RawSiteToSiteClient::establish() { + if (peer_state_ != IDLE) { + logger_->log_error("Site2Site peer state is not idle while try to establish"); + return false; + } + + bool ret = peer_->Open(); + + if (!ret) { + logger_->log_error("Site2Site peer socket open failed"); + return false; + } + + // Negotiate the version + ret = initiateResourceNegotiation(); + + if (!ret) { + logger_->log_error("Site2Site Protocol Version Negotiation failed"); + return false; + } + + logger_->log_info("Site2Site socket established"); + peer_state_ = ESTABLISHED; + + return true; +} + +bool RawSiteToSiteClient::initiateResourceNegotiation() { + // Negotiate the version + if (peer_state_ != IDLE) { + logger_->log_error("Site2Site peer state is not idle while initiateResourceNegotiation"); + return false; + } + + logger_->log_info("Negotiate protocol version with destination port %s current version %d", port_id_str_, _currentVersion); + + int ret = peer_->writeUTF(getResourceName()); + + logger_->log_info("result of writing resource name is %i", ret); + if (ret <= 0) { + logger_->log_debug("result of writing resource name is %i", ret); + // tearDown(); + return false; + } + + ret = peer_->write(_currentVersion); + + if (ret <= 0) { + logger_->log_info("result of writing version is %i", ret); + return false; + } + + uint8_t statusCode; + ret = peer_->read(statusCode); + + if (ret <= 0) { + logger_->log_info("result of writing version status code %i", ret); + return false; + } + logger_->log_info("status code is %i", statusCode); + switch (statusCode) { + case RESOURCE_OK: + logger_->log_info("Site2Site Protocol Negotiate protocol version OK"); + return true; + case DIFFERENT_RESOURCE_VERSION: + uint32_t serverVersion; + ret = peer_->read(serverVersion); + if (ret <= 0) { + return false; + } + logger_->log_info("Site2Site Server Response asked for a different protocol version %d", serverVersion); + for (unsigned int i = (_currentVersionIndex + 1); i < sizeof(_supportedVersion) / sizeof(uint32_t); i++) { + if (serverVersion >= _supportedVersion[i]) { + _currentVersion = _supportedVersion[i]; + _currentVersionIndex = i; + return initiateResourceNegotiation(); + } + } + ret = -1; + return false; + case NEGOTIATED_ABORT: + logger_->log_info("Site2Site Negotiate protocol response ABORT"); + ret = -1; + return false; + default: + logger_->log_info("Negotiate protocol response unknown code %d", statusCode); + return true; + } + + return true; +} + +bool RawSiteToSiteClient::initiateCodecResourceNegotiation() { + // Negotiate the version + if (peer_state_ != HANDSHAKED) { + logger_->log_error("Site2Site peer state is not handshaked while initiateCodecResourceNegotiation"); + return false; + } + + logger_->log_info("Negotiate Codec version with destination port %s current version %d", port_id_str_, _currentCodecVersion); + + int ret = peer_->writeUTF(getCodecResourceName()); + + if (ret <= 0) { + logger_->log_debug("result of getCodecResourceName is %i", ret); + return false; + } + + ret = peer_->write(_currentCodecVersion); + + if (ret <= 0) { + logger_->log_debug("result of _currentCodecVersion is %i", ret); + return false; + } + + uint8_t statusCode; + ret = peer_->read(statusCode); + + if (ret <= 0) { + return false; + } + + switch (statusCode) { + case RESOURCE_OK: + logger_->log_info("Site2Site Codec Negotiate version OK"); + return true; + case DIFFERENT_RESOURCE_VERSION: + uint32_t serverVersion; + ret = peer_->read(serverVersion); + if (ret <= 0) { + return false; + } + logger_->log_info("Site2Site Server Response asked for a different codec version %d", serverVersion); + for (unsigned int i = (_currentCodecVersionIndex + 1); i < sizeof(_supportedCodecVersion) / sizeof(uint32_t); i++) { + if (serverVersion >= _supportedCodecVersion[i]) { + _currentCodecVersion = _supportedCodecVersion[i]; + _currentCodecVersionIndex = i; + return initiateCodecResourceNegotiation(); + } + } + ret = -1; + return false; + case NEGOTIATED_ABORT: + logger_->log_info("Site2Site Codec Negotiate response ABORT"); + ret = -1; + return false; + default: + logger_->log_info("Negotiate Codec response unknown code %d", statusCode); + return true; + } + + return true; +} + +bool RawSiteToSiteClient::handShake() { + if (peer_state_ != ESTABLISHED) { + logger_->log_error("Site2Site peer state is not established while handshake"); + return false; + } + logger_->log_info("Site2Site Protocol Perform hand shake with destination port %s", port_id_str_); + uuid_t uuid; + // Generate the global UUID for the com identify + id_generator_->generate(uuid); + char uuidStr[37]; + uuid_unparse_lower(uuid, uuidStr); + _commsIdentifier = uuidStr; + + int ret = peer_->writeUTF(_commsIdentifier); + + if (ret <= 0) { + return false; + } + + std::map properties; + properties[HandShakePropertyStr[GZIP]] = "false"; + properties[HandShakePropertyStr[PORT_IDENTIFIER]] = port_id_str_; + properties[HandShakePropertyStr[REQUEST_EXPIRATION_MILLIS]] = std::to_string(_timeOut); + if (_currentVersion >= 5) { + if (_batchCount > 0) + properties[HandShakePropertyStr[BATCH_COUNT]] = std::to_string(_batchCount); + if (_batchSize > 0) + properties[HandShakePropertyStr[BATCH_SIZE]] = std::to_string(_batchSize); + if (_batchDuration > 0) + properties[HandShakePropertyStr[BATCH_DURATION]] = std::to_string(_batchDuration); + } + + if (_currentVersion >= 3) { + ret = peer_->writeUTF(peer_->getURL()); + if (ret <= 0) { + return false; + } + } + + uint32_t size = properties.size(); + ret = peer_->write(size); + if (ret <= 0) { + return false; + } + + std::map::iterator it; + for (it = properties.begin(); it != properties.end(); it++) { + ret = peer_->writeUTF(it->first); + if (ret <= 0) { + return false; + } + ret = peer_->writeUTF(it->second); + if (ret <= 0) { + return false; + } + logger_->log_info("Site2Site Protocol Send handshake properties %s %s", it->first.c_str(), it->second.c_str()); + } + + RespondCode code; + std::string message; + + ret = readRespond(nullptr, code, message); + + if (ret <= 0) { + return false; + } + + switch (code) { + case PROPERTIES_OK: + logger_->log_info("Site2Site HandShake Completed"); + peer_state_ = HANDSHAKED; + return true; + case PORT_NOT_IN_VALID_STATE: + case UNKNOWN_PORT: + case PORTS_DESTINATION_FULL: + logger_->log_error("Site2Site HandShake Failed because destination port, %s, is either invalid or full", port_id_str_); + ret = -1; + return false; + default: + logger_->log_info("HandShake Failed because of unknown respond code %d", code); + ret = -1; + return false; + } + + return false; +} + +void RawSiteToSiteClient::tearDown() { + if (peer_state_ >= ESTABLISHED) { + logger_->log_info("Site2Site Protocol tearDown"); + // need to write shutdown request + writeRequestType(SHUTDOWN); + } + + known_transactions_.clear(); + peer_->Close(); + peer_state_ = IDLE; +} + +bool RawSiteToSiteClient::getPeerList(std::vector &peers) { + if (establish() && handShake()) { + int status = writeRequestType(REQUEST_PEER_LIST); + + if (status <= 0) { + tearDown(); + return false; + } + + uint32_t number; + status = peer_->read(number); + + if (status <= 0) { + tearDown(); + return false; + } + + for (int i = 0; i < number; i++) { + std::string host; + status = peer_->readUTF(host); + if (status <= 0) { + tearDown(); + return false; + } + uint32_t port; + status = peer_->read(port); + if (status <= 0) { + tearDown(); + return false; + } + uint8_t secure; + status = peer_->read(secure); + if (status <= 0) { + tearDown(); + return false; + } + uint32_t count; + status = peer_->read(count); + if (status <= 0) { + tearDown(); + return false; + } + PeerStatus status(std::make_shared(port_id_, host, port, secure), count, true); + peers.push_back(std::move(status)); + logger_->log_info("Site2Site Peer host %s, port %d, Secure %d", host, port, secure); + } + + tearDown(); + return true; + } else { + tearDown(); + return false; + } +} + +int RawSiteToSiteClient::writeRequestType(RequestType type) { + if (type >= MAX_REQUEST_TYPE) + return -1; + + return peer_->writeUTF(RequestTypeStr[type]); +} + +int RawSiteToSiteClient::readRequestType(RequestType &type) { + std::string requestTypeStr; + + int ret = peer_->readUTF(requestTypeStr); + + if (ret <= 0) + return ret; + + for (int i = NEGOTIATE_FLOWFILE_CODEC; i <= SHUTDOWN; i++) { + if (RequestTypeStr[i] == requestTypeStr) { + type = (RequestType) i; + return ret; + } + } + + return -1; +} + +int RawSiteToSiteClient::readRespond(const std::shared_ptr &transaction, RespondCode &code, std::string &message) { + uint8_t firstByte; + + int ret = peer_->read(firstByte); + + if (ret <= 0 || firstByte != CODE_SEQUENCE_VALUE_1) + return -1; + + uint8_t secondByte; + + ret = peer_->read(secondByte); + + if (ret <= 0 || secondByte != CODE_SEQUENCE_VALUE_2) + return -1; + + uint8_t thirdByte; + + ret = peer_->read(thirdByte); + + if (ret <= 0) + return ret; + + code = (RespondCode) thirdByte; + + RespondCodeContext *resCode = getRespondCodeContext(code); + + if (resCode == NULL) { + // Not a valid respond code + return -1; + } + if (resCode->hasDescription) { + ret = peer_->readUTF(message); + if (ret <= 0) + return -1; + } + return 3 + message.size(); +} + +int RawSiteToSiteClient::writeRespond(const std::shared_ptr &transaction, RespondCode code, std::string message) { + RespondCodeContext *resCode = getRespondCodeContext(code); + + if (resCode == NULL) { + // Not a valid respond code + return -1; + } + + uint8_t codeSeq[3]; + codeSeq[0] = CODE_SEQUENCE_VALUE_1; + codeSeq[1] = CODE_SEQUENCE_VALUE_2; + codeSeq[2] = (uint8_t) code; + + int ret = peer_->write(codeSeq, 3); + + if (ret != 3) + return -1; + + if (resCode->hasDescription) { + ret = peer_->writeUTF(message); + if (ret > 0) { + return (3 + ret); + } else { + return ret; + } + } else { + return 3; + } +} + +bool RawSiteToSiteClient::negotiateCodec() { + if (peer_state_ != HANDSHAKED) { + logger_->log_error("Site2Site peer state is not handshaked while negotiate codec"); + return false; + } + + logger_->log_info("Site2Site Protocol Negotiate Codec with destination port %s", port_id_str_); + + int status = writeRequestType(NEGOTIATE_FLOWFILE_CODEC); + + if (status <= 0) { + return false; + } + + // Negotiate the codec version + bool ret = initiateCodecResourceNegotiation(); + + if (!ret) { + logger_->log_error("Site2Site Codec Version Negotiation failed"); + return false; + } + + logger_->log_info("Site2Site Codec Completed and move to READY state for data transfer"); + peer_state_ = READY; + + return true; +} + +bool RawSiteToSiteClient::bootstrap() { + if (peer_state_ == READY) + return true; + + tearDown(); + + if (establish() && handShake() && negotiateCodec()) { + logger_->log_info("Site2Site Ready For data transaction"); + return true; + } else { + peer_->yield(); + tearDown(); + return false; + } +} + +std::shared_ptr RawSiteToSiteClient::createTransaction(std::string &transactionID, TransferDirection direction) { + int ret; + bool dataAvailable; + std::shared_ptr transaction = nullptr; + + if (peer_state_ != READY) { + bootstrap(); + } + + if (peer_state_ != READY) { + return transaction; + } + + if (direction == RECEIVE) { + ret = writeRequestType(RECEIVE_FLOWFILES); + + if (ret <= 0) { + return transaction; + } + + RespondCode code; + std::string message; + + ret = readRespond(nullptr, code, message); + + if (ret <= 0) { + return transaction; + } + + org::apache::nifi::minifi::io::CRCStream crcstream(peer_.get()); + switch (code) { + case MORE_DATA: + dataAvailable = true; + logger_->log_info("Site2Site peer indicates that data is available"); + transaction = std::make_shared(direction, crcstream); + known_transactions_[transaction->getUUIDStr()] = transaction; + transactionID = transaction->getUUIDStr(); + transaction->setDataAvailable(dataAvailable); + logger_->log_info("Site2Site create transaction %s", transaction->getUUIDStr().c_str()); + return transaction; + case NO_MORE_DATA: + dataAvailable = false; + logger_->log_info("Site2Site peer indicates that no data is available"); + transaction = std::make_shared(direction, crcstream); + known_transactions_[transaction->getUUIDStr()] = transaction; + transactionID = transaction->getUUIDStr(); + transaction->setDataAvailable(dataAvailable); + logger_->log_info("Site2Site create transaction %s", transaction->getUUIDStr().c_str()); + return transaction; + default: + logger_->log_info("Site2Site got unexpected response %d when asking for data", code); + return NULL; + } + } else { + ret = writeRequestType(SEND_FLOWFILES); + + if (ret <= 0) { + return NULL; + } else { + org::apache::nifi::minifi::io::CRCStream crcstream(peer_.get()); + transaction = std::make_shared(direction, crcstream); + known_transactions_[transaction->getUUIDStr()] = transaction; + transactionID = transaction->getUUIDStr(); + logger_->log_info("Site2Site create transaction %s", transaction->getUUIDStr().c_str()); + return transaction; + } + } +} + +bool RawSiteToSiteClient::transmitPayload(const std::shared_ptr &context, const std::shared_ptr &session, const std::string &payload, + std::map attributes) { + std::shared_ptr transaction = NULL; + + if (payload.length() <= 0) + return false; + + if (peer_state_ != READY) { + if (!bootstrap()) { + return false; + } + } + + if (peer_state_ != READY) { + context->yield(); + tearDown(); + throw Exception(SITE2SITE_EXCEPTION, "Can not establish handshake with peer"); + } + + // Create the transaction + std::string transactionID; + transaction = createTransaction(transactionID, SEND); + + if (transaction == NULL) { + context->yield(); + tearDown(); + throw Exception(SITE2SITE_EXCEPTION, "Can not create transaction"); + } + + try { + DataPacket packet(getLogger(), transaction, attributes, payload); + + int16_t resp = send(transactionID, &packet, nullptr, session); + if (resp == -1) { + throw Exception(SITE2SITE_EXCEPTION, "Send Failed"); + } + logger_->log_info("Site2Site transaction %s send bytes length %d", transactionID.c_str(), payload.length()); + + if (!confirm(transactionID)) { + throw Exception(SITE2SITE_EXCEPTION, "Confirm Failed"); + } + if (!complete(transactionID)) { + throw Exception(SITE2SITE_EXCEPTION, "Complete Failed"); + } + logger_->log_info("Site2Site transaction %s successfully send flow record %d, content bytes %d", transactionID.c_str(), transaction->current_transfers_, transaction->_bytes); + } catch (std::exception &exception) { + if (transaction) + deleteTransaction(transactionID); + context->yield(); + tearDown(); + logger_->log_debug("Caught Exception %s", exception.what()); + throw; + } catch (...) { + if (transaction) + deleteTransaction(transactionID); + context->yield(); + tearDown(); + logger_->log_debug("Caught Exception during RawSiteToSiteClient::transferBytes"); + throw; + } + + deleteTransaction(transactionID); + + return true; +} + +} /* namespace sitetosite */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/sitetosite/SiteToSiteClient.cpp b/libminifi/src/sitetosite/SiteToSiteClient.cpp new file mode 100644 index 0000000000..e3a47193aa --- /dev/null +++ b/libminifi/src/sitetosite/SiteToSiteClient.cpp @@ -0,0 +1,773 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "sitetosite/SiteToSiteClient.h" +#include +#include +#include +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace sitetosite { + +int SiteToSiteClient::writeRequestType(RequestType type) { + if (type >= MAX_REQUEST_TYPE) + return -1; + + return peer_->writeUTF(RequestTypeStr[type]); +} + +int SiteToSiteClient::readRequestType(RequestType &type) { + std::string requestTypeStr; + + int ret = peer_->readUTF(requestTypeStr); + + if (ret <= 0) + return ret; + + for (int i = NEGOTIATE_FLOWFILE_CODEC; i <= SHUTDOWN; i++) { + if (RequestTypeStr[i] == requestTypeStr) { + type = (RequestType) i; + return ret; + } + } + + return -1; +} + +int SiteToSiteClient::readResponse(const std::shared_ptr &transaction, RespondCode &code, std::string &message) { + uint8_t firstByte; + + int ret = peer_->read(firstByte); + + if (ret <= 0 || firstByte != CODE_SEQUENCE_VALUE_1) + return -1; + + uint8_t secondByte; + + ret = peer_->read(secondByte); + + if (ret <= 0 || secondByte != CODE_SEQUENCE_VALUE_2) + return -1; + + uint8_t thirdByte; + + ret = peer_->read(thirdByte); + + if (ret <= 0) + return ret; + + code = (RespondCode) thirdByte; + + RespondCodeContext *resCode = this->getRespondCodeContext(code); + + if (resCode == NULL) { + // Not a valid respond code + return -1; + } + if (resCode->hasDescription) { + ret = peer_->readUTF(message); + if (ret <= 0) + return -1; + } + return 3 + message.size(); +} + +void SiteToSiteClient::deleteTransaction(std::string transactionID) { + std::shared_ptr transaction = NULL; + + std::map >::iterator it = this->known_transactions_.find(transactionID); + + if (it == known_transactions_.end()) { + return; + } else { + transaction = it->second; + } + + logger_->log_info("Site2Site delete transaction %s", transaction->getUUIDStr().c_str()); + known_transactions_.erase(transactionID); +} + +int SiteToSiteClient::writeResponse(const std::shared_ptr &transaction, RespondCode code, std::string message) { + RespondCodeContext *resCode = this->getRespondCodeContext(code); + + if (resCode == NULL) { + // Not a valid respond code + return -1; + } + + uint8_t codeSeq[3]; + codeSeq[0] = CODE_SEQUENCE_VALUE_1; + codeSeq[1] = CODE_SEQUENCE_VALUE_2; + codeSeq[2] = (uint8_t) code; + + int ret = peer_->write(codeSeq, 3); + + if (ret != 3) + return -1; + + if (resCode->hasDescription) { + ret = peer_->writeUTF(message); + if (ret > 0) { + return (3 + ret); + } else { + return ret; + } + } else { + return 3; + } +} + +void SiteToSiteClient::tearDown() { + if (peer_state_ >= ESTABLISHED) { + logger_->log_info("Site2Site Protocol tearDown"); + // need to write shutdown request + writeRequestType(SHUTDOWN); + } + + known_transactions_.clear(); + peer_->Close(); + peer_state_ = IDLE; +} + +bool SiteToSiteClient::transferFlowFiles(const std::shared_ptr &context, const std::shared_ptr &session) { + std::shared_ptr flow = std::static_pointer_cast(session->get()); + + std::shared_ptr transaction = NULL; + + if (!flow) { + return false; + } + + if (peer_state_ != READY) { + if (!bootstrap()) + return false; + } + + if (peer_state_ != READY) { + context->yield(); + tearDown(); + throw Exception(SITE2SITE_EXCEPTION, "Can not establish handshake with peer"); + } + + // Create the transaction + std::string transactionID; + transaction = createTransaction(transactionID, SEND); + if (transaction == NULL) { + context->yield(); + tearDown(); + throw Exception(SITE2SITE_EXCEPTION, "Can not create transaction"); + } + + bool continueTransaction = true; + uint64_t startSendingNanos = getTimeNano(); + + try { + while (continueTransaction) { + uint64_t startTime = getTimeMillis(); + std::string payload; + DataPacket packet(getLogger(), transaction, flow->getAttributes(), payload); + + int16_t resp = send(transactionID, &packet, flow, session); + if (resp == -1) { + throw Exception(SITE2SITE_EXCEPTION, "Send Failed"); + } + + logger_->log_info("Site2Site transaction %s send flow record %s", transactionID.c_str(), flow->getUUIDStr().c_str()); + if (resp == 0) { + uint64_t endTime = getTimeMillis(); + std::string transitUri = peer_->getURL() + "/" + flow->getUUIDStr(); + std::string details = "urn:nifi:" + flow->getUUIDStr() + "Remote Host=" + peer_->getHostName(); + session->getProvenanceReporter()->send(flow, transitUri, details, endTime - startTime, false); + } + session->remove(flow); + + uint64_t transferNanos = getTimeNano() - startSendingNanos; + if (transferNanos > _batchSendNanos) + break; + + flow = std::static_pointer_cast(session->get()); + + if (!flow) { + continueTransaction = false; + } + } // while true + + if (!confirm(transactionID)) { + std::stringstream ss; + ss << "Confirm Failed for " << transactionID; + throw Exception(SITE2SITE_EXCEPTION, ss.str().c_str()); + } + if (!complete(transactionID)) { + std::stringstream ss; + ss << "Complete Failed for " << transactionID; + throw Exception(SITE2SITE_EXCEPTION, ss.str().c_str()); + } + logger_->log_info("Site2Site transaction %s successfully send flow record %d, content bytes %d", transactionID.c_str(), transaction->total_transfers_, transaction->_bytes); + } catch (std::exception &exception) { + if (transaction) + deleteTransaction(transactionID); + context->yield(); + tearDown(); + logger_->log_debug("Caught Exception %s", exception.what()); + throw; + } catch (...) { + if (transaction) + deleteTransaction(transactionID); + context->yield(); + tearDown(); + logger_->log_debug("Caught Exception during SiteToSiteClient::transferFlowFiles"); + throw; + } + + deleteTransaction(transactionID); + + return true; +} + +bool SiteToSiteClient::confirm(std::string transactionID) { + int ret; + std::shared_ptr transaction = NULL; + + if (peer_state_ != READY) { + bootstrap(); + } + + if (peer_state_ != READY) { + return false; + } + + std::map >::iterator it = this->known_transactions_.find(transactionID); + + if (it == known_transactions_.end()) { + return false; + } else { + transaction = it->second; + } + + if (transaction->getState() == TRANSACTION_STARTED && !transaction->isDataAvailable() && transaction->getDirection() == RECEIVE) { + transaction->_state = TRANSACTION_CONFIRMED; + return true; + } + + if (transaction->getState() != DATA_EXCHANGED) + return false; + + if (transaction->getDirection() == RECEIVE) { + if (transaction->isDataAvailable()) + return false; + // we received a FINISH_TRANSACTION indicator. Send back a CONFIRM_TRANSACTION message + // to peer so that we can verify that the connection is still open. This is a two-phase commit, + // which helps to prevent the chances of data duplication. Without doing this, we may commit the + // session and then when we send the response back to the peer, the peer may have timed out and may not + // be listening. As a result, it will re-send the data. By doing this two-phase commit, we narrow the + // Critical Section involved in this transaction so that rather than the Critical Section being the + // time window involved in the entire transaction, it is reduced to a simple round-trip conversation. + int64_t crcValue = transaction->getCRC(); + std::string crc = std::to_string(crcValue); + logger_->log_info("Site2Site Send confirm with CRC %d to transaction %s", transaction->getCRC(), transactionID.c_str()); + ret = writeResponse(transaction, CONFIRM_TRANSACTION, crc); + if (ret <= 0) + return false; + RespondCode code; + std::string message; + readResponse(transaction, code, message); + if (ret <= 0) + return false; + + if (code == CONFIRM_TRANSACTION) { + logger_->log_info("Site2Site transaction %s peer confirm transaction", transactionID.c_str()); + transaction->_state = TRANSACTION_CONFIRMED; + return true; + } else if (code == BAD_CHECKSUM) { + logger_->log_info("Site2Site transaction %s peer indicate bad checksum", transactionID.c_str()); + return false; + } else { + logger_->log_info("Site2Site transaction %s peer unknown respond code %d", transactionID.c_str(), code); + return false; + } + } else { + logger_->log_info("Site2Site Send FINISH TRANSACTION for transaction %s", transactionID.c_str()); + ret = writeResponse(transaction, FINISH_TRANSACTION, "FINISH_TRANSACTION"); + if (ret <= 0) + return false; + RespondCode code; + std::string message; + readResponse(transaction, code, message); + + // we've sent a FINISH_TRANSACTION. Now we'll wait for the peer to send a 'Confirm Transaction' response + if (code == CONFIRM_TRANSACTION) { + logger_->log_info("Site2Site transaction %s peer confirm transaction with CRC %s", transactionID.c_str(), message.c_str()); + if (this->_currentVersion > 3) { + int64_t crcValue = transaction->getCRC(); + std::string crc = std::to_string(crcValue); + if (message == crc) { + logger_->log_info("Site2Site transaction %s CRC matched", transactionID.c_str()); + ret = writeResponse(transaction, CONFIRM_TRANSACTION, "CONFIRM_TRANSACTION"); + if (ret <= 0) + return false; + transaction->_state = TRANSACTION_CONFIRMED; + return true; + } else { + logger_->log_info("Site2Site transaction %s CRC not matched %s", transactionID.c_str(), crc.c_str()); + ret = writeResponse(transaction, BAD_CHECKSUM, "BAD_CHECKSUM"); + return false; + } + } + ret = writeResponse(transaction, CONFIRM_TRANSACTION, "CONFIRM_TRANSACTION"); + if (ret <= 0) + return false; + transaction->_state = TRANSACTION_CONFIRMED; + return true; + } else { + logger_->log_info("Site2Site transaction %s peer unknown respond code %d", transactionID.c_str(), code); + return false; + } + return false; + } +} + +void SiteToSiteClient::cancel(std::string transactionID) { + std::shared_ptr transaction = NULL; + + if (peer_state_ != READY) { + return; + } + + std::map >::iterator it = this->known_transactions_.find(transactionID); + + if (it == known_transactions_.end()) { + return; + } else { + transaction = it->second; + } + + if (transaction->getState() == TRANSACTION_CANCELED || transaction->getState() == TRANSACTION_COMPLETED || transaction->getState() == TRANSACTION_ERROR) { + return; + } + + this->writeResponse(transaction, CANCEL_TRANSACTION, "Cancel"); + transaction->_state = TRANSACTION_CANCELED; + + tearDown(); + return; +} + +void SiteToSiteClient::error(std::string transactionID) { + std::shared_ptr transaction = NULL; + + std::map >::iterator it = this->known_transactions_.find(transactionID); + + if (it == known_transactions_.end()) { + return; + } else { + transaction = it->second; + } + + transaction->_state = TRANSACTION_ERROR; + tearDown(); + return; +} + +// Complete the transaction +bool SiteToSiteClient::complete(std::string transactionID) { + int ret; + std::shared_ptr transaction = NULL; + + if (peer_state_ != READY) { + bootstrap(); + } + + if (peer_state_ != READY) { + return false; + } + + auto it = this->known_transactions_.find(transactionID); + + if (it == known_transactions_.end()) { + return false; + } else { + transaction = it->second; + } + + if (transaction->total_transfers_ > 0 && transaction->getState() != TRANSACTION_CONFIRMED) { + return false; + } + if (transaction->getDirection() == RECEIVE) { + if (transaction->current_transfers_ == 0) { + transaction->_state = TRANSACTION_COMPLETED; + return true; + } else { + logger_->log_info("Site2Site transaction %s send finished", transactionID.c_str()); + ret = this->writeResponse(transaction, TRANSACTION_FINISHED, "Finished"); + if (ret <= 0) { + return false; + } else { + transaction->_state = TRANSACTION_COMPLETED; + return true; + } + } + } else { + RespondCode code; + std::string message; + int ret; + + ret = readResponse(transaction, code, message); + + if (ret <= 0) + return false; + + if (code == TRANSACTION_FINISHED) { + logger_->log_info("Site2Site transaction %s peer finished transaction", transactionID.c_str()); + transaction->_state = TRANSACTION_COMPLETED; + return true; + } else { + logger_->log_info("Site2Site transaction %s peer unknown respond code %d", transactionID.c_str(), code); + return false; + } + } +} + +int16_t SiteToSiteClient::send(std::string transactionID, DataPacket *packet, const std::shared_ptr &flowFile, const std::shared_ptr &session) { + int ret; + std::shared_ptr transaction = NULL; + + if (flowFile && !flowFile->getResourceClaim()->exists()) { + logger_->log_info("Claim %s does not exist for FlowFile %s", flowFile->getResourceClaim()->getContentFullPath(), flowFile->getUUIDStr()); + return -2; + } + if (peer_state_ != READY) { + bootstrap(); + } + + if (peer_state_ != READY) { + return -1; + } + std::map >::iterator it = this->known_transactions_.find(transactionID); + + if (it == known_transactions_.end()) { + return -1; + } else { + transaction = it->second; + } + + if (transaction->getState() != TRANSACTION_STARTED && transaction->getState() != DATA_EXCHANGED) { + logger_->log_info("Site2Site transaction %s is not at started or exchanged state", transactionID.c_str()); + return -1; + } + + if (transaction->getDirection() != SEND) { + logger_->log_info("Site2Site transaction %s direction is wrong", transactionID.c_str()); + return -1; + } + + if (transaction->current_transfers_ > 0) { + ret = writeResponse(transaction, CONTINUE_TRANSACTION, "CONTINUE_TRANSACTION"); + if (ret <= 0) { + return -1; + } + } + // start to read the packet + uint32_t numAttributes = packet->_attributes.size(); + ret = transaction->getStream().write(numAttributes); + if (ret != 4) { + return -1; + } + + std::map::iterator itAttribute; + for (itAttribute = packet->_attributes.begin(); itAttribute != packet->_attributes.end(); itAttribute++) { + ret = transaction->getStream().writeUTF(itAttribute->first, true); + + if (ret <= 0) { + return -1; + } + ret = transaction->getStream().writeUTF(itAttribute->second, true); + if (ret <= 0) { + return -1; + } + logger_->log_info("Site2Site transaction %s send attribute key %s value %s", transactionID.c_str(), itAttribute->first.c_str(), itAttribute->second.c_str()); + } + + uint64_t len = 0; + if (flowFile) { + len = flowFile->getSize(); + ret = transaction->getStream().write(len); + if (ret != 8) { + logger_->log_info("ret != 8"); + return -1; + } + if (flowFile->getSize() > 0) { + sitetosite::ReadCallback callback(packet); + session->read(flowFile, &callback); + if (flowFile->getSize() != packet->_size) { + logger_->log_info("MisMatched sizes %d %d", flowFile->getSize(), packet->_size); + return -2; + } + } + if (packet->payload_.length() == 0 && len == 0) { + if (flowFile->getResourceClaim() == nullptr) + logger_->log_debug("no claim"); + else + logger_->log_debug("Flowfile empty %s", flowFile->getResourceClaim()->getContentFullPath()); + } + } else if (packet->payload_.length() > 0) { + len = packet->payload_.length(); + + ret = transaction->getStream().write(len); + if (ret != 8) { + return -1; + } + + ret = transaction->getStream().writeData(reinterpret_cast(const_cast(packet->payload_.c_str())), len); + if (ret != len) { + logger_->log_info("ret != len"); + return -1; + } + packet->_size += len; + } + + transaction->current_transfers_++; + transaction->total_transfers_++; + transaction->_state = DATA_EXCHANGED; + transaction->_bytes += len; + logger_->log_info("Site2Site transaction %s send flow record %d, total length %d", transactionID.c_str(), transaction->total_transfers_, transaction->_bytes); + + return 0; +} + +bool SiteToSiteClient::receive(std::string transactionID, DataPacket *packet, bool &eof) { + int ret; + std::shared_ptr transaction = NULL; + + if (peer_state_ != READY) { + bootstrap(); + } + + if (peer_state_ != READY) { + return false; + } + + auto it = this->known_transactions_.find(transactionID); + + if (it == known_transactions_.end()) { + return false; + } else { + transaction = it->second; + } + + if (transaction->getState() != TRANSACTION_STARTED && transaction->getState() != DATA_EXCHANGED) { + logger_->log_info("Site2Site transaction %s is not at started or exchanged state", transactionID.c_str()); + return false; + } + + if (transaction->getDirection() != RECEIVE) { + logger_->log_info("Site2Site transaction %s direction is wrong", transactionID.c_str()); + return false; + } + + if (!transaction->isDataAvailable()) { + eof = true; + return true; + } + + if (transaction->current_transfers_ > 0) { + // if we already has transfer before, check to see whether another one is available + RespondCode code; + std::string message; + + ret = readResponse(transaction, code, message); + + if (ret <= 0) { + return false; + } + if (code == CONTINUE_TRANSACTION) { + logger_->log_info("Site2Site transaction %s peer indicate continue transaction", transactionID.c_str()); + transaction->_dataAvailable = true; + } else if (code == FINISH_TRANSACTION) { + logger_->log_info("Site2Site transaction %s peer indicate finish transaction", transactionID.c_str()); + transaction->_dataAvailable = false; + eof = true; + return true; + } else { + logger_->log_info("Site2Site transaction %s peer indicate wrong respond code %d", transactionID.c_str(), code); + return false; + } + } + + if (!transaction->isDataAvailable()) { + eof = true; + return true; + } + + // start to read the packet + uint32_t numAttributes; + ret = transaction->getStream().read(numAttributes); + if (ret <= 0 || numAttributes > MAX_NUM_ATTRIBUTES) { + return false; + } + + // read the attributes + logger_->log_info("Site2Site transaction %s receives attribute key %llu", transactionID.c_str(), numAttributes); + for (unsigned int i = 0; i < numAttributes; i++) { + std::string key; + std::string value; + ret = transaction->getStream().readUTF(key, true); + if (ret <= 0) { + return false; + } + ret = transaction->getStream().readUTF(value, true); + if (ret <= 0) { + return false; + } + packet->_attributes[key] = value; + logger_->log_info("Site2Site transaction %s receives attribute key %s value %s", transactionID.c_str(), key.c_str(), value.c_str()); + } + + uint64_t len; + ret = transaction->getStream().read(len); + if (ret <= 0) { + return false; + } + + packet->_size = len; + if (len > 0) { + transaction->current_transfers_++; + transaction->total_transfers_++; + } else { + logger_->log_info("Site2Site transaction %s receives attribute ?", transactionID); + transaction->_dataAvailable = false; + eof = true; + return true; + } + transaction->_state = DATA_EXCHANGED; + transaction->_bytes += len; + logger_->log_info("Site2Site transaction %s receives flow record %d, total length %d", transactionID.c_str(), transaction->total_transfers_, transaction->_bytes); + + return true; +} + +bool SiteToSiteClient::receiveFlowFiles(const std::shared_ptr &context, const std::shared_ptr &session) { + uint64_t bytes = 0; + int transfers = 0; + std::shared_ptr transaction = NULL; + + if (peer_state_ != READY) { + if (!bootstrap()) { + return false; + } + } + + if (peer_state_ != READY) { + context->yield(); + tearDown(); + throw Exception(SITE2SITE_EXCEPTION, "Can not establish handshake with peer"); + } + + // Create the transaction + std::string transactionID; + transaction = createTransaction(transactionID, RECEIVE); + + if (transaction == NULL) { + context->yield(); + tearDown(); + throw Exception(SITE2SITE_EXCEPTION, "Can not create transaction"); + } + + try { + while (true) { + std::map empty; + uint64_t startTime = getTimeMillis(); + std::string payload; + DataPacket packet(getLogger(), transaction, empty, payload); + bool eof = false; + + if (!receive(transactionID, &packet, eof)) { + throw Exception(SITE2SITE_EXCEPTION, "Receive Failed"); + } + if (eof) { + // transaction done + break; + } + std::shared_ptr flowFile = std::static_pointer_cast(session->create()); + + if (!flowFile) { + throw Exception(SITE2SITE_EXCEPTION, "Flow File Creation Failed"); + } + std::map::iterator it; + std::string sourceIdentifier; + for (it = packet._attributes.begin(); it != packet._attributes.end(); it++) { + if (it->first == FlowAttributeKey(UUID)) + sourceIdentifier = it->second; + flowFile->addAttribute(it->first, it->second); + } + + if (packet._size > 0) { + sitetosite::WriteCallback callback(&packet); + session->write(flowFile, &callback); + if (flowFile->getSize() != packet._size) { + throw Exception(SITE2SITE_EXCEPTION, "Receive Size Not Right"); + } + } + core::Relationship relation; // undefined relationship + uint64_t endTime = getTimeMillis(); + std::string transitUri = peer_->getURL() + "/" + sourceIdentifier; + std::string details = "urn:nifi:" + sourceIdentifier + "Remote Host=" + peer_->getHostName(); + session->getProvenanceReporter()->receive(flowFile, transitUri, sourceIdentifier, details, endTime - startTime); + session->transfer(flowFile, relation); + // receive the transfer for the flow record + bytes += packet._size; + transfers++; + } // while true + + if (transfers > 0 && !confirm(transactionID)) { + throw Exception(SITE2SITE_EXCEPTION, "Confirm Transaction Failed"); + } + if (!complete(transactionID)) { + std::stringstream transaction_str; + transaction_str << "Complete Transaction " << transactionID << " Failed"; + throw Exception(SITE2SITE_EXCEPTION, transaction_str.str().c_str()); + } + logger_->log_info("Site2Site transaction %s successfully receive flow record %d, content bytes %d", transactionID.c_str(), transfers, bytes); + // we yield the receive if we did not get anything + if (transfers == 0) + context->yield(); + } catch (std::exception &exception) { + if (transaction) + deleteTransaction(transactionID); + context->yield(); + tearDown(); + logger_->log_debug("Caught Exception %s", exception.what()); + throw; + } catch (...) { + if (transaction) + deleteTransaction(transactionID); + context->yield(); + tearDown(); + logger_->log_debug("Caught Exception during RawSiteToSiteClient::receiveFlowFiles"); + throw; + } + + deleteTransaction(transactionID); + + return true; +} +} /* namespace sitetosite */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/utils/ByteArrayCallback.cpp b/libminifi/src/utils/ByteArrayCallback.cpp new file mode 100644 index 0000000000..84edfde0cf --- /dev/null +++ b/libminifi/src/utils/ByteArrayCallback.cpp @@ -0,0 +1,155 @@ +/** + * @file SiteToSiteProvenanceReportingTask.cpp + * SiteToSiteProvenanceReportingTask class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "utils/ByteArrayCallback.h" +#include +#include +#include +#include +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace utils { + +int64_t ByteOutputCallback::process(std::shared_ptr stream) { + stream->seek(0); + if (stream->getSize() > 0) { + std::unique_ptr buffer = std::unique_ptr(new char[stream->getSize()]); + readFully(buffer.get(), stream->getSize()); + stream->readData(reinterpret_cast(buffer.get()), stream->getSize()); + return stream->getSize(); + } + return size_.load(); +} + +const std::vector ByteOutputCallback::to_string() { + std::vector buffer; + buffer.resize(size_.load()); + readFully(buffer.data(), size_.load()); + return buffer; +} + +void ByteOutputCallback::close() { + is_alive_ = false; + spinner_.notify_all(); +} + +size_t ByteOutputCallback::getSize() { + return size_; +} + +bool ByteOutputCallback::waitingOps() { + if (vector_lock_.try_lock()) { + vector_lock_.unlock(); + return false; + } + return true; +} + +void ByteOutputCallback::write(char *data, size_t size) { + size_t amount_to_write = size; + size_t pos = 0; + do { + if (size_ > max_size_) { + std::unique_lock lock(vector_lock_); + if (size_ > max_size_) { + spinner_.wait(lock, [&] { + return size_ < max_size_ || !is_alive_;}); + } + // if we're not alive, we will let the write continue in the event that + // we do not wish to lose this data. In the event taht we don't care, we've simply + // spent wasted cycles on locking and notification. + } + { + std::lock_guard lock(vector_lock_); + vec.push(std::string(data + pos, size)); + size_ += size; + pos += size; + amount_to_write -= size; + spinner_.notify_all(); + } + } while (amount_to_write > 0); +} + +size_t ByteOutputCallback::readFully(char *buffer, size_t size) { + return read_current_str(buffer, size); +} + +size_t ByteOutputCallback::read_current_str(char *buffer, size_t size) { + size_t amount_to_read = size; + size_t curr_buf_pos = 0; + do { + { + std::lock_guard lock(vector_lock_); + + if (current_str_pos < current_str.length() && current_str.length() > 0) { + size_t str_remaining = current_str.length() - current_str_pos; + size_t current_str_read = str_remaining; + if (str_remaining > amount_to_read) { + current_str_read = amount_to_read; + } + memcpy(buffer + curr_buf_pos, current_str.data() + current_str_pos, current_str_read); + curr_buf_pos += current_str_read; + amount_to_read -= current_str_read; + current_str_pos += current_str_read; + size_ -= current_str_read; + if (current_str.length() - current_str_read <= 0) { + preload_next_str(); + } + } else { + preload_next_str(); + } + } + if (size_ < amount_to_read) { + { + std::unique_lock lock(vector_lock_); + if (size_ < amount_to_read) { + spinner_.wait(lock, [&] { + return size_ >= amount_to_read || !is_alive_;}); + } + + if (size_ == 0 && !is_alive_) { + return 0; + } + } + std::lock_guard lock(vector_lock_); + preload_next_str(); + } + } while (amount_to_read > 0 && (is_alive_ || size_ > 0) && current_str.length() > 0); + + spinner_.notify_all(); + return size - amount_to_read; +} + +void ByteOutputCallback::preload_next_str() { + // reset the current str. + current_str = ""; + if (!vec.empty()) { + current_str = std::move(vec.front()); + current_str_pos = 0; + vec.pop(); + } else { + } +} +} /* namespace utils */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/test/TestBase.cpp b/libminifi/test/TestBase.cpp index c18153c557..d61bdac22f 100644 --- a/libminifi/test/TestBase.cpp +++ b/libminifi/test/TestBase.cpp @@ -144,7 +144,8 @@ void TestPlan::reset() { } } -bool TestPlan::runNextProcessor(std::function verify) { + +bool TestPlan::runNextProcessor(std::function, const std::shared_ptr)> verify) { if (!finalized) { finalize(); } @@ -164,7 +165,7 @@ bool TestPlan::runNextProcessor(std::functionincrementActiveTasks(); processor->setScheduledState(core::ScheduledState::RUNNING); if (verify != nullptr) { - verify(context.get(), current_session.get()); + verify(context, current_session); } else { logger_->log_info("Running %s", processor->getName()); processor->onTrigger(context, current_session); diff --git a/libminifi/test/TestBase.h b/libminifi/test/TestBase.h index 4eba0a84eb..c4bc7d7a7d 100644 --- a/libminifi/test/TestBase.h +++ b/libminifi/test/TestBase.h @@ -115,7 +115,7 @@ class LogTestController { std::ostringstream log_output; std::shared_ptr logger_; - private: + private: class TestBootstrapLogger : public logging::Logger { public: TestBootstrapLogger(std::shared_ptr logger) @@ -164,7 +164,7 @@ class TestPlan { void reset(); - bool runNextProcessor(std::function verify = nullptr); + bool runNextProcessor(std::function, const std::shared_ptr)> verify = nullptr); std::set getProvenanceRecords(); @@ -184,7 +184,6 @@ class TestPlan { protected: - std::shared_ptr logger_; void finalize(); @@ -230,8 +229,7 @@ class TestController { utils::IdGenerator::getIdGenerator()->initialize(std::make_shared()); } - std::shared_ptr createPlan() - { + std::shared_ptr createPlan() { std::shared_ptr configuration = std::make_shared(); std::shared_ptr content_repo = std::make_shared(); @@ -242,16 +240,15 @@ class TestController { return std::make_shared(content_repo, flow_repo, repo); } - void runSession(std::shared_ptr &plan, bool runToCompletion = true, std::function verify = nullptr) { - while (plan->runNextProcessor(verify) && runToCompletion) - { + void runSession(std::shared_ptr &plan, bool runToCompletion = true, std::function&, const std::shared_ptr&)> verify = + nullptr) { + + while (plan->runNextProcessor(verify) && runToCompletion) { } } - - ~TestController() { for (auto dir : directories) { DIR *created_dir; @@ -282,8 +279,6 @@ class TestController { protected: - - std::mutex test_mutex; //std::map diff --git a/libminifi/test/curl-tests/C2NullConfiguration.cpp b/libminifi/test/curl-tests/C2NullConfiguration.cpp index de8d3b8ced..d163a21929 100644 --- a/libminifi/test/curl-tests/C2NullConfiguration.cpp +++ b/libminifi/test/curl-tests/C2NullConfiguration.cpp @@ -92,11 +92,9 @@ class VerifyC2Server : public IntegrationBase { std::string url = ""; inv->getProperty(minifi::processors::InvokeHTTP::URL.getName(), url); - std::cout << "url is " << url << std::endl; std::string port, scheme, path; parse_http_components(url, port, scheme, path); - std::cout << "path is " << path << std::endl; configuration->set("c2.agent.protocol.class", "null"); configuration->set("c2.rest.url", ""); configuration->set("c2.rest.url.ack", ""); diff --git a/libminifi/test/curl-tests/C2VerifyServeResults.cpp b/libminifi/test/curl-tests/C2VerifyServeResults.cpp index 101fc592d3..625a5f277b 100644 --- a/libminifi/test/curl-tests/C2VerifyServeResults.cpp +++ b/libminifi/test/curl-tests/C2VerifyServeResults.cpp @@ -92,11 +92,9 @@ class VerifyC2Server : public IntegrationBase { std::string url = ""; inv->getProperty(minifi::processors::InvokeHTTP::URL.getName(), url); - std::cout << "url is " << url << std::endl; std::string port, scheme, path; parse_http_components(url, port, scheme, path); - std::cout << "path is " << path << std::endl; configuration->set("c2.agent.heartbeat.reporter.classes", "RESTReceiver"); configuration->set("c2.rest.listener.port", port); configuration->set("c2.agent.heartbeat.period", "10"); diff --git a/libminifi/test/curl-tests/CMakeLists.txt b/libminifi/test/curl-tests/CMakeLists.txt index 37e8b25d39..e162c7c376 100644 --- a/libminifi/test/curl-tests/CMakeLists.txt +++ b/libminifi/test/curl-tests/CMakeLists.txt @@ -29,6 +29,7 @@ FOREACH(testfile ${CURL_INTEGRATION_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/http-curl/client/") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/http-curl/processors/") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/http-curl/protocols/") + target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/http-curl/sitetosite/") target_link_libraries(${testfilename} ${CURL_LIBRARIES} ) createTests("${testfilename}") if (APPLE) @@ -46,8 +47,8 @@ add_test(NAME C2UpdateTest COMMAND C2UpdateTest "${TEST_RESOURCES}/TestHTTPGet.y add_test(NAME HttpGetIntegrationTestSecure COMMAND HttpGetIntegrationTest "${TEST_RESOURCES}/TestHTTPGetSecure.yml" "${TEST_RESOURCES}/") add_test(NAME HttpPostIntegrationTest COMMAND HttpPostIntegrationTest "${TEST_RESOURCES}/TestHTTPPost.yml" "${TEST_RESOURCES}/") add_test(NAME HttpPostIntegrationTestChunked COMMAND HttpPostIntegrationTest "${TEST_RESOURCES}/TestHTTPPostChunkedEncoding.yml" "${TEST_RESOURCES}/") -add_test(NAME C2VerifyServeResults COMMAND C2VerifyServeResults "${TEST_RESOURCES}/TestHTTPGet.yml" "${TEST_RESOURCES}/") -add_test(NAME C2VerifyHeartbeatAndStop COMMAND C2VerifyHeartbeatAndStop "${TEST_RESOURCES}/TestHTTPGet.yml" "${TEST_RESOURCES}/") -add_test(NAME SiteToSiteRestTest COMMAND SiteToSiteRestTest "${TEST_RESOURCES}/TestSite2SiteRest.yml" "${TEST_RESOURCES}/" "http://localhost:8082/nifi-api/controller") +add_test(NAME C2VerifyServeResults COMMAND C2VerifyServeResults "${TEST_RESOURCES}/C2VerifyServeResults.yml" "${TEST_RESOURCES}/") +add_test(NAME C2VerifyHeartbeatAndStop COMMAND C2VerifyHeartbeatAndStop "${TEST_RESOURCES}/C2VerifyHeartbeatAndStop.yml" "${TEST_RESOURCES}/") +add_test(NAME SiteToSiteRestTest COMMAND SiteToSiteRestTest "${TEST_RESOURCES}/TestSite2SiteRest.yml" "${TEST_RESOURCES}/" "http://localhost:8077/nifi-api/controller") add_test(NAME ControllerServiceIntegrationTests COMMAND ControllerServiceIntegrationTests "${TEST_RESOURCES}/TestControllerServices.yml" "${TEST_RESOURCES}/") add_test(NAME ThreadPoolAdjust COMMAND ThreadPoolAdjust "${TEST_RESOURCES}/TestHTTPPostChunkedEncoding.yml" "${TEST_RESOURCES}/") diff --git a/libminifi/test/curl-tests/ControllerServiceIntegrationTests.cpp b/libminifi/test/curl-tests/ControllerServiceIntegrationTests.cpp index 7a4ee35070..d34d65efe3 100644 --- a/libminifi/test/curl-tests/ControllerServiceIntegrationTests.cpp +++ b/libminifi/test/curl-tests/ControllerServiceIntegrationTests.cpp @@ -123,7 +123,6 @@ int main(int argc, char **argv) { ssl_client = std::static_pointer_cast(ssl_client_cont->getControllerServiceImplementation()); } assert(ssl_client->getCACertificate().length() > 0); - std::cout << "Disabling ID" << std::endl; // now let's disable one of the controller services. std::shared_ptr cs_id = controller->getControllerServiceNode("ID"); assert(cs_id != nullptr); @@ -133,7 +132,6 @@ int main(int argc, char **argv) { disabled = true; waitToVerifyProcessor(); } - std::cout << "Disabled ID" << std::endl; { std::lock_guard lock(control_mutex); controller->enableControllerService(cs_id); @@ -141,7 +139,6 @@ int main(int argc, char **argv) { waitToVerifyProcessor(); } std::shared_ptr mock_cont = controller->getControllerServiceNode("MockItLikeIts1995"); - std::cout << "Disabling MockItLikeIts1995" << std::endl; assert(cs_id->enabled()); { std::lock_guard lock(control_mutex); @@ -149,7 +146,6 @@ int main(int argc, char **argv) { disabled = true; waitToVerifyProcessor(); } -std::cout << "Disabled MockItLikeIts1995" << std::endl; assert(cs_id->enabled() == false); { std::lock_guard lock(control_mutex); @@ -157,7 +153,6 @@ std::cout << "Disabled MockItLikeIts1995" << std::endl; disabled = false; waitToVerifyProcessor(); } -std::cout << "Enabled ref for MockItLikeIts1995" << std::endl; assert(cs_id->enabled() == true); controller->waitUnload(60000); diff --git a/libminifi/test/curl-tests/HTTPSiteToSiteTests.cpp b/libminifi/test/curl-tests/HTTPSiteToSiteTests.cpp new file mode 100644 index 0000000000..775739dc7d --- /dev/null +++ b/libminifi/test/curl-tests/HTTPSiteToSiteTests.cpp @@ -0,0 +1,262 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define CURLOPT_SSL_VERIFYPEER_DISABLE 1 +#include +#undef NDEBUG +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "HTTPClient.h" +#include "CivetServer.h" +#include "sitetosite/HTTPProtocol.h" +#include "InvokeHTTP.h" +#include "../TestBase.h" +#include "utils/StringUtils.h" +#include "core/Core.h" +#include "../include/core/logging/Logger.h" +#include "core/ProcessGroup.h" +#include "core/yaml/YamlConfiguration.h" +#include "FlowController.h" +#include "properties/Configure.h" +#include "io/StreamFactory.h" +#include "RemoteProcessorGroupPort.h" +#include "core/ConfigurableComponent.h" +#include "../TestServer.h" +#include "../integration/IntegrationBase.h" +#include "sitetositehttp/HTTPHandlers.h" +#include "client/HTTPStream.h" + +class SiteToSiteTestHarness : public IntegrationBase { + public: + explicit SiteToSiteTestHarness(bool isSecure) + : isSecure(isSecure) { + char format[] = "/tmp/ssth.XXXXXX"; + dir = testController.createTempDirectory(format); + } + + void testSetup() { + LogTestController::getInstance().setDebug(); + LogTestController::getInstance().setDebug(); + LogTestController::getInstance().setDebug(); + LogTestController::getInstance().setDebug(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setInfo(); + LogTestController::getInstance().setDebug(); + + std::fstream file; + ss << dir << "/" << "tstFile.ext"; + file.open(ss.str(), std::ios::out); + file << "tempFile"; + file.close(); + + configuration->set("nifi.c2.enable", "false"); + configuration->set("nifi.remote.input.http.enabled", "true"); + configuration->set("nifi.remote.input.socket.port", "8082"); + } + + virtual void waitToVerifyProcessor() { + std::this_thread::sleep_for(std::chrono::seconds(3)); + } + + void cleanup() { + unlink(ss.str().c_str()); + } + + void runAssertions() { + } + + protected: + bool isSecure; + char *dir; + std::stringstream ss; + TestController testController; +}; + +struct test_profile { + test_profile() + : flow_url_broken(false), + transaction_url_broken(false), + empty_transaction_url(false), + no_delete(false), + invalid_checksum(false) { + } + + bool allFalse() const { + return !flow_url_broken && !transaction_url_broken && !empty_transaction_url && !no_delete && !invalid_checksum; + } + // tests for a broken flow file url + bool flow_url_broken; + // transaction url will return incorrect information + bool transaction_url_broken; + // Location will be absent within the + bool empty_transaction_url; + // delete url is not supported. + bool no_delete; + // invalid checksum error + bool invalid_checksum; +}; + +void run_variance(std::string test_file_location, bool isSecure, std::string url, const struct test_profile &profile) { + SiteToSiteTestHarness harness(isSecure); + + SiteToSiteLocationResponder responder(isSecure); + + TransactionResponder transaction_response(url, "471deef6-2a6e-4a7d-912a-81cc17e3a204", true, profile.transaction_url_broken, profile.empty_transaction_url); + + std::string transaction_id = transaction_response.getTransactionId(); + + harness.setKeyDir(""); + + std::string controller_loc = url + "/controller"; + + harness.setUrl(controller_loc, &responder); + + std::string transaction_url = url + "/data-transfer/input-ports/471deef6-2a6e-4a7d-912a-81cc17e3a204/transactions"; + std::string action_url = url + "/site-to-site/input-ports/471deef6-2a6e-4a7d-912a-81cc17e3a204/transactions"; + + std::string transaction_output_url = url + "/data-transfer/output-ports/471deef6-2a6e-4a7d-912a-81cc17e3a203/transactions"; + std::string action_output_url = url + "/site-to-site/output-ports/471deef6-2a6e-4a7d-912a-81cc17e3a203/transactions"; + + harness.setUrl(transaction_url, &transaction_response); + + std::string peer_url = url + "/site-to-site/peers"; + + PeerResponder peer_response(url); + + harness.setUrl(peer_url, &peer_response); + + std::string flow_url = action_url + "/" + transaction_id + "/flow-files"; + + FlowFileResponder flowResponder(true, profile.flow_url_broken, profile.invalid_checksum); + flowResponder.setFlowUrl(flow_url); + auto producedFlows = flowResponder.getFlows(); + + TransactionResponder transaction_response_output(url, "471deef6-2a6e-4a7d-912a-81cc17e3a203", false, profile.transaction_url_broken, profile.empty_transaction_url); + std::string transaction_output_id = transaction_response_output.getTransactionId(); + transaction_response_output.setFeed(producedFlows); + + harness.setUrl(transaction_output_url, &transaction_response_output); + + std::string flow_output_url = action_output_url + "/" + transaction_output_id + "/flow-files"; + + FlowFileResponder flowOutputResponder(false, profile.flow_url_broken, profile.invalid_checksum); + flowOutputResponder.setFlowUrl(flow_output_url); + flowOutputResponder.setFeed(producedFlows); + + harness.setUrl(flow_url, &flowResponder); + harness.setUrl(flow_output_url, &flowOutputResponder); + + if (!profile.no_delete) { + std::string delete_url = transaction_url + "/" + transaction_id; + DeleteTransactionResponder deleteResponse(delete_url, "201 OK", 12); + harness.setUrl(delete_url, &deleteResponse); + + std::string delete_output_url = transaction_output_url + "/" + transaction_output_id; + DeleteTransactionResponder deleteOutputResponse(delete_output_url, "201 OK", producedFlows); + harness.setUrl(delete_output_url, &deleteOutputResponse); + } + + harness.run(test_file_location); + + std::stringstream assertStr; + if (profile.allFalse()) { + assertStr << "Site2Site transaction " << transaction_id << " peer finished transaction"; + assert(LogTestController::getInstance().contains(assertStr.str()) == true); + } else if (profile.empty_transaction_url) { + assert(LogTestController::getInstance().contains("Location is empty") == true); + } else if (profile.transaction_url_broken) { + assert(LogTestController::getInstance().contains("Could not create transaction, intent is ohstuff") == true); + } else if (profile.invalid_checksum) { + assertStr << "Site2Site transaction " << transaction_id << " peer confirm transaction with CRC Imawrongchecksumshortandstout"; + assert(LogTestController::getInstance().contains(assertStr.str()) == true); + assertStr.str(std::string()); + assertStr << "Site2Site transaction " << transaction_id << " CRC not matched"; + assert(LogTestController::getInstance().contains(assertStr.str()) == true); + assertStr.str(std::string()); + assertStr << "Site2Site delete transaction " << transaction_id; + assert(LogTestController::getInstance().contains(assertStr.str()) == true); + } else if (profile.no_delete) { + assert(LogTestController::getInstance().contains("Received 401 response code from delete") == true); + } else { + assertStr << "Site2Site transaction " << transaction_id << " peer unknown respond code 254"; + assert(LogTestController::getInstance().contains(assertStr.str()) == true); + } + LogTestController::getInstance().reset(); +} + +int main(int argc, char **argv) { + transaction_id = 0; + transaction_id_output = 0; + std::string key_dir, test_file_location, url; + if (argc > 1) { + test_file_location = argv[1]; + key_dir = argv[2]; + url = argv[3]; + } + + bool isSecure = false; + if (url.find("https") != std::string::npos) { + isSecure = true; + } + + { + struct test_profile profile; + run_variance(test_file_location, isSecure, url, profile); + } + + { + struct test_profile profile; + profile.flow_url_broken = true; + run_variance(test_file_location, isSecure, url, profile); + } + + { + struct test_profile profile; + profile.empty_transaction_url = true; + run_variance(test_file_location, isSecure, url, profile); + } + + { + struct test_profile profile; + profile.transaction_url_broken = true; + run_variance(test_file_location, isSecure, url, profile); + } + + { + struct test_profile profile; + profile.no_delete = true; + run_variance(test_file_location, isSecure, url, profile); + } + + { + struct test_profile profile; + profile.invalid_checksum = true; + run_variance(test_file_location, isSecure, url, profile); + } + + return 0; +} diff --git a/libminifi/test/curl-tests/sitetositehttp/CivetStream.h b/libminifi/test/curl-tests/sitetositehttp/CivetStream.h new file mode 100644 index 0000000000..571b0ca1ee --- /dev/null +++ b/libminifi/test/curl-tests/sitetositehttp/CivetStream.h @@ -0,0 +1,138 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef EXTENSIONS_HTTP_CURL_CLIENT_CIVETSTREAM_H_ +#define EXTENSIONS_HTTP_CURL_CLIENT_CIVETSTREAM_H_ + +#include +#include +#include +#include +#include + +#include "io/BaseStream.h" +#include "civetweb.h" +#include "CivetServer.h" +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace io { + +class CivetStream : public io::BaseStream { + public: + /** + * File Stream constructor that accepts an fstream shared pointer. + * It must already be initialized for read and write. + */ + explicit CivetStream(struct mg_connection *conn) + : io::BaseStream(), conn(conn) { + + } + + virtual ~CivetStream() { + } + /** + * Skip to the specified offset. + * @param offset offset to which we will skip + */ + void seek(uint64_t offset){ + + } + + const uint64_t getSize() const { + return BaseStream::readBuffer; + } + + // data stream extensions + /** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ + virtual int readData(std::vector &buf, int buflen) { + if (buf.capacity() < buflen) { + buf.resize(buflen); + } + int ret = readData(reinterpret_cast(&buf[0]), buflen); + + if (ret < buflen) { + buf.resize(ret); + } + return ret; + } + + /** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ + virtual int readData(uint8_t *buf, int buflen) { + return mg_read(conn,buf,buflen); + } + + /** + * Write value to the stream using std::vector + * @param buf incoming buffer + * @param buflen buffer to write + * + */ + virtual int writeData(std::vector &buf, int buflen) { + return 0; + } + + /** + * writes value to stream + * @param value value to write + * @param size size of value + */ + virtual int writeData(uint8_t *value, int size) { + return 0; + } + + protected: + + /** + * Creates a vector and returns the vector using the provided + * type name. + * @param t incoming object + * @returns vector. + */ + template + inline std::vector readBuffer(const T& t) { + std::vector buf; + buf.resize(sizeof t); + readData(reinterpret_cast(&buf[0]), sizeof(t)); + return buf; + } + + void reset(); + + //size_t pos; + struct mg_connection *conn; + + private: + + std::shared_ptr logger_; +}; +} /* namespace io */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif /* EXTENSIONS_HTTP_CURL_CLIENT_CIVETSTREAM_H_ */ diff --git a/libminifi/test/curl-tests/sitetositehttp/HTTPHandlers.h b/libminifi/test/curl-tests/sitetositehttp/HTTPHandlers.h new file mode 100644 index 0000000000..3fe4623dfe --- /dev/null +++ b/libminifi/test/curl-tests/sitetositehttp/HTTPHandlers.h @@ -0,0 +1,322 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "civetweb.h" +#include "CivetServer.h" +#include "concurrentqueue.h" + + + +#include "CivetStream.h" +#include "io/CRCStream.h" +#ifndef LIBMINIFI_TEST_CURL_TESTS_SITETOSITEHTTP_HTTPHANDLERS_H_ +#define LIBMINIFI_TEST_CURL_TESTS_SITETOSITEHTTP_HTTPHANDLERS_H_ +static std::atomic transaction_id; +static std::atomic transaction_id_output; + +class FlowObj { + public: + FlowObj() + : total_size(0) { + + } + explicit FlowObj(const FlowObj &&other) + : attributes(std::move(other.attributes)), + total_size(std::move(other.total_size)), + data(std::move(other.data)) { + + } + uint64_t total_size; + std::map attributes; + std::vector data; + +}; + +class SiteToSiteLocationResponder : public CivetHandler { + public: + explicit SiteToSiteLocationResponder(bool isSecure) + : isSecure(isSecure) { + } + bool handleGet(CivetServer *server, struct mg_connection *conn) { + std::string site2site_rest_resp = "{" + "\"revision\": {" + "\"clientId\": \"483d53eb-53ec-4e93-b4d4-1fc3d23dae6f\"" + "}," + "\"controller\": {" + "\"id\": \"fe4a3a42-53b6-4af1-a80d-6fdfe60de97f\"," + "\"name\": \"NiFi Flow\"," + "\"siteToSiteSecure\": "; + site2site_rest_resp += (isSecure ? "true" : "false"); + site2site_rest_resp += "}}"; + mg_printf(conn, "HTTP/1.1 200 OK\r\nContent-Type: " + "text/plain\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", + site2site_rest_resp.length()); + mg_printf(conn, "%s", site2site_rest_resp.c_str()); + return true; + } + + protected: + bool isSecure; +}; + +class PeerResponder : public CivetHandler { + public: + + explicit PeerResponder(const std::string base_url) + : base_url(base_url) { + } + + bool handleGet(CivetServer *server, struct mg_connection *conn) { + std::string site2site_rest_resp = "{\"peers\" : [{ \"hostname\": \"localhost\", \"port\": 8082, \"secure\": false, \"flowFileCount\" : 0 }] }"; + std::stringstream headers; + headers << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << site2site_rest_resp.length() << "\r\nConnection: close\r\n\r\n"; + mg_printf(conn, "%s", headers.str().c_str()); + mg_printf(conn, "%s", site2site_rest_resp.c_str()); + return true; + } + + protected: + std::string base_url; +}; + +class TransactionResponder : public CivetHandler { + public: + + explicit TransactionResponder(const std::string base_url, std::string port_id, bool input_port, bool wrong_uri, bool empty_transaction_uri) + : base_url(base_url), + wrong_uri(wrong_uri), + empty_transaction_uri(empty_transaction_uri), + input_port(input_port), + port_id(port_id), + flow_files_feed_(nullptr) { + + if (input_port) { + transaction_id_str = "fe4a3a42-53b6-4af1-a80d-6fdfe60de96"; + transaction_id_str += std::to_string(transaction_id.load()); + transaction_id++; + } else { + transaction_id_str = "fe4a3a42-53b6-4af1-a80d-6fdfe60de95"; + transaction_id_str += std::to_string(transaction_id_output.load()); + transaction_id_output++; + } + } + + bool handlePost(CivetServer *server, struct mg_connection *conn) { + std::string site2site_rest_resp = ""; + std::stringstream headers; + headers << "HTTP/1.1 201 OK\r\nContent-Type: application/json\r\nContent-Length: " << site2site_rest_resp.length() << "\r\nx-location-uri-intent: "; + if (wrong_uri) + headers << "ohstuff\r\n"; + else + headers << "transaction-url\r\n"; + + std::string port_type; + + if (input_port) + port_type = "input-ports"; + else + port_type = "output-ports"; + if (!empty_transaction_uri) + headers << "Location: " << base_url << "/site-to-site/" << port_type << "/" << port_id << "/transactions/" << transaction_id_str << "\r\n"; + headers << "Connection: close\r\n\r\n"; + mg_printf(conn, "%s", headers.str().c_str()); + mg_printf(conn, "%s", site2site_rest_resp.c_str()); + return true; + } + + void setFeed(moodycamel::ConcurrentQueue> *feed) { + flow_files_feed_ = feed; + } + + std::string getTransactionId() { + return transaction_id_str; + } + protected: + moodycamel::ConcurrentQueue> *flow_files_feed_; + std::string transaction_id_str; + std::string base_url;bool wrong_uri;bool empty_transaction_uri;bool input_port; + std::string port_id; +}; + +class FlowFileResponder : public CivetHandler { + + moodycamel::ConcurrentQueue> flow_files_; + moodycamel::ConcurrentQueue> *flow_files_feed_; + + public: + + explicit FlowFileResponder(bool input_port, bool wrong_uri, bool invalid_checksum) + : wrong_uri(wrong_uri), + input_port(input_port), + flow_files_feed_(nullptr), + invalid_checksum(invalid_checksum) { + } + + moodycamel::ConcurrentQueue> *getFlows() { + return &flow_files_; + } + + void setFeed(moodycamel::ConcurrentQueue> *feed) { + flow_files_feed_ = feed; + } + + bool handlePost(CivetServer *server, struct mg_connection *conn) { + std::string site2site_rest_resp = ""; + std::stringstream headers; + + if (!wrong_uri) { + minifi::io::CivetStream civet_stream(conn); + minifi::io::CRCStream stream(&civet_stream); + uint32_t num_attributes; + uint64_t total_size = 0; + total_size += stream.read(num_attributes); + + auto flow = std::make_shared(); + + for (int i = 0; i < num_attributes; i++) { + std::string name, value; + total_size += stream.readUTF(name, true); + total_size += stream.readUTF(value, true); + flow->attributes[name] = value; + } + uint64_t length; + total_size += stream.read(length); + + total_size += length; + flow->data.resize(length); + flow->total_size = total_size; + + assert(stream.readData(flow->data.data(), length) == length); + + assert(flow->attributes["path"] == "."); + assert(!flow->attributes["uuid"].empty()); + assert(!flow->attributes["filename"].empty()); + + if (!invalid_checksum) { + site2site_rest_resp = std::to_string(stream.getCRC()); + flow_files_.enqueue(flow); + } else { + site2site_rest_resp = "Imawrongchecksumshortandstout"; + } + + headers << "HTTP/1.1 202 OK\r\nContent-Type: application/json\r\nContent-Length: " << site2site_rest_resp.length() << "\r\nConnection: close\r\n\r\n"; + } else { + headers << "HTTP/1.1 404\r\nConnection: close\r\n\r\n"; + } + + mg_printf(conn, "%s", headers.str().c_str()); + mg_printf(conn, "%s", site2site_rest_resp.c_str()); + return true; + } + + bool handleGet(CivetServer *server, struct mg_connection *conn) { + + if (flow_files_feed_->size_approx() > 0) { + std::shared_ptr flow; + uint8_t buf[1]; + std::vector> flows; + uint64_t total = 0; + + while (flow_files_feed_->try_dequeue(flow)) { + flows.push_back(flow); + total += flow->total_size; + } + mg_printf(conn, "HTTP/1.1 200 OK\r\n" + "Content-Length: %llu\r\n" + "Content-Type: application/octet-stream\r\n" + "Connection: close\r\n\r\n", + total); + minifi::io::BaseStream serializer; + minifi::io::CRCStream stream(&serializer); + for (auto flow : flows) { + uint32_t num_attributes = flow->attributes.size(); + stream.write(num_attributes); + for (auto entry : flow->attributes) { + stream.writeUTF(entry.first); + stream.writeUTF(entry.second); + } + uint64_t length = flow->data.size(); + stream.write(length); + stream.writeData(flow->data.data(), length); + } + auto ret = mg_write(conn, serializer.getBuffer(), total); + } else { + std::cout << "Nothing to transfer feed" << std::endl; + mg_printf(conn, "HTTP/1.1 200 OK\r\nConnection: " + "close\r\nContent-Length: 0\r\n"); + mg_printf(conn, "Content-Type: text/plain\r\n\r\n"); + + } + return true; + } + + void setFlowUrl(std::string flowUrl) { + base_url = flowUrl; + } + + protected: +// base url + std::string base_url; +// set the wrong url + bool wrong_uri; +// we are running an input port + bool input_port; +// invalid checksum is returned. + bool invalid_checksum; +}; + +class DeleteTransactionResponder : public CivetHandler { + public: + + explicit DeleteTransactionResponder(const std::string base_url, std::string response_code, int expected_resp_code) + : base_url(base_url), + response_code(response_code), + flow_files_feed_(nullptr) { + expected_resp_code_str = std::to_string(expected_resp_code); + } + + explicit DeleteTransactionResponder(const std::string base_url, std::string response_code, moodycamel::ConcurrentQueue> *feed) + : base_url(base_url), + response_code(response_code), + flow_files_feed_(feed) { + } + + bool handleDelete(CivetServer *server, struct mg_connection *conn) { + + std::string site2site_rest_resp = ""; + std::stringstream headers; + std::string resp; + CivetServer::getParam(conn, "responseCode", resp); + headers << "HTTP/1.1 " << response_code << "\r\nContent-Type: application/json\r\nContent-Length: " << site2site_rest_resp.length() << "\r\n"; + headers << "Connection: close\r\n\r\n"; + mg_printf(conn, "%s", headers.str().c_str()); + mg_printf(conn, "%s", site2site_rest_resp.c_str()); + return true; + } + + void setFeed(moodycamel::ConcurrentQueue> *feed) { + flow_files_feed_ = feed; + } + + protected: + moodycamel::ConcurrentQueue> *flow_files_feed_; + std::string base_url; + std::string expected_resp_code_str; + std::string response_code; +}; + +#endif /* LIBMINIFI_TEST_CURL_TESTS_SITETOSITEHTTP_HTTPHANDLERS_H_ */ diff --git a/libminifi/test/integration/IntegrationBase.h b/libminifi/test/integration/IntegrationBase.h index a854508fd4..7626cdf25d 100644 --- a/libminifi/test/integration/IntegrationBase.h +++ b/libminifi/test/integration/IntegrationBase.h @@ -41,9 +41,7 @@ int ssl_enable(void *ssl_context, void *user_data) { return 0; } -void waitToVerifyProcessor() { - std::this_thread::sleep_for(std::chrono::seconds(3)); -} + class IntegrationBase { public: @@ -66,6 +64,10 @@ class IntegrationBase { virtual void runAssertions() = 0; + virtual void waitToVerifyProcessor() { + std::this_thread::sleep_for(std::chrono::seconds(3)); + } + protected: virtual void queryRootProcessGroup(std::shared_ptr pg) { @@ -160,6 +162,11 @@ void IntegrationBase::setUrl(std::string url, CivetHandler *handler) { parse_http_components(url, port, scheme, path); struct mg_callbacks callback; if (url.find("localhost") != std::string::npos) { + + if (server != nullptr){ + server->addHandler(path,handler); + return; + } if (scheme == "https" && !key_dir.empty()) { std::string cert = ""; cert = key_dir + "nifi-cert.pem"; diff --git a/libminifi/test/integration/ProvenanceReportingTest.cpp b/libminifi/test/integration/ProvenanceReportingTest.cpp index 5845767a36..7db235f844 100644 --- a/libminifi/test/integration/ProvenanceReportingTest.cpp +++ b/libminifi/test/integration/ProvenanceReportingTest.cpp @@ -73,7 +73,7 @@ int main(int argc, char **argv) { std::shared_ptr pg = std::shared_ptr(ptr.get()); ptr.release(); std::shared_ptr socket_context = std::make_shared(std::make_shared()); - org::apache::nifi::minifi::io::Socket server(socket_context, "localhost", 10001, 1); + org::apache::nifi::minifi::io::Socket server(socket_context, "localhost", 10005, 1); controller->load(); controller->start(); diff --git a/libminifi/test/resources/C2VerifyHeartbeatAndStop.yml b/libminifi/test/resources/C2VerifyHeartbeatAndStop.yml new file mode 100644 index 0000000000..a2b0747814 --- /dev/null +++ b/libminifi/test/resources/C2VerifyHeartbeatAndStop.yml @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +Flow Controller: + name: MiNiFi Flow + id: 2438e3c8-015a-1000-79ca-83af40ec1990 +Processors: + - name: invoke + id: 2438e3c8-015a-1000-79ca-83af40ec1991 + class: org.apache.nifi.processors.standard.InvokeHTTP + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 1 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + Properties: + HTTP Method: GET + Remote URL: http://localhost:10015/geturl + - name: LogAttribute + id: 2438e3c8-015a-1000-79ca-83af40ec1992 + class: org.apache.nifi.processors.standard.LogAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 1 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: response + Properties: + Log Level: info + Log Payload: true + +Connections: + - name: TransferFilesToRPG + id: 2438e3c8-015a-1000-79ca-83af40ec1997 + source name: invoke + source id: 2438e3c8-015a-1000-79ca-83af40ec1991 + source relationship name: success + destination name: LogAttribute + destination id: 2438e3c8-015a-1000-79ca-83af40ec1992 + max work queue size: 0 + max work queue data size: 1 MB + flowfile expiration: 60 sec + - name: TransferFilesToRPG2 + id: 2438e3c8-015a-1000-79ca-83af40ec1917 + source name: LogAttribute + source id: 2438e3c8-015a-1000-79ca-83af40ec1992 + destination name: LogAttribute + destination id: 2438e3c8-015a-1000-79ca-83af40ec1992 + source relationship name: success + max work queue size: 0 + max work queue data size: 1 MB + flowfile expiration: 60 sec + +Remote Processing Groups: + diff --git a/libminifi/test/resources/C2VerifyServeResults.yml b/libminifi/test/resources/C2VerifyServeResults.yml new file mode 100644 index 0000000000..18db3ddb4b --- /dev/null +++ b/libminifi/test/resources/C2VerifyServeResults.yml @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +Flow Controller: + name: MiNiFi Flow + id: 2438e3c8-015a-1000-79ca-83af40ec1990 +Processors: + - name: invoke + id: 2438e3c8-015a-1000-79ca-83af40ec1991 + class: org.apache.nifi.processors.standard.InvokeHTTP + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 1 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + Properties: + HTTP Method: GET + Remote URL: http://localhost:10013/geturl + - name: LogAttribute + id: 2438e3c8-015a-1000-79ca-83af40ec1992 + class: org.apache.nifi.processors.standard.LogAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 1 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: response + Properties: + Log Level: info + Log Payload: true + +Connections: + - name: TransferFilesToRPG + id: 2438e3c8-015a-1000-79ca-83af40ec1997 + source name: invoke + source id: 2438e3c8-015a-1000-79ca-83af40ec1991 + source relationship name: success + destination name: LogAttribute + destination id: 2438e3c8-015a-1000-79ca-83af40ec1992 + max work queue size: 0 + max work queue data size: 1 MB + flowfile expiration: 60 sec + - name: TransferFilesToRPG2 + id: 2438e3c8-015a-1000-79ca-83af40ec1917 + source name: LogAttribute + source id: 2438e3c8-015a-1000-79ca-83af40ec1992 + destination name: LogAttribute + destination id: 2438e3c8-015a-1000-79ca-83af40ec1992 + source relationship name: success + max work queue size: 0 + max work queue data size: 1 MB + flowfile expiration: 60 sec + +Remote Processing Groups: + diff --git a/libminifi/test/resources/TestHTTPGetSecure.yml b/libminifi/test/resources/TestHTTPGetSecure.yml index 1ac82bdd6e..a5a0beeb92 100644 --- a/libminifi/test/resources/TestHTTPGetSecure.yml +++ b/libminifi/test/resources/TestHTTPGetSecure.yml @@ -33,7 +33,7 @@ Processors: Properties: SSL Context Service: SSLContextService HTTP Method: GET - Remote URL: https://localhost:10003/geturl + Remote URL: https://localhost:10004/geturl Disable Peer Verification: true - name: LogAttribute id: 2438e3c8-015a-1000-79ca-83af40ec1992 diff --git a/libminifi/test/resources/TestHTTPPost.yml b/libminifi/test/resources/TestHTTPPost.yml index 32e4f42e7e..e4dbf88309 100644 --- a/libminifi/test/resources/TestHTTPPost.yml +++ b/libminifi/test/resources/TestHTTPPost.yml @@ -43,7 +43,7 @@ Processors: auto-terminated relationships list: Properties: Base Path: urlofchampions - Listening Port: 10004 + Listening Port: 10007 - name: Invoke id: 2438e3c8-015a-1000-79ca-83af40ec1992 class: org.apache.nifi.processors.standard.InvokeHTTP @@ -58,7 +58,7 @@ Processors: Properties: HTTP Method: POST Content-type: text/html - Remote URL: http://localhost:10004/urlofchampions + Remote URL: http://localhost:10007/urlofchampions - name: Loggit id: 2438e3c8-015a-1000-79ca-83af40ec1993 class: org.apache.nifi.processors.standard.LogAttribute diff --git a/libminifi/test/resources/TestHTTPPostChunkedEncoding.yml b/libminifi/test/resources/TestHTTPPostChunkedEncoding.yml index 110783c1bb..f31f3e7935 100644 --- a/libminifi/test/resources/TestHTTPPostChunkedEncoding.yml +++ b/libminifi/test/resources/TestHTTPPostChunkedEncoding.yml @@ -43,7 +43,7 @@ Processors: auto-terminated relationships list: Properties: Base Path: urlofchampions - Listening Port: 10004 + Listening Port: 10006 - name: Invoke id: 2438e3c8-015a-1000-79ca-83af40ec1992 class: org.apache.nifi.processors.standard.InvokeHTTP @@ -59,7 +59,7 @@ Processors: HTTP Method: POST Use Chunked Encoding: true Content-type: text/html - Remote URL: http://localhost:10004/urlofchampions + Remote URL: http://localhost:10006/urlofchampions - name: Loggit id: 2438e3c8-015a-1000-79ca-83af40ec1993 class: org.apache.nifi.processors.standard.LogAttribute diff --git a/libminifi/test/resources/TestHTTPSiteToSite.yml b/libminifi/test/resources/TestHTTPSiteToSite.yml new file mode 100644 index 0000000000..582f48badb --- /dev/null +++ b/libminifi/test/resources/TestHTTPSiteToSite.yml @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Flow Controller: + id: 471deef6-2a6e-4a7d-912a-81cc17e3a205 + name: MiNiFi Flow + +Processors: + - name: GetFile + id: 471deef6-2a6e-4a7d-912a-81cc17e3a206 + class: org.apache.nifi.processors.standard.GenerateFlowFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 1 sec + penalization period: 30 sec + yield period: 10 sec + run duration nanos: 0 + auto-terminated relationships list: + Properties: + Input Directory: /tmp/site2siteGetFile + Keep Source File: true + - name: GetFile + id: 471deef6-2a6e-4a7d-912a-81cc17e3a205 + class: org.apache.nifi.processors.standard.LogAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 1 sec + penalization period: 30 sec + yield period: 10 sec + run duration nanos: 0 + auto-terminated relationships list: + Properties: + + +Connections: + - name: GenerateFlowFileS2S + id: 471deef6-2a6e-4a7d-912a-81cc17e3a207 + source id: 471deef6-2a6e-4a7d-912a-81cc17e3a206 + source relationship name: success + destination id: 471deef6-2a6e-4a7d-912a-81cc17e3a204 + max work queue size: 0 + max work queue data size: 1 MB + flowfile expiration: 60 sec + queue prioritizer class: org.apache.nifi.prioritizer.NewestFlowFileFirstPrioritizer + - name: GenerateFlowFileS2S + id: 471deef6-2a6e-4a7d-912a-81cc17e3a207 + source id: 471deef6-2a6e-4a7d-912a-81cc17e3a203 + source relationship name: success + destination id: 471deef6-2a6e-4a7d-912a-81cc17e3a205 + max work queue size: 0 + max work queue data size: 1 MB + flowfile expiration: 60 sec + queue prioritizer class: org.apache.nifi.prioritizer.NewestFlowFileFirstPrioritizer + +Remote Processing Groups: + - name: NiFi Flow + id: 471deef6-2a6e-4a7d-912a-81cc17e3a208 + url: http://localhost:8082/nifi + timeout: 30 secs + yield period: 1 sec + Input Ports: + - id: 471deef6-2a6e-4a7d-912a-81cc17e3a204 + name: From Node A + max concurrent tasks: 1 + use compression: false + Properties: # Deviates from spec and will later be removed when this is autonegotiated + Port UUID: 471deef6-2a6e-4a7d-912a-81cc17e3a204 + Port: 8082 + Host Name: 127.0.0.1 + Output Ports: + - id: 471deef6-2a6e-4a7d-912a-81cc17e3a203 + name: To Node A + max concurrent tasks: 1 + use compression: false + Properties: # Deviates from spec and will later be removed when this is autonegotiated + Port UUID: 471deef6-2a6e-4a7d-912a-81cc17e3a203 + Port: 8082 + Host Name: 127.0.0.1 + diff --git a/libminifi/test/resources/TestSite2SiteRest.yml b/libminifi/test/resources/TestSite2SiteRest.yml index ca751b5351..87534a8a82 100644 --- a/libminifi/test/resources/TestSite2SiteRest.yml +++ b/libminifi/test/resources/TestSite2SiteRest.yml @@ -46,7 +46,7 @@ Connections: Remote Processing Groups: - name: NiFi Flow id: 471deef6-2a6e-4a7d-912a-81cc17e3a208 - url: http://localhost:8082/nifi + url: http://localhost:8077/nifi timeout: 30 secs yield period: 10 sec Input Ports: @@ -56,3 +56,5 @@ Remote Processing Groups: use compression: false Properties: # Deviates from spec and will later be removed when this is autonegotiated Port UUID: 471deef6-2a6e-4a7d-912a-81cc17e3a204 + Port: 8077 + Host Name: 127.0.0.1 diff --git a/libminifi/test/resources/TestSite2SiteRestSecure.yml b/libminifi/test/resources/TestSite2SiteRestSecure.yml index fd530ae96a..5fa572c590 100644 --- a/libminifi/test/resources/TestSite2SiteRestSecure.yml +++ b/libminifi/test/resources/TestSite2SiteRestSecure.yml @@ -46,7 +46,7 @@ Connections: Remote Processing Groups: - name: NiFi Flow id: 471deef6-2a6e-4a7d-912a-81cc17e3a208 - url: https://localhost:8082/nifi + url: https://localhost:8072/nifi timeout: 30 secs yield period: 10 sec Input Ports: diff --git a/libminifi/test/resources/ThreadPoolAdjust.yml b/libminifi/test/resources/ThreadPoolAdjust.yml new file mode 100644 index 0000000000..8b3c989776 --- /dev/null +++ b/libminifi/test/resources/ThreadPoolAdjust.yml @@ -0,0 +1,97 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +Flow Controller: + name: MiNiFi Flow + id: 2438e3c8-015a-1000-79ca-83af40ec1990 +Processors: + - name: generate + id: 2438e3c8-015a-1000-79ca-83af40ec1991 + class: org.apache.nifi.processors.standard.GenerateFlowFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 1 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + Properties: + - name: listen + id: 2438e3c8-015a-1000-79ca-83af40ec1994 + class: org.apache.nifi.processors.standard.ListenHTTP + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 1 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + Properties: + Base Path: urlofchampions + Listening Port: 10016 + - name: Invoke + id: 2438e3c8-015a-1000-79ca-83af40ec1992 + class: org.apache.nifi.processors.standard.InvokeHTTP + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 1 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - success + Properties: + HTTP Method: POST + Use Chunked Encoding: true + Content-type: text/html + Remote URL: http://localhost:10016/urlofchampions + - name: Loggit + id: 2438e3c8-015a-1000-79ca-83af40ec1993 + class: org.apache.nifi.processors.standard.LogAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 1 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - success + Properties: + LogLevel: debug + +Connections: + - name: GenerateFlowFile/Invoke + id: 2438e3c8-015a-1000-79ca-83af40ec1997 + source name: invoke + source id: 2438e3c8-015a-1000-79ca-83af40ec1991 + source relationship name: success + destination name: LogAttribute + destination id: 2438e3c8-015a-1000-79ca-83af40ec1992 + max work queue size: 0 + max work queue data size: 1 MB + flowfile expiration: 60 sec + - name: Listen/Loggit + id: 2438e3c8-015a-1000-79ca-83af40ec1918 + source id: 2438e3c8-015a-1000-79ca-83af40ec1994 + destination id: 2438e3c8-015a-1000-79ca-83af40ec1993 + source relationship name: success + max work queue size: 0 + max work queue data size: 1 MB + flowfile expiration: 60 sec +Remote Processing Groups: + diff --git a/libminifi/test/unit/GetTCPTests.cpp b/libminifi/test/unit/GetTCPTests.cpp index 67f42a4fab..c09da68162 100644 --- a/libminifi/test/unit/GetTCPTests.cpp +++ b/libminifi/test/unit/GetTCPTests.cpp @@ -160,7 +160,7 @@ TEST_CASE("GetTCPWithOEM", "[GetTCP2]") { TestController testController; - org::apache::nifi::minifi::io::Socket server(socket_context, "localhost", 9183, 1); + org::apache::nifi::minifi::io::Socket server(socket_context, "localhost", 9182, 1); REQUIRE(-1 != server.initialize()); @@ -211,7 +211,7 @@ TEST_CASE("GetTCPWithOEM", "[GetTCP2]") { std::shared_ptr controller_services_provider = nullptr; std::shared_ptr context = std::make_shared(node, controller_services_provider, repo, repo, content_repo); std::shared_ptr context2 = std::make_shared(node2, controller_services_provider, repo, repo, content_repo); - context->setProperty(org::apache::nifi::minifi::processors::GetTCP::EndpointList, "localhost:9183"); + context->setProperty(org::apache::nifi::minifi::processors::GetTCP::EndpointList, "localhost:9182"); context->setProperty(org::apache::nifi::minifi::processors::GetTCP::ReconnectInterval, "100 msec"); // we're using new lines above context->setProperty(org::apache::nifi::minifi::processors::GetTCP::EndOfMessageByte, "10"); @@ -286,7 +286,7 @@ TEST_CASE("GetTCPWithOnlyOEM", "[GetTCP3]") { LogTestController::getInstance().setDebug(); - org::apache::nifi::minifi::io::Socket server(socket_context, "localhost", 9183, 1); + org::apache::nifi::minifi::io::Socket server(socket_context, "localhost", 9182, 1); REQUIRE(-1 != server.initialize()); @@ -336,7 +336,7 @@ TEST_CASE("GetTCPWithOnlyOEM", "[GetTCP3]") { std::shared_ptr controller_services_provider = nullptr; std::shared_ptr context = std::make_shared(node, controller_services_provider, repo, repo, content_repo); std::shared_ptr context2 = std::make_shared(node2, controller_services_provider, repo, repo, content_repo); - context->setProperty(org::apache::nifi::minifi::processors::GetTCP::EndpointList, "localhost:9183"); + context->setProperty(org::apache::nifi::minifi::processors::GetTCP::EndpointList, "localhost:9182"); context->setProperty(org::apache::nifi::minifi::processors::GetTCP::ReconnectInterval, "100 msec"); // we're using new lines above context->setProperty(org::apache::nifi::minifi::processors::GetTCP::EndOfMessageByte, "10"); @@ -402,7 +402,7 @@ TEST_CASE("GetTCPEmptyNoConnect", "[GetTCP3]") { plan->addProcessor("LogAttribute", "logattribute", core::Relationship("success", "description"), true); - plan->setProperty(getfile, org::apache::nifi::minifi::processors::GetTCP::EndpointList.getName(), "localhost:9183"); + plan->setProperty(getfile, org::apache::nifi::minifi::processors::GetTCP::EndpointList.getName(), "localhost:9182"); plan->setProperty(getfile, org::apache::nifi::minifi::processors::GetTCP::ReconnectInterval.getName(), "100 msec"); // we're using new lines above plan->setProperty(getfile, org::apache::nifi::minifi::processors::GetTCP::EndOfMessageByte.getName(), "10"); @@ -413,7 +413,7 @@ TEST_CASE("GetTCPEmptyNoConnect", "[GetTCP3]") { REQUIRE(record == nullptr); REQUIRE(records.size() == 0); - REQUIRE(true == LogTestController::getInstance().contains("Could not create socket during initialization for localhost:9183")); + REQUIRE(true == LogTestController::getInstance().contains("Could not create socket during initialization for localhost:9182")); LogTestController::getInstance().reset(); } diff --git a/libminifi/test/unit/LoggerTests.cpp b/libminifi/test/unit/LoggerTests.cpp index 9daf33e4c0..b39b233577 100644 --- a/libminifi/test/unit/LoggerTests.cpp +++ b/libminifi/test/unit/LoggerTests.cpp @@ -75,5 +75,4 @@ TEST_CASE("Test log Levels change", "[ttl5]") { } TEST_CASE("Test Demangle template", "[ttl6]") { - std::cout << core::getClassName>() << std::endl; } diff --git a/libminifi/test/unit/ProcessorTests.cpp b/libminifi/test/unit/ProcessorTests.cpp index 25376c969a..99b9a07c43 100644 --- a/libminifi/test/unit/ProcessorTests.cpp +++ b/libminifi/test/unit/ProcessorTests.cpp @@ -122,13 +122,11 @@ TEST_CASE("Test GetFileMultiple", "[getfileCreate3]") { session->commit(); std::shared_ptr ffr = session->get(); - std::cout << repo->getRepoMap().size() << std::endl; REQUIRE(repo->getRepoMap().size() == (prev + 1)); prev++; } } - TEST_CASE("LogAttributeTest", "[getfileCreate3]") { TestController testController; LogTestController::getInstance().setDebug(); @@ -240,12 +238,13 @@ TEST_CASE("Test Find file", "[getfileCreate3]") { std::string jsonStr; std::size_t deserialized = 0; repo->DeSerialize(recordsReport, deserialized); - std::function verifyReporter = [&](core::ProcessContext *context, core::ProcessSession *session) { - taskReport->getJsonReport(context, session, recordsReport, jsonStr); - REQUIRE(recordsReport.size() == 1); - REQUIRE(taskReport->getName() == std::string(org::apache::nifi::minifi::core::reporting::SiteToSiteProvenanceReportingTask::ReportTaskName)); - REQUIRE(jsonStr.find("\"componentType\" : \"getfileCreate2\"") != std::string::npos); - }; + std::function &, const std::shared_ptr&)> verifyReporter = + [&](const std::shared_ptr &context, const std::shared_ptr &session) { + taskReport->getJsonReport(context, session, recordsReport, jsonStr); + REQUIRE(recordsReport.size() == 1); + REQUIRE(taskReport->getName() == std::string(org::apache::nifi::minifi::core::reporting::SiteToSiteProvenanceReportingTask::ReportTaskName)); + REQUIRE(jsonStr.find("\"componentType\" : \"getfileCreate2\"") != std::string::npos); + }; testController.runSession(plan, false, verifyReporter); } diff --git a/libminifi/test/unit/SerializationTests.cpp b/libminifi/test/unit/SerializationTests.cpp index a66376d052..c0852a2952 100644 --- a/libminifi/test/unit/SerializationTests.cpp +++ b/libminifi/test/unit/SerializationTests.cpp @@ -17,8 +17,6 @@ */ #define CATCH_CONFIG_MAIN // This tells Catch to provide a main() - only do this in one cpp file #include "io/BaseStream.h" -#include "Site2SitePeer.h" -#include "Site2SiteClientProtocol.h" #include #include "SiteToSiteHelper.h" #include diff --git a/libminifi/test/unit/Site2SiteTests.cpp b/libminifi/test/unit/Site2SiteTests.cpp index c17ffb3a20..d79a95029d 100644 --- a/libminifi/test/unit/Site2SiteTests.cpp +++ b/libminifi/test/unit/Site2SiteTests.cpp @@ -23,8 +23,8 @@ #include #include #include "io/BaseStream.h" -#include "Site2SitePeer.h" -#include "Site2SiteClientProtocol.h" +#include "sitetosite/Peer.h" +#include "sitetosite/RawSocketProtocol.h" #include #include "../TestBase.h" #include "../unit/SiteToSiteHelper.h" @@ -32,10 +32,10 @@ #define FMT_DEFAULT fmt_lower TEST_CASE("TestSetPortId", "[S2S1]") { - std::unique_ptr peer = std::unique_ptr( - new minifi::Site2SitePeer(std::unique_ptr(new org::apache::nifi::minifi::io::DataStream()), "fake_host", 65433)); + std::unique_ptr peer = std::unique_ptr( + new minifi::sitetosite::SiteToSitePeer(std::unique_ptr(new org::apache::nifi::minifi::io::DataStream()), "fake_host", 65433)); - minifi::Site2SiteClientProtocol protocol(std::move(peer)); + minifi::sitetosite::RawSiteToSiteClient protocol(std::move(peer)); std::string uuid_str = "c56a4180-65aa-42ec-a945-5fd21dec0538"; @@ -49,10 +49,10 @@ TEST_CASE("TestSetPortId", "[S2S1]") { } TEST_CASE("TestSetPortIdUppercase", "[S2S2]") { - std::unique_ptr peer = std::unique_ptr( - new minifi::Site2SitePeer(std::unique_ptr(new org::apache::nifi::minifi::io::DataStream()), "fake_host", 65433)); + std::unique_ptr peer = std::unique_ptr( + new minifi::sitetosite::SiteToSitePeer(std::unique_ptr(new org::apache::nifi::minifi::io::DataStream()), "fake_host", 65433)); - minifi::Site2SiteClientProtocol protocol(std::move(peer)); + minifi::sitetosite::RawSiteToSiteClient protocol(std::move(peer)); std::string uuid_str = "C56A4180-65AA-42EC-A945-5FD21DEC0538"; @@ -94,10 +94,10 @@ TEST_CASE("TestSiteToSiteVerifySend", "[S2S3]") { sunny_path_bootstrap(collector); - std::unique_ptr peer = std::unique_ptr( - new minifi::Site2SitePeer(std::unique_ptr(new org::apache::nifi::minifi::io::BaseStream(collector)), "fake_host", 65433)); + std::unique_ptr peer = std::unique_ptr( + new minifi::sitetosite::SiteToSitePeer(std::unique_ptr(new org::apache::nifi::minifi::io::BaseStream(collector)), "fake_host", 65433)); - minifi::Site2SiteClientProtocol protocol(std::move(peer)); + minifi::sitetosite::RawSiteToSiteClient protocol(std::move(peer)); std::string uuid_str = "C56A4180-65AA-42EC-A945-5FD21DEC0538"; @@ -140,12 +140,13 @@ TEST_CASE("TestSiteToSiteVerifySend", "[S2S3]") { // Create the transaction std::string transactionID; std::string payload = "Test MiNiFi payload"; - minifi::Transaction *transaction; - transaction = protocol.createTransaction(transactionID, minifi::SEND); + std::shared_ptr transaction; + transaction = protocol.createTransaction(transactionID, minifi::sitetosite::SEND); collector->get_next_client_response(); REQUIRE(collector->get_next_client_response() == "SEND_FLOWFILES"); std::map attributes; - minifi::DataPacket packet(&protocol, transaction, attributes, payload); + std::shared_ptr logger = nullptr; + minifi::sitetosite::DataPacket packet(logger, transaction, attributes, payload); REQUIRE(protocol.send(transactionID, &packet, nullptr, nullptr) == 0); collector->get_next_client_response(); collector->get_next_client_response(); @@ -162,10 +163,10 @@ TEST_CASE("TestSiteToSiteVerifyNegotiationFail", "[S2S4]") { collector->push_response(resp_code); collector->push_response(resp_code); - std::unique_ptr peer = std::unique_ptr( - new minifi::Site2SitePeer(std::unique_ptr(new org::apache::nifi::minifi::io::BaseStream(collector)), "fake_host", 65433)); + std::unique_ptr peer = std::unique_ptr( + new minifi::sitetosite::SiteToSitePeer(std::unique_ptr(new org::apache::nifi::minifi::io::BaseStream(collector)), "fake_host", 65433)); - minifi::Site2SiteClientProtocol protocol(std::move(peer)); + minifi::sitetosite::RawSiteToSiteClient protocol(std::move(peer)); std::string uuid_str = "C56A4180-65AA-42EC-A945-5FD21DEC0538";