From a8edfb0c3e0d6cda2bf3ab50793b95f82fbc7e65 Mon Sep 17 00:00:00 2001 From: Marc Parisi Date: Tue, 14 Feb 2017 08:38:05 -0500 Subject: [PATCH 1/3] MINIFI-206: Implement Extensible Comm Mechanism Create initial communication mechanism that's initially based on BSD sockets. MINIFI-201: Change how hostname is resolved and ensure that it is not null MINIFI-199: Utilize zlib's crc32 implementation, which is more portable and faster. we can explore alternatives when that time arrives. MINIFI-200: Put host "get" into initialization. Any change wouldn't be picked up by a running process most likely. --- CMakeLists.txt | 4 +- libminifi/CMakeLists.txt | 17 +- libminifi/include/AppendHostInfo.h | 4 +- libminifi/include/Configure.h | 16 +- libminifi/include/Connection.h | 4 +- libminifi/include/Exception.h | 1 - libminifi/include/ExecuteProcess.h | 4 +- libminifi/include/FlowControlProtocol.h | 20 +- libminifi/include/FlowController.h | 38 +- libminifi/include/FlowFileRecord.h | 4 +- libminifi/include/GetFile.h | 4 +- libminifi/include/ListenSyslog.h | 6 +- libminifi/include/LogAttribute.h | 4 +- libminifi/include/Logger.h | 38 +- libminifi/include/ProcessContext.h | 4 +- libminifi/include/ProcessGroup.h | 58 +-- libminifi/include/ProcessSession.h | 6 +- libminifi/include/Processor.h | 6 +- libminifi/include/Property.h | 196 +++----- libminifi/include/Provenance.h | 39 +- libminifi/include/PutFile.h | 6 +- libminifi/include/RealTimeDataCollector.h | 4 +- libminifi/include/RemoteProcessorGroupPort.h | 33 +- libminifi/include/ResourceClaim.h | 4 +- libminifi/include/SchedulingAgent.h | 10 +- libminifi/include/Site2SiteClientProtocol.h | 437 +++++++++--------- libminifi/include/Site2SitePeer.h | 342 +++++--------- libminifi/include/TailFile.h | 4 +- libminifi/include/io/BaseStream.h | 150 ++++++ libminifi/include/io/CRCStream.h | 318 +++++++++++++ libminifi/include/io/ClientSocket.h | 245 ++++++++++ libminifi/include/io/DataStream.h | 120 +++++ libminifi/include/io/EndianCheck.h | 47 ++ libminifi/include/{ => io}/Serializable.h | 124 +---- libminifi/include/io/SocketFactory.h | 87 ++++ libminifi/include/io/TLSSocket.h | 185 ++++++++ libminifi/include/io/validation.h | 77 ++++ libminifi/include/utils/FailurePolicy.h | 45 ++ libminifi/include/utils/StringUtils.h | 125 +++++ libminifi/include/{ => utils}/TimeUtil.h | 66 ++- libminifi/src/AppendHostInfo.cpp | 14 +- libminifi/src/Configure.cpp | 34 +- libminifi/src/Connection.cpp | 12 +- libminifi/src/ExecuteProcess.cpp | 21 +- libminifi/src/FlowControlProtocol.cpp | 68 +-- libminifi/src/FlowController.cpp | 190 +++----- libminifi/src/FlowFileRecord.cpp | 8 +- libminifi/src/GenerateFlowFile.cpp | 3 +- libminifi/src/GetFile.cpp | 29 +- libminifi/src/ListenSyslog.cpp | 17 +- libminifi/src/LogAttribute.cpp | 15 +- libminifi/src/Logger.cpp | 2 +- libminifi/src/ProcessGroup.cpp | 259 +++++------ libminifi/src/ProcessSession.cpp | 46 +- libminifi/src/Processor.cpp | 40 +- libminifi/src/Provenance.cpp | 26 +- libminifi/src/PutFile.cpp | 23 +- libminifi/src/RealTimeDataCollector.cpp | 77 ++-- libminifi/src/RemoteProcessorGroupPort.cpp | 55 ++- libminifi/src/ResourceClaim.cpp | 6 +- libminifi/src/SchedulingAgent.cpp | 6 +- libminifi/src/Site2SiteClientProtocol.cpp | 228 ++++----- libminifi/src/Site2SitePeer.cpp | 461 +------------------ libminifi/src/TailFile.cpp | 19 +- libminifi/src/ThreadedSchedulingAgent.cpp | 26 +- libminifi/src/io/BaseStream.cpp | 153 ++++++ libminifi/src/io/CRCStream.cpp | 24 + libminifi/src/io/ClientSocket.cpp | 425 +++++++++++++++++ libminifi/src/io/DataStream.cpp | 128 +++++ libminifi/src/io/EndianCheck.cpp | 22 + libminifi/src/{ => io}/Serializable.cpp | 124 +---- libminifi/src/io/SocketFactory.cpp | 24 + libminifi/src/io/TLSSocket.cpp | 236 ++++++++++ libminifi/test/unit/CRCTests.h | 78 ++++ libminifi/test/unit/PropertyTests.h | 106 +++++ libminifi/test/unit/SerializationTests.h | 5 +- libminifi/test/unit/SocketTests.h | 190 ++++++++ libminifi/test/unit/Tests.cpp | 4 + libminifi/test/unit/TimeUtilsTest.h | 30 ++ 79 files changed, 3992 insertions(+), 2144 deletions(-) create mode 100644 libminifi/include/io/BaseStream.h create mode 100644 libminifi/include/io/CRCStream.h create mode 100644 libminifi/include/io/ClientSocket.h create mode 100644 libminifi/include/io/DataStream.h create mode 100644 libminifi/include/io/EndianCheck.h rename libminifi/include/{ => io}/Serializable.h (69%) create mode 100644 libminifi/include/io/SocketFactory.h create mode 100644 libminifi/include/io/TLSSocket.h create mode 100644 libminifi/include/io/validation.h create mode 100644 libminifi/include/utils/FailurePolicy.h create mode 100644 libminifi/include/utils/StringUtils.h rename libminifi/include/{ => utils}/TimeUtil.h (54%) create mode 100644 libminifi/src/io/BaseStream.cpp create mode 100644 libminifi/src/io/CRCStream.cpp create mode 100644 libminifi/src/io/ClientSocket.cpp create mode 100644 libminifi/src/io/DataStream.cpp create mode 100644 libminifi/src/io/EndianCheck.cpp rename libminifi/src/{ => io}/Serializable.cpp (67%) create mode 100644 libminifi/src/io/SocketFactory.cpp create mode 100644 libminifi/src/io/TLSSocket.cpp create mode 100644 libminifi/test/unit/CRCTests.h create mode 100644 libminifi/test/unit/PropertyTests.h create mode 100644 libminifi/test/unit/SocketTests.h create mode 100644 libminifi/test/unit/TimeUtilsTest.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ae1f8ee97..0f2c31b18b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,8 +33,8 @@ IF(POLICY CMP0048) ENDIF(POLICY CMP0048) include(CheckCXXCompilerFlag) -CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11) -CHECK_CXX_COMPILER_FLAG("-std=c++0x" COMPILER_SUPPORTS_CXX0X) +CHECK_CXX_COMPILER_FLAG("-std=c++11 " COMPILER_SUPPORTS_CXX11) +CHECK_CXX_COMPILER_FLAG("-std=c++0x " COMPILER_SUPPORTS_CXX0X) if(COMPILER_SUPPORTS_CXX11) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") elseif(COMPILER_SUPPORTS_CXX0X) diff --git a/libminifi/CMakeLists.txt b/libminifi/CMakeLists.txt index 269492cd17..cada08b0b1 100644 --- a/libminifi/CMakeLists.txt +++ b/libminifi/CMakeLists.txt @@ -36,12 +36,23 @@ ENDIF(POLICY CMP0048) set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED ON) +include(CheckCXXCompilerFlag) +CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11) +CHECK_CXX_COMPILER_FLAG("-std=c++0x" COMPILER_SUPPORTS_CXX0X) +if(COMPILER_SUPPORTS_CXX11) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -g") +elseif(COMPILER_SUPPORTS_CXX0X) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x") +else() + message(STATUS "The compiler ${CMAKE_CXX_COMPILER} has no C++11 support. Please use a different C++ compiler.") +endif() + include_directories(../include) include_directories(../thirdparty/yaml-cpp-yaml-cpp-0.5.3/include) include_directories(../thirdparty/civetweb-1.9.1/include) include_directories(include) -file(GLOB SOURCES "src/*.cpp") +file(GLOB_RECURSE SOURCES "src/*.cpp") file(GLOB SPD_SOURCES "../include/spdlog/*") # Workaround the limitations of having a @@ -52,6 +63,10 @@ add_library(minifi STATIC ${SOURCES}) # Include Boost System find_package(Boost COMPONENTS system REQUIRED) target_link_libraries(minifi ${Boost_SYSTEM_LIBRARY}) +find_package(ZLIB REQUIRED) + +include_directories(${ZLIB_INCLUDE_DIRS}) +target_link_libraries (minifi ${ZLIB_LIBRARIES}) # Include LevelDB find_package (Leveldb REQUIRED) diff --git a/libminifi/include/AppendHostInfo.h b/libminifi/include/AppendHostInfo.h index 8cc1d4174a..5d62aae4c6 100644 --- a/libminifi/include/AppendHostInfo.h +++ b/libminifi/include/AppendHostInfo.h @@ -35,7 +35,7 @@ class AppendHostInfo : public Processor AppendHostInfo(std::string name, uuid_t uuid = NULL) : Processor(name, uuid) { - _logger = Logger::getLogger(); + logger_ = Logger::getLogger(); } //! Destructor virtual ~AppendHostInfo() @@ -61,7 +61,7 @@ class AppendHostInfo : public Processor private: //! Logger - Logger *_logger; + Logger *logger_; }; #endif diff --git a/libminifi/include/Configure.h b/libminifi/include/Configure.h index f4aae3ce68..a60a6d5fb3 100644 --- a/libminifi/include/Configure.h +++ b/libminifi/include/Configure.h @@ -34,11 +34,11 @@ class Configure { //! Get the singleton logger instance static Configure * getConfigure() { - if (!_configure) + if (!configure_) { - _configure = new Configure(); + configure_ = new Configure(); } - return _configure; + return configure_; } //! nifi.flow.configuration.file static const char *nifi_flow_configuration_file; @@ -79,10 +79,6 @@ class Configure { } //! Get the config value bool get(std::string key, std::string &value); - // Trim String utils - std::string trim(const std::string& s); - std::string trimLeft(const std::string& s); - std::string trimRight(const std::string& s); //! Parse one line in configure file like key=value void parseConfigureFileLine(char *buf); //! Load Configure File @@ -105,19 +101,19 @@ class Configure { //! Mutex for protection std::mutex _mtx; //! Logger - Logger *_logger; + Logger *logger_; //! Home location for this executable std::string _minifiHome; Configure() { - _logger = Logger::getLogger(); + logger_ = Logger::getLogger(); } virtual ~Configure() { } - static Configure *_configure; + static Configure *configure_; protected: std::map _properties; diff --git a/libminifi/include/Connection.h b/libminifi/include/Connection.h index dc6b94bea0..d372b77d17 100644 --- a/libminifi/include/Connection.h +++ b/libminifi/include/Connection.h @@ -43,7 +43,7 @@ class Connection /*! * Create a new processor */ - Connection(std::string name, uuid_t uuid = NULL, uuid_t srcUUID = NULL, uuid_t destUUID = NULL); + explicit Connection(std::string name, uuid_t uuid = NULL, uuid_t srcUUID = NULL, uuid_t destUUID = NULL); //! Destructor virtual ~Connection() {} //! Set Connection Name @@ -190,7 +190,7 @@ class Connection //! Queue for the Flow File std::queue _queue; //! Logger - Logger *_logger; + Logger *logger_; // Prevent default copy constructor and assignment operation // Only support pass by reference or pointer Connection(const Connection &parent); diff --git a/libminifi/include/Exception.h b/libminifi/include/Exception.h index d32145488c..1e02fa5ea6 100644 --- a/libminifi/include/Exception.h +++ b/libminifi/include/Exception.h @@ -80,7 +80,6 @@ class Exception : public std::exception return _whatStr.c_str(); } -protected: private: //! Exception type diff --git a/libminifi/include/ExecuteProcess.h b/libminifi/include/ExecuteProcess.h index dce287ad59..83cf23cfd3 100644 --- a/libminifi/include/ExecuteProcess.h +++ b/libminifi/include/ExecuteProcess.h @@ -46,7 +46,7 @@ class ExecuteProcess : public Processor ExecuteProcess(std::string name, uuid_t uuid = NULL) : Processor(name, uuid) { - _logger = Logger::getLogger(); + logger_ = Logger::getLogger(); _redirectErrorStream = false; _batchDuration = 0; _workingDir = "."; @@ -94,7 +94,7 @@ class ExecuteProcess : public Processor private: //! Logger - Logger *_logger; + Logger *logger_; //! Property std::string _command; std::string _commandArgument; diff --git a/libminifi/include/FlowControlProtocol.h b/libminifi/include/FlowControlProtocol.h index ebf3c8ae1e..b3b068692e 100644 --- a/libminifi/include/FlowControlProtocol.h +++ b/libminifi/include/FlowControlProtocol.h @@ -176,8 +176,8 @@ class FlowControlProtocol */ FlowControlProtocol(FlowController *controller) { _controller = controller; - _logger = Logger::getLogger(); - _configure = Configure::getConfigure(); + logger_ = Logger::getLogger(); + configure_ = Configure::getConfigure(); _socket = 0; _serverName = "localhost"; _serverPort = DEFAULT_NIFI_SERVER_PORT; @@ -190,22 +190,22 @@ class FlowControlProtocol std::string value; - if (_configure->get(Configure::nifi_server_name, value)) + if (configure_->get(Configure::nifi_server_name, value)) { _serverName = value; - _logger->log_info("NiFi Server Name %s", _serverName.c_str()); + logger_->log_info("NiFi Server Name %s", _serverName.c_str()); } - if (_configure->get(Configure::nifi_server_port, value) && Property::StringToInt(value, _serverPort)) + if (configure_->get(Configure::nifi_server_port, value) && Property::StringToInt(value, _serverPort)) { - _logger->log_info("NiFi Server Port: [%d]", _serverPort); + logger_->log_info("NiFi Server Port: [%d]", _serverPort); } - if (_configure->get(Configure::nifi_server_report_interval, value)) + if (configure_->get(Configure::nifi_server_report_interval, value)) { TimeUnit unit; if (Property::StringToTime(value, _reportInterval, unit) && Property::ConvertTimeUnitToMS(_reportInterval, unit, _reportInterval)) { - _logger->log_info("NiFi server report interval: [%d] ms", _reportInterval); + logger_->log_info("NiFi server report interval: [%d] ms", _reportInterval); } } else @@ -304,9 +304,9 @@ class FlowControlProtocol //! Mutex for protection std::mutex _mtx; //! Logger - Logger *_logger = NULL; + Logger *logger_ = NULL; //! Configure - Configure *_configure = NULL; + Configure *configure_ = NULL; //! NiFi server Name std::string _serverName; //! NiFi server port diff --git a/libminifi/include/FlowController.h b/libminifi/include/FlowController.h index 561cd1bf01..56dba038ed 100644 --- a/libminifi/include/FlowController.h +++ b/libminifi/include/FlowController.h @@ -85,30 +85,9 @@ class FlowController { static const int DEFAULT_MAX_TIMER_DRIVEN_THREAD = 10; static const int DEFAULT_MAX_EVENT_DRIVEN_THREAD = 5; - //! passphase for the private file callback - static int pemPassWordCb(char *buf, int size, int rwflag, void *userdata) - { - std::string passphrase; - - if (Configure::getConfigure()->get(Configure::nifi_security_client_pass_phrase, passphrase)) - { - std::ifstream file(passphrase.c_str(), std::ifstream::in); - if (!file.good()) - { - memset(buf, 0, size); - return 0; - } - memset(buf, 0, size); - file.getline(buf, size); - return (int) strlen(buf); - } - return 0; - } //! Destructor virtual ~FlowController(){ - if (_ctx) - SSL_CTX_free(_ctx); } //! Set FlowController Name virtual void setName(std::string name) { @@ -193,18 +172,9 @@ class FlowController { _protocol->setSerialNumber(number); } - - //! getSSLContext - virtual SSL_CTX *getSSLContext() - { - return _ctx; - } protected: - //! SSL context - SSL_CTX *_ctx; - //! A global unique identifier uuid_t _uuid; //! FlowController Name @@ -243,13 +213,13 @@ class FlowController { FlowController() : _root(0), _maxTimerDrivenThreads(0), _maxEventDrivenThreads(0), _running( false), _initialized(false), _provenanceRepo(0), _protocol( - 0), _logger(Logger::getLogger()), _ctx(NULL){ + 0), logger_(Logger::getLogger()){ } private: //! Logger - Logger *_logger; + Logger *logger_; }; @@ -311,8 +281,8 @@ class FlowControllerImpl: public FlowController { //! Mutex for protection std::mutex _mtx; //! Logger - Logger *_logger; - Configure *_configure; + Logger *logger_; + Configure *configure_; //! Process Processor Node YAML void parseProcessorNodeYaml(YAML::Node processorNode, ProcessGroup *parent); diff --git a/libminifi/include/FlowFileRecord.h b/libminifi/include/FlowFileRecord.h index 9382996fb0..66265724bb 100644 --- a/libminifi/include/FlowFileRecord.h +++ b/libminifi/include/FlowFileRecord.h @@ -31,7 +31,7 @@ #include #include -#include "TimeUtil.h" +#include "utils/TimeUtil.h" #include "Logger.h" #include "ResourceClaim.h" @@ -216,7 +216,7 @@ class FlowFileRecord //! Orginal connection queue that this flow file was dequeued from Connection *_orginalConnection; //! Logger - Logger *_logger; + Logger *logger_; //! Snapshot flow record for session rollback bool _snapshot; // Prevent default copy constructor and assignment operation diff --git a/libminifi/include/GetFile.h b/libminifi/include/GetFile.h index 4a8775fb1a..0d74110fe7 100644 --- a/libminifi/include/GetFile.h +++ b/libminifi/include/GetFile.h @@ -35,7 +35,7 @@ class GetFile : public Processor GetFile(std::string name, uuid_t uuid = NULL) : Processor(name, uuid) { - _logger = Logger::getLogger(); + logger_ = Logger::getLogger(); _directory = "."; _recursive = true; _keepSourceFile = false; @@ -82,7 +82,7 @@ class GetFile : public Processor private: //! Logger - Logger *_logger; + Logger *logger_; //! Queue for store directory list std::queue _dirList; //! Get Listing size diff --git a/libminifi/include/ListenSyslog.h b/libminifi/include/ListenSyslog.h index 81bc92c25d..309a3ad457 100644 --- a/libminifi/include/ListenSyslog.h +++ b/libminifi/include/ListenSyslog.h @@ -53,7 +53,7 @@ class ListenSyslog : public Processor ListenSyslog(std::string name, uuid_t uuid = NULL) : Processor(name, uuid) { - _logger = Logger::getLogger(); + logger_ = Logger::getLogger(); _eventQueueByteSize = 0; _serverSocket = 0; _recvBufSize = 65507; @@ -87,7 +87,7 @@ class ListenSyslog : public Processor _clientSockets.clear(); if (_serverSocket > 0) { - _logger->log_info("ListenSysLog Server socket %d close", _serverSocket); + logger_->log_info("ListenSysLog Server socket %d close", _serverSocket); close(_serverSocket); _serverSocket = 0; } @@ -130,7 +130,7 @@ class ListenSyslog : public Processor private: //! Logger - Logger *_logger; + Logger *logger_; //! Run function for the thread static void run(ListenSyslog *process); //! Run Thread diff --git a/libminifi/include/LogAttribute.h b/libminifi/include/LogAttribute.h index 125ebf3397..2478c7a763 100644 --- a/libminifi/include/LogAttribute.h +++ b/libminifi/include/LogAttribute.h @@ -35,7 +35,7 @@ class LogAttribute : public Processor LogAttribute(std::string name, uuid_t uuid = NULL) : Processor(name, uuid) { - _logger = Logger::getLogger(); + logger_ = Logger::getLogger(); } //! Destructor virtual ~LogAttribute() @@ -122,7 +122,7 @@ class LogAttribute : public Processor private: //! Logger - Logger *_logger; + Logger *logger_; }; #endif diff --git a/libminifi/include/Logger.h b/libminifi/include/Logger.h index 189ce6c1e0..91d0e72a2e 100644 --- a/libminifi/include/Logger.h +++ b/libminifi/include/Logger.h @@ -65,14 +65,14 @@ class Logger { //! Get the singleton logger instance static Logger * getLogger() { - if (!_logger) - _logger = new Logger(); - return _logger; + if (!logger_) + logger_ = new Logger(); + return logger_; } void setLogLevel(LOG_LEVEL_E level) { - if (_spdlog == NULL) + if (spdlog_ == NULL) return; - _spdlog->set_level((spdlog::level::level_enum) level); + spdlog_->set_level((spdlog::level::level_enum) level); } void setLogLevel(const std::string &level,LOG_LEVEL_E defaultLevel = info ) @@ -112,10 +112,10 @@ class Logger { * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match */ void log_error(const char *const format, ...) { - if(_spdlog == NULL || !_spdlog->should_log(spdlog::level::level_enum::err)) + if(spdlog_ == NULL || !spdlog_->should_log(spdlog::level::level_enum::err)) return; FILL_BUFFER - _spdlog->error(buffer); + spdlog_->error(buffer); } /** * @brief Log warn message @@ -123,10 +123,10 @@ class Logger { * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match */ void log_warn(const char *const format, ...) { - if(_spdlog == NULL || !_spdlog->should_log(spdlog::level::level_enum::warn)) + if(spdlog_ == NULL || !spdlog_->should_log(spdlog::level::level_enum::warn)) return; FILL_BUFFER - _spdlog->warn(buffer); + spdlog_->warn(buffer); } /** * @brief Log info message @@ -134,10 +134,10 @@ class Logger { * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match */ void log_info(const char *const format, ...) { - if(_spdlog == NULL || !_spdlog->should_log(spdlog::level::level_enum::info)) + if(spdlog_ == NULL || !spdlog_->should_log(spdlog::level::level_enum::info)) return; FILL_BUFFER - _spdlog->info(buffer); + spdlog_->info(buffer); } /** * @brief Log debug message @@ -145,10 +145,10 @@ class Logger { * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match */ void log_debug(const char *const format, ...) { - if(_spdlog == NULL || !_spdlog->should_log(spdlog::level::level_enum::debug)) + if(spdlog_ == NULL || !spdlog_->should_log(spdlog::level::level_enum::debug)) return; FILL_BUFFER - _spdlog->debug(buffer); + spdlog_->debug(buffer); } /** * @brief Log trace message @@ -156,10 +156,10 @@ class Logger { * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match */ void log_trace(const char *const format, ...) { - if(_spdlog == NULL || !_spdlog->should_log(spdlog::level::level_enum::trace)) + if(spdlog_ == NULL || !spdlog_->should_log(spdlog::level::level_enum::trace)) return; FILL_BUFFER - _spdlog->trace(buffer); + spdlog_->trace(buffer); } protected: @@ -174,14 +174,14 @@ class Logger { * Create a logger * */ Logger(const std::string logger_name = LOG_NAME, const std::string filename = LOG_FILE_NAME, size_t max_file_size = DEFAULT_LOG_FILE_SIZE, size_t max_files = DEFAULT_LOG_FILE_NUMBER, bool force_flush = true) { - _spdlog = rotating_logger_mt(logger_name, filename, max_file_size, max_files); - _spdlog->set_level((spdlog::level::level_enum) debug); + spdlog_ = rotating_logger_mt(logger_name, filename, max_file_size, max_files); + spdlog_->set_level((spdlog::level::level_enum) debug); } //! spdlog - std::shared_ptr _spdlog; + std::shared_ptr spdlog_; //! Singleton logger instance - static Logger *_logger; + static Logger *logger_; }; #endif diff --git a/libminifi/include/ProcessContext.h b/libminifi/include/ProcessContext.h index 5c3288e510..469c87e0e5 100644 --- a/libminifi/include/ProcessContext.h +++ b/libminifi/include/ProcessContext.h @@ -40,7 +40,7 @@ class ProcessContext * Create a new process context associated with the processor/controller service/state manager */ ProcessContext(Processor *processor = NULL) : _processor(processor) { - _logger = Logger::getLogger(); + logger_ = Logger::getLogger(); } //! Destructor virtual ~ProcessContext() {} @@ -106,7 +106,7 @@ class ProcessContext //! Processor Processor *_processor; //! Logger - Logger *_logger; + Logger *logger_; }; diff --git a/libminifi/include/ProcessGroup.h b/libminifi/include/ProcessGroup.h index 304cfe6964..28661ae3fb 100644 --- a/libminifi/include/ProcessGroup.h +++ b/libminifi/include/ProcessGroup.h @@ -56,56 +56,56 @@ class ProcessGroup virtual ~ProcessGroup(); //! Set Processor Name void setName(std::string name) { - _name = name; + name_ = name; } //! Get Process Name std::string getName(void) { - return (_name); + return (name_); } //! Set URL void setURL(std::string url) { - _url = url; + url_ = url; } //! Get URL std::string getURL(void) { - return (_url); + return (url_); } //! SetTransmitting void setTransmitting(bool val) { - _transmitting = val; + transmitting_ = val; } //! Get Transmitting bool getTransmitting() { - return _transmitting; + return transmitting_; } //! setTimeOut void setTimeOut(uint64_t time) { - _timeOut = time; + timeOut_ = time; } uint64_t getTimeOut() { - return _timeOut; + return timeOut_; } //! Set Processor yield period in MilliSecond void setYieldPeriodMsec(uint64_t period) { - _yieldPeriodMsec = period; + yield_period_msec_ = period; } //! Get Processor yield period in MilliSecond uint64_t getYieldPeriodMsec(void) { - return(_yieldPeriodMsec); + return(yield_period_msec_); } //! Set UUID void setUUID(uuid_t uuid) { - uuid_copy(_uuid, uuid); + uuid_copy(uuid_, uuid); } //! Get UUID bool getUUID(uuid_t uuid) { if (uuid) { - uuid_copy(uuid, _uuid); + uuid_copy(uuid, uuid_); return true; } else @@ -121,13 +121,13 @@ class ProcessGroup bool isRootProcessGroup(); //! set parent process group void setParent(ProcessGroup *parent) { - std::lock_guard lock(_mtx); - _parentProcessGroup = parent; + std::lock_guard lock(mtx_); + parent_process_group_ = parent; } //! get parent process group ProcessGroup *getParent(void) { - std::lock_guard lock(_mtx); - return _parentProcessGroup; + std::lock_guard lock(mtx_); + return parent_process_group_; } //! Add processor void addProcessor(Processor *processor); @@ -150,32 +150,32 @@ class ProcessGroup protected: //! A global unique identifier - uuid_t _uuid; + uuid_t uuid_; //! Processor Group Name - std::string _name; + std::string name_; //! Process Group Type - ProcessGroupType _type; + ProcessGroupType type_; //! Processors (ProcessNode) inside this process group which include Input/Output Port, Remote Process Group input/Output port - std::set _processors; - std::set _childProcessGroups; + std::set processors_; + std::set child_process_groups_; //! Connections between the processor inside the group; - std::set _connections; + std::set connections_; //! Parent Process Group - ProcessGroup* _parentProcessGroup; + ProcessGroup* parent_process_group_; //! Yield Period in Milliseconds - std::atomic _yieldPeriodMsec; - std::atomic _timeOut; + std::atomic yield_period_msec_; + std::atomic timeOut_; //! URL - std::string _url; + std::string url_; //! Transmitting - std::atomic _transmitting; + std::atomic transmitting_; private: //! Mutex for protection - std::mutex _mtx; + std::mutex mtx_; //! Logger - Logger *_logger; + Logger *logger_; // Prevent default copy constructor and assignment operation // Only support pass by reference or pointer ProcessGroup(const ProcessGroup &parent); diff --git a/libminifi/include/ProcessSession.h b/libminifi/include/ProcessSession.h index 44032c35c3..71465533f6 100644 --- a/libminifi/include/ProcessSession.h +++ b/libminifi/include/ProcessSession.h @@ -45,8 +45,8 @@ class ProcessSession * Create a new process session */ ProcessSession(ProcessContext *processContext = NULL) : _processContext(processContext) { - _logger = Logger::getLogger(); - _logger->log_trace("ProcessSession created for %s", _processContext->getProcessor()->getName().c_str()); + logger_ = Logger::getLogger(); + logger_->log_trace("ProcessSession created for %s", _processContext->getProcessor()->getName().c_str()); _provenanceReport = new ProvenanceReporter(_processContext->getProcessor()->getUUIDStr(), _processContext->getProcessor()->getName()); } @@ -121,7 +121,7 @@ class ProcessSession //! ProcessContext ProcessContext *_processContext; //! Logger - Logger *_logger; + Logger *logger_; //! Provenance Report ProvenanceReporter *_provenanceReport; diff --git a/libminifi/include/Processor.h b/libminifi/include/Processor.h index 95f47c15d7..b675b7c518 100644 --- a/libminifi/include/Processor.h +++ b/libminifi/include/Processor.h @@ -33,7 +33,7 @@ #include #include -#include "TimeUtil.h" +#include "utils/TimeUtil.h" #include "Property.h" #include "Relationship.h" #include "Connection.h" @@ -119,7 +119,9 @@ class Processor return true; } else + { return false; + } } //! Set the supported processor properties while the process is not running bool setSupportedProperties(std::set properties); @@ -352,7 +354,7 @@ class Processor //! Check all incoming connections for work bool isWorkAvailable(); //! Logger - Logger *_logger; + Logger *logger_; // Prevent default copy constructor and assignment operation // Only support pass by reference or pointer Processor(const Processor &parent); diff --git a/libminifi/include/Property.h b/libminifi/include/Property.h index a724394ae1..93f9c0e406 100644 --- a/libminifi/include/Property.h +++ b/libminifi/include/Property.h @@ -20,24 +20,22 @@ #ifndef __PROPERTY_H__ #define __PROPERTY_H__ +#include +#include #include #include #include #include #include #include +#include #include #include #include //! Time Unit enum TimeUnit { - DAY, - HOUR, - MINUTE, - SECOND, - MILLISECOND, - NANOSECOND + DAY, HOUR, MINUTE, SECOND, MILLISECOND, NANOSECOND }; //! Property Class @@ -48,12 +46,15 @@ class Property { /*! * Create a new property */ - Property(const std::string name, const std::string description, const std::string value) - : _name(name), _description(description), _value(value) { + Property(const std::string name, const std::string description, + const std::string value) : + _name(name), _description(description), _value(value) { + } + Property() { } - Property() {} //! Destructor - virtual ~Property() {} + virtual ~Property() { + } //! Get Name for the property std::string getName() { return _name; @@ -71,84 +72,60 @@ class Property { _value = value; } //! Compare - bool operator < (const Property & right) const { + bool operator <(const Property & right) const { return _name < right._name; } //! Convert TimeUnit to MilliSecond - static bool ConvertTimeUnitToMS(int64_t input, TimeUnit unit, int64_t &out) - { - if (unit == MILLISECOND) - { + static bool ConvertTimeUnitToMS(int64_t input, TimeUnit unit, + int64_t &out) { + if (unit == MILLISECOND) { out = input; return true; - } - else if (unit == SECOND) - { + } else if (unit == SECOND) { out = input * 1000; return true; - } - else if (unit == MINUTE) - { + } else if (unit == MINUTE) { out = input * 60 * 1000; return true; - } - else if (unit == HOUR) - { + } else if (unit == HOUR) { out = input * 60 * 60 * 1000; return true; - } - else if (unit == DAY) - { + } else if (unit == DAY) { out = 24 * 60 * 60 * 1000; return true; - } - else if (unit == NANOSECOND) - { - out = input/1000/1000; + } else if (unit == NANOSECOND) { + out = input / 1000 / 1000; return true; - } - else - { + } else { return false; } } //! Convert TimeUnit to NanoSecond - static bool ConvertTimeUnitToNS(int64_t input, TimeUnit unit, int64_t &out) - { - if (unit == MILLISECOND) - { + static bool ConvertTimeUnitToNS(int64_t input, TimeUnit unit, + int64_t &out) { + if (unit == MILLISECOND) { out = input * 1000 * 1000; return true; - } - else if (unit == SECOND) - { + } else if (unit == SECOND) { out = input * 1000 * 1000 * 1000; return true; - } - else if (unit == MINUTE) - { + } else if (unit == MINUTE) { out = input * 60 * 1000 * 1000 * 1000; return true; - } - else if (unit == HOUR) - { + } else if (unit == HOUR) { out = input * 60 * 60 * 1000 * 1000 * 1000; return true; - } - else if (unit == NANOSECOND) - { + } else if (unit == NANOSECOND) { out = input; return true; - } - else - { + } else { return false; } } //! Convert String - static bool StringToTime(std::string input, int64_t &output, TimeUnit &timeunit) - { + static bool StringToTime(std::string input, int64_t &output, + TimeUnit &timeunit) { if (input.size() == 0) { return false; } @@ -157,62 +134,52 @@ class Property { char *pEnd; long int ival = strtol(cvalue, &pEnd, 0); - if (pEnd[0] == '\0') - { + if (pEnd[0] == '\0') { return false; } - while (*pEnd == ' ') - { + while (*pEnd == ' ') { // Skip the space pEnd++; } std::string unit(pEnd); - if (unit == "sec" || unit == "s" || unit == "second" || unit == "seconds" || unit == "secs") - { + if (unit == "sec" || unit == "s" || unit == "second" + || unit == "seconds" || unit == "secs") { timeunit = SECOND; output = ival; return true; - } - else if (unit == "min" || unit == "m" || unit == "mins" || unit == "minute" || unit == "minutes") - { + } else if (unit == "min" || unit == "m" || unit == "mins" + || unit == "minute" || unit == "minutes") { timeunit = MINUTE; output = ival; return true; - } - else if (unit == "ns" || unit == "nano" || unit == "nanos" || unit == "nanoseconds") - { + } else if (unit == "ns" || unit == "nano" || unit == "nanos" + || unit == "nanoseconds") { timeunit = NANOSECOND; output = ival; return true; - } - else if (unit == "ms" || unit == "milli" || unit == "millis" || unit == "milliseconds") - { + } else if (unit == "ms" || unit == "milli" || unit == "millis" + || unit == "milliseconds") { timeunit = MILLISECOND; output = ival; return true; - } - else if (unit == "h" || unit == "hr" || unit == "hour" || unit == "hrs" || unit == "hours") - { + } else if (unit == "h" || unit == "hr" || unit == "hour" + || unit == "hrs" || unit == "hours") { timeunit = HOUR; output = ival; return true; - } - else if (unit == "d" || unit == "day" || unit == "days") - { + } else if (unit == "d" || unit == "day" || unit == "days") { timeunit = DAY; output = ival; return true; - } - else + } else return false; } //! Convert String to Integer - static bool StringToInt(std::string input, int64_t &output) - { + static bool StringToInt(std::string input, int64_t &output) { if (input.size() == 0) { return false; } @@ -221,26 +188,23 @@ class Property { char *pEnd; long int ival = strtol(cvalue, &pEnd, 0); - if (pEnd[0] == '\0') - { + if (pEnd[0] == '\0') { output = ival; return true; } - while (*pEnd == ' ') - { + while (*pEnd == ' ') { // Skip the space pEnd++; } char end0 = toupper(pEnd[0]); - if ( (end0 == 'K') || (end0 == 'M') || (end0 == 'G') || (end0 == 'T') || (end0 == 'P') ) - { - if (pEnd[1] == '\0') - { + if ((end0 == 'K') || (end0 == 'M') || (end0 == 'G') || (end0 == 'T') + || (end0 == 'P')) { + if (pEnd[1] == '\0') { unsigned long int multiplier = 1000; - if ( (end0 != 'K') ) { + if ((end0 != 'K')) { multiplier *= 1000; if (end0 != 'M') { multiplier *= 1000; @@ -255,11 +219,12 @@ class Property { output = ival * multiplier; return true; - } else if ((pEnd[1] == 'b' || pEnd[1] == 'B') && (pEnd[2] == '\0')) { + } else if ((pEnd[1] == 'b' || pEnd[1] == 'B') + && (pEnd[2] == '\0')) { unsigned long int multiplier = 1024; - if ( (end0 != 'K') ) { + if ((end0 != 'K')) { multiplier *= 1024; if (end0 != 'M') { multiplier *= 1024; @@ -278,56 +243,7 @@ class Property { return false; } - //! Convert String to Float - static bool StringToFloat(std::string input, float &output) - { - const char *cvalue = input.c_str(); - char *pEnd; - float val = strtof(cvalue, &pEnd); - - if (pEnd[0] == '\0') - { - output = val; - return true; - } - else - return false; - } - //! Convert String to Bool - static bool StringToBool(std::string input, bool &output) - { - if (input == "true" || input == "True" || input == "TRUE") - { - output = true; - return true; - } - if (input == "false" || input == "False" || input == "FALSE") - { - output = false; - return true; - } - return false; - } - // Trim String utils - static std::string trim(const std::string& s) - { - return trimRight(trimLeft(s)); - } - - static std::string trimLeft(const std::string& s) - { - const char *WHITESPACE = " \n\r\t"; - size_t startpos = s.find_first_not_of(WHITESPACE); - return (startpos == std::string::npos) ? "" : s.substr(startpos); - } - - static std::string trimRight(const std::string& s) - { - const char *WHITESPACE = " \n\r\t"; - size_t endpos = s.find_last_not_of(WHITESPACE); - return (endpos == std::string::npos) ? "" : s.substr(0, endpos+1); - } protected: //! Name diff --git a/libminifi/include/Provenance.h b/libminifi/include/Provenance.h index 822b3cf3e2..10945c31bb 100644 --- a/libminifi/include/Provenance.h +++ b/libminifi/include/Provenance.h @@ -42,8 +42,8 @@ #include "Logger.h" #include "Property.h" #include "ResourceClaim.h" -#include "TimeUtil.h" -#include "Serializable.h" +#include "io/Serializable.h" +#include "utils/TimeUtil.h" // Provenance Event Record Serialization Seg Size #define PROVENANCE_EVENT_RECORD_SEG_SIZE 2048 @@ -173,12 +173,12 @@ class ProvenanceEventRecord : protected Serializable uuid_generate(_eventId); uuid_unparse_lower(_eventId, eventIdStr); _eventIdStr = eventIdStr; - _logger = Logger::getLogger(); + logger_ = Logger::getLogger(); } ProvenanceEventRecord() { _eventTime = getTimeMillis(); - _logger = Logger::getLogger(); + logger_ = Logger::getLogger(); } //! Destructor @@ -451,7 +451,7 @@ class ProvenanceEventRecord : protected Serializable private: //! Logger - Logger *_logger; + Logger *logger_; // Prevent default copy constructor and assignment operation // Only support pass by reference or pointer @@ -470,7 +470,7 @@ class ProvenanceReporter * Create a new provenance reporter associated with the process session */ ProvenanceReporter(std::string componentId, std::string componentType) { - _logger = Logger::getLogger(); + logger_ = Logger::getLogger(); _componentId = componentId; _componentType = componentType; } @@ -555,7 +555,7 @@ class ProvenanceReporter //! Incoming connection Iterator std::set _events; //! Logger - Logger *_logger; + Logger *logger_; // Prevent default copy constructor and assignment operation // Only support pass by reference or pointer @@ -577,8 +577,8 @@ class ProvenanceRepository * Create a new provenance repository */ ProvenanceRepository() { - _logger = Logger::getLogger(); - _configure = Configure::getConfigure(); + logger_ = Logger::getLogger(); + configure_ = Configure::getConfigure(); _directory = PROVENANCE_DIRECTORY; _maxPartitionMillis = MAX_PROVENANCE_ENTRY_LIFE_TIME; _purgePeriod = PROVENANCE_PURGE_PERIOD; @@ -601,17 +601,17 @@ class ProvenanceRepository virtual bool initialize() { std::string value; - if (_configure->get(Configure::nifi_provenance_repository_directory_default, value)) + if (configure_->get(Configure::nifi_provenance_repository_directory_default, value)) { _directory = value; } - _logger->log_info("NiFi Provenance Repository Directory %s", _directory.c_str()); - if (_configure->get(Configure::nifi_provenance_repository_max_storage_size, value)) + logger_->log_info("NiFi Provenance Repository Directory %s", _directory.c_str()); + if (configure_->get(Configure::nifi_provenance_repository_max_storage_size, value)) { Property::StringToInt(value, _maxPartitionBytes); } - _logger->log_info("NiFi Provenance Max Partition Bytes %d", _maxPartitionBytes); - if (_configure->get(Configure::nifi_provenance_repository_max_storage_time, value)) + logger_->log_info("NiFi Provenance Max Partition Bytes %d", _maxPartitionBytes); + if (configure_->get(Configure::nifi_provenance_repository_max_storage_time, value)) { TimeUnit unit; if (Property::StringToTime(value, _maxPartitionMillis, unit) && @@ -619,17 +619,17 @@ class ProvenanceRepository { } } - _logger->log_info("NiFi Provenance Max Storage Time: [%d] ms", _maxPartitionMillis); + logger_->log_info("NiFi Provenance Max Storage Time: [%d] ms", _maxPartitionMillis); leveldb::Options options; options.create_if_missing = true; leveldb::Status status = leveldb::DB::Open(options, _directory.c_str(), &_db); if (status.ok()) { - _logger->log_info("NiFi Provenance Repository database open %s success", _directory.c_str()); + logger_->log_info("NiFi Provenance Repository database open %s success", _directory.c_str()); } else { - _logger->log_error("NiFi Provenance Repository database open %s fail", _directory.c_str()); + logger_->log_error("NiFi Provenance Repository database open %s fail", _directory.c_str()); return false; } @@ -640,6 +640,7 @@ class ProvenanceRepository //! Put virtual bool Put(std::string key, uint8_t *buf, int bufLen) { + // persistent to the DB leveldb::Slice value((const char *) buf, bufLen); leveldb::Status status; @@ -709,10 +710,10 @@ class ProvenanceRepository //! repository directory std::string _directory; //! Logger - Logger *_logger; + Logger *logger_; //! Configure //! max db entry life time - Configure *_configure; + Configure *configure_; int64_t _maxPartitionMillis; //! max db size int64_t _maxPartitionBytes; diff --git a/libminifi/include/PutFile.h b/libminifi/include/PutFile.h index 9f1375df71..bf48f79e60 100644 --- a/libminifi/include/PutFile.h +++ b/libminifi/include/PutFile.h @@ -40,7 +40,7 @@ class PutFile : public Processor PutFile(std::string name, uuid_t uuid = NULL) : Processor(name, uuid) { - _logger = Logger::getLogger(); + logger_ = Logger::getLogger(); } //! Destructor virtual ~PutFile() @@ -69,7 +69,7 @@ class PutFile : public Processor bool commit(); private: - Logger *_logger; + Logger *logger_; std::ofstream _tmpFileOs; bool _writeSucceeded = false; std::string _tmpFile; @@ -80,7 +80,7 @@ class PutFile : public Processor private: //! Logger - Logger *_logger; + Logger *logger_; bool putFile(ProcessSession *session, FlowFileRecord *flowFile, const std::string &tmpFile, const std::string &destFile); }; diff --git a/libminifi/include/RealTimeDataCollector.h b/libminifi/include/RealTimeDataCollector.h index 760b566a22..6458f6dcf2 100644 --- a/libminifi/include/RealTimeDataCollector.h +++ b/libminifi/include/RealTimeDataCollector.h @@ -47,7 +47,7 @@ class RealTimeDataCollector : public Processor { _realTimeSocket = 0; _batchSocket = 0; - _logger = Logger::getLogger(); + logger_ = Logger::getLogger(); _firstInvoking = false; _realTimeAccumulated = 0; _batchAcccumulated = 0; @@ -113,7 +113,7 @@ class RealTimeDataCollector : public Processor int _realTimeSocket; int _batchSocket; //! Logger - Logger *_logger; + Logger *logger_; //! Mutex for protection std::mutex _mtx; //! Queued data size diff --git a/libminifi/include/RemoteProcessorGroupPort.h b/libminifi/include/RemoteProcessorGroupPort.h index cd99e50e0a..73e8ac2c8f 100644 --- a/libminifi/include/RemoteProcessorGroupPort.h +++ b/libminifi/include/RemoteProcessorGroupPort.h @@ -20,6 +20,7 @@ #ifndef __REMOTE_PROCESSOR_GROUP_PORT_H__ #define __REMOTE_PROCESSOR_GROUP_PORT_H__ +#include #include "FlowFileRecord.h" #include "Processor.h" #include "ProcessSession.h" @@ -34,18 +35,16 @@ class RemoteProcessorGroupPort : public Processor * Create a new processor */ RemoteProcessorGroupPort(std::string name, uuid_t uuid = NULL) - : Processor(name, uuid) + : Processor(name, uuid), direction_(SEND),transmitting_(false), peer_() { - _logger = Logger::getLogger(); - _peer = new Site2SitePeer("", 9999); - _protocol = new Site2SiteClientProtocol(_peer); - _protocol->setPortId(uuid); + logger_ = Logger::getLogger(); + protocol_ = std::unique_ptr(new Site2SiteClientProtocol(0)); + protocol_->setPortId(uuid); } //! Destructor virtual ~RemoteProcessorGroupPort() { - delete _protocol; - delete _peer; + } //! Processor Name static const std::string ProcessorName; @@ -62,34 +61,34 @@ class RemoteProcessorGroupPort : public Processor //! Set Direction void setDirection(TransferDirection direction) { - _direction = direction; - if (_direction == RECEIVE) + direction_ = direction; + if (direction_ == RECEIVE) this->setTriggerWhenEmpty(true); } //! Set Timeout void setTimeOut(uint64_t timeout) { - _protocol->setTimeOut(timeout); + protocol_->setTimeOut(timeout); } //! SetTransmitting void setTransmitting(bool val) { - _transmitting = val; + transmitting_ = val; } protected: private: //! Logger - Logger *_logger; - //! Peer Connection - Site2SitePeer *_peer; + Logger *logger_; + + Site2SitePeer peer_; //! Peer Protocol - Site2SiteClientProtocol *_protocol; + std::unique_ptr protocol_; //! Transaction Direction - TransferDirection _direction; + TransferDirection direction_; //! Transmitting - bool _transmitting; + bool transmitting_; }; diff --git a/libminifi/include/ResourceClaim.h b/libminifi/include/ResourceClaim.h index b4085f24a3..dcbb558ee8 100644 --- a/libminifi/include/ResourceClaim.h +++ b/libminifi/include/ResourceClaim.h @@ -81,9 +81,9 @@ class ResourceClaim { private: //! Configure - Configure *_configure; + Configure *configure_; //! Logger - Logger *_logger; + Logger *logger_; // Prevent default copy constructor and assignment operation // Only support pass by reference or pointer ResourceClaim(const ResourceClaim &parent); diff --git a/libminifi/include/SchedulingAgent.h b/libminifi/include/SchedulingAgent.h index 1f94585647..c814fa3533 100644 --- a/libminifi/include/SchedulingAgent.h +++ b/libminifi/include/SchedulingAgent.h @@ -27,7 +27,7 @@ #include #include #include -#include "TimeUtil.h" +#include "utils/TimeUtil.h" #include "Logger.h" #include "Configure.h" #include "FlowFileRecord.h" @@ -43,8 +43,8 @@ class SchedulingAgent * Create a new processor */ SchedulingAgent() { - _configure = Configure::getConfigure(); - _logger = Logger::getLogger(); + configure_ = Configure::getConfigure(); + logger_ = Logger::getLogger(); _running = false; } //! Destructor @@ -75,9 +75,9 @@ class SchedulingAgent protected: //! Logger - Logger *_logger; + Logger *logger_; //! Configure - Configure *_configure; + Configure *configure_; //! Mutex for protection std::mutex _mtx; //! Whether it is running diff --git a/libminifi/include/Site2SiteClientProtocol.h b/libminifi/include/Site2SiteClientProtocol.h index 11634366b1..fc1c411461 100644 --- a/libminifi/include/Site2SiteClientProtocol.h +++ b/libminifi/include/Site2SiteClientProtocol.h @@ -41,6 +41,7 @@ #include "FlowFileRecord.h" #include "ProcessContext.h" #include "ProcessSession.h" +#include "io/CRCStream.h" //! Resource Negotiated Status Code #define RESOURCE_OK 20 @@ -64,7 +65,6 @@ typedef enum { RECEIVE } TransferDirection; - //! Peer State typedef enum { /** @@ -116,55 +116,50 @@ typedef enum { //! Request Type typedef enum { - NEGOTIATE_FLOWFILE_CODEC = 0, - REQUEST_PEER_LIST, - SEND_FLOWFILES, - RECEIVE_FLOWFILES, - SHUTDOWN, + NEGOTIATE_FLOWFILE_CODEC = 0, + REQUEST_PEER_LIST, + SEND_FLOWFILES, + RECEIVE_FLOWFILES, + SHUTDOWN, MAX_REQUEST_TYPE } RequestType; //! Request Type Str -static const char *RequestTypeStr[MAX_REQUEST_TYPE] = -{ - "NEGOTIATE_FLOWFILE_CODEC", - "REQUEST_PEER_LIST", - "SEND_FLOWFILES", - "RECEIVE_FLOWFILES", - "SHUTDOWN" -}; +static const char *RequestTypeStr[MAX_REQUEST_TYPE] = { + "NEGOTIATE_FLOWFILE_CODEC", "REQUEST_PEER_LIST", "SEND_FLOWFILES", + "RECEIVE_FLOWFILES", "SHUTDOWN" }; //! Respond Code typedef enum { - RESERVED = 0, - // ResponseCode, so that we can indicate a 0 followed by some other bytes - - // handshaking properties - PROPERTIES_OK = 1, - UNKNOWN_PROPERTY_NAME = 230, - ILLEGAL_PROPERTY_VALUE = 231, - MISSING_PROPERTY = 232, - // transaction indicators - CONTINUE_TRANSACTION = 10, - FINISH_TRANSACTION = 11, - CONFIRM_TRANSACTION = 12, // "Explanation" of this code is the checksum - TRANSACTION_FINISHED = 13, - TRANSACTION_FINISHED_BUT_DESTINATION_FULL = 14, - CANCEL_TRANSACTION = 15, - BAD_CHECKSUM = 19, - // data availability indicators - MORE_DATA = 20, - NO_MORE_DATA = 21, - // port state indicators - UNKNOWN_PORT = 200, - PORT_NOT_IN_VALID_STATE = 201, - PORTS_DESTINATION_FULL = 202, - // authorization - UNAUTHORIZED = 240, - // error indicators - ABORT = 250, - UNRECOGNIZED_RESPONSE_CODE = 254, - END_OF_STREAM = 255 + RESERVED = 0, + // ResponseCode, so that we can indicate a 0 followed by some other bytes + + // handshaking properties + PROPERTIES_OK = 1, + UNKNOWN_PROPERTY_NAME = 230, + ILLEGAL_PROPERTY_VALUE = 231, + MISSING_PROPERTY = 232, + // transaction indicators + CONTINUE_TRANSACTION = 10, + FINISH_TRANSACTION = 11, + CONFIRM_TRANSACTION = 12, // "Explanation" of this code is the checksum + TRANSACTION_FINISHED = 13, + TRANSACTION_FINISHED_BUT_DESTINATION_FULL = 14, + CANCEL_TRANSACTION = 15, + BAD_CHECKSUM = 19, + // data availability indicators + MORE_DATA = 20, + NO_MORE_DATA = 21, + // port state indicators + UNKNOWN_PORT = 200, + PORT_NOT_IN_VALID_STATE = 201, + PORTS_DESTINATION_FULL = 202, + // authorization + UNAUTHORIZED = 240, + // error indicators + ABORT = 250, + UNRECOGNIZED_RESPONSE_CODE = 254, + END_OF_STREAM = 255 } RespondCode; //! Respond Code Class @@ -175,30 +170,26 @@ typedef struct { } RespondCodeContext; //! Respond Code Context -static RespondCodeContext respondCodeContext[] = -{ - {RESERVED, "Reserved for Future Use", false}, - {PROPERTIES_OK, "Properties OK", false}, - {UNKNOWN_PROPERTY_NAME, "Unknown Property Name", true}, - {ILLEGAL_PROPERTY_VALUE, "Illegal Property Value", true}, - {MISSING_PROPERTY, "Missing Property", true}, - {CONTINUE_TRANSACTION, "Continue Transaction", false}, - {FINISH_TRANSACTION, "Finish Transaction", false}, - {CONFIRM_TRANSACTION, "Confirm Transaction", true}, - {TRANSACTION_FINISHED, "Transaction Finished", false}, - {TRANSACTION_FINISHED_BUT_DESTINATION_FULL, "Transaction Finished But Destination is Full", false}, - {CANCEL_TRANSACTION, "Cancel Transaction", true}, - {BAD_CHECKSUM, "Bad Checksum", false}, - {MORE_DATA, "More Data Exists", false}, - {NO_MORE_DATA, "No More Data Exists", false}, - {UNKNOWN_PORT, "Unknown Port", false}, - {PORT_NOT_IN_VALID_STATE, "Port Not in a Valid State", true}, - {PORTS_DESTINATION_FULL, "Port's Destination is Full", false}, - {UNAUTHORIZED, "User Not Authorized", true}, - {ABORT, "Abort", true}, - {UNRECOGNIZED_RESPONSE_CODE, "Unrecognized Response Code", false}, - {END_OF_STREAM, "End of Stream", false} -}; +static RespondCodeContext respondCodeContext[] = { { RESERVED, + "Reserved for Future Use", false }, { PROPERTIES_OK, "Properties OK", + false }, { UNKNOWN_PROPERTY_NAME, "Unknown Property Name", true }, { + ILLEGAL_PROPERTY_VALUE, "Illegal Property Value", true }, { + MISSING_PROPERTY, "Missing Property", true }, { CONTINUE_TRANSACTION, + "Continue Transaction", false }, { FINISH_TRANSACTION, + "Finish Transaction", false }, { CONFIRM_TRANSACTION, + "Confirm Transaction", true }, { TRANSACTION_FINISHED, + "Transaction Finished", false }, { + TRANSACTION_FINISHED_BUT_DESTINATION_FULL, + "Transaction Finished But Destination is Full", false }, { + CANCEL_TRANSACTION, "Cancel Transaction", true }, { BAD_CHECKSUM, + "Bad Checksum", false }, { MORE_DATA, "More Data Exists", false }, { + NO_MORE_DATA, "No More Data Exists", false }, { UNKNOWN_PORT, + "Unknown Port", false }, { PORT_NOT_IN_VALID_STATE, + "Port Not in a Valid State", true }, { PORTS_DESTINATION_FULL, + "Port's Destination is Full", false }, { UNAUTHORIZED, + "User Not Authorized", true }, { ABORT, "Abort", true }, { + UNRECOGNIZED_RESPONSE_CODE, "Unrecognized Response Code", false }, { + END_OF_STREAM, "End of Stream", false } }; //! Respond Code Sequence Pattern static const uint8_t CODE_SEQUENCE_VALUE_1 = (uint8_t) 'R'; @@ -209,94 +200,92 @@ static const uint8_t CODE_SEQUENCE_VALUE_2 = (uint8_t) 'C'; * Protocol. */ typedef enum { - /** - * Boolean value indicating whether or not the contents of a FlowFile should - * be GZipped when transferred. - */ - GZIP, - /** - * The unique identifier of the port to communicate with - */ - PORT_IDENTIFIER, - /** - * Indicates the number of milliseconds after the request was made that the - * client will wait for a response. If no response has been received by the - * time this value expires, the server can move on without attempting to - * service the request because the client will have already disconnected. - */ - REQUEST_EXPIRATION_MILLIS, - /** - * The preferred number of FlowFiles that the server should send to the - * client when pulling data. This property was introduced in version 5 of - * the protocol. - */ - BATCH_COUNT, - /** - * The preferred number of bytes that the server should send to the client - * when pulling data. This property was introduced in version 5 of the - * protocol. - */ - BATCH_SIZE, - /** - * The preferred amount of time that the server should send data to the - * client when pulling data. This property was introduced in version 5 of - * the protocol. Value is in milliseconds. - */ - BATCH_DURATION, - MAX_HANDSHAKE_PROPERTY + /** + * Boolean value indicating whether or not the contents of a FlowFile should + * be GZipped when transferred. + */ + GZIP, + /** + * The unique identifier of the port to communicate with + */ + PORT_IDENTIFIER, + /** + * Indicates the number of milliseconds after the request was made that the + * client will wait for a response. If no response has been received by the + * time this value expires, the server can move on without attempting to + * service the request because the client will have already disconnected. + */ + REQUEST_EXPIRATION_MILLIS, + /** + * The preferred number of FlowFiles that the server should send to the + * client when pulling data. This property was introduced in version 5 of + * the protocol. + */ + BATCH_COUNT, + /** + * The preferred number of bytes that the server should send to the client + * when pulling data. This property was introduced in version 5 of the + * protocol. + */ + BATCH_SIZE, + /** + * The preferred amount of time that the server should send data to the + * client when pulling data. This property was introduced in version 5 of + * the protocol. Value is in milliseconds. + */ + BATCH_DURATION, MAX_HANDSHAKE_PROPERTY } HandshakeProperty; //! HandShakeProperty Str -static const char *HandShakePropertyStr[MAX_HANDSHAKE_PROPERTY] = -{ - /** - * Boolean value indicating whether or not the contents of a FlowFile should - * be GZipped when transferred. - */ - "GZIP", - /** - * The unique identifier of the port to communicate with - */ - "PORT_IDENTIFIER", - /** - * Indicates the number of milliseconds after the request was made that the - * client will wait for a response. If no response has been received by the - * time this value expires, the server can move on without attempting to - * service the request because the client will have already disconnected. - */ - "REQUEST_EXPIRATION_MILLIS", - /** - * The preferred number of FlowFiles that the server should send to the - * client when pulling data. This property was introduced in version 5 of - * the protocol. - */ - "BATCH_COUNT", - /** - * The preferred number of bytes that the server should send to the client - * when pulling data. This property was introduced in version 5 of the - * protocol. - */ - "BATCH_SIZE", - /** - * The preferred amount of time that the server should send data to the - * client when pulling data. This property was introduced in version 5 of - * the protocol. Value is in milliseconds. - */ - "BATCH_DURATION" -}; +static const char *HandShakePropertyStr[MAX_HANDSHAKE_PROPERTY] = { +/** + * Boolean value indicating whether or not the contents of a FlowFile should + * be GZipped when transferred. + */ +"GZIP", +/** + * The unique identifier of the port to communicate with + */ +"PORT_IDENTIFIER", +/** + * Indicates the number of milliseconds after the request was made that the + * client will wait for a response. If no response has been received by the + * time this value expires, the server can move on without attempting to + * service the request because the client will have already disconnected. + */ +"REQUEST_EXPIRATION_MILLIS", +/** + * The preferred number of FlowFiles that the server should send to the + * client when pulling data. This property was introduced in version 5 of + * the protocol. + */ +"BATCH_COUNT", +/** + * The preferred number of bytes that the server should send to the client + * when pulling data. This property was introduced in version 5 of the + * protocol. + */ +"BATCH_SIZE", +/** + * The preferred amount of time that the server should send data to the + * client when pulling data. This property was introduced in version 5 of + * the protocol. Value is in milliseconds. + */ +"BATCH_DURATION" }; class Site2SiteClientProtocol; //! Transaction Class -class Transaction -{ +class Transaction { friend class Site2SiteClientProtocol; public: //! Constructor /*! * Create a new transaction */ - Transaction(TransferDirection direction) { + explicit Transaction(TransferDirection direction, + CRCStream &stream) : + crcStream(std::move(stream)) { _state = TRANSACTION_STARTED; _direction = direction; _dataAvailable = false; @@ -311,48 +300,50 @@ class Transaction _uuidStr = uuidStr; } //! Destructor - virtual ~Transaction() - { + virtual ~Transaction() { } //! getUUIDStr - std::string getUUIDStr() - { + std::string getUUIDStr() { return _uuidStr; } //! getState - TransactionState getState() - { + TransactionState getState() { return _state; } //! isDataAvailable - bool isDataAvailable() - { + bool isDataAvailable() { return _dataAvailable; } //! setDataAvailable() - void setDataAvailable(bool value) - { + void setDataAvailable(bool value) { _dataAvailable = value; } //! getDirection - TransferDirection getDirection() - { + TransferDirection getDirection() { return _direction; } //! getCRC - long getCRC() - { - return _crc.getCRC(); + long getCRC() { + return crcStream.getCRC(); } //! updateCRC - void updateCRC(uint8_t *buffer, uint32_t length) - { - _crc.update(buffer, length); + void updateCRC(uint8_t *buffer, uint32_t length) { + crcStream.updateCRC(buffer, length); } + CRCStream &getStream() { + return crcStream; + } + + + Transaction(const Transaction &parent) = delete; + Transaction &operator=(const Transaction &parent) = delete; + protected: private: + + CRCStream crcStream; //! Transaction State TransactionState _state; //! Transaction Direction @@ -367,21 +358,14 @@ class Transaction int _transfers; //! Number of content bytes uint64_t _bytes; - //! CRC32 - CRC32 _crc; - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - Transaction(const Transaction &parent); - Transaction &operator=(const Transaction &parent); }; /** * Represents a piece of data that is to be sent to or that was received from a * NiFi instance. */ -class DataPacket -{ +class DataPacket { public: DataPacket(Site2SiteClientProtocol *protocol, Transaction *transaction, std::map attributes) { @@ -397,17 +381,16 @@ class DataPacket }; //! Site2SiteClientProtocol Class -class Site2SiteClientProtocol -{ +class Site2SiteClientProtocol { public: //! Constructor /*! * Create a new control protocol */ Site2SiteClientProtocol(Site2SitePeer *peer) { - _logger = Logger::getLogger(); - _configure = Configure::getConfigure(); - _peer = peer; + logger_ = Logger::getLogger(); + configure_ = Configure::getConfigure(); + peer_ = peer; _batchSize = 0; _batchCount = 0; _batchDuration = 0; @@ -426,34 +409,34 @@ class Site2SiteClientProtocol _currentCodecVersionIndex = 0; } //! Destructor - virtual ~Site2SiteClientProtocol() - { + virtual ~Site2SiteClientProtocol() { } public: //! setBatchSize - void setBatchSize(uint64_t size) - { + void setBatchSize(uint64_t size) { _batchSize = size; } //! setBatchCount - void setBatchCount(uint64_t count) - { + void setBatchCount(uint64_t count) { _batchCount = count; } //! setBatchDuration - void setBatchDuration(uint64_t duration) - { + void setBatchDuration(uint64_t duration) { _batchDuration = duration; } //! setTimeOut - void setTimeOut(uint64_t time) - { + void setTimeOut(uint64_t time) { _timeOut = time; - if (_peer) - _peer->setTimeOut(time); + if (peer_) + peer_->setTimeOut(time); } + + void setPeer(Site2SitePeer *peer) + { + peer_ = peer; + } /** * Provides a reference to the time out * @returns timeout @@ -470,21 +453,18 @@ class Site2SiteClientProtocol return _portIdStr; } //! setPortId - void setPortId(uuid_t id) - { + void setPortId(uuid_t id) { uuid_copy(_portId, id); char idStr[37]; uuid_unparse_lower(id, idStr); _portIdStr = idStr; } //! getResourceName - std::string getResourceName() - { + std::string getResourceName() { return "SocketFlowFileProtocol"; } //! getCodecResourceName - std::string getCodecResourceName() - { + std::string getCodecResourceName() { return "StandardFlowFileCodec"; } //! bootstrap the protocol to the ready for transaction state by going through the state machine @@ -510,31 +490,31 @@ class Site2SiteClientProtocol //! write respond int writeRespond(RespondCode code, std::string message); //! getRespondCodeContext - RespondCodeContext *getRespondCodeContext(RespondCode code) - { - for (unsigned int i = 0; i < sizeof(respondCodeContext)/sizeof(RespondCodeContext); i++) - { - if (respondCodeContext[i].code == code) - { + RespondCodeContext *getRespondCodeContext(RespondCode code) { + for (unsigned int i = 0; + i < sizeof(respondCodeContext) / sizeof(RespondCodeContext); + i++) { + if (respondCodeContext[i].code == code) { return &respondCodeContext[i]; } } return NULL; } //! getPeer - Site2SitePeer *getPeer() - { - return _peer; + Site2SitePeer *getPeer() { + return peer_; } //! Creation of a new transaction, return the transaction ID if success, //! Return NULL when any error occurs - Transaction *createTransaction(std::string &transactionID, TransferDirection direction); + Transaction *createTransaction(std::string &transactionID, + TransferDirection direction); //! Receive the data packet from the transaction //! Return false when any error occurs bool receive(std::string transactionID, DataPacket *packet, bool &eof); //! Send the data packet from the transaction //! Return false when any error occurs - bool send(std::string transactionID, DataPacket *packet, FlowFileRecord *flowFile, ProcessSession *session); + bool send(std::string transactionID, DataPacket *packet, + FlowFileRecord *flowFile, ProcessSession *session); //! Confirm the data that was sent or received by comparing CRC32's of the data sent and the data received. bool confirm(std::string transactionID); //! Cancel the transaction @@ -550,22 +530,22 @@ class Site2SiteClientProtocol //! deleteTransaction void deleteTransaction(std::string transactionID); //! Nest Callback Class for write stream - class WriteCallback : public OutputStreamCallback - { - public: - WriteCallback(DataPacket *packet) - : _packet(packet) {} + class WriteCallback: public OutputStreamCallback { + public: + WriteCallback(DataPacket *packet) : + _packet(packet) { + } DataPacket *_packet; void process(std::ofstream *stream) { uint8_t buffer[8192]; int len = _packet->_size; - while (len > 0) - { + while (len > 0) { int size = std::min(len, (int) sizeof(buffer)); - int ret = _packet->_protocol->_peer->readData(buffer, size, &_packet->_transaction->_crc); - if (ret != size) - { - _packet->_protocol->_logger->log_error("Site2Site Receive Flow Size %d Failed %d", size, ret); + int ret = _packet->_transaction->getStream().readData(buffer,size); + if (ret != size) { + _packet->_protocol->logger_->log_error( + "Site2Site Receive Flow Size %d Failed %d", size, + ret); break; } stream->write((const char *) buffer, size); @@ -574,26 +554,26 @@ class Site2SiteClientProtocol } }; //! Nest Callback Class for read stream - class ReadCallback : public InputStreamCallback - { - public: - ReadCallback(DataPacket *packet) - : _packet(packet) {} + class ReadCallback: public InputStreamCallback { + public: + ReadCallback(DataPacket *packet) : + _packet(packet) { + } DataPacket *_packet; void process(std::ifstream *stream) { _packet->_size = 0; uint8_t buffer[8192]; int readSize; - while (stream->good()) - { - if (!stream->read((char *)buffer, 8192)) + while (stream->good()) { + if (!stream->read((char *) buffer, 8192)) readSize = stream->gcount(); else readSize = 8192; - int ret = _packet->_protocol->_peer->write(buffer, readSize, &_packet->_transaction->_crc); - if (ret != readSize) - { - _packet->_protocol->_logger->log_error("Site2Site Send Flow Size %d Failed %d", readSize, ret); + int ret = _packet->_transaction->getStream().writeData(buffer,readSize); + if (ret != readSize) { + _packet->_protocol->logger_->log_error( + "Site2Site Send Flow Size %d Failed %d", readSize, + ret); break; } _packet->_size += readSize; @@ -604,12 +584,13 @@ class Site2SiteClientProtocol protected: private: + //! Mutex for protection std::mutex _mtx; //! Logger - Logger *_logger; + Logger *logger_; //! Configure - Configure *_configure; + Configure *configure_; //! Batch Count std::atomic _batchCount; //! Batch Size @@ -619,7 +600,7 @@ class Site2SiteClientProtocol //! Timeout in msec std::atomic _timeOut; //! Peer Connection - Site2SitePeer *_peer; + Site2SitePeer *peer_; //! portId uuid_t _portId; //! portIDStr diff --git a/libminifi/include/Site2SitePeer.h b/libminifi/include/Site2SitePeer.h index cd082f6cea..bd49e55d94 100644 --- a/libminifi/include/Site2SitePeer.h +++ b/libminifi/include/Site2SitePeer.h @@ -21,11 +21,6 @@ #define __SITE2SITE_PEER_H__ #include -#include -#include -#include -#include -#include #include #include #include @@ -33,83 +28,49 @@ #include #include #include -#include "TimeUtil.h" +#include #include "Logger.h" #include "Configure.h" #include "Property.h" -// OpenSSL related -#include -#include +#include "io/ClientSocket.h" +#include "io/BaseStream.h" +#include "utils/TimeUtil.h" -class CRC32 -{ -public: - CRC32() { - crc = 0; - - if (tableInit) - return; - - tableInit = true; - unsigned int poly = 0xedb88320; - unsigned int temp = 0; - for(unsigned int i = 0; i < 256; ++i) { - temp = i; - for(int j = 8; j > 0; --j) { - if((temp & 1) == 1) { - temp = (unsigned int)((temp >> 1) ^ poly); - }else { - temp >>= 1; - } - } - table[i] = temp; - } - } - - unsigned int update(uint8_t * bytes, size_t size) { - crc = crc ^ ~0U; - for(unsigned int i = 0; i < size; ++i) { - uint8_t index = (uint8_t)(((crc) & 0xff) ^ bytes[i]); - crc = (unsigned int)((crc >> 8) ^ table[index]); - } - crc = crc ^ ~0U; - return crc; - } - - long getCRC() - { - return crc; - } - -private: - static unsigned int table[256]; - static std::atomic tableInit; - unsigned int crc; -}; -static const char MAGIC_BYTES[] = {'N', 'i', 'F', 'i'}; +static const char MAGIC_BYTES[] = { 'N', 'i', 'F', 'i' }; //! Site2SitePeer Class -class Site2SitePeer -{ +class Site2SitePeer : public BaseStream{ public: - //! Constructor - /*! + + + Site2SitePeer() : stream_(nullptr),host_(""),port_(-1){ + + } + /* * Create a new site2site peer */ - Site2SitePeer(std::string host, uint16_t port) { - _logger = Logger::getLogger(); - _configure = Configure::getConfigure(); - _socket = 0; - _host = host; - _port = port; + explicit Site2SitePeer(std::unique_ptr injected_socket, const std::string host_, uint16_t port_ ) : + host_(host_), port_(port_), stream_(injected_socket.release()){ + logger_ = Logger::getLogger(); + configure_ = Configure::getConfigure(); _yieldExpiration = 0; _timeOut = 30000; // 30 seconds - _url = "nifi://" + _host + ":" + std::to_string(_port); - _ssl = NULL; + _url = "nifi://" + host_ + ":" + std::to_string(port_); + } + + explicit Site2SitePeer(Site2SitePeer &&ss) : stream_( ss.stream_.release()), host_( std::move(ss.host_) ), port_ (std::move(ss.port_) ) + { + logger_ = Logger::getLogger(); + configure_ = Configure::getConfigure(); + _yieldExpiration.store(ss._yieldExpiration); + _timeOut.store(ss._timeOut); + _url = std::move(ss._url); } //! Destructor - virtual ~Site2SitePeer() { Close();} + virtual ~Site2SitePeer() { + Close(); + } //! Set Processor yield period in MilliSecond void setYieldPeriodMsec(uint64_t period) { _yieldPeriodMsec = period; @@ -120,240 +81,179 @@ class Site2SitePeer } //! Get Processor yield period in MilliSecond uint64_t getYieldPeriodMsec(void) { - return(_yieldPeriodMsec); + return (_yieldPeriodMsec); } //! Yield based on the yield period - void yield() - { + void yield() { _yieldExpiration = (getTimeMillis() + _yieldPeriodMsec); } //! setHostName - void setHostName(std::string host) - { - _host = host; - _url = "nifi://" + _host + ":" + std::to_string(_port); + void setHostName(std::string host_) { + this->host_ = host_; + _url = "nifi://" + host_ + ":" + std::to_string(port_); } //! setPort - void setPort(uint16_t port) - { - _port = port; - _url = "nifi://" + _host + ":" + std::to_string(_port); + void setPort(uint16_t port_) { + this->port_ = port_; + _url = "nifi://" + host_ + ":" + std::to_string(port_); } //! getHostName - std::string getHostName() - { - return _host; + std::string getHostName() { + return host_; } //! getPort - uint16_t getPort() - { - return _port; + uint16_t getPort() { + return port_; } //! Yield based on the input time - void yield(uint64_t time) - { + void yield(uint64_t time) { _yieldExpiration = (getTimeMillis() + time); } //! whether need be to yield - bool isYield() - { + bool isYield() { if (_yieldExpiration > 0) return (_yieldExpiration >= getTimeMillis()); else return false; } // clear yield expiration - void clearYield() - { + void clearYield() { _yieldExpiration = 0; } //! Yield based on the yield period - void yield(std::string portId) - { + void yield(std::string portId) { std::lock_guard lock(_mtx); uint64_t yieldExpiration = (getTimeMillis() + _yieldPeriodMsec); _yieldExpirationPortIdMap[portId] = yieldExpiration; } //! Yield based on the input time - void yield(std::string portId, uint64_t time) - { + void yield(std::string portId, uint64_t time) { std::lock_guard lock(_mtx); uint64_t yieldExpiration = (getTimeMillis() + time); _yieldExpirationPortIdMap[portId] = yieldExpiration; } //! whether need be to yield - bool isYield(std::string portId) - { + bool isYield(std::string portId) { std::lock_guard lock(_mtx); - std::map::iterator it = this->_yieldExpirationPortIdMap.find(portId); - if (it != _yieldExpirationPortIdMap.end()) - { + std::map::iterator it = + this->_yieldExpirationPortIdMap.find(portId); + if (it != _yieldExpirationPortIdMap.end()) { uint64_t yieldExpiration = it->second; return (yieldExpiration >= getTimeMillis()); - } - else - { + } else { return false; } } //! clear yield expiration - void clearYield(std::string portId) - { + void clearYield(std::string portId) { std::lock_guard lock(_mtx); - std::map::iterator it = this->_yieldExpirationPortIdMap.find(portId); - if (it != _yieldExpirationPortIdMap.end()) - { + std::map::iterator it = + this->_yieldExpirationPortIdMap.find(portId); + if (it != _yieldExpirationPortIdMap.end()) { _yieldExpirationPortIdMap.erase(portId); } } //! setTimeOut - void setTimeOut(uint64_t time) - { + void setTimeOut(uint64_t time) { _timeOut = time; } //! getTimeOut - uint64_t getTimeOut() - { + uint64_t getTimeOut() { return _timeOut; } - int write(uint8_t value, CRC32 *crc = NULL) - { - return sendData(&value, 1, crc); + int write(uint8_t value) { + return Serializable::write(value,stream_.get()); } - int write(char value, CRC32 *crc = NULL) - { - return sendData((uint8_t *)&value, 1, crc); + int write(char value) { + return Serializable::write(value,stream_.get()); } - int write(uint32_t value, CRC32 *crc = NULL) - { - uint8_t temp[4]; + int write(uint32_t value) { + + return Serializable::write(value,stream_.get()); - temp[0] = (value & 0xFF000000) >> 24; - temp[1] = (value & 0x00FF0000) >> 16; - temp[2] = (value & 0x0000FF00) >> 8; - temp[3] = (value & 0x000000FF); - return sendData(temp, 4, crc); } - int write(uint16_t value, CRC32 *crc = NULL) - { - uint8_t temp[2]; - temp[0] = (value & 0xFF00) >> 8; - temp[1] = (value & 0xFF); - return sendData(temp, 2, crc); + int write(uint16_t value) { + return Serializable::write(value,stream_.get()); } - int write(uint8_t *value, int len, CRC32 *crc = NULL) - { - return sendData(value, len, crc); + int write(uint8_t *value, int len) { + return Serializable::write(value,len,stream_.get()); } - int write(uint64_t value, CRC32 *crc = NULL) - { - uint8_t temp[8]; - - temp[0] = (value >> 56) & 0xFF; - temp[1] = (value >> 48) & 0xFF; - temp[2] = (value >> 40) & 0xFF; - temp[3] = (value >> 32) & 0xFF; - temp[4] = (value >> 24) & 0xFF; - temp[5] = (value >> 16) & 0xFF; - temp[6] = (value >> 8) & 0xFF; - temp[7] = (value >> 0) & 0xFF; - return sendData(temp, 8, crc); - } - int write(bool value, CRC32 *crc = NULL) - { + int write(uint64_t value) { + return Serializable::write(value,stream_.get()); + } + int write(bool value) { uint8_t temp = value; - return write(temp, crc); + return Serializable::write(temp,stream_.get()); } - int writeUTF(std::string str, bool widen = false, CRC32 *crc = NULL); - int read(uint8_t &value, CRC32 *crc = NULL) - { - uint8_t buf; - - int ret = readData(&buf, 1, crc); - if (ret == 1) - value = buf; - return ret; + int writeUTF(std::string str, bool widen = false){ + return Serializable::writeUTF(str,stream_.get(),widen); } - int read(uint16_t &value, CRC32 *crc = NULL) - { - uint8_t buf[2]; - - int ret = readData(buf, 2, crc); - if (ret == 2) - value = (buf[0] << 8) | buf[1]; - return ret; + int read(uint8_t &value) { + return Serializable::read(value,stream_.get()); } - int read(char &value, CRC32 *crc = NULL) - { - uint8_t buf; - - int ret = readData(&buf, 1, crc); - if (ret == 1) - value = (char) buf; - return ret; + int read(uint16_t &value) { + return Serializable::read(value,stream_.get()); } - int read(uint8_t *value, int len, CRC32 *crc = NULL) - { - return readData(value, len, crc); + int read(char &value) { + return Serializable::read(value,stream_.get()); } - int read(uint32_t &value, CRC32 *crc = NULL) - { - uint8_t buf[4]; - - int ret = readData(buf, 4, crc); - if (ret == 4) - value = (buf[0] << 24) | (buf[1] << 16) | (buf[2] << 8) | buf[3]; - return ret; + int read(uint8_t *value, int len) { + return Serializable::read(value,len,stream_.get()); + } + int read(uint32_t &value) { + return Serializable::read(value,stream_.get()); } - int read(uint64_t &value, CRC32 *crc = NULL) + int read(uint64_t &value) { + return Serializable::read(value,stream_.get()); + } + int readUTF(std::string &str, bool widen = false) { - uint8_t buf[8]; - - int ret = readData(buf, 8, crc); - if (ret == 8) - { - value = ((uint64_t) buf[0] << 56) | - ((uint64_t) (buf[1] & 255) << 48) | - ((uint64_t) (buf[2] & 255) << 40) | - ((uint64_t) (buf[3] & 255) << 32) | - ((uint64_t) (buf[4] & 255) << 24) | - ((uint64_t) (buf[5] & 255) << 16) | - ((uint64_t) (buf[6] & 255) << 8) | - ((uint64_t) (buf[7] & 255) << 0); - } - return ret; + return Serializable::readUTF(str,stream_.get(),widen); } - int readUTF(std::string &str, bool widen = false, CRC32 *crc = NULL); //! open connection to the peer bool Open(); //! close connection to the peer void Close(); - //! Send Data via the socket, return -1 for failure - int sendData(uint8_t *buf, int buflen, CRC32 *crc = NULL); - //! Read length into buf, return -1 for failure and 0 for EOF - int readData(uint8_t *buf, int buflen, CRC32 *crc = NULL); - //! Select on the socket - int Select(int msec); + + /** + * Move assignment operator. + */ + Site2SitePeer& operator=(Site2SitePeer&& other) + { + stream_ = std::unique_ptr( other.stream_.release()); + host_ = std::move(other.host_); + port_ = std::move(other.port_); + logger_ = Logger::getLogger(); + configure_ = Configure::getConfigure(); + _yieldExpiration = 0; + _timeOut = 30000; // 30 seconds + _url = "nifi://" + host_ + ":" + std::to_string(port_); + + return *this; + } + + Site2SitePeer(const Site2SitePeer &parent) = delete; + Site2SitePeer &operator=(const Site2SitePeer &parent) = delete; protected: private: + + std::unique_ptr stream_; + + std::string host_; + uint16_t port_; + //! Mutex for protection std::mutex _mtx; - //! S2S server Name - std::string _host; - //! S2S server port - uint16_t _port; - //! socket to server - int _socket; //! URL std::string _url; //! socket timeout; std::atomic _timeOut; //! Logger - Logger *_logger; + Logger *logger_; //! Configure - Configure *_configure; + Configure *configure_; //! Yield Period in Milliseconds std::atomic _yieldPeriodMsec; //! Yield Expiration @@ -361,11 +261,9 @@ class Site2SitePeer //! Yield Expiration per destination PortID std::map _yieldExpirationPortIdMap; //! OpenSSL connection state - SSL* _ssl; // Prevent default copy constructor and assignment operation // Only support pass by reference or pointer - Site2SitePeer(const Site2SitePeer &parent); - Site2SitePeer &operator=(const Site2SitePeer &parent); + }; #endif diff --git a/libminifi/include/TailFile.h b/libminifi/include/TailFile.h index 5c4ba098c1..c2fe555fd0 100644 --- a/libminifi/include/TailFile.h +++ b/libminifi/include/TailFile.h @@ -35,7 +35,7 @@ class TailFile : public Processor TailFile(std::string name, uuid_t uuid = NULL) : Processor(name, uuid) { - _logger = Logger::getLogger(); + logger_ = Logger::getLogger(); _stateRecovered = false; } //! Destructor @@ -65,7 +65,7 @@ class TailFile : public Processor private: //! Logger - Logger *_logger; + Logger *logger_; std::string _fileLocation; //! Property Specified Tailed File Name std::string _fileName; diff --git a/libminifi/include/io/BaseStream.h b/libminifi/include/io/BaseStream.h new file mode 100644 index 0000000000..c3ebe4277b --- /dev/null +++ b/libminifi/include/io/BaseStream.h @@ -0,0 +1,150 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBMINIFI_INCLUDE_IO_BASESTREAM_H_ +#define LIBMINIFI_INCLUDE_IO_BASESTREAM_H_ + + +#include +#include "EndianCheck.h" +#include "DataStream.h" +#include "Serializable.h" +class BaseStream: public DataStream, public Serializable { + +public: + BaseStream() { + + } + virtual ~BaseStream() { + + } + /** + * write 4 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ + virtual int write(uint32_t base_value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * write 2 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ + virtual int write(uint16_t base_value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * write valueto stream + * @param value non encoded value + * @param len length of value + * @param strema output stream + * @return resulting write size + **/ + virtual int write(uint8_t *value, int len); + + /** + * write 8 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ + virtual int write(uint64_t base_value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * write bool to stream + * @param value non encoded value + * @return resulting write size + **/ + virtual int write(bool value); + + /** + * write UTF string to stream + * @param str string to write + * @return resulting write size + **/ + virtual int writeUTF(std::string str, bool widen = false); + + /** + * reads a byte from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ + virtual int read(uint8_t &value); + + /** + * reads two bytes from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ + virtual int read(uint16_t &base_value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * reads a byte from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ + virtual int read(char &value); + + /** + * reads a byte array from the stream + * @param value reference in which will set the result + * @param len length to read + * @param stream stream from which we will read + * @return resulting read size + **/ + virtual int read(uint8_t *value, int len); + + /** + * reads four bytes from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ + virtual int read(uint32_t &value, + bool is_little_endian = EndiannessCheck::IS_LITTLE); + + /** + * reads eight byte from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ + virtual int read(uint64_t &value, + bool is_little_endian = EndiannessCheck::IS_LITTLE); + + /** + * read UTF from stream + * @param str reference string + * @param stream stream from which we will read + * @return resulting read size + **/ + virtual int readUTF(std::string &str, bool widen = false); +}; + +#endif /* LIBMINIFI_INCLUDE_IO_BASESTREAM_H_ */ diff --git a/libminifi/include/io/CRCStream.h b/libminifi/include/io/CRCStream.h new file mode 100644 index 0000000000..7fdfb2ba1d --- /dev/null +++ b/libminifi/include/io/CRCStream.h @@ -0,0 +1,318 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBMINIFI_INCLUDE_IO_CRCSTREAM_H_ +#define LIBMINIFI_INCLUDE_IO_CRCSTREAM_H_ + +#include +#include +#include + +#include "BaseStream.h" +#include "Serializable.h" + +#define htonll_r(x) ((((uint64_t)htonl(x)) << 32) + htonl((x) >> 32)) + + +template +class CRCStream: public BaseStream { +public: + /** + * Raw pointer because the caller guarantees that + * it will exceed our lifetime. + */ + explicit CRCStream(T *stream); + + explicit CRCStream( CRCStream &&move ); + + virtual ~CRCStream() { + + } + + /** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ + virtual int readData(std::vector &buf, int buflen); + /** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ + virtual int readData(uint8_t *buf, int buflen); + + /** + * Write value to the stream using std::vector + * @param buf incoming buffer + * @param buflen buffer to write + * + */ + virtual int writeData(std::vector &buf, int buflen); + + /** + * writes value to stream + * @param value value to write + * @param size size of value + */ + virtual int writeData(uint8_t *value, int size); + + /** + * write 4 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ + virtual int write(uint32_t base_value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + /** + * write 2 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ + virtual int write(uint16_t base_value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + + /** + * write 8 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ + virtual int write(uint64_t base_value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + + + /** + * Reads a system word + * @param value value to write + */ + virtual int read(uint64_t &value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * Reads a uint32_t + * @param value value to write + */ + virtual int read(uint32_t &value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * Reads a system short + * @param value value to write + */ + virtual int read(uint16_t &value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + + virtual short initialize() { + child_stream->initialize(); + reset(); + return 0; + } + + + void updateCRC(uint8_t *buffer, uint32_t length); + + uint64_t getCRC() { + return crc; + } + + void reset(); +protected: + + /** + * Creates a vector and returns the vector using the provided + * type name. + * @param t incoming object + * @returns vector. + */ + template + std::vector readBuffer(const K& t){ + std::vector buf; + buf.resize(sizeof t); + readData((uint8_t*) &buf[0], sizeof(t)); + return buf; + } + + template + void updateCRCPrimitive(const K &t) { + uint8_t bytes[sizeof t]; + std::copy(static_cast(static_cast(&t)), + static_cast(static_cast(&t)) + sizeof t, + bytes); + updateCRC(bytes, sizeof t); + } + + + uint64_t crc; + T *child_stream; +}; + + +template +CRCStream::CRCStream(T *other) : + child_stream(other) { + crc = crc32(0L, Z_NULL, 0); +} + +template +CRCStream::CRCStream(CRCStream &&move) : + crc(std::move(move.crc)), child_stream(std::move(move.child_stream)) { + +} + +template +int CRCStream::readData(std::vector &buf, int buflen) { + + if (buf.capacity() < buflen) + buf.resize(buflen); + return readData((uint8_t*) &buf[0], buflen); +} + +template +int CRCStream::readData(uint8_t *buf, int buflen) { + int ret = child_stream->read(buf, buflen); + crc = crc32(crc, buf, buflen); + return ret; +} + +template +int CRCStream::writeData(std::vector &buf, int buflen) { + + if (buf.capacity() < buflen) + buf.resize(buflen); + return writeData((uint8_t*) &buf[0], buflen); +} + +template +int CRCStream::writeData(uint8_t *value, int size) { + + int ret = child_stream->write(value, size); + crc = crc32(crc, value, size); + return ret; + +} +template +void CRCStream::reset() { + crc = crc32(0L, Z_NULL, 0); +} +template +void CRCStream::updateCRC(uint8_t *buffer, uint32_t length) { + crc = crc32(crc, buffer, length); +} + +template +int CRCStream::write(uint64_t base_value, bool is_little_endian){ + + updateCRCPrimitive(base_value); + const uint64_t value = + is_little_endian == 1 ? htonll_r(base_value) : base_value; + uint8_t bytes[sizeof value]; + std::copy(static_cast(static_cast(&value)), + static_cast(static_cast(&value)) + sizeof value, + bytes); + return child_stream->write(bytes,sizeof value); +} + + +template +int CRCStream::write(uint32_t base_value, bool is_little_endian){ + updateCRCPrimitive(base_value); + const uint32_t value = is_little_endian ? htonl(base_value) : base_value; + uint8_t bytes[sizeof value]; + std::copy(static_cast(static_cast(&value)), + static_cast(static_cast(&value)) + sizeof value, + bytes); + return child_stream->write(bytes,sizeof value); +} + +template +int CRCStream::write(uint16_t base_value, bool is_little_endian){ + updateCRCPrimitive(base_value); + const uint16_t value = + is_little_endian == 1 ? htons(base_value) : base_value; + uint8_t bytes[sizeof value]; + std::copy(static_cast(static_cast(&value)), + static_cast(static_cast(&value)) + sizeof value, + bytes); + return child_stream->write(bytes,sizeof value); +} + + +template +int CRCStream::read(uint64_t &value, bool is_little_endian) { + + auto buf = readBuffer(value); + + if (is_little_endian) { + value = ((uint64_t) buf[0] << 56) | ((uint64_t) (buf[1] & 255) << 48) + | ((uint64_t) (buf[2] & 255) << 40) + | ((uint64_t) (buf[3] & 255) << 32) + | ((uint64_t) (buf[4] & 255) << 24) + | ((uint64_t) (buf[5] & 255) << 16) + | ((uint64_t) (buf[6] & 255) << 8) + | ((uint64_t) (buf[7] & 255) << 0); + } else { + value = ((uint64_t) buf[0] << 0) | ((uint64_t) (buf[1] & 255) << 8) + | ((uint64_t) (buf[2] & 255) << 16) + | ((uint64_t) (buf[3] & 255) << 24) + | ((uint64_t) (buf[4] & 255) << 32) + | ((uint64_t) (buf[5] & 255) << 40) + | ((uint64_t) (buf[6] & 255) << 48) + | ((uint64_t) (buf[7] & 255) << 56); + } + return sizeof(value); +} + +template +int CRCStream::read(uint32_t &value, bool is_little_endian) { + + auto buf = readBuffer(value); + + if (is_little_endian) { + value = (buf[0] << 24) | (buf[1] << 16) | (buf[2] << 8) | buf[3]; + } else { + value = buf[0] | buf[1] << 8 | buf[2] << 16 | buf[3] << 24; + + } + + return sizeof(value); +} + +template +int CRCStream::read(uint16_t &value, bool is_little_endian) { + + auto buf = readBuffer(value); + + if (is_little_endian) { + value = (buf[0] << 8) | buf[1]; + } else { + value = buf[0] | buf[1] << 8; + + } + return sizeof(value); +} + + +#endif /* LIBMINIFI_INCLUDE_IO_CRCSTREAM_H_ */ diff --git a/libminifi/include/io/ClientSocket.h b/libminifi/include/io/ClientSocket.h new file mode 100644 index 0000000000..f73b0b4aa2 --- /dev/null +++ b/libminifi/include/io/ClientSocket.h @@ -0,0 +1,245 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBMINIFI_INCLUDE_IO_CLIENTSOCKET_H_ +#define LIBMINIFI_INCLUDE_IO_CLIENTSOCKET_H_ + +#include +#include +#include +#include +#include +#include +#include +#include "io/BaseStream.h" +#include "Logger.h" + +#include "io/validation.h" + +/** + * Socket class. + * Purpose: Provides a general purpose socket interface that abstracts + * connecting information from users + * Design: Extends DataStream and allows us to perform most streaming + * operations against a BSD socket + * + * + */ +class Socket: public BaseStream { +public: + /** + * Constructor that accepts host name, port and listeners. With this + * contructor we will be creating a server socket + * @param hostname our host name + * @param port connecting port + * @param listeners number of listeners in the queue + */ + explicit Socket(const std::string &hostname, const uint16_t port, + const uint16_t listeners); + + /** + * Constructor that creates a client socket. + * @param hostname hostname we are connecting to. + * @param port port we are connecting to. + */ + explicit Socket(const std::string &hostname, const uint16_t port); + + /** + * Move constructor. + */ + explicit Socket(const Socket &&); + + static std::string HOSTNAME; + + /** + * Static function to return the current machine's host name + */ + static std::string getMyHostName(std::string *str = &HOSTNAME) { + if (__builtin_expect(!IsNullOrEmpty(str), 0)) + return *str; + else { + char hostname[1024]; + gethostname(hostname, 1024); + Socket mySock(hostname, 0); + mySock.initialize(); + return mySock.getHostname(); + } + } + + /** + * Destructor + */ + + virtual ~Socket(); + + virtual void closeStream(); + /** + * Initializes the socket + * @return result of the creation operation. + */ + virtual short initialize(); + + std::string getHostname() const; + + /** + * Return the port for this socket + * @returns port + */ + uint16_t getPort(); + + // data stream extensions + /** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ + virtual int readData(std::vector &buf, int buflen); + /** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ + virtual int readData(uint8_t *buf, int buflen); + + /** + * Write value to the stream using std::vector + * @param buf incoming buffer + * @param buflen buffer to write + * + */ + virtual int writeData(std::vector &buf, int buflen); + + /** + * writes value to stream + * @param value value to write + * @param size size of value + */ + virtual int writeData(uint8_t *value, int size); + + + + /** + * Writes a system word + * @param value value to write + */ + virtual int write(uint64_t value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * Writes a uint32_t + * @param value value to write + */ + virtual int write(uint32_t value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * Writes a system short + * @param value value to write + */ + virtual int write(uint16_t value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + + /** + * Reads a system word + * @param value value to write + */ + virtual int read(uint64_t &value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * Reads a uint32_t + * @param value value to write + */ + virtual int read(uint32_t &value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * Reads a system short + * @param value value to write + */ + virtual int read(uint16_t &value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * Returns the underlying buffer + * @return vector's array + **/ + const uint8_t *getBuffer() const { + return ::DataStream::getBuffer(); + } + + /** + * Retrieve size of data stream + * @return size of data stream + **/ + const uint32_t getSize() const { + return ::DataStream::getSize(); + } + +protected: + + /** + * Creates a vector and returns the vector using the provided + * type name. + * @param t incoming object + * @returns vector. + */ + template + std::vector readBuffer(const T&); + + /** + * Creates a connection using the address info object. + * @param p addrinfo structure. + * @returns fd. + */ + virtual int8_t createConnection(const addrinfo *p); + + /** + * Sets socket options depending on the instance. + * @param sock socket file descriptor. + */ + virtual short setSocketOptions(const int sock); + + /** + * Attempt to select the socket file descriptor + * @param msec timeout interval to wait + * @returns file descriptor + */ + virtual short select_descriptor(const uint16_t msec); + + Logger *logger_; + + addrinfo *addr_info_; + + std::recursive_mutex selection_mutex_; + + std::string requested_hostname_; + std::string canonical_hostname_; + uint16_t port_; + + // connection information + int32_t socket_file_descriptor_; + + fd_set total_list_; + fd_set read_fds_; + std::atomic socket_max_; + uint16_t listeners_; + +}; + +#endif /* LIBMINIFI_INCLUDE_IO_CLIENTSOCKET_H_ */ diff --git a/libminifi/include/io/DataStream.h b/libminifi/include/io/DataStream.h new file mode 100644 index 0000000000..2f793ff410 --- /dev/null +++ b/libminifi/include/io/DataStream.h @@ -0,0 +1,120 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBMINIFI_INCLUDE_IO_DATASTREAM_H_ +#define LIBMINIFI_INCLUDE_IO_DATASTREAM_H_ + +#include +#include +#include "EndianCheck.h" + +/** + * DataStream defines the mechanism through which + * binary data will be written to a sink + */ +class DataStream { +public: + + DataStream() : + readBuffer(0) { + + } + + /** + * Constructor + **/ + explicit DataStream(const uint8_t *buf, const uint32_t buflen) : + DataStream() { + writeData((uint8_t*) buf, buflen); + + } + + virtual short initialize() { + buffer.clear(); + readBuffer = 0; + return 0; + } + + virtual void closeStream() + { + + } + /** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ + virtual int readData(std::vector &buf, int buflen); + /** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ + virtual int readData(uint8_t *buf, int buflen); + + /** + * writes valiue to buffer + * @param value value to write + * @param size size of value + */ + virtual int writeData(uint8_t *value, int size); + + /** + * Reads a system word + * @param value value to write + */ + virtual int read(uint64_t &value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * Reads a uint32_t + * @param value value to write + */ + virtual int read(uint32_t &value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * Reads a system short + * @param value value to write + */ + virtual int read(uint16_t &value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * Returns the underlying buffer + * @return vector's array + **/ + const uint8_t *getBuffer() const { + return &buffer[0]; + } + + /** + * Retrieve size of data stream + * @return size of data stream + **/ + const uint32_t getSize() const { + return buffer.size(); + } + +protected: + // All serialization related method and internal buf + std::vector buffer; + uint32_t readBuffer; +}; + +#endif /* LIBMINIFI_INCLUDE_IO_DATASTREAM_H_ */ diff --git a/libminifi/include/io/EndianCheck.h b/libminifi/include/io/EndianCheck.h new file mode 100644 index 0000000000..ef900e0eed --- /dev/null +++ b/libminifi/include/io/EndianCheck.h @@ -0,0 +1,47 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#ifndef LIBMINIFI_INCLUDE_IO_ENDIANCHECK_H_ +#define LIBMINIFI_INCLUDE_IO_ENDIANCHECK_H_ + + + +/** + * Mechanism to determine endianness of host. + * Accounts for only BIG/LITTLE/BIENDIAN + **/ +class EndiannessCheck +{ +public: + static bool IS_LITTLE; +private: + + static bool is_little_endian() { + /* do whatever is needed at static init time */ + unsigned int x = 1; + char *c = (char*) &x; + IS_LITTLE=*c==1; + return IS_LITTLE; + } +}; + + + + +#endif /* LIBMINIFI_INCLUDE_IO_ENDIANCHECK_H_ */ diff --git a/libminifi/include/Serializable.h b/libminifi/include/io/Serializable.h similarity index 69% rename from libminifi/include/Serializable.h rename to libminifi/include/io/Serializable.h index c32dee0990..59d3a7338e 100644 --- a/libminifi/include/Serializable.h +++ b/libminifi/include/io/Serializable.h @@ -1,5 +1,5 @@ /** - * + * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -15,124 +15,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef __SERIALIZABLE_H__ -#define __SERIALIZABLE_H__ - -#include -#include -#include - - -/** - * Mechanism to determine endianness of host. - * Accounts for only BIG/LITTLE/BIENDIAN - **/ -class EndiannessCheck -{ -public: - static bool IS_LITTLE; -private: - - static bool is_little_endian() { - /* do whatever is needed at static init time */ - unsigned int x = 1; - char *c = (char*) &x; - IS_LITTLE=*c==1; - return IS_LITTLE; - } -}; - - -/** - * DataStream defines the mechanism through which - * binary data will be written to a sink - */ -class DataStream -{ -public: - - DataStream() : readBuffer(0) - { - - } - - /** - * Constructor - **/ - DataStream(const uint8_t *buf, const uint32_t buflen) : DataStream() - { - writeData((uint8_t*)buf,buflen); - - } - /** - * Reads data and places it into buf - * @param buf buffer in which we extract data - * @param buflen - */ - int readData(std::vector &buf, int buflen); - /** - * Reads data and places it into buf - * @param buf buffer in which we extract data - * @param buflen - */ - int readData(uint8_t *buf, int buflen); +#ifndef LIBMINIFI_INCLUDE_IO_SERIALIZABLE_H_ +#define LIBMINIFI_INCLUDE_IO_SERIALIZABLE_H_ - /** - * writes valiue to buffer - * @param value value to write - * @param size size of value - */ - int writeData(uint8_t *value, int size); - - - /** - * Reads a system word - * @param value value to write - */ - inline int readLongLong(uint64_t &value, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - - /** - * Reads a uint32_t - * @param value value to write - */ - inline int readLong(uint32_t &value, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - - /** - * Reads a system short - * @param value value to write - */ - inline int readShort(uint16_t &value,bool is_little_endian = - EndiannessCheck::IS_LITTLE); +#include +#include "EndianCheck.h" +#include "DataStream.h" - /** - * Returns the underlying buffer - * @return vector's array - **/ - const uint8_t *getBuffer() const - { - return &buffer[0]; - } - - /** - * Retrieve size of data stream - * @return size of data stream - **/ - const uint32_t getSize() const - { - return buffer.size(); - } - -private: - // All serialization related method and internal buf - std::vector buffer; - uint32_t readBuffer; -}; /** * Serializable instances provide base functionality to @@ -291,4 +182,5 @@ class Serializable { }; -#endif + +#endif /* LIBMINIFI_INCLUDE_IO_SERIALIZABLE_H_ */ diff --git a/libminifi/include/io/SocketFactory.h b/libminifi/include/io/SocketFactory.h new file mode 100644 index 0000000000..e7988fc8bf --- /dev/null +++ b/libminifi/include/io/SocketFactory.h @@ -0,0 +1,87 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SOCKET_FACTORY_H +#define SOCKET_FACTORY_H + +#include "ClientSocket.h" +#include "TLSSocket.h" +#include "ClientSocket.h" +#include "Configure.h" +#include "utils/StringUtils.h" + +/** + Purpose: Due to the current design this is the only mechanism by which we can + inject different socket types + +**/ +class SocketFactory{ +public: + + /** + * Build an instance, creating a memory fence, which + * allows us to avoid locking. This is tantamount to double checked locking. + * @returns new SocketFactory; + */ + static SocketFactory *getInstance() { + SocketFactory* atomic_context = context_instance_.load( + std::memory_order_relaxed); + std::atomic_thread_fence(std::memory_order_acquire); + if (atomic_context == nullptr) { + std::lock_guard lock(context_mutex_); + atomic_context = context_instance_.load(std::memory_order_relaxed); + if (atomic_context == nullptr) { + atomic_context = new SocketFactory(); + std::atomic_thread_fence(std::memory_order_release); + context_instance_.store(atomic_context, + std::memory_order_relaxed); + } + } + return atomic_context; + } + + /** + * Creates a socket and returns a unique ptr + * + */ + std::unique_ptr createSocket(const std::string &host, const uint16_t port) { + Socket *socket = 0; + if (is_secure_) { + socket = new TLSSocket(host, port); + } else { + socket = new Socket(host, port); + } + return std::unique_ptr(socket); + } +protected: + SocketFactory() : + configure_(Configure::getConfigure()) { + std::string secureStr; + is_secure_ = false; + if (configure_->get(Configure::nifi_remote_input_secure, secureStr)) { + StringUtils::StringToBool(secureStr, is_secure_); + } + } + + bool is_secure_; + static std::atomic context_instance_; + static std::mutex context_mutex_; + + Configure *configure_; +}; + +#endif diff --git a/libminifi/include/io/TLSSocket.h b/libminifi/include/io/TLSSocket.h new file mode 100644 index 0000000000..909cfc2d29 --- /dev/null +++ b/libminifi/include/io/TLSSocket.h @@ -0,0 +1,185 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBMINIFI_INCLUDE_IO_TLSSOCKET_H_ +#define LIBMINIFI_INCLUDE_IO_TLSSOCKET_H_ + +#include +#include "ClientSocket.h" +#include +#include + +#include "Configure.h" +#include +#include + +#define TLS_ERROR_CONTEXT 1 +#define TLS_ERROR_PEM_MISSING 2 +#define TLS_ERROR_CERT_MISSING 3 +#define TLS_ERROR_KEY_ERROR 4 +#define TLS_ERROR_CERT_ERROR 5 + +class TLSContext { + +public: + + /** + * Build an instance, creating a memory fence, which + * allows us to avoid locking. This is tantamount to double checked locking. + * @returns new TLSContext; + */ + static TLSContext *getInstance() { + TLSContext* atomic_context = context_instance.load( + std::memory_order_relaxed); + std::atomic_thread_fence(std::memory_order_acquire); + if (atomic_context == nullptr) { + std::lock_guard lock(context_mutex); + atomic_context = context_instance.load(std::memory_order_relaxed); + if (atomic_context == nullptr) { + atomic_context = new TLSContext(); + atomic_context->initialize(); + std::atomic_thread_fence(std::memory_order_release); + context_instance.store(atomic_context, + std::memory_order_relaxed); + } + } + return atomic_context; + } + + virtual ~TLSContext() { + if (0 != ctx) + SSL_CTX_free(ctx); + } + + SSL_CTX *getContext() { + return ctx; + } + + short getError() { + return error_value; + } + + short initialize(); + +private: + + static int pemPassWordCb(char *buf, int size, int rwflag, void *userdata) { + std::string passphrase; + + if (Configure::getConfigure()->get( + Configure::nifi_security_client_pass_phrase, passphrase)) { + + std::ifstream file(passphrase.c_str(), std::ifstream::in); + if (!file.good()) { + memset(buf, 0x00, size); + return 0; + } + + std::string password; + password.assign((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + file.close(); + memset(buf,0x00,size); + memcpy(buf, password.c_str(), password.length()-1); + + return password.length()-1; + } + return 0; + } + + TLSContext(); + + Logger *logger; + Configure *configuration; + SSL_CTX *ctx; + + short error_value; + + static std::atomic context_instance; + static std::mutex context_mutex; +}; + +class TLSSocket: public Socket { +public: + + /** + * Constructor that accepts host name, port and listeners. With this + * contructor we will be creating a server socket + * @param hostname our host name + * @param port connecting port + * @param listeners number of listeners in the queue + */ + explicit TLSSocket(const std::string &hostname, const uint16_t port, + const uint16_t listeners); + + /** + * Constructor that creates a client socket. + * @param hostname hostname we are connecting to. + * @param port port we are connecting to. + */ + explicit TLSSocket(const std::string &hostname, const uint16_t port); + + /** + * Move constructor. + */ + explicit TLSSocket(const TLSSocket &&); + + virtual ~TLSSocket(); + + /** + * Initializes the socket + * @return result of the creation operation. + */ + short initialize(); + + /** + * Attempt to select the socket file descriptor + * @param msec timeout interval to wait + * @returns file descriptor + */ + virtual short select_descriptor(const uint16_t msec); + + /** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ + virtual int readData(uint8_t *buf, int buflen); + + /** + * Write value to the stream using std::vector + * @param buf incoming buffer + * @param buflen buffer to write + * + */ + int writeData(std::vector &buf, int buflen); + + /** + * Write value to the stream using uint8_t ptr + * @param buf incoming buffer + * @param buflen buffer to write + * + */ + int writeData(uint8_t *value, int size); + +protected: + + SSL* ssl; + +}; + +#endif /* LIBMINIFI_INCLUDE_IO_TLSSOCKET_H_ */ diff --git a/libminifi/include/io/validation.h b/libminifi/include/io/validation.h new file mode 100644 index 0000000000..e1b4bb6f29 --- /dev/null +++ b/libminifi/include/io/validation.h @@ -0,0 +1,77 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef VALIDATION_H +#define VALIDATION_H +#include +#include + +/** + * A checker that will, at compile time, tell us + * if the declared type has a size method. + */ +template +class size_function_functor_checker { + typedef char hasit; + typedef long doesnothaveit; + + // look for the declared type + template static hasit test(decltype(&O::size)); + template static doesnothaveit test(...); + +public: + enum { + has_size_function = sizeof(test(0)) == sizeof(char) + }; +}; + +/** + * Determines if the variable is null or ::size() == 0 + */ +template +static auto IsNullOrEmpty( + T &object) -> typename std::enable_if::has_size_function==1, bool>::type { + return object.size() == 0; +} + +/** + * Determines if the variable is null or ::size() == 0 + */ +template +static auto IsNullOrEmpty( + T *object) -> typename std::enable_if::has_size_function==1, bool>::type { + return (0 == object || object->size() == 0); +} + +/** + * Determines if the variable is null or ::size() == 0 + */ +template +static auto IsNullOrEmpty( + T *object) -> typename std::enable_if::has_size_function , bool>::type { + return (0 == object); +} +/** + * Determines if the variable is null or strlen(str) == 0 + */ +static auto IsNullOrEmpty(char *str)-> decltype(NULL !=str, bool()) { + return (NULL == str || strlen(str) == 0); +} + + +#endif diff --git a/libminifi/include/utils/FailurePolicy.h b/libminifi/include/utils/FailurePolicy.h new file mode 100644 index 0000000000..a4a7f9e225 --- /dev/null +++ b/libminifi/include/utils/FailurePolicy.h @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBMINIFI_INCLUDE_UTILS_FAILUREPOLICY_H_ +#define LIBMINIFI_INCLUDE_UTILS_FAILUREPOLICY_H_ + +/** + * Basic failure policy enumeration + * + */ +enum FailurePolicy { + + /** + * DO NOTHING + */ + NOTHING, + /** + * Return a response code from the executing function + */ + RETURN, + /** + * Throw an exception for flow control. + */ + EXCEPT, + /** + * Exit the program. This should only be used when something + * precludes us from continuing + */ + EXIT +}; + +#endif /* LIBMINIFI_INCLUDE_UTILS_FAILUREPOLICY_H_ */ diff --git a/libminifi/include/utils/StringUtils.h b/libminifi/include/utils/StringUtils.h new file mode 100644 index 0000000000..30858c8ea3 --- /dev/null +++ b/libminifi/include/utils/StringUtils.h @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBMINIFI_INCLUDE_IO_STRINGUTILS_H_ +#define LIBMINIFI_INCLUDE_IO_STRINGUTILS_H_ + +#include +#include +#include "../utils/FailurePolicy.h" + +/** + * Stateless String utility class. + * + * Design: Static class, with no member variables + * + * Purpose: Houses many useful string utilities. + */ +class StringUtils { +public: + /** + * Converts a string to a boolean + * Better handles mixed case. + * @param input input string + * @param output output string. + */ + static bool StringToBool(std::string input, bool &output) { + + std::transform(input.begin(), input.end(), input.begin(), ::tolower); + std::istringstream(input) >> std::boolalpha >> output; + return output; + } + + // Trim String utils + + /** + * Trims a string left to right + * @param s incoming string + * @returns modified string + */ + static std::string trim(std::string s) { + return trimRight(trimLeft(s)); + } + + /** + * Trims left most part of a string + * @param s incoming string + * @returns modified string + */ + static inline std::string trimLeft(std::string s) { + s.erase(s.begin(), + std::find_if(s.begin(), s.end(), + std::not1( + std::pointer_to_unary_function( + std::isspace)))); + return s; + } + + /** + * Trims a string on the right + * @param s incoming string + * @returns modified string + */ + + static inline std::string trimRight(std::string s) { + s.erase( + std::find_if(s.rbegin(), s.rend(), + std::not1( + std::pointer_to_unary_function( + std::isspace))).base(), s.end()); + return s; + } + + /** + * Converts a string to a float + * @param input input string + * @param output output float + * @param cp failure policy + */ + static bool StringToFloat(std::string input, float &output, + FailurePolicy cp = RETURN) { + try { + output = std::stof(input); + } catch (const std::invalid_argument &ie) { + switch (cp) { + case RETURN: + case NOTHING: + return false; + case EXIT: + exit(1); + case EXCEPT: + throw ie; + } + } catch (const std::out_of_range &ofr) { + switch (cp) { + case RETURN: + case NOTHING: + return false; + case EXIT: + exit(1); + case EXCEPT: + throw ofr; + + } + } + + return true; + + } + +}; + +#endif /* LIBMINIFI_INCLUDE_IO_STRINGUTILS_H_ */ diff --git a/libminifi/include/TimeUtil.h b/libminifi/include/utils/TimeUtil.h similarity index 54% rename from libminifi/include/TimeUtil.h rename to libminifi/include/utils/TimeUtil.h index b02424564c..76b334bdc2 100644 --- a/libminifi/include/TimeUtil.h +++ b/libminifi/include/utils/TimeUtil.h @@ -1,7 +1,4 @@ /** - * @file TimeUtil.h - * Basic Time Utility - * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -25,51 +22,45 @@ #include #include #include -#include - -#ifdef __MACH__ -#include -#include -#endif - -inline uint64_t getTimeMillis() -{ - uint64_t value; +#include +#include +#include - timeval time; - gettimeofday(&time, NULL); - value = ((uint64_t) (time.tv_sec) * 1000) + (time.tv_usec / 1000); +#define TIME_FORMAT "%Y-%m-%d %H:%M:%S" - return value; +/** + * Gets the current time in milliseconds + * @returns milliseconds since epoch + */ +inline uint64_t getTimeMillis() { + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); } -inline uint64_t getTimeNano() -{ - struct timespec ts; - -#ifdef __MACH__ // OS X does not have clock_gettime, use clock_get_time - clock_serv_t cclock; - mach_timespec_t mts; - host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock); - clock_get_time(cclock, &mts); - mach_port_deallocate(mach_task_self(), cclock); - ts.tv_sec = mts.tv_sec; - ts.tv_nsec = mts.tv_nsec; -#else - clock_gettime(CLOCK_REALTIME, &ts); -#endif +/** + * Gets the current time in nanoseconds + * @returns nanoseconds since epoch + */ +inline uint64_t getTimeNano() { + + return std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); - return ((uint64_t) (ts.tv_sec) * 1000000000 + ts.tv_nsec); } -//! Convert millisecond since UTC to a time display string -inline std::string getTimeStr(uint64_t msec) +/** + * Returns a string based on TIME_FORMAT, converting + * the parameter to a string + * @param msec milliseconds since epoch + * @returns string representing the time + */ +inline std::string getTimeStr(uint64_t msec, bool enforce_locale = false) { char date[120]; time_t second = (time_t) (msec/1000); msec = msec % 1000; - strftime(date, sizeof(date) / sizeof(*date), "%Y-%m-%d %H:%M:%S", - localtime(&second)); + strftime(date, sizeof(date) / sizeof(*date), TIME_FORMAT, + ( enforce_locale==true ? gmtime(&second) : localtime(&second))); std::string ret = date; date[0] = '\0'; @@ -79,4 +70,5 @@ inline std::string getTimeStr(uint64_t msec) return ret; } + #endif diff --git a/libminifi/src/AppendHostInfo.cpp b/libminifi/src/AppendHostInfo.cpp index c9ce9325fa..d0769c17a1 100644 --- a/libminifi/src/AppendHostInfo.cpp +++ b/libminifi/src/AppendHostInfo.cpp @@ -31,6 +31,8 @@ #include #include +#include "io/ClientSocket.h" + #define __USE_POSIX #include @@ -66,14 +68,10 @@ void AppendHostInfo::onTrigger(ProcessContext *context, ProcessSession *session) return; //Get Hostname - char hostname[HOST_NAME_MAX]; - hostname[HOST_NAME_MAX-1] = '\0'; - gethostname(hostname, HOST_NAME_MAX-1); - struct hostent* h; - h = gethostbyname(hostname); - std::string hostAttribute; - context->getProperty(HostAttribute.getName(), hostAttribute); - flow->addAttribute(hostAttribute.c_str(), h->h_name); + + std::string hostAttribute = ""; + context->getProperty(HostAttribute.getName(), hostAttribute); + flow->addAttribute(hostAttribute.c_str(), Socket::getMyHostName()); //Get IP address for the specified interface std::string iface; diff --git a/libminifi/src/Configure.cpp b/libminifi/src/Configure.cpp index 61e782b478..dac8408f36 100644 --- a/libminifi/src/Configure.cpp +++ b/libminifi/src/Configure.cpp @@ -18,8 +18,9 @@ * limitations under the License. */ #include "Configure.h" +#include "utils/StringUtils.h" -Configure *Configure::_configure(NULL); +Configure *Configure::configure_(NULL); const char *Configure::nifi_flow_configuration_file = "nifi.flow.configuration.file"; const char *Configure::nifi_administrative_yield_duration = "nifi.administrative.yield.duration"; const char *Configure::nifi_bored_yield_duration = "nifi.bored.yield.duration"; @@ -42,7 +43,7 @@ const char *Configure::nifi_security_client_ca_certificate = "nifi.security.clie bool Configure::get(std::string key, std::string &value) { std::lock_guard lock(_mtx); - std::map::iterator it = _properties.find(key); + auto it = _properties.find(key); if (it != _properties.end()) { @@ -55,25 +56,6 @@ bool Configure::get(std::string key, std::string &value) } } -// Trim String utils -std::string Configure::trim(const std::string& s) -{ - return trimRight(trimLeft(s)); -} - -std::string Configure::trimLeft(const std::string& s) -{ - const char *WHITESPACE = " \n\r\t"; - size_t startpos = s.find_first_not_of(WHITESPACE); - return (startpos == std::string::npos) ? "" : s.substr(startpos); -} - -std::string Configure::trimRight(const std::string& s) -{ - const char *WHITESPACE = " \n\r\t"; - size_t endpos = s.find_last_not_of(WHITESPACE); - return (endpos == std::string::npos) ? "" : s.substr(0, endpos+1); -} //! Parse one line in configure file like key=value void Configure::parseConfigureFileLine(char *buf) @@ -109,8 +91,8 @@ void Configure::parseConfigureFileLine(char *buf) } std::string value = equal; - key = trimRight(key); - value = trimRight(value); + key = StringUtils::trimRight(key); + value = StringUtils::trimRight(value); set(key, value); } @@ -124,7 +106,7 @@ void Configure::loadConfigureFile(const char *fileName) // perform a naive determination if this is a relative path if (fileName[0] != '/') { - adjustedFilename = adjustedFilename + _configure->getHome() + "/" + fileName; + adjustedFilename = adjustedFilename + configure_->getHome() + "/" + fileName; } else { @@ -134,12 +116,12 @@ void Configure::loadConfigureFile(const char *fileName) char *path = NULL; char full_path[PATH_MAX]; path = realpath(adjustedFilename.c_str(), full_path); - _logger->log_info("Using configuration file located at %s", path); + logger_->log_info("Using configuration file located at %s", path); std::ifstream file(path, std::ifstream::in); if (!file.good()) { - _logger->log_error("load configure file failed %s", path); + logger_->log_error("load configure file failed %s", path); return; } this->clear(); diff --git a/libminifi/src/Connection.cpp b/libminifi/src/Connection.cpp index 7beaf7a9f6..791fa638e2 100644 --- a/libminifi/src/Connection.cpp +++ b/libminifi/src/Connection.cpp @@ -51,9 +51,9 @@ Connection::Connection(std::string name, uuid_t uuid, uuid_t srcUUID, uuid_t des _expiredDuration = 0; _queuedDataSize = 0; - _logger = Logger::getLogger(); + logger_ = Logger::getLogger(); - _logger->log_info("Connection %s created", _name.c_str()); + logger_->log_info("Connection %s created", _name.c_str()); } bool Connection::isEmpty() @@ -89,7 +89,7 @@ void Connection::put(FlowFileRecord *flow) _queuedDataSize += flow->getSize(); - _logger->log_debug("Enqueue flow file UUID %s to connection %s", + logger_->log_debug("Enqueue flow file UUID %s to connection %s", flow->getUUIDStr().c_str(), _name.c_str()); } @@ -129,7 +129,7 @@ FlowFileRecord* Connection::poll(std::set &expiredFlowRecords) break; } item->setOriginalConnection(this); - _logger->log_debug("Dequeue flow file UUID %s from connection %s", + logger_->log_debug("Dequeue flow file UUID %s from connection %s", item->getUUIDStr().c_str(), _name.c_str()); return item; } @@ -145,7 +145,7 @@ FlowFileRecord* Connection::poll(std::set &expiredFlowRecords) break; } item->setOriginalConnection(this); - _logger->log_debug("Dequeue flow file UUID %s from connection %s", + logger_->log_debug("Dequeue flow file UUID %s from connection %s", item->getUUIDStr().c_str(), _name.c_str()); return item; } @@ -165,5 +165,5 @@ void Connection::drain() delete item; } - _logger->log_debug("Drain connection %s", _name.c_str()); + logger_->log_debug("Drain connection %s", _name.c_str()); } diff --git a/libminifi/src/ExecuteProcess.cpp b/libminifi/src/ExecuteProcess.cpp index 569c985c7c..61f96d52a3 100644 --- a/libminifi/src/ExecuteProcess.cpp +++ b/libminifi/src/ExecuteProcess.cpp @@ -17,11 +17,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "TimeUtil.h" #include "ExecuteProcess.h" #include "ProcessContext.h" #include "ProcessSession.h" #include +#include "utils/StringUtils.h" +#include "utils/TimeUtil.h" const std::string ExecuteProcess::ProcessorName("ExecuteProcess"); Property ExecuteProcess::Command("Command", "Specifies the command to be executed; if just the name of an executable is provided, it must be in the user's environment PATH.", ""); @@ -78,7 +79,7 @@ void ExecuteProcess::onTrigger(ProcessContext *context, ProcessSession *session) } if (context->getProperty(RedirectErrorStream.getName(), value)) { - Property::StringToBool(value, _redirectErrorStream); + StringUtils::StringToBool(value, _redirectErrorStream); } this->_fullCommand = _command + " " + _commandArgument; if (_fullCommand.length() == 0) @@ -91,12 +92,12 @@ void ExecuteProcess::onTrigger(ProcessContext *context, ProcessSession *session) // change to working directory if (chdir(_workingDir.c_str()) != 0) { - _logger->log_error("Execute Command can not chdir %s", _workingDir.c_str()); + logger_->log_error("Execute Command can not chdir %s", _workingDir.c_str()); yield(); return; } } - _logger->log_info("Execute Command %s", _fullCommand.c_str()); + logger_->log_info("Execute Command %s", _fullCommand.c_str()); // split the command into array char cstr[_fullCommand.length()+1]; std::strcpy(cstr, _fullCommand.c_str()); @@ -125,7 +126,7 @@ void ExecuteProcess::onTrigger(ProcessContext *context, ProcessSession *session) switch (_pid = fork()) { case -1: - _logger->log_error("Execute Process fork failed"); + logger_->log_error("Execute Process fork failed"); _processRunning = false; close(_pipefd[0]); close(_pipefd[1]); @@ -153,7 +154,7 @@ void ExecuteProcess::onTrigger(ProcessContext *context, ProcessSession *session) int numRead = read(_pipefd[0], buffer, sizeof(buffer)); if (numRead <= 0) break; - _logger->log_info("Execute Command Respond %d", numRead); + logger_->log_info("Execute Command Respond %d", numRead); ExecuteProcess::WriteCallback callback(buffer, numRead); FlowFileRecord *flowFile = session->create(); if (!flowFile) @@ -178,7 +179,7 @@ void ExecuteProcess::onTrigger(ProcessContext *context, ProcessSession *session) { if (totalRead > 0) { - _logger->log_info("Execute Command Respond %d", totalRead); + logger_->log_info("Execute Command Respond %d", totalRead); // child exits and close the pipe ExecuteProcess::WriteCallback callback(buffer, totalRead); if (!flowFile) @@ -203,7 +204,7 @@ void ExecuteProcess::onTrigger(ProcessContext *context, ProcessSession *session) if (numRead == (sizeof(buffer) - totalRead)) { // we reach the max buffer size - _logger->log_info("Execute Command Max Respond %d", sizeof(buffer)); + logger_->log_info("Execute Command Max Respond %d", sizeof(buffer)); ExecuteProcess::WriteCallback callback(buffer, sizeof(buffer)); if (!flowFile) { @@ -234,11 +235,11 @@ void ExecuteProcess::onTrigger(ProcessContext *context, ProcessSession *session) died= wait(&status); if (WIFEXITED(status)) { - _logger->log_info("Execute Command Complete %s status %d pid %d", _fullCommand.c_str(), WEXITSTATUS(status), _pid); + logger_->log_info("Execute Command Complete %s status %d pid %d", _fullCommand.c_str(), WEXITSTATUS(status), _pid); } else { - _logger->log_info("Execute Command Complete %s status %d pid %d", _fullCommand.c_str(), WTERMSIG(status), _pid); + logger_->log_info("Execute Command Complete %s status %d pid %d", _fullCommand.c_str(), WTERMSIG(status), _pid); } close(_pipefd[0]); diff --git a/libminifi/src/FlowControlProtocol.cpp b/libminifi/src/FlowControlProtocol.cpp index a74e32ec07..22ef1f90c2 100644 --- a/libminifi/src/FlowControlProtocol.cpp +++ b/libminifi/src/FlowControlProtocol.cpp @@ -45,7 +45,7 @@ int FlowControlProtocol::connectServer(const char *host, uint16_t port) sock = socket(AF_INET, SOCK_STREAM, 0); if (sock < 0) { - _logger->log_error("Could not create socket to hostName %s", host); + logger_->log_error("Could not create socket to hostName %s", host); return 0; } @@ -57,14 +57,14 @@ int FlowControlProtocol::connectServer(const char *host, uint16_t port) { if (setsockopt(sock, SOL_TCP, TCP_NODELAY, (void *)&opt, sizeof(opt)) < 0) { - _logger->log_error("setsockopt() TCP_NODELAY failed"); + logger_->log_error("setsockopt() TCP_NODELAY failed"); close(sock); return 0; } if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *)&opt, sizeof(opt)) < 0) { - _logger->log_error("setsockopt() SO_REUSEADDR failed"); + logger_->log_error("setsockopt() SO_REUSEADDR failed"); close(sock); return 0; } @@ -73,7 +73,7 @@ int FlowControlProtocol::connectServer(const char *host, uint16_t port) int sndsize = 256*1024; if (setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (char *)&sndsize, (int)sizeof(sndsize)) < 0) { - _logger->log_error("setsockopt() SO_SNDBUF failed"); + logger_->log_error("setsockopt() SO_SNDBUF failed"); close(sock); return 0; } @@ -90,7 +90,7 @@ int FlowControlProtocol::connectServer(const char *host, uint16_t port) socklen = sizeof(sa); if (bind(sock, (struct sockaddr *)&sa, socklen) < 0) { - _logger->log_error("socket bind failed"); + logger_->log_error("socket bind failed"); close(sock); return 0; } @@ -105,12 +105,12 @@ int FlowControlProtocol::connectServer(const char *host, uint16_t port) if (status < 0) { - _logger->log_error("socket connect failed to %s %d", host, port); + logger_->log_error("socket connect failed to %s %d", host, port); close(sock); return 0; } - _logger->log_info("Flow Control Protocol socket %d connect to server %s port %d success", sock, host, port); + logger_->log_info("Flow Control Protocol socket %d connect to server %s port %d success", sock, host, port); return sock; } @@ -220,7 +220,7 @@ void FlowControlProtocol::start() if (_running) return; _running = true; - _logger->log_info("FlowControl Protocol Start"); + logger_->log_info("FlowControl Protocol Start"); _thread = new std::thread(run, this); _thread->detach(); } @@ -230,7 +230,7 @@ void FlowControlProtocol::stop() if (!_running) return; _running = false; - _logger->log_info("FlowControl Protocol Stop"); + logger_->log_info("FlowControl Protocol Stop"); } void FlowControlProtocol::run(FlowControlProtocol *protocol) @@ -253,7 +253,7 @@ int FlowControlProtocol::sendRegisterReq() { if (_registered) { - _logger->log_info("Already registered"); + logger_->log_info("Already registered"); return -1; } @@ -299,7 +299,7 @@ int FlowControlProtocol::sendRegisterReq() { close(_socket); _socket = 0; - _logger->log_error("Flow Control Protocol Send Register Req failed"); + logger_->log_error("Flow Control Protocol Send Register Req failed"); return -1; } @@ -310,26 +310,26 @@ int FlowControlProtocol::sendRegisterReq() { close(_socket); _socket = 0; - _logger->log_error("Flow Control Protocol Read Register Resp header failed"); + logger_->log_error("Flow Control Protocol Read Register Resp header failed"); return -1; } - _logger->log_info("Flow Control Protocol receive MsgType %s", FlowControlMsgTypeToStr((FlowControlMsgType) hdr.msgType)); - _logger->log_info("Flow Control Protocol receive Seq Num %d", hdr.seqNumber); - _logger->log_info("Flow Control Protocol receive Resp Code %s", FlowControlRespCodeToStr((FlowControlRespCode) hdr.status)); - _logger->log_info("Flow Control Protocol receive Payload len %d", hdr.payloadLen); + logger_->log_info("Flow Control Protocol receive MsgType %s", FlowControlMsgTypeToStr((FlowControlMsgType) hdr.msgType)); + logger_->log_info("Flow Control Protocol receive Seq Num %d", hdr.seqNumber); + logger_->log_info("Flow Control Protocol receive Resp Code %s", FlowControlRespCodeToStr((FlowControlRespCode) hdr.status)); + logger_->log_info("Flow Control Protocol receive Payload len %d", hdr.payloadLen); if (hdr.status == RESP_SUCCESS && hdr.seqNumber == this->_seqNumber) { this->_registered = true; this->_seqNumber++; - _logger->log_info("Flow Control Protocol Register success"); + logger_->log_info("Flow Control Protocol Register success"); uint8_t *payload = new uint8_t[hdr.payloadLen]; uint8_t *payloadPtr = payload; status = readData(payload, hdr.payloadLen); if (status <= 0) { delete[] payload; - _logger->log_info("Flow Control Protocol Register Read Payload fail"); + logger_->log_info("Flow Control Protocol Register Read Payload fail"); close(_socket); _socket = 0; return -1; @@ -343,7 +343,7 @@ int FlowControlProtocol::sendRegisterReq() // Fixed 4 bytes uint32_t reportInterval; payloadPtr = this->decode(payloadPtr, reportInterval); - _logger->log_info("Flow Control Protocol receive report interval %d ms", reportInterval); + logger_->log_info("Flow Control Protocol receive report interval %d ms", reportInterval); this->_reportInterval = reportInterval; } else @@ -358,7 +358,7 @@ int FlowControlProtocol::sendRegisterReq() } else { - _logger->log_info("Flow Control Protocol Register fail"); + logger_->log_info("Flow Control Protocol Register fail"); close(_socket); _socket = 0; return -1; @@ -406,7 +406,7 @@ int FlowControlProtocol::sendReportReq() { close(_socket); _socket = 0; - _logger->log_error("Flow Control Protocol Send Report Req failed"); + logger_->log_error("Flow Control Protocol Send Report Req failed"); return -1; } @@ -417,13 +417,13 @@ int FlowControlProtocol::sendReportReq() { close(_socket); _socket = 0; - _logger->log_error("Flow Control Protocol Read Report Resp header failed"); + logger_->log_error("Flow Control Protocol Read Report Resp header failed"); return -1; } - _logger->log_info("Flow Control Protocol receive MsgType %s", FlowControlMsgTypeToStr((FlowControlMsgType) hdr.msgType)); - _logger->log_info("Flow Control Protocol receive Seq Num %d", hdr.seqNumber); - _logger->log_info("Flow Control Protocol receive Resp Code %s", FlowControlRespCodeToStr((FlowControlRespCode) hdr.status)); - _logger->log_info("Flow Control Protocol receive Payload len %d", hdr.payloadLen); + logger_->log_info("Flow Control Protocol receive MsgType %s", FlowControlMsgTypeToStr((FlowControlMsgType) hdr.msgType)); + logger_->log_info("Flow Control Protocol receive Seq Num %d", hdr.seqNumber); + logger_->log_info("Flow Control Protocol receive Resp Code %s", FlowControlRespCodeToStr((FlowControlRespCode) hdr.status)); + logger_->log_info("Flow Control Protocol receive Payload len %d", hdr.payloadLen); if (hdr.status == RESP_SUCCESS && hdr.seqNumber == this->_seqNumber) { @@ -434,7 +434,7 @@ int FlowControlProtocol::sendReportReq() if (status <= 0) { delete[] payload; - _logger->log_info("Flow Control Protocol Report Resp Read Payload fail"); + logger_->log_info("Flow Control Protocol Report Resp Read Payload fail"); close(_socket); _socket = 0; return -1; @@ -452,7 +452,7 @@ int FlowControlProtocol::sendReportReq() payloadPtr = this->decode(payloadPtr, len); processor = (const char *) payloadPtr; payloadPtr += len; - _logger->log_info("Flow Control Protocol receive report resp processor %s", processor.c_str()); + logger_->log_info("Flow Control Protocol receive report resp processor %s", processor.c_str()); } else if (((FlowControlMsgID) msgID) == PROPERTY_NAME) { @@ -460,7 +460,7 @@ int FlowControlProtocol::sendReportReq() payloadPtr = this->decode(payloadPtr, len); propertyName = (const char *) payloadPtr; payloadPtr += len; - _logger->log_info("Flow Control Protocol receive report resp property name %s", propertyName.c_str()); + logger_->log_info("Flow Control Protocol receive report resp property name %s", propertyName.c_str()); } else if (((FlowControlMsgID) msgID) == PROPERTY_VALUE) { @@ -468,7 +468,7 @@ int FlowControlProtocol::sendReportReq() payloadPtr = this->decode(payloadPtr, len); propertyValue = (const char *) payloadPtr; payloadPtr += len; - _logger->log_info("Flow Control Protocol receive report resp property value %s", propertyValue.c_str()); + logger_->log_info("Flow Control Protocol receive report resp property value %s", propertyValue.c_str()); this->_controller->updatePropertyValue(processor, propertyName, propertyValue); } else @@ -483,7 +483,7 @@ int FlowControlProtocol::sendReportReq() } else if (hdr.status == RESP_TRIGGER_REGISTER && hdr.seqNumber == this->_seqNumber) { - _logger->log_info("Flow Control Protocol trigger reregister"); + logger_->log_info("Flow Control Protocol trigger reregister"); this->_registered = false; this->_seqNumber++; close(_socket); @@ -492,7 +492,7 @@ int FlowControlProtocol::sendReportReq() } else if (hdr.status == RESP_STOP_FLOW_CONTROLLER && hdr.seqNumber == this->_seqNumber) { - _logger->log_info("Flow Control Protocol stop flow controller"); + logger_->log_info("Flow Control Protocol stop flow controller"); this->_controller->stop(true); this->_seqNumber++; close(_socket); @@ -501,7 +501,7 @@ int FlowControlProtocol::sendReportReq() } else if (hdr.status == RESP_START_FLOW_CONTROLLER && hdr.seqNumber == this->_seqNumber) { - _logger->log_info("Flow Control Protocol start flow controller"); + logger_->log_info("Flow Control Protocol start flow controller"); this->_controller->start(); this->_seqNumber++; close(_socket); @@ -510,7 +510,7 @@ int FlowControlProtocol::sendReportReq() } else { - _logger->log_info("Flow Control Protocol Report fail"); + logger_->log_info("Flow Control Protocol Report fail"); close(_socket); _socket = 0; return -1; diff --git a/libminifi/src/FlowController.cpp b/libminifi/src/FlowController.cpp index 98bf1a13ba..4f5faf83c6 100644 --- a/libminifi/src/FlowController.cpp +++ b/libminifi/src/FlowController.cpp @@ -31,6 +31,7 @@ #include #include "FlowController.h" #include "ProcessContext.h" +#include "utils/StringUtils.h" FlowController *FlowControllerFactory::_flowController(NULL); @@ -45,14 +46,14 @@ FlowControllerImpl::FlowControllerImpl(std::string name) { _running = false; _initialized = false; _root = NULL; - _logger = Logger::getLogger(); + logger_ = Logger::getLogger(); _protocol = new FlowControlProtocol(this); // NiFi config properties - _configure = Configure::getConfigure(); + configure_ = Configure::getConfigure(); std::string rawConfigFileString; - _configure->get(Configure::nifi_flow_configuration_file, + configure_->get(Configure::nifi_flow_configuration_file, rawConfigFileString); if (!rawConfigFileString.empty()) { @@ -66,7 +67,7 @@ FlowControllerImpl::FlowControllerImpl(std::string name) { if (!_configurationFileName.empty()) { // perform a naive determination if this is a relative path if (_configurationFileName.c_str()[0] != '/') { - adjustedFilename = adjustedFilename + _configure->getHome() + "/" + adjustedFilename = adjustedFilename + configure_->getHome() + "/" + _configurationFileName; } else { adjustedFilename = _configurationFileName; @@ -77,7 +78,7 @@ FlowControllerImpl::FlowControllerImpl(std::string name) { std::string pathString(path); _configurationFileName = pathString; - _logger->log_info("FlowController NiFi Configuration file %s", pathString.c_str()); + logger_->log_info("FlowController NiFi Configuration file %s", pathString.c_str()); // Create the content repo directory if needed struct stat contentDirStat; @@ -85,101 +86,22 @@ FlowControllerImpl::FlowControllerImpl(std::string name) { if (stat(ResourceClaim::default_directory_path.c_str(), &contentDirStat) != -1 && S_ISDIR(contentDirStat.st_mode)) { path = realpath(ResourceClaim::default_directory_path.c_str(), full_path); - _logger->log_info("FlowController content directory %s", full_path); + logger_->log_info("FlowController content directory %s", full_path); } else { if (mkdir(ResourceClaim::default_directory_path.c_str(), 0777) == -1) { - _logger->log_error("FlowController content directory creation failed"); + logger_->log_error("FlowController content directory creation failed"); exit(1); } } - std::string secureStr; - bool isSecure = false; - if (_configure->get(Configure::nifi_remote_input_secure, secureStr)) - { - Property::StringToBool(secureStr, isSecure); - } + std::string clientAuthStr; - bool needClientCert = true; - if (!(_configure->get(Configure::nifi_security_need_ClientAuth, clientAuthStr) - && Property::StringToBool(clientAuthStr, needClientCert))) - { - needClientCert = true; - } - _logger->log_info("Site2Site secure setting is %d, needClientCert %d", isSecure, needClientCert); - _ctx = NULL; - if (isSecure) - { - // create SSL context - SSL_library_init(); - const SSL_METHOD *method; - - OpenSSL_add_all_algorithms(); - SSL_load_error_strings(); - method = TLSv1_2_client_method(); - _ctx = SSL_CTX_new(method); - if ( _ctx == NULL ) - { - ERR_print_errors_fp(stderr); - _logger->log_error("Could not create SSL context, Exiting."); - exit(1); - } - if (needClientCert) - { - std::string certificate; - std::string privatekey; - std::string passphrase; - std::string caCertificate; - - if (!(_configure->get(Configure::nifi_security_client_certificate, certificate) && - _configure->get(Configure::nifi_security_client_private_key, privatekey))) - { - _logger->log_error("Certificate and Private Key PEM file not configured, Exiting."); - exit(1); - } - // load certificates and private key in PEM format - if ( SSL_CTX_use_certificate_file(_ctx, certificate.c_str(), SSL_FILETYPE_PEM) <= 0 ) - { - ERR_print_errors_fp(stderr); - _logger->log_error("Could not create load certificate, Exiting."); - exit(1); - } - if (_configure->get(Configure::nifi_security_client_pass_phrase, passphrase)) - { - // if the private key has passphase - SSL_CTX_set_default_passwd_cb(_ctx, FlowController::pemPassWordCb); - } - if ( SSL_CTX_use_PrivateKey_file(_ctx, privatekey.c_str(), SSL_FILETYPE_PEM) <= 0 ) - { - ERR_print_errors_fp(stderr); - _logger->log_error("Could not create load private key, Exiting."); - exit(1); - } - /* verify private key */ - if ( !SSL_CTX_check_private_key(_ctx) ) - { - _logger->log_error("Private key does not match the public certificate"); - exit(1); - } - /* load CA certificates */ - if (_configure->get(Configure::nifi_security_client_ca_certificate, caCertificate)) - { - if (!SSL_CTX_load_verify_locations(_ctx, caCertificate.c_str(), 0)) - { - _logger->log_error("Can not load CA certificate, Exiting."); - exit(1); - } - } - - _logger->log_info("Load/Verify Client Certificate OK."); - } - } if (!path) { - _logger->log_error( + logger_->log_error( "Could not locate path from provided configuration file name (%s). Exiting.", full_path); exit(1); @@ -208,7 +130,7 @@ void FlowControllerImpl::stop(bool force) { // immediately indicate that we are not running _running = false; - _logger->log_info("Stop Flow Controller"); + logger_->log_info("Stop Flow Controller"); this->_timerScheduler.stop(); this->_eventScheduler.stop(); // Wait for sometime for thread stop @@ -253,7 +175,7 @@ void FlowControllerImpl::unload() { stop(true); } if (_initialized) { - _logger->log_info("Unload Flow Controller"); + logger_->log_info("Unload Flow Controller"); if (_root) delete _root; _root = NULL; @@ -287,7 +209,7 @@ Processor *FlowControllerImpl::createProcessor(std::string name, uuid_t uuid) { } else if (name == AppendHostInfo::ProcessorName) { processor = new AppendHostInfo(name, uuid); } else { - _logger->log_error("No Processor defined for %s", name.c_str()); + logger_->log_error("No Processor defined for %s", name.c_str()); return NULL; } @@ -323,8 +245,8 @@ void FlowControllerImpl::parseRootProcessGroupYaml(YAML::Node rootFlowNode) { char uuidStr[37]; uuid_unparse_lower(_uuid, uuidStr); - _logger->log_debug("parseRootProcessGroup: id => [%s]", uuidStr); - _logger->log_debug("parseRootProcessGroup: name => [%s]", flowName.c_str()); + logger_->log_debug("parseRootProcessGroup: id => [%s]", uuidStr); + logger_->log_debug("parseRootProcessGroup: name => [%s]", flowName.c_str()); group = this->createRootProcessGroup(flowName, uuid); this->_root = group; this->_name = flowName; @@ -340,7 +262,7 @@ void FlowControllerImpl::parseProcessorNodeYaml(YAML::Node processorsNode, Processor *processor = NULL; if (!parentGroup) { - _logger->log_error("parseProcessNodeYaml: no parent group exists"); + logger_->log_error("parseProcessNodeYaml: no parent group exists"); return; } @@ -356,10 +278,10 @@ void FlowControllerImpl::parseProcessorNodeYaml(YAML::Node processorsNode, YAML::Node procNode = iter->as(); procCfg.name = procNode["name"].as(); - _logger->log_debug("parseProcessorNode: name => [%s]", + logger_->log_debug("parseProcessorNode: name => [%s]", procCfg.name.c_str()); procCfg.javaClass = procNode["class"].as(); - _logger->log_debug("parseProcessorNode: class => [%s]", + logger_->log_debug("parseProcessorNode: class => [%s]", procCfg.javaClass.c_str()); char uuidStr[37]; @@ -379,7 +301,7 @@ void FlowControllerImpl::parseProcessorNodeYaml(YAML::Node processorsNode, } if (!processor) { - _logger->log_error( + logger_->log_error( "Could not create a processor %s with name %s", procCfg.name.c_str(), uuidStr); throw std::invalid_argument( @@ -389,31 +311,31 @@ void FlowControllerImpl::parseProcessorNodeYaml(YAML::Node processorsNode, procCfg.maxConcurrentTasks = procNode["max concurrent tasks"].as(); - _logger->log_debug( + logger_->log_debug( "parseProcessorNode: max concurrent tasks => [%s]", procCfg.maxConcurrentTasks.c_str()); procCfg.schedulingStrategy = procNode["scheduling strategy"].as< std::string>(); - _logger->log_debug( + logger_->log_debug( "parseProcessorNode: scheduling strategy => [%s]", procCfg.schedulingStrategy.c_str()); procCfg.schedulingPeriod = procNode["scheduling period"].as< std::string>(); - _logger->log_debug( + logger_->log_debug( "parseProcessorNode: scheduling period => [%s]", procCfg.schedulingPeriod.c_str()); procCfg.penalizationPeriod = procNode["penalization period"].as< std::string>(); - _logger->log_debug( + logger_->log_debug( "parseProcessorNode: penalization period => [%s]", procCfg.penalizationPeriod.c_str()); procCfg.yieldPeriod = procNode["yield period"].as(); - _logger->log_debug("parseProcessorNode: yield period => [%s]", + logger_->log_debug("parseProcessorNode: yield period => [%s]", procCfg.yieldPeriod.c_str()); procCfg.yieldPeriod = procNode["run duration nanos"].as< std::string>(); - _logger->log_debug( + logger_->log_debug( "parseProcessorNode: run duration nanos => [%s]", procCfg.runDurationNanos.c_str()); @@ -447,7 +369,7 @@ void FlowControllerImpl::parseProcessorNodeYaml(YAML::Node processorsNode, schedulingPeriod, unit) && Property::ConvertTimeUnitToNS(schedulingPeriod, unit, schedulingPeriod)) { - _logger->log_debug( + logger_->log_debug( "convert: parseProcessorNode: schedulingPeriod => [%d] ns", schedulingPeriod); processor->setSchedulingPeriodNano(schedulingPeriod); @@ -457,7 +379,7 @@ void FlowControllerImpl::parseProcessorNodeYaml(YAML::Node processorsNode, penalizationPeriod, unit) && Property::ConvertTimeUnitToMS(penalizationPeriod, unit, penalizationPeriod)) { - _logger->log_debug( + logger_->log_debug( "convert: parseProcessorNode: penalizationPeriod => [%d] ms", penalizationPeriod); processor->setPenalizationPeriodMsec(penalizationPeriod); @@ -467,7 +389,7 @@ void FlowControllerImpl::parseProcessorNodeYaml(YAML::Node processorsNode, unit) && Property::ConvertTimeUnitToMS(yieldPeriod, unit, yieldPeriod)) { - _logger->log_debug( + logger_->log_debug( "convert: parseProcessorNode: yieldPeriod => [%d] ms", yieldPeriod); processor->setYieldPeriodMsec(yieldPeriod); @@ -478,15 +400,15 @@ void FlowControllerImpl::parseProcessorNodeYaml(YAML::Node processorsNode, if (procCfg.schedulingStrategy == "TIMER_DRIVEN") { processor->setSchedulingStrategy(TIMER_DRIVEN); - _logger->log_debug("setting scheduling strategy as %s", + logger_->log_debug("setting scheduling strategy as %s", procCfg.schedulingStrategy.c_str()); } else if (procCfg.schedulingStrategy == "EVENT_DRIVEN") { processor->setSchedulingStrategy(EVENT_DRIVEN); - _logger->log_debug("setting scheduling strategy as %s", + logger_->log_debug("setting scheduling strategy as %s", procCfg.schedulingStrategy.c_str()); } else { processor->setSchedulingStrategy(CRON_DRIVEN); - _logger->log_debug("setting scheduling strategy as %s", + logger_->log_debug("setting scheduling strategy as %s", procCfg.schedulingStrategy.c_str()); } @@ -494,7 +416,7 @@ void FlowControllerImpl::parseProcessorNodeYaml(YAML::Node processorsNode, int64_t maxConcurrentTasks; if (Property::StringToInt(procCfg.maxConcurrentTasks, maxConcurrentTasks)) { - _logger->log_debug( + logger_->log_debug( "parseProcessorNode: maxConcurrentTasks => [%d]", maxConcurrentTasks); processor->setMaxConcurrentTasks(maxConcurrentTasks); @@ -502,7 +424,7 @@ void FlowControllerImpl::parseProcessorNodeYaml(YAML::Node processorsNode, if (Property::StringToInt(procCfg.runDurationNanos, runDurationNanos)) { - _logger->log_debug( + logger_->log_debug( "parseProcessorNode: runDurationNanos => [%d]", runDurationNanos); processor->setRunDurationNano(runDurationNanos); @@ -511,7 +433,7 @@ void FlowControllerImpl::parseProcessorNodeYaml(YAML::Node processorsNode, std::set autoTerminatedRelationships; for (auto &&relString : procCfg.autoTerminatedRelationships) { Relationship relationship(relString, ""); - _logger->log_debug( + logger_->log_debug( "parseProcessorNode: autoTerminatedRelationship => [%s]", relString.c_str()); autoTerminatedRelationships.insert(relationship); @@ -534,7 +456,7 @@ void FlowControllerImpl::parseRemoteProcessGroupYaml(YAML::Node *rpgNode, uuid_t uuid; if (!parentGroup) { - _logger->log_error( + logger_->log_error( "parseRemoteProcessGroupYaml: no parent group exists"); return; } @@ -546,21 +468,21 @@ void FlowControllerImpl::parseRemoteProcessGroupYaml(YAML::Node *rpgNode, YAML::Node rpgNode = iter->as(); auto name = rpgNode["name"].as(); - _logger->log_debug("parseRemoteProcessGroupYaml: name => [%s]", + logger_->log_debug("parseRemoteProcessGroupYaml: name => [%s]", name.c_str()); std::string url = rpgNode["url"].as(); - _logger->log_debug("parseRemoteProcessGroupYaml: url => [%s]", + logger_->log_debug("parseRemoteProcessGroupYaml: url => [%s]", url.c_str()); std::string timeout = rpgNode["timeout"].as(); - _logger->log_debug( + logger_->log_debug( "parseRemoteProcessGroupYaml: timeout => [%s]", timeout.c_str()); std::string yieldPeriod = rpgNode["yield period"].as(); - _logger->log_debug( + logger_->log_debug( "parseRemoteProcessGroupYaml: yield period => [%s]", yieldPeriod.c_str()); @@ -587,7 +509,7 @@ void FlowControllerImpl::parseRemoteProcessGroupYaml(YAML::Node *rpgNode, if (Property::StringToTime(yieldPeriod, yieldPeriodValue, unit) && Property::ConvertTimeUnitToMS(yieldPeriodValue, unit, yieldPeriodValue) && group) { - _logger->log_debug( + logger_->log_debug( "parseRemoteProcessGroupYaml: yieldPeriod => [%d] ms", yieldPeriodValue); group->setYieldPeriodMsec(yieldPeriodValue); @@ -596,7 +518,7 @@ void FlowControllerImpl::parseRemoteProcessGroupYaml(YAML::Node *rpgNode, if (Property::StringToTime(timeout, timeoutValue, unit) && Property::ConvertTimeUnitToMS(timeoutValue, unit, timeoutValue) && group) { - _logger->log_debug( + logger_->log_debug( "parseRemoteProcessGroupYaml: timeoutValue => [%d] ms", timeoutValue); group->setTimeOut(timeoutValue); @@ -608,7 +530,7 @@ void FlowControllerImpl::parseRemoteProcessGroupYaml(YAML::Node *rpgNode, if (inputPorts && inputPorts.IsSequence()) { for (YAML::const_iterator portIter = inputPorts.begin(); portIter != inputPorts.end(); ++portIter) { - _logger->log_debug("Got a current port, iterating..."); + logger_->log_debug("Got a current port, iterating..."); YAML::Node currPort = portIter->as(); @@ -618,7 +540,7 @@ void FlowControllerImpl::parseRemoteProcessGroupYaml(YAML::Node *rpgNode, if (outputPorts && outputPorts.IsSequence()) { for (YAML::const_iterator portIter = outputPorts.begin(); portIter != outputPorts.end(); ++portIter) { - _logger->log_debug("Got a current port, iterating..."); + logger_->log_debug("Got a current port, iterating..."); YAML::Node currPort = portIter->as(); @@ -637,7 +559,7 @@ void FlowControllerImpl::parseConnectionYaml(YAML::Node *connectionsNode, Connection *connection = NULL; if (!parent) { - _logger->log_error("parseProcessNode: no parent group was provided"); + logger_->log_error("parseProcessNode: no parent group was provided"); return; } @@ -658,7 +580,7 @@ void FlowControllerImpl::parseConnectionYaml(YAML::Node *connectionsNode, char uuidStr[37]; uuid_unparse_lower(_uuid, uuidStr); - _logger->log_debug( + logger_->log_debug( "Created connection with UUID %s and name %s", uuidStr, name.c_str()); connection = this->createConnection(name, uuid); @@ -666,7 +588,7 @@ void FlowControllerImpl::parseConnectionYaml(YAML::Node *connectionsNode, connectionNode["source relationship name"].as< std::string>(); Relationship relationship(rawRelationship, ""); - _logger->log_debug("parseConnection: relationship => [%s]", + logger_->log_debug("parseConnection: relationship => [%s]", rawRelationship.c_str()); if (connection) connection->setRelationship(relationship); @@ -677,7 +599,7 @@ void FlowControllerImpl::parseConnectionYaml(YAML::Node *connectionsNode, connectionSrcProcName); if (!srcProcessor) { - _logger->log_error( + logger_->log_error( "Could not locate a source with name %s to create a connection", connectionSrcProcName.c_str()); throw std::invalid_argument( @@ -723,7 +645,7 @@ void FlowControllerImpl::parsePortYaml(YAML::Node *portNode, RemoteProcessorGroupPort *port = NULL; if (!parent) { - _logger->log_error("parseProcessNode: no parent group existed"); + logger_->log_error("parseProcessNode: no parent group existed"); return; } @@ -760,7 +682,7 @@ void FlowControllerImpl::parsePortYaml(YAML::Node *portNode, if (Property::StringToInt(rawMaxConcurrentTasks, maxConcurrentTasks)) { processor->setMaxConcurrentTasks(maxConcurrentTasks); } - _logger->log_debug("parseProcessorNode: maxConcurrentTasks => [%d]", + logger_->log_debug("parseProcessorNode: maxConcurrentTasks => [%d]", maxConcurrentTasks); processor->setMaxConcurrentTasks(maxConcurrentTasks); @@ -776,7 +698,7 @@ void FlowControllerImpl::parsePropertiesNodeYaml(YAML::Node *propertiesNode, if (!propertyValueNode.IsNull() && propertyValueNode.IsDefined()) { std::string rawValueString = propertyValueNode.as(); if (!processor->setProperty(propertyName, rawValueString)) { - _logger->log_warn( + logger_->log_warn( "Received property %s with value %s but is not one of the properties for %s", propertyName.c_str(), rawValueString.c_str(), processor->getName().c_str()); @@ -790,7 +712,7 @@ void FlowControllerImpl::load() { stop(true); } if (!_initialized) { - _logger->log_info("Load Flow Controller from file %s", _configurationFileName.c_str()); + logger_->log_info("Load Flow Controller from file %s", _configurationFileName.c_str()); YAML::Node flow = YAML::LoadFile(_configurationFileName); @@ -813,7 +735,7 @@ void FlowControllerImpl::load() { void FlowControllerImpl::reload(std::string yamlFile) { - _logger->log_info("Starting to reload Flow Controller with yaml %s", yamlFile.c_str()); + logger_->log_info("Starting to reload Flow Controller with yaml %s", yamlFile.c_str()); stop(true); unload(); std::string oldYamlFile = this->_configurationFileName; @@ -823,7 +745,7 @@ void FlowControllerImpl::reload(std::string yamlFile) if (!this->_root) { this->_configurationFileName = oldYamlFile; - _logger->log_info("Rollback Flow Controller to YAML %s", oldYamlFile.c_str()); + logger_->log_info("Rollback Flow Controller to YAML %s", oldYamlFile.c_str()); stop(true); unload(); load(); @@ -833,13 +755,13 @@ void FlowControllerImpl::reload(std::string yamlFile) bool FlowControllerImpl::start() { if (!_initialized) { - _logger->log_error( + logger_->log_error( "Can not start Flow Controller because it has not been initialized"); return false; } else { if (!_running) { - _logger->log_info("Starting Flow Controller"); + logger_->log_info("Starting Flow Controller"); this->_timerScheduler.start(); this->_eventScheduler.start(); if (this->_root) @@ -847,7 +769,7 @@ bool FlowControllerImpl::start() { &this->_eventScheduler); _running = true; this->_protocol->start(); - _logger->log_info("Started Flow Controller"); + logger_->log_info("Started Flow Controller"); } return true; } diff --git a/libminifi/src/FlowFileRecord.cpp b/libminifi/src/FlowFileRecord.cpp index a5a58d67af..1d509f531e 100644 --- a/libminifi/src/FlowFileRecord.cpp +++ b/libminifi/src/FlowFileRecord.cpp @@ -70,22 +70,22 @@ FlowFileRecord::FlowFileRecord(std::map attributes, Re if (_claim) // Increase the flow file record owned count for the resource claim _claim->increaseFlowFileRecordOwnedCount(); - _logger = Logger::getLogger(); + logger_ = Logger::getLogger(); } FlowFileRecord::~FlowFileRecord() { if (!_snapshot) - _logger->log_debug("Delete FlowFile UUID %s", _uuidStr.c_str()); + logger_->log_debug("Delete FlowFile UUID %s", _uuidStr.c_str()); else - _logger->log_debug("Delete SnapShot FlowFile UUID %s", _uuidStr.c_str()); + logger_->log_debug("Delete SnapShot FlowFile UUID %s", _uuidStr.c_str()); if (_claim) { // Decrease the flow file record owned count for the resource claim _claim->decreaseFlowFileRecordOwnedCount(); if (_claim->getFlowFileRecordOwnedCount() <= 0) { - _logger->log_debug("Delete Resource Claim %s", _claim->getContentFullPath().c_str()); + logger_->log_debug("Delete Resource Claim %s", _claim->getContentFullPath().c_str()); std::remove(_claim->getContentFullPath().c_str()); delete _claim; } diff --git a/libminifi/src/GenerateFlowFile.cpp b/libminifi/src/GenerateFlowFile.cpp index 4b0603de17..12d7f7095e 100644 --- a/libminifi/src/GenerateFlowFile.cpp +++ b/libminifi/src/GenerateFlowFile.cpp @@ -26,6 +26,7 @@ #include #include #include +#include "utils/StringUtils.h" #include "GenerateFlowFile.h" #include "ProcessContext.h" @@ -73,7 +74,7 @@ void GenerateFlowFile::onTrigger(ProcessContext *context, ProcessSession *sessio } if (context->getProperty(UniqueFlowFiles.getName(), value)) { - Property::StringToBool(value, uniqueFlowFile); + StringUtils::StringToBool(value, uniqueFlowFile); } if (!uniqueFlowFile) diff --git a/libminifi/src/GetFile.cpp b/libminifi/src/GetFile.cpp index 53d285da9d..40dd387a51 100644 --- a/libminifi/src/GetFile.cpp +++ b/libminifi/src/GetFile.cpp @@ -37,8 +37,9 @@ #include #endif #endif +#include "utils/StringUtils.h" #include -#include "TimeUtil.h" +#include "utils/TimeUtil.h" #include "GetFile.h" #include "ProcessContext.h" #include "ProcessSession.h" @@ -86,7 +87,7 @@ void GetFile::onTrigger(ProcessContext *context, ProcessSession *session) { std::string value; - _logger->log_info("onTrigger GetFile"); + logger_->log_info("onTrigger GetFile"); if (context->getProperty(Directory.getName(), value)) { _directory = value; @@ -97,14 +98,14 @@ void GetFile::onTrigger(ProcessContext *context, ProcessSession *session) } if (context->getProperty(IgnoreHiddenFile.getName(), value)) { - Property::StringToBool(value, _ignoreHiddenFile); + StringUtils::StringToBool(value, _ignoreHiddenFile); } if (context->getProperty(KeepSourceFile.getName(), value)) { - Property::StringToBool(value, _keepSourceFile); + StringUtils::StringToBool(value, _keepSourceFile); } - _logger->log_info("onTrigger GetFile"); + logger_->log_info("onTrigger GetFile"); if (context->getProperty(MaxAge.getName(), value)) { TimeUnit unit; @@ -142,7 +143,7 @@ void GetFile::onTrigger(ProcessContext *context, ProcessSession *session) } if (context->getProperty(Recurse.getName(), value)) { - Property::StringToBool(value, _recursive); + StringUtils::StringToBool(value, _recursive); } if (context->getProperty(FileFilter.getName(), value)) @@ -151,7 +152,7 @@ void GetFile::onTrigger(ProcessContext *context, ProcessSession *session) } // Perform directory list - _logger->log_info("Is listing empty %i",isListingEmpty()); + logger_->log_info("Is listing empty %i",isListingEmpty()); if (isListingEmpty()) { if (_pollInterval == 0 || (getTimeMillis() - _lastDirectoryListingTime) > _pollInterval) @@ -159,7 +160,7 @@ void GetFile::onTrigger(ProcessContext *context, ProcessSession *session) performListing(_directory); } } - _logger->log_info("Is listing empty %i",isListingEmpty()); + logger_->log_info("Is listing empty %i",isListingEmpty()); if (!isListingEmpty()) { @@ -172,7 +173,7 @@ void GetFile::onTrigger(ProcessContext *context, ProcessSession *session) std::string fileName = list.front(); list.pop(); - _logger->log_info("GetFile process %s", fileName.c_str()); + logger_->log_info("GetFile process %s", fileName.c_str()); FlowFileRecord *flowFile = session->create(); if (!flowFile) return; @@ -188,7 +189,7 @@ void GetFile::onTrigger(ProcessContext *context, ProcessSession *session) } catch (std::exception &exception) { - _logger->log_debug("GetFile Caught Exception %s", exception.what()); + logger_->log_debug("GetFile Caught Exception %s", exception.what()); throw; } catch (...) @@ -275,13 +276,13 @@ bool GetFile::acceptFile(std::string fullName, std::string name) return false; } } catch (std::regex_error e) { - _logger->log_error("Invalid File Filter regex: %s.", e.what()); + logger_->log_error("Invalid File Filter regex: %s.", e.what()); return false; } #endif #endif #else - _logger->log_info("Cannot support regex filtering"); + logger_->log_info("Cannot support regex filtering"); #endif return true; } @@ -291,13 +292,13 @@ bool GetFile::acceptFile(std::string fullName, std::string name) void GetFile::performListing(std::string dir) { - _logger->log_info("Performing file listing against %s",dir.c_str()); + logger_->log_info("Performing file listing against %s",dir.c_str()); DIR *d; d = opendir(dir.c_str()); if (!d) return; // only perform a listing while we are not empty - _logger->log_info("Performing file listing against %s",dir.c_str()); + logger_->log_info("Performing file listing against %s",dir.c_str()); while (isRunning()) { struct dirent *entry; diff --git a/libminifi/src/ListenSyslog.cpp b/libminifi/src/ListenSyslog.cpp index ace37d775f..f2901e049e 100644 --- a/libminifi/src/ListenSyslog.cpp +++ b/libminifi/src/ListenSyslog.cpp @@ -20,7 +20,8 @@ #include #include #include -#include "TimeUtil.h" +#include "utils/TimeUtil.h" +#include "utils/StringUtils.h" #include "ListenSyslog.h" #include "ProcessContext.h" #include "ProcessSession.h" @@ -65,7 +66,7 @@ void ListenSyslog::startSocketThread() if (_thread != NULL) return; - _logger->log_info("ListenSysLog Socket Thread Start"); + logger_->log_info("ListenSysLog Socket Thread Start"); _serverTheadRunning = true; _thread = new std::thread(run, this); _thread->detach(); @@ -109,7 +110,7 @@ void ListenSyslog::runThread() sockfd = socket(AF_INET, SOCK_DGRAM, 0); if (sockfd < 0) { - _logger->log_info("ListenSysLog Server socket creation failed"); + logger_->log_info("ListenSysLog Server socket creation failed"); break; } bzero((char *) &serv_addr, sizeof(serv_addr)); @@ -119,13 +120,13 @@ void ListenSyslog::runThread() if (bind(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { - _logger->log_error("ListenSysLog Server socket bind failed"); + logger_->log_error("ListenSysLog Server socket bind failed"); break; } if (_protocol == "TCP") listen(sockfd,5); _serverSocket = sockfd; - _logger->log_error("ListenSysLog Server socket %d bind OK to port %d", _serverSocket, portno); + logger_->log_error("ListenSysLog Server socket %d bind OK to port %d", _serverSocket, portno); } FD_ZERO(&_readfds); FD_SET(_serverSocket, &_readfds); @@ -166,7 +167,7 @@ void ListenSyslog::runThread() if (_clientSockets.size() < _maxConnections) { _clientSockets.push_back(newsockfd); - _logger->log_info("ListenSysLog new client socket %d connection", newsockfd); + logger_->log_info("ListenSysLog new client socket %d connection", newsockfd); continue; } else @@ -200,7 +201,7 @@ void ListenSyslog::runThread() if (recvlen <= 0) { close(clientSocket); - _logger->log_info("ListenSysLog client socket %d close", clientSocket); + logger_->log_info("ListenSysLog client socket %d close", clientSocket); it = _clientSockets.erase(it); } else @@ -285,7 +286,7 @@ void ListenSyslog::onTrigger(ProcessContext *context, ProcessSession *session) } if (context->getProperty(ParseMessages.getName(), value)) { - Property::StringToBool(value, _parseMessages); + StringUtils::StringToBool(value, _parseMessages); } if (context->getProperty(Port.getName(), value)) { diff --git a/libminifi/src/LogAttribute.cpp b/libminifi/src/LogAttribute.cpp index 82130f8789..345eb699b1 100644 --- a/libminifi/src/LogAttribute.cpp +++ b/libminifi/src/LogAttribute.cpp @@ -27,7 +27,8 @@ #include #include -#include "TimeUtil.h" +#include "utils/TimeUtil.h" +#include "utils/StringUtils.h" #include "LogAttribute.h" #include "ProcessContext.h" #include "ProcessSession.h" @@ -81,7 +82,7 @@ void LogAttribute::onTrigger(ProcessContext *context, ProcessSession *session) } if (context->getProperty(LogPayload.getName(), value)) { - Property::StringToBool(value, logPayload); + StringUtils::StringToBool(value, logPayload); } message << "Logging for flow file " << "\n"; @@ -128,19 +129,19 @@ void LogAttribute::onTrigger(ProcessContext *context, ProcessSession *session) switch (level) { case LogAttrLevelInfo: - _logger->log_info("%s", output.c_str()); + logger_->log_info("%s", output.c_str()); break; case LogAttrLevelDebug: - _logger->log_debug("%s", output.c_str()); + logger_->log_debug("%s", output.c_str()); break; case LogAttrLevelError: - _logger->log_error("%s", output.c_str()); + logger_->log_error("%s", output.c_str()); break; case LogAttrLevelTrace: - _logger->log_trace("%s", output.c_str()); + logger_->log_trace("%s", output.c_str()); break; case LogAttrLevelWarn: - _logger->log_warn("%s", output.c_str()); + logger_->log_warn("%s", output.c_str()); break; default: break; diff --git a/libminifi/src/Logger.cpp b/libminifi/src/Logger.cpp index 984f609ab7..fb3023ded1 100644 --- a/libminifi/src/Logger.cpp +++ b/libminifi/src/Logger.cpp @@ -23,5 +23,5 @@ #include "Logger.h" -Logger *Logger::_logger(NULL); +Logger *Logger::logger_(NULL); diff --git a/libminifi/src/ProcessGroup.cpp b/libminifi/src/ProcessGroup.cpp index b3cb8ac973..f451838f78 100644 --- a/libminifi/src/ProcessGroup.cpp +++ b/libminifi/src/ProcessGroup.cpp @@ -29,118 +29,103 @@ #include "ProcessGroup.h" #include "Processor.h" -ProcessGroup::ProcessGroup(ProcessGroupType type, std::string name, uuid_t uuid, ProcessGroup *parent) -: _name(name), - _type(type), - _parentProcessGroup(parent) -{ +ProcessGroup::ProcessGroup(ProcessGroupType type, std::string name, uuid_t uuid, + ProcessGroup *parent) : + name_(name), type_(type), parent_process_group_(parent) { if (!uuid) // Generate the global UUID for the flow record - uuid_generate(_uuid); + uuid_generate(uuid_); else - uuid_copy(_uuid, uuid); + uuid_copy(uuid_, uuid); - _yieldPeriodMsec = 0; - _transmitting = false; + yield_period_msec_ = 0; + transmitting_ = false; - _logger = Logger::getLogger(); - _logger->log_info("ProcessGroup %s created", _name.c_str()); + logger_ = Logger::getLogger(); + logger_->log_info("ProcessGroup %s created", name_.c_str()); } -ProcessGroup::~ProcessGroup() -{ - for (std::set::iterator it = _connections.begin(); it != _connections.end(); ++it) - { +ProcessGroup::~ProcessGroup() { + for (std::set::iterator it = connections_.begin(); + it != connections_.end(); ++it) { Connection *connection = *it; connection->drain(); delete connection; } - for (std::set::iterator it = _childProcessGroups.begin(); it != _childProcessGroups.end(); ++it) - { + for (std::set::iterator it = child_process_groups_.begin(); + it != child_process_groups_.end(); ++it) { ProcessGroup *processGroup(*it); delete processGroup; } - for (std::set::iterator it = _processors.begin(); it != _processors.end(); ++it) - { + for (std::set::iterator it = processors_.begin(); + it != processors_.end(); ++it) { Processor *processor(*it); delete processor; } } -bool ProcessGroup::isRootProcessGroup() -{ - std::lock_guard lock(_mtx); - return (_type == ROOT_PROCESS_GROUP); +bool ProcessGroup::isRootProcessGroup() { + std::lock_guard lock(mtx_); + return (type_ == ROOT_PROCESS_GROUP); } -void ProcessGroup::addProcessor(Processor *processor) -{ - std::lock_guard lock(_mtx); +void ProcessGroup::addProcessor(Processor *processor) { + std::lock_guard lock(mtx_); - if (_processors.find(processor) == _processors.end()) - { + if (processors_.find(processor) == processors_.end()) { // We do not have the same processor in this process group yet - _processors.insert(processor); - _logger->log_info("Add processor %s into process group %s", - processor->getName().c_str(), _name.c_str()); + processors_.insert(processor); + logger_->log_info("Add processor %s into process group %s", + processor->getName().c_str(), name_.c_str()); } } -void ProcessGroup::removeProcessor(Processor *processor) -{ - std::lock_guard lock(_mtx); +void ProcessGroup::removeProcessor(Processor *processor) { + std::lock_guard lock(mtx_); - if (_processors.find(processor) != _processors.end()) - { + if (processors_.find(processor) != processors_.end()) { // We do have the same processor in this process group yet - _processors.erase(processor); - _logger->log_info("Remove processor %s from process group %s", - processor->getName().c_str(), _name.c_str()); + processors_.erase(processor); + logger_->log_info("Remove processor %s from process group %s", + processor->getName().c_str(), name_.c_str()); } } -void ProcessGroup::addProcessGroup(ProcessGroup *child) -{ - std::lock_guard lock(_mtx); +void ProcessGroup::addProcessGroup(ProcessGroup *child) { + std::lock_guard lock(mtx_); - if (_childProcessGroups.find(child) == _childProcessGroups.end()) - { + if (child_process_groups_.find(child) == child_process_groups_.end()) { // We do not have the same child process group in this process group yet - _childProcessGroups.insert(child); - _logger->log_info("Add child process group %s into process group %s", - child->getName().c_str(), _name.c_str()); + child_process_groups_.insert(child); + logger_->log_info("Add child process group %s into process group %s", + child->getName().c_str(), name_.c_str()); } } -void ProcessGroup::removeProcessGroup(ProcessGroup *child) -{ - std::lock_guard lock(_mtx); +void ProcessGroup::removeProcessGroup(ProcessGroup *child) { + std::lock_guard lock(mtx_); - if (_childProcessGroups.find(child) != _childProcessGroups.end()) - { + if (child_process_groups_.find(child) != child_process_groups_.end()) { // We do have the same child process group in this process group yet - _childProcessGroups.erase(child); - _logger->log_info("Remove child process group %s from process group %s", - child->getName().c_str(), _name.c_str()); + child_process_groups_.erase(child); + logger_->log_info("Remove child process group %s from process group %s", + child->getName().c_str(), name_.c_str()); } } void ProcessGroup::startProcessing(TimerDrivenSchedulingAgent *timeScheduler, - EventDrivenSchedulingAgent *eventScheduler) -{ - std::lock_guard lock(_mtx); + EventDrivenSchedulingAgent *eventScheduler) { + std::lock_guard lock(mtx_); - try - { + try { // Start all the processor node, input and output ports - for(auto processor : _processors) - { - _logger->log_debug("Starting %s",processor->getName().c_str()); + for (auto processor : processors_) { + logger_->log_debug("Starting %s", processor->getName().c_str()); - if (!processor->isRunning() && processor->getScheduledState() != DISABLED) - { + if (!processor->isRunning() + && processor->getScheduledState() != DISABLED) { if (processor->getSchedulingStrategy() == TIMER_DRIVEN) timeScheduler->schedule(processor); else if (processor->getSchedulingStrategy() == EVENT_DRIVEN) @@ -148,78 +133,76 @@ void ProcessGroup::startProcessing(TimerDrivenSchedulingAgent *timeScheduler, } } // Start processing the group - for(auto processGroup : _childProcessGroups) - { + for (auto processGroup : child_process_groups_) { processGroup->startProcessing(timeScheduler, eventScheduler); } - } - catch (std::exception &exception) - { - _logger->log_debug("Caught Exception %s", exception.what()); + } catch (std::exception &exception) { + logger_->log_debug("Caught Exception %s", exception.what()); throw; - } - catch (...) - { - _logger->log_debug("Caught Exception during process group start processing"); + } catch (...) { + logger_->log_debug( + "Caught Exception during process group start processing"); throw; } } void ProcessGroup::stopProcessing(TimerDrivenSchedulingAgent *timeScheduler, - EventDrivenSchedulingAgent *eventScheduler) -{ - std::lock_guard lock(_mtx); + EventDrivenSchedulingAgent *eventScheduler) { + std::lock_guard lock(mtx_); - try - { + try { // Stop all the processor node, input and output ports - for (std::set::iterator it = _processors.begin(); it != _processors.end(); ++it) - { + for (std::set::iterator it = processors_.begin(); + it != processors_.end(); ++it) { Processor *processor(*it); if (processor->getSchedulingStrategy() == TIMER_DRIVEN) - timeScheduler->unschedule(processor); + timeScheduler->unschedule(processor); else if (processor->getSchedulingStrategy() == EVENT_DRIVEN) - eventScheduler->unschedule(processor); + eventScheduler->unschedule(processor); } - for (std::set::iterator it = _childProcessGroups.begin(); it != _childProcessGroups.end(); ++it) - { + for (std::set::iterator it = + child_process_groups_.begin(); it != child_process_groups_.end(); + ++it) { ProcessGroup *processGroup(*it); processGroup->stopProcessing(timeScheduler, eventScheduler); } - } - catch (std::exception &exception) - { - _logger->log_debug("Caught Exception %s", exception.what()); + } catch (std::exception &exception) { + logger_->log_debug("Caught Exception %s", exception.what()); throw; - } - catch (...) - { - _logger->log_debug("Caught Exception during process group stop processing"); + } catch (...) { + logger_->log_debug( + "Caught Exception during process group stop processing"); throw; } } -Processor *ProcessGroup::findProcessor(uuid_t uuid) -{ +Processor *ProcessGroup::findProcessor(uuid_t uuid) { Processor *ret = NULL; // std::lock_guard lock(_mtx); - for (std::set::iterator it = _processors.begin(); it != _processors.end(); ++it) - { - Processor *processor(*it); - _logger->log_info("find processor %s",processor->getName().c_str()); + for(auto processor : processors_){ + logger_->log_info("find processor %s", processor->getName().c_str()); uuid_t processorUUID; - if (processor->getUUID(processorUUID) && uuid_compare(processorUUID, uuid) == 0) - return processor; - } - for (std::set::iterator it = _childProcessGroups.begin(); it != _childProcessGroups.end(); ++it) - { + if (processor->getUUID(processorUUID)) { - ProcessGroup *processGroup(*it); - _logger->log_info("find processor child %s",processGroup->getName().c_str()); + char uuid_str[37]; // ex. "1b4e28ba-2fa1-11d2-883f-0016d3cca427" + "\0" + uuid_unparse_lower(processorUUID, uuid_str); + std::string processorUUIDstr = uuid_str; + uuid_unparse_lower(uuid, uuid_str); + std::string uuidStr = uuid_str; + if (processorUUIDstr == uuidStr) { + return processor; + } + } + + } + for(auto processGroup : child_process_groups_){ + + logger_->log_info("find processor child %s", + processGroup->getName().c_str()); Processor *processor = processGroup->findProcessor(uuid); if (processor) return processor; @@ -228,21 +211,17 @@ Processor *ProcessGroup::findProcessor(uuid_t uuid) return ret; } -Processor *ProcessGroup::findProcessor(std::string processorName) -{ +Processor *ProcessGroup::findProcessor(std::string processorName) { Processor *ret = NULL; - for (std::set::iterator it = _processors.begin(); it != _processors.end(); ++it) - { - Processor *processor(*it); - _logger->log_debug("Current processor is %s", processor->getName().c_str()); + for(auto processor : processors_){ + logger_->log_debug("Current processor is %s", + processor->getName().c_str()); if (processor->getName() == processorName) return processor; } - for (std::set::iterator it = _childProcessGroups.begin(); it != _childProcessGroups.end(); ++it) - { - ProcessGroup *processGroup(*it); + for(auto processGroup : child_process_groups_){ Processor *processor = processGroup->findProcessor(processorName); if (processor) return processor; @@ -251,38 +230,32 @@ Processor *ProcessGroup::findProcessor(std::string processorName) return ret; } -void ProcessGroup::updatePropertyValue(std::string processorName, std::string propertyName, std::string propertyValue) -{ - std::lock_guard lock(_mtx); +void ProcessGroup::updatePropertyValue(std::string processorName, + std::string propertyName, std::string propertyValue) { + std::lock_guard lock(mtx_); - for (std::set::iterator it = _processors.begin(); it != _processors.end(); ++it) - { - Processor *processor(*it); - if (processor->getName() == processorName) - { + for(auto processor : processors_){ + if (processor->getName() == processorName) { processor->setProperty(propertyName, propertyValue); } } - for (std::set::iterator it = _childProcessGroups.begin(); it != _childProcessGroups.end(); ++it) - { - ProcessGroup *processGroup(*it); - processGroup->updatePropertyValue(processorName, propertyName, propertyValue); + for(auto processGroup : child_process_groups_){ + processGroup->updatePropertyValue(processorName, propertyName, + propertyValue); } return; } -void ProcessGroup::addConnection(Connection *connection) -{ - std::lock_guard lock(_mtx); +void ProcessGroup::addConnection(Connection *connection) { + std::lock_guard lock(mtx_); - if (_connections.find(connection) == _connections.end()) - { + if (connections_.find(connection) == connections_.end()) { // We do not have the same connection in this process group yet - _connections.insert(connection); - _logger->log_info("Add connection %s into process group %s", - connection->getName().c_str(), _name.c_str()); + connections_.insert(connection); + logger_->log_info("Add connection %s into process group %s", + connection->getName().c_str(), name_.c_str()); uuid_t sourceUUID; Processor *source = NULL; connection->getSourceProcessorUUID(sourceUUID); @@ -298,16 +271,14 @@ void ProcessGroup::addConnection(Connection *connection) } } -void ProcessGroup::removeConnection(Connection *connection) -{ - std::lock_guard lock(_mtx); +void ProcessGroup::removeConnection(Connection *connection) { + std::lock_guard lock(mtx_); - if (_connections.find(connection) != _connections.end()) - { + if (connections_.find(connection) != connections_.end()) { // We do not have the same connection in this process group yet - _connections.erase(connection); - _logger->log_info("Remove connection %s into process group %s", - connection->getName().c_str(), _name.c_str()); + connections_.erase(connection); + logger_->log_info("Remove connection %s into process group %s", + connection->getName().c_str(), name_.c_str()); uuid_t sourceUUID; Processor *source = NULL; connection->getSourceProcessorUUID(sourceUUID); diff --git a/libminifi/src/ProcessSession.cpp b/libminifi/src/ProcessSession.cpp index f3d769e33f..c67abcc501 100644 --- a/libminifi/src/ProcessSession.cpp +++ b/libminifi/src/ProcessSession.cpp @@ -37,7 +37,7 @@ FlowFileRecord* ProcessSession::create() if (record) { _addedFlowFiles[record->getUUIDStr()] = record; - _logger->log_debug("Create FlowFile with UUID %s", record->getUUIDStr().c_str()); + logger_->log_debug("Create FlowFile with UUID %s", record->getUUIDStr().c_str()); std::string details = _processContext->getProcessor()->getName() + " creates flow record " + record->getUUIDStr(); _provenanceReport->create(record, details); } @@ -53,7 +53,7 @@ FlowFileRecord* ProcessSession::create(FlowFileRecord *parent) if (record) { _addedFlowFiles[record->getUUIDStr()] = record; - _logger->log_debug("Create FlowFile with UUID %s", record->getUUIDStr().c_str()); + logger_->log_debug("Create FlowFile with UUID %s", record->getUUIDStr().c_str()); } if (record) @@ -104,7 +104,7 @@ FlowFileRecord* ProcessSession::cloneDuringTransfer(FlowFileRecord *parent) if (record) { this->_clonedFlowFiles[record->getUUIDStr()] = record; - _logger->log_debug("Clone FlowFile with UUID %s during transfer", record->getUUIDStr().c_str()); + logger_->log_debug("Clone FlowFile with UUID %s during transfer", record->getUUIDStr().c_str()); // Copy attributes std::map parentAttributes = parent->getAttributes(); std::map::iterator it; @@ -145,7 +145,7 @@ FlowFileRecord* ProcessSession::clone(FlowFileRecord *parent, long offset, long if ((offset + size) > (long) parent->_size) { // Set offset and size - _logger->log_error("clone offset %d and size %d exceed parent size %d", + logger_->log_error("clone offset %d and size %d exceed parent size %d", offset, size, parent->_size); // Remove the Add FlowFile for the session std::map::iterator it = @@ -228,7 +228,7 @@ void ProcessSession::write(FlowFileRecord *flow, OutputStreamCallback *callback) flow->_claim = claim; claim->increaseFlowFileRecordOwnedCount(); /* - _logger->log_debug("Write offset %d length %d into content %s for FlowFile UUID %s", + logger_->log_debug("Write offset %d length %d into content %s for FlowFile UUID %s", flow->_offset, flow->_size, flow->_claim->getContentFullPath().c_str(), flow->getUUIDStr().c_str()); */ fs.close(); std::string details = _processContext->getProcessor()->getName() + " modify flow record content " + flow->getUUIDStr(); @@ -255,7 +255,7 @@ void ProcessSession::write(FlowFileRecord *flow, OutputStreamCallback *callback) } if (claim) delete claim; - _logger->log_debug("Caught Exception %s", exception.what()); + logger_->log_debug("Caught Exception %s", exception.what()); throw; } catch (...) @@ -267,7 +267,7 @@ void ProcessSession::write(FlowFileRecord *flow, OutputStreamCallback *callback) } if (claim) delete claim; - _logger->log_debug("Caught Exception during process session write"); + logger_->log_debug("Caught Exception during process session write"); throw; } } @@ -299,7 +299,7 @@ void ProcessSession::append(FlowFileRecord *flow, OutputStreamCallback *callback uint64_t appendSize = fs.tellp() - oldPos; flow->_size += appendSize; /* - _logger->log_debug("Append offset %d extra length %d to new size %d into content %s for FlowFile UUID %s", + logger_->log_debug("Append offset %d extra length %d to new size %d into content %s for FlowFile UUID %s", flow->_offset, appendSize, flow->_size, claim->getContentFullPath().c_str(), flow->getUUIDStr().c_str()); */ fs.close(); std::string details = _processContext->getProcessor()->getName() + " modify flow record content " + flow->getUUIDStr(); @@ -319,12 +319,12 @@ void ProcessSession::append(FlowFileRecord *flow, OutputStreamCallback *callback } catch (std::exception &exception) { - _logger->log_debug("Caught Exception %s", exception.what()); + logger_->log_debug("Caught Exception %s", exception.what()); throw; } catch (...) { - _logger->log_debug("Caught Exception during process session append"); + logger_->log_debug("Caught Exception during process session append"); throw; } } @@ -351,7 +351,7 @@ void ProcessSession::read(FlowFileRecord *flow, InputStreamCallback *callback) { callback->process(&fs); /* - _logger->log_debug("Read offset %d size %d content %s for FlowFile UUID %s", + logger_->log_debug("Read offset %d size %d content %s for FlowFile UUID %s", flow->_offset, flow->_size, claim->getContentFullPath().c_str(), flow->getUUIDStr().c_str()); */ fs.close(); } @@ -368,12 +368,12 @@ void ProcessSession::read(FlowFileRecord *flow, InputStreamCallback *callback) } catch (std::exception &exception) { - _logger->log_debug("Caught Exception %s", exception.what()); + logger_->log_debug("Caught Exception %s", exception.what()); throw; } catch (...) { - _logger->log_debug("Caught Exception during process session read"); + logger_->log_debug("Caught Exception during process session read"); throw; } } @@ -421,7 +421,7 @@ void ProcessSession::import(std::string source, FlowFileRecord *flow, bool keepS flow->_claim = claim; claim->increaseFlowFileRecordOwnedCount(); - _logger->log_debug("Import offset %d length %d into content %s for FlowFile UUID %s", + logger_->log_debug("Import offset %d length %d into content %s for FlowFile UUID %s", flow->_offset, flow->_size, flow->_claim->getContentFullPath().c_str(), flow->getUUIDStr().c_str()); fs.close(); @@ -455,7 +455,7 @@ void ProcessSession::import(std::string source, FlowFileRecord *flow, bool keepS } if (claim) delete claim; - _logger->log_debug("Caught Exception %s", exception.what()); + logger_->log_debug("Caught Exception %s", exception.what()); delete[] buf; throw; } @@ -468,7 +468,7 @@ void ProcessSession::import(std::string source, FlowFileRecord *flow, bool keepS } if (claim) delete claim; - _logger->log_debug("Caught Exception during process session write"); + logger_->log_debug("Caught Exception during process session write"); delete[] buf; throw; } @@ -654,16 +654,16 @@ void ProcessSession::commit() _originalFlowFiles.clear(); // persistent the provenance report this->_provenanceReport->commit(); - _logger->log_trace("ProcessSession committed for %s", _processContext->getProcessor()->getName().c_str()); + logger_->log_trace("ProcessSession committed for %s", _processContext->getProcessor()->getName().c_str()); } catch (std::exception &exception) { - _logger->log_debug("Caught Exception %s", exception.what()); + logger_->log_debug("Caught Exception %s", exception.what()); throw; } catch (...) { - _logger->log_debug("Caught Exception during process session commit"); + logger_->log_debug("Caught Exception during process session commit"); throw; } } @@ -706,16 +706,16 @@ void ProcessSession::rollback() } _updatedFlowFiles.clear(); _deletedFlowFiles.clear(); - _logger->log_trace("ProcessSession rollback for %s", _processContext->getProcessor()->getName().c_str()); + logger_->log_trace("ProcessSession rollback for %s", _processContext->getProcessor()->getName().c_str()); } catch (std::exception &exception) { - _logger->log_debug("Caught Exception %s", exception.what()); + logger_->log_debug("Caught Exception %s", exception.what()); throw; } catch (...) { - _logger->log_debug("Caught Exception during process session roll back"); + logger_->log_debug("Caught Exception during process session roll back"); throw; } } @@ -751,7 +751,7 @@ FlowFileRecord *ProcessSession::get() _updatedFlowFiles[ret->getUUIDStr()] = ret; std::map empty; FlowFileRecord *snapshot = new FlowFileRecord(empty); - _logger->log_debug("Create Snapshot FlowFile with UUID %s", snapshot->getUUIDStr().c_str()); + logger_->log_debug("Create Snapshot FlowFile with UUID %s", snapshot->getUUIDStr().c_str()); snapshot->duplicate(ret); // save a snapshot _originalFlowFiles[snapshot->getUUIDStr()] = snapshot; diff --git a/libminifi/src/Processor.cpp b/libminifi/src/Processor.cpp index a683e4be33..94aaa20711 100644 --- a/libminifi/src/Processor.cpp +++ b/libminifi/src/Processor.cpp @@ -59,8 +59,8 @@ Processor::Processor(std::string name, uuid_t uuid) _activeTasks = 0; _yieldExpiration = 0; _incomingConnectionsIter = this->_incomingConnections.begin(); - _logger = Logger::getLogger(); - _logger->log_info("Processor %s created UUID %s", _name.c_str(), _uuidStr.c_str()); + logger_ = Logger::getLogger(); + logger_->log_info("Processor %s created UUID %s", _name.c_str(), _uuidStr.c_str()); } Processor::~Processor() @@ -82,7 +82,7 @@ bool Processor::setSupportedProperties(std::set properties) { if (isRunning()) { - _logger->log_info("Can not set processor property while the process %s is running", + logger_->log_info("Can not set processor property while the process %s is running", _name.c_str()); return false; } @@ -93,7 +93,7 @@ bool Processor::setSupportedProperties(std::set properties) for (auto item : properties) { _properties[item.getName()] = item; - _logger->log_info("Processor %s supported property name %s", _name.c_str(), item.getName().c_str()); + logger_->log_info("Processor %s supported property name %s", _name.c_str(), item.getName().c_str()); } return true; @@ -103,7 +103,7 @@ bool Processor::setSupportedRelationships(std::set relationships) { if (isRunning()) { - _logger->log_info("Can not set processor supported relationship while the process %s is running", + logger_->log_info("Can not set processor supported relationship while the process %s is running", _name.c_str()); return false; } @@ -114,7 +114,7 @@ bool Processor::setSupportedRelationships(std::set relationships) for(auto item : relationships) { _relationships[item.getName()] = item; - _logger->log_info("Processor %s supported relationship name %s", _name.c_str(), item.getName().c_str()); + logger_->log_info("Processor %s supported relationship name %s", _name.c_str(), item.getName().c_str()); } return true; @@ -124,7 +124,7 @@ bool Processor::setAutoTerminatedRelationships(std::set relationsh { if (isRunning()) { - _logger->log_info("Can not set processor auto terminated relationship while the process %s is running", + logger_->log_info("Can not set processor auto terminated relationship while the process %s is running", _name.c_str()); return false; } @@ -135,7 +135,7 @@ bool Processor::setAutoTerminatedRelationships(std::set relationsh for(auto item : relationships) { _autoTerminatedRelationships[item.getName()] = item; - _logger->log_info("Processor %s auto terminated relationship name %s", _name.c_str(), item.getName().c_str()); + logger_->log_info("Processor %s auto terminated relationship name %s", _name.c_str(), item.getName().c_str()); } return true; @@ -212,7 +212,7 @@ bool Processor::setProperty(std::string name, std::string value) Property item = it->second; item.setValue(value); _properties[item.getName()] = item; - _logger->log_info("Processor %s property name %s value %s", _name.c_str(), item.getName().c_str(), value.c_str()); + logger_->log_info("Processor %s property name %s value %s", _name.c_str(), item.getName().c_str(), value.c_str()); return true; } else @@ -231,7 +231,7 @@ bool Processor::setProperty(Property prop, std::string value) { Property item = it->second; item.setValue(value); _properties[item.getName()] = item; - _logger->log_info("Processor %s property name %s value %s", + logger_->log_info("Processor %s property name %s value %s", _name.c_str(), item.getName().c_str(), value.c_str()); return true; } else { @@ -267,7 +267,7 @@ bool Processor::addConnection(Connection *connection) if (isRunning()) { - _logger->log_info("Can not add connection while the process %s is running", + logger_->log_info("Can not add connection while the process %s is running", _name.c_str()); return false; } @@ -294,7 +294,7 @@ bool Processor::addConnection(Connection *connection) { _incomingConnections.insert(connection); connection->setDestinationProcessor(this); - _logger->log_info("Add connection %s into Processor %s incoming connection", + logger_->log_info("Add connection %s into Processor %s incoming connection", connection->getName().c_str(), _name.c_str()); _incomingConnectionsIter = this->_incomingConnections.begin(); ret = true; @@ -318,7 +318,7 @@ bool Processor::addConnection(Connection *connection) existedConnection.insert(connection); connection->setSourceProcessor(this); _outGoingConnections[relationship] = existedConnection; - _logger->log_info("Add connection %s into Processor %s outgoing connection for relationship %s", + logger_->log_info("Add connection %s into Processor %s outgoing connection for relationship %s", connection->getName().c_str(), _name.c_str(), relationship.c_str()); ret = true; } @@ -331,7 +331,7 @@ bool Processor::addConnection(Connection *connection) newConnection.insert(connection); connection->setSourceProcessor(this); _outGoingConnections[relationship] = newConnection; - _logger->log_info("Add connection %s into Processor %s outgoing connection for relationship %s", + logger_->log_info("Add connection %s into Processor %s outgoing connection for relationship %s", connection->getName().c_str(), _name.c_str(), relationship.c_str()); ret = true; } @@ -345,7 +345,7 @@ void Processor::removeConnection(Connection *connection) { if (isRunning()) { - _logger->log_info("Can not remove connection while the process %s is running", + logger_->log_info("Can not remove connection while the process %s is running", _name.c_str()); return; } @@ -365,7 +365,7 @@ void Processor::removeConnection(Connection *connection) { _incomingConnections.erase(connection); connection->setDestinationProcessor(NULL); - _logger->log_info("Remove connection %s into Processor %s incoming connection", + logger_->log_info("Remove connection %s into Processor %s incoming connection", connection->getName().c_str(), _name.c_str()); _incomingConnectionsIter = this->_incomingConnections.begin(); } @@ -387,7 +387,7 @@ void Processor::removeConnection(Connection *connection) { _outGoingConnections[relationship].erase(connection); connection->setSourceProcessor(NULL); - _logger->log_info("Remove connection %s into Processor %s outgoing connection for relationship %s", + logger_->log_info("Remove connection %s into Processor %s outgoing connection for relationship %s", connection->getName().c_str(), _name.c_str(), relationship.c_str()); } } @@ -458,13 +458,13 @@ void Processor::onTrigger(ProcessContext *context, ProcessSessionFactory *sessio } catch (std::exception &exception) { - _logger->log_debug("Caught Exception %s", exception.what()); + logger_->log_debug("Caught Exception %s", exception.what()); session->rollback(); throw; } catch (...) { - _logger->log_debug("Caught Exception Processor::onTrigger"); + logger_->log_debug("Caught Exception Processor::onTrigger"); session->rollback(); throw; } @@ -519,7 +519,7 @@ bool Processor::isWorkAvailable() } catch (...) { - _logger->log_error("Caught an exception while checking if work is available; unless it was positively determined that work is available, assuming NO work is available!"); + logger_->log_error("Caught an exception while checking if work is available; unless it was positively determined that work is available, assuming NO work is available!"); } return hasWork; diff --git a/libminifi/src/Provenance.cpp b/libminifi/src/Provenance.cpp index c21de76d7f..ace3b45f66 100644 --- a/libminifi/src/Provenance.cpp +++ b/libminifi/src/Provenance.cpp @@ -20,7 +20,8 @@ #include #include #include -#include "Serializable.h" +#include "io/DataStream.h" +#include "io/Serializable.h" #include "Provenance.h" #include "Relationship.h" #include "Logger.h" @@ -35,11 +36,11 @@ bool ProvenanceEventRecord::DeSerialize(ProvenanceRepository *repo, ret = repo->Get(key, value); if (!ret) { - _logger->log_error("NiFi Provenance Store event %s can not found", + logger_->log_error("NiFi Provenance Store event %s can not found", key.c_str()); return false; } else - _logger->log_debug("NiFi Provenance Read event %s length %d", + logger_->log_debug("NiFi Provenance Read event %s length %d", key.c_str(), value.length()); @@ -48,11 +49,11 @@ bool ProvenanceEventRecord::DeSerialize(ProvenanceRepository *repo, ret = DeSerialize(stream); if (ret) { - _logger->log_debug( + logger_->log_debug( "NiFi Provenance retrieve event %s size %d eventType %d success", _eventIdStr.c_str(), stream.getSize(), _eventType); } else { - _logger->log_debug( + logger_->log_debug( "NiFi Provenance retrieve event %s size %d eventType %d fail", _eventIdStr.c_str(), stream.getSize(), _eventType); } @@ -221,11 +222,12 @@ bool ProvenanceEventRecord::Serialize(ProvenanceRepository *repo) { } // Persistent to the DB + if (repo->Put(_eventIdStr, const_cast(outStream.getBuffer()), outStream.getSize())) { - _logger->log_debug("NiFi Provenance Store event %s size %d success", + logger_->log_debug("NiFi Provenance Store event %s size %d success", _eventIdStr.c_str(), outStream.getSize()); } else { - _logger->log_error("NiFi Provenance Store event %s size %d fail", + logger_->log_error("NiFi Provenance Store event %s size %d fail", _eventIdStr.c_str(), outStream.getSize()); } @@ -393,7 +395,7 @@ void ProvenanceReporter::commit() { event->Serialize( FlowControllerFactory::getFlowController()->getProvenanceRepository()); } else { - _logger->log_debug("Provenance Repository is full"); + logger_->log_debug("Provenance Repository is full"); } } } @@ -566,7 +568,7 @@ void ProvenanceRepository::start() { if (_running) return; _running = true; - _logger->log_info("ProvenanceRepository Monitor Thread Start"); + logger_->log_info("ProvenanceRepository Monitor Thread Start"); _thread = new std::thread(run, this); _thread->detach(); } @@ -575,7 +577,7 @@ void ProvenanceRepository::stop() { if (!_running) return; _running = false; - _logger->log_info("ProvenanceRepository Monitor Thread Stop"); + logger_->log_info("ProvenanceRepository Monitor Thread Stop"); } void ProvenanceRepository::run(ProvenanceRepository *repo) { @@ -599,7 +601,7 @@ void ProvenanceRepository::run(ProvenanceRepository *repo) { > repo->_maxPartitionMillis) purgeList.push_back(key); } else { - repo->_logger->log_debug( + repo->logger_->log_debug( "NiFi Provenance retrieve event %s fail", key.c_str()); purgeList.push_back(key); @@ -610,7 +612,7 @@ void ProvenanceRepository::run(ProvenanceRepository *repo) { for (itPurge = purgeList.begin(); itPurge != purgeList.end(); itPurge++) { std::string eventId = *itPurge; - repo->_logger->log_info("ProvenanceRepository Repo Purge %s", + repo->logger_->log_info("ProvenanceRepository Repo Purge %s", eventId.c_str()); repo->Delete(eventId); } diff --git a/libminifi/src/PutFile.cpp b/libminifi/src/PutFile.cpp index 79a3c5af1c..d7cc83ad60 100644 --- a/libminifi/src/PutFile.cpp +++ b/libminifi/src/PutFile.cpp @@ -24,7 +24,8 @@ #include #include -#include "TimeUtil.h" +#include "utils/StringUtils.h" +#include "utils/TimeUtil.h" #include "PutFile.h" #include "ProcessContext.h" #include "ProcessSession.h" @@ -61,7 +62,7 @@ void PutFile::onTrigger(ProcessContext *context, ProcessSession *session) if (!context->getProperty(Directory.getName(), directory)) { - _logger->log_error("Directory attribute is missing or invalid"); + logger_->log_error("Directory attribute is missing or invalid"); return; } @@ -69,7 +70,7 @@ void PutFile::onTrigger(ProcessContext *context, ProcessSession *session) if (!context->getProperty(ConflictResolution.getName(), conflictResolution)) { - _logger->log_error("Conflict Resolution Strategy attribute is missing or invalid"); + logger_->log_error("Conflict Resolution Strategy attribute is missing or invalid"); return; } @@ -92,21 +93,21 @@ void PutFile::onTrigger(ProcessContext *context, ProcessSession *session) std::stringstream tmpFileSs; tmpFileSs << directory << "/." << filename << "." << tmpFileUuidStr; std::string tmpFile = tmpFileSs.str(); - _logger->log_info("PutFile using temporary file %s", tmpFile.c_str()); + logger_->log_info("PutFile using temporary file %s", tmpFile.c_str()); // Determine dest full file paths std::stringstream destFileSs; destFileSs << directory << "/" << filename; std::string destFile = destFileSs.str(); - _logger->log_info("PutFile writing file %s into directory %s", filename.c_str(), directory.c_str()); + logger_->log_info("PutFile writing file %s into directory %s", filename.c_str(), directory.c_str()); // If file exists, apply conflict resolution strategy struct stat statResult; if (stat(destFile.c_str(), &statResult) == 0) { - _logger->log_info("Destination file %s exists; applying Conflict Resolution Strategy: %s", destFile.c_str(), conflictResolution.c_str()); + logger_->log_info("Destination file %s exists; applying Conflict Resolution Strategy: %s", destFile.c_str(), conflictResolution.c_str()); if (conflictResolution == CONFLICT_RESOLUTION_STRATEGY_REPLACE) { @@ -150,7 +151,7 @@ PutFile::ReadCallback::ReadCallback(const std::string &tmpFile, const std::strin , _tmpFileOs(tmpFile) , _destFile(destFile) { - _logger = Logger::getLogger(); + logger_ = Logger::getLogger(); } // Copy the entire file contents to the temporary file @@ -168,7 +169,7 @@ bool PutFile::ReadCallback::commit() { bool success = false; - _logger->log_info("PutFile committing put file operation to %s", _destFile.c_str()); + logger_->log_info("PutFile committing put file operation to %s", _destFile.c_str()); if (_writeSucceeded) { @@ -176,17 +177,17 @@ bool PutFile::ReadCallback::commit() if (rename(_tmpFile.c_str(), _destFile.c_str())) { - _logger->log_info("PutFile commit put file operation to %s failed because rename() call failed", _destFile.c_str()); + logger_->log_info("PutFile commit put file operation to %s failed because rename() call failed", _destFile.c_str()); } else { success = true; - _logger->log_info("PutFile commit put file operation to %s succeeded", _destFile.c_str()); + logger_->log_info("PutFile commit put file operation to %s succeeded", _destFile.c_str()); } } else { - _logger->log_error("PutFile commit put file operation to %s failed because write failed", _destFile.c_str()); + logger_->log_error("PutFile commit put file operation to %s failed because write failed", _destFile.c_str()); } return success; diff --git a/libminifi/src/RealTimeDataCollector.cpp b/libminifi/src/RealTimeDataCollector.cpp index c7118ff5e6..7dd64693a0 100644 --- a/libminifi/src/RealTimeDataCollector.cpp +++ b/libminifi/src/RealTimeDataCollector.cpp @@ -28,6 +28,7 @@ #include #include +#include "utils/StringUtils.h" #include "RealTimeDataCollector.h" #include "ProcessContext.h" #include "ProcessSession.h" @@ -88,7 +89,7 @@ int RealTimeDataCollector::connectServer(const char *host, uint16_t port) sock = socket(AF_INET, SOCK_STREAM, 0); if (sock < 0) { - _logger->log_error("Could not create socket to hostName %s", host); + logger_->log_error("Could not create socket to hostName %s", host); return 0; } @@ -100,14 +101,14 @@ int RealTimeDataCollector::connectServer(const char *host, uint16_t port) { if (setsockopt(sock, SOL_TCP, TCP_NODELAY, (void *)&opt, sizeof(opt)) < 0) { - _logger->log_error("setsockopt() TCP_NODELAY failed"); + logger_->log_error("setsockopt() TCP_NODELAY failed"); close(sock); return 0; } if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *)&opt, sizeof(opt)) < 0) { - _logger->log_error("setsockopt() SO_REUSEADDR failed"); + logger_->log_error("setsockopt() SO_REUSEADDR failed"); close(sock); return 0; } @@ -116,7 +117,7 @@ int RealTimeDataCollector::connectServer(const char *host, uint16_t port) int sndsize = 256*1024; if (setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (char *)&sndsize, (int)sizeof(sndsize)) < 0) { - _logger->log_error("setsockopt() SO_SNDBUF failed"); + logger_->log_error("setsockopt() SO_SNDBUF failed"); close(sock); return 0; } @@ -134,7 +135,7 @@ int RealTimeDataCollector::connectServer(const char *host, uint16_t port) socklen = sizeof(sa); if (bind(sock, (struct sockaddr *)&sa, socklen) < 0) { - _logger->log_error("socket bind failed"); + logger_->log_error("socket bind failed"); close(sock); return 0; } @@ -149,12 +150,12 @@ int RealTimeDataCollector::connectServer(const char *host, uint16_t port) if (status < 0) { - _logger->log_error("socket connect failed to %s %d", host, port); + logger_->log_error("socket connect failed to %s %d", host, port); close(sock); return 0; } - _logger->log_info("socket %d connect to server %s port %d success", sock, host, port); + logger_->log_info("socket %d connect to server %s port %d success", sock, host, port); return sock; } @@ -175,7 +176,7 @@ int RealTimeDataCollector::sendData(int socket, const char *buf, int buflen) } if (ret) - _logger->log_debug("Send data size %d over socket %d", buflen, socket); + logger_->log_debug("Send data size %d over socket %d", buflen, socket); return ret; } @@ -188,41 +189,39 @@ void RealTimeDataCollector::onTriggerRealTime(ProcessContext *context, ProcessSe if (this->getProperty(REALTIMEMSGID.getName(), value)) { this->_realTimeMsgID.clear(); - this->_logger->log_info("Real Time Msg IDs %s", value.c_str()); + this->logger_->log_info("Real Time Msg IDs %s", value.c_str()); std::stringstream lineStream(value); std::string cell; while(std::getline(lineStream, cell, ',')) { this->_realTimeMsgID.push_back(cell); - // this->_logger->log_debug("Real Time Msg ID %s", cell.c_str()); + // this->logger_->log_debug("Real Time Msg ID %s", cell.c_str()); } } if (this->getProperty(BATCHMSGID.getName(), value)) { this->_batchMsgID.clear(); - this->_logger->log_info("Batch Msg IDs %s", value.c_str()); + this->logger_->log_info("Batch Msg IDs %s", value.c_str()); std::stringstream lineStream(value); std::string cell; while(std::getline(lineStream, cell, ',')) { - cell = Property::trim(cell); + cell = StringUtils::trim(cell); this->_batchMsgID.push_back(cell); - // this->_logger->log_debug("Batch Msg ID %s", cell.c_str()); } } - // _logger->log_info("onTriggerRealTime"); // Open the file if (!this->_fileStream.is_open()) { _fileStream.open(this->_fileName.c_str(), std::ifstream::in); if (this->_fileStream.is_open()) - _logger->log_debug("open %s", _fileName.c_str()); + logger_->log_debug("open %s", _fileName.c_str()); } if (!_fileStream.good()) { - _logger->log_error("load data file failed %s", _fileName.c_str()); + logger_->log_error("load data file failed %s", _fileName.c_str()); return; } if (this->_fileStream.is_open()) @@ -236,7 +235,7 @@ void RealTimeDataCollector::onTriggerRealTime(ProcessContext *context, ProcessSe std::string cell; if (std::getline(lineStream, cell, ',')) { - cell = Property::trim(cell); + cell = StringUtils::trim(cell); // Check whether it match to the batch traffic for (std::vector::iterator it = _batchMsgID.begin(); it != _batchMsgID.end(); ++it) { @@ -248,12 +247,12 @@ void RealTimeDataCollector::onTriggerRealTime(ProcessContext *context, ProcessSe { std::string item = _queue.front(); _queuedDataSize -= item.size(); - _logger->log_debug("Pop item size %d from batch queue, queue buffer size %d", item.size(), _queuedDataSize); + logger_->log_debug("Pop item size %d from batch queue, queue buffer size %d", item.size(), _queuedDataSize); _queue.pop(); } _queue.push(line); _queuedDataSize += line.size(); - _logger->log_debug("Push batch msg ID %s into batch queue, queue buffer size %d", cell.c_str(), _queuedDataSize); + logger_->log_debug("Push batch msg ID %s into batch queue, queue buffer size %d", cell.c_str(), _queuedDataSize); } } bool findRealTime = false; @@ -287,12 +286,12 @@ void RealTimeDataCollector::onTriggerRealTime(ProcessContext *context, ProcessSe { std::string item = _queue.front(); _queuedDataSize -= item.size(); - _logger->log_debug("Pop item size %d from batch queue, queue buffer size %d", item.size(), _queuedDataSize); + logger_->log_debug("Pop item size %d from batch queue, queue buffer size %d", item.size(), _queuedDataSize); _queue.pop(); } _queue.push(line); _queuedDataSize += line.size(); - _logger->log_debug("Push real time msg ID %s into batch queue, queue buffer size %d", cell.c_str(), _queuedDataSize); + logger_->log_debug("Push real time msg ID %s into batch queue, queue buffer size %d", cell.c_str(), _queuedDataSize); } // find real time findRealTime = true; @@ -317,7 +316,7 @@ void RealTimeDataCollector::onTriggerBatch(ProcessContext *context, ProcessSessi { if (_batchAcccumulated >= this->_batchInterval) { - // _logger->log_info("onTriggerBatch"); + // logger_->log_info("onTriggerBatch"); // dequeue the batch and send over WIFI int status = 0; if (this->_batchSocket <= 0) @@ -367,35 +366,35 @@ void RealTimeDataCollector::onTrigger(ProcessContext *context, ProcessSession *s if (this->getProperty(FILENAME.getName(), value)) { this->_fileName = value; - this->_logger->log_info("Data Collector File Name %s", _fileName.c_str()); + this->logger_->log_info("Data Collector File Name %s", _fileName.c_str()); } this->_realTimeServerName = "localhost"; if (this->getProperty(REALTIMESERVERNAME.getName(), value)) { this->_realTimeServerName = value; - this->_logger->log_info("Real Time Server Name %s", this->_realTimeServerName.c_str()); + this->logger_->log_info("Real Time Server Name %s", this->_realTimeServerName.c_str()); } this->_realTimeServerPort = 10000; if (this->getProperty(REALTIMESERVERPORT.getName(), value)) { Property::StringToInt(value, _realTimeServerPort); - this->_logger->log_info("Real Time Server Port %d", _realTimeServerPort); + this->logger_->log_info("Real Time Server Port %d", _realTimeServerPort); } if (this->getProperty(BATCHSERVERNAME.getName(), value)) { this->_batchServerName = value; - this->_logger->log_info("Batch Server Name %s", this->_batchServerName.c_str()); + this->logger_->log_info("Batch Server Name %s", this->_batchServerName.c_str()); } this->_batchServerPort = 10001; if (this->getProperty(BATCHSERVERPORT.getName(), value)) { Property::StringToInt(value, _batchServerPort); - this->_logger->log_info("Batch Server Port %d", _batchServerPort); + this->logger_->log_info("Batch Server Port %d", _batchServerPort); } if (this->getProperty(ITERATION.getName(), value)) { - Property::StringToBool(value, this->_iteration); - _logger->log_info("Iteration %d", _iteration); + StringUtils::StringToBool(value, this->_iteration); + logger_->log_info("Iteration %d", _iteration); } this->_realTimeInterval = 10000000; //10 msec if (this->getProperty(REALTIMEINTERVAL.getName(), value)) @@ -404,7 +403,7 @@ void RealTimeDataCollector::onTrigger(ProcessContext *context, ProcessSession *s if (Property::StringToTime(value, _realTimeInterval, unit) && Property::ConvertTimeUnitToNS(_realTimeInterval, unit, _realTimeInterval)) { - _logger->log_info("Real Time Interval: [%d] ns", _realTimeInterval); + logger_->log_info("Real Time Interval: [%d] ns", _realTimeInterval); } } this->_batchInterval = 100000000; //100 msec @@ -414,38 +413,38 @@ void RealTimeDataCollector::onTrigger(ProcessContext *context, ProcessSession *s if (Property::StringToTime(value, _batchInterval, unit) && Property::ConvertTimeUnitToNS(_batchInterval, unit, _batchInterval)) { - _logger->log_info("Batch Time Interval: [%d] ns", _batchInterval); + logger_->log_info("Batch Time Interval: [%d] ns", _batchInterval); } } this->_batchMaxBufferSize = 256*1024; if (this->getProperty(BATCHMAXBUFFERSIZE.getName(), value)) { Property::StringToInt(value, _batchMaxBufferSize); - this->_logger->log_info("Batch Max Buffer Size %d", _batchMaxBufferSize); + this->logger_->log_info("Batch Max Buffer Size %d", _batchMaxBufferSize); } if (this->getProperty(REALTIMEMSGID.getName(), value)) { - this->_logger->log_info("Real Time Msg IDs %s", value.c_str()); + this->logger_->log_info("Real Time Msg IDs %s", value.c_str()); std::stringstream lineStream(value); std::string cell; while(std::getline(lineStream, cell, ',')) { this->_realTimeMsgID.push_back(cell); - this->_logger->log_info("Real Time Msg ID %s", cell.c_str()); + this->logger_->log_info("Real Time Msg ID %s", cell.c_str()); } } if (this->getProperty(BATCHMSGID.getName(), value)) { - this->_logger->log_info("Batch Msg IDs %s", value.c_str()); + this->logger_->log_info("Batch Msg IDs %s", value.c_str()); std::stringstream lineStream(value); std::string cell; while(std::getline(lineStream, cell, ',')) { - cell = Property::trim(cell); + cell = StringUtils::trim(cell); this->_batchMsgID.push_back(cell); - this->_logger->log_info("Batch Msg ID %s", cell.c_str()); + this->logger_->log_info("Batch Msg ID %s", cell.c_str()); } } // Connect the LTE socket @@ -462,12 +461,12 @@ void RealTimeDataCollector::onTrigger(ProcessContext *context, ProcessSession *s _fileStream.open(this->_fileName.c_str(), std::ifstream::in); if (!_fileStream.good()) { - _logger->log_error("load data file failed %s", _fileName.c_str()); + logger_->log_error("load data file failed %s", _fileName.c_str()); return; } else { - _logger->log_debug("open %s", _fileName.c_str()); + logger_->log_debug("open %s", _fileName.c_str()); } _realTimeThreadId = id; this->_firstInvoking = true; diff --git a/libminifi/src/RemoteProcessorGroupPort.cpp b/libminifi/src/RemoteProcessorGroupPort.cpp index 9d849ae4a4..dd1d035fbc 100644 --- a/libminifi/src/RemoteProcessorGroupPort.cpp +++ b/libminifi/src/RemoteProcessorGroupPort.cpp @@ -27,8 +27,12 @@ #include #include -#include "TimeUtil.h" #include "RemoteProcessorGroupPort.h" + +#include "io/ClientSocket.h" +#include "io/SocketFactory.h" + +#include "utils/TimeUtil.h" #include "ProcessContext.h" #include "ProcessSession.h" @@ -39,6 +43,7 @@ Relationship RemoteProcessorGroupPort::relation; void RemoteProcessorGroupPort::initialize() { + //! Set the supported properties std::set properties; properties.insert(hostName); @@ -48,20 +53,20 @@ void RemoteProcessorGroupPort::initialize() std::set relationships; relationships.insert(relation); setSupportedRelationships(relationships); + } void RemoteProcessorGroupPort::onTrigger(ProcessContext *context, ProcessSession *session) { std::string value; - if (!_transmitting) + if (!transmitting_) return; - - std::string host = _peer->getHostName(); - uint16_t sport = _peer->getPort(); + + std::string host = peer_.getHostName(); + uint16_t sport = peer_.getPort(); int64_t lvalue; - bool needReset = false; - + if (context->getProperty(hostName.getName(), value)) { host = value; @@ -70,31 +75,47 @@ void RemoteProcessorGroupPort::onTrigger(ProcessContext *context, ProcessSession { sport = (uint16_t) lvalue; } - if (host != _peer->getHostName()) + + if (host != peer_.getHostName() || sport != peer_.getPort()) + + { + + std::unique_ptr str = std::unique_ptr(SocketFactory::getInstance()->createSocket(host,sport)); + peer_ = std::move(Site2SitePeer (std::move(str), host, sport)); + protocol_->setPeer(&peer_); + + } + + + + bool needReset = false; + + + if (host != peer_.getHostName()) { - _peer->setHostName(host); + peer_.setHostName(host); needReset= true; } - if (sport != _peer->getPort()) + if (sport != peer_.getPort()) { - _peer->setPort(sport); + peer_.setPort(sport); needReset = true; } if (needReset) - _protocol->tearDown(); + protocol_->tearDown(); - if (!_protocol->bootstrap()) + if (!protocol_->bootstrap()) { // bootstrap the client protocol if needeed context->yield(); - _logger->log_error("Site2Site bootstrap failed yield period %d peer timeout %d", context->getProcessor()->getYieldPeriodMsec(), _protocol->getPeer()->getTimeOut()); + logger_->log_error("Site2Site bootstrap failed yield period %d peer ", context->getProcessor()->getYieldPeriodMsec()); return; } - if (_direction == RECEIVE) - _protocol->receiveFlowFiles(context, session); + if (direction_ == RECEIVE) + protocol_->receiveFlowFiles(context, session); else - _protocol->transferFlowFiles(context, session); + protocol_->transferFlowFiles(context, session); return; } diff --git a/libminifi/src/ResourceClaim.cpp b/libminifi/src/ResourceClaim.cpp index 41c60e2a55..ce1f2489e3 100644 --- a/libminifi/src/ResourceClaim.cpp +++ b/libminifi/src/ResourceClaim.cpp @@ -43,7 +43,7 @@ ResourceClaim::ResourceClaim(const std::string contentDirectory) // Create the full content path for the content _contentFullPath = contentDirectory + "/" + uuidStr; - _configure = Configure::getConfigure(); - _logger = Logger::getLogger(); - _logger->log_debug("Resource Claim created %s", _contentFullPath.c_str()); + configure_ = Configure::getConfigure(); + logger_ = Logger::getLogger(); + logger_->log_debug("Resource Claim created %s", _contentFullPath.c_str()); } diff --git a/libminifi/src/SchedulingAgent.cpp b/libminifi/src/SchedulingAgent.cpp index 8e6249cb52..984abdcc45 100644 --- a/libminifi/src/SchedulingAgent.cpp +++ b/libminifi/src/SchedulingAgent.cpp @@ -65,18 +65,18 @@ bool SchedulingAgent::onTrigger(Processor *processor, ProcessContext *processCon catch (Exception &exception) { // Normal exception - _logger->log_debug("Caught Exception %s", exception.what()); + logger_->log_debug("Caught Exception %s", exception.what()); processor->decrementActiveTask(); } catch (std::exception &exception) { - _logger->log_debug("Caught Exception %s", exception.what()); + logger_->log_debug("Caught Exception %s", exception.what()); processor->yield(_administrativeYieldDuration); processor->decrementActiveTask(); } catch (...) { - _logger->log_debug("Caught Exception during SchedulingAgent::onTrigger"); + logger_->log_debug("Caught Exception during SchedulingAgent::onTrigger"); processor->yield(_administrativeYieldDuration); processor->decrementActiveTask(); } diff --git a/libminifi/src/Site2SiteClientProtocol.cpp b/libminifi/src/Site2SiteClientProtocol.cpp index 11d5ef721b..bd4de975ef 100644 --- a/libminifi/src/Site2SiteClientProtocol.cpp +++ b/libminifi/src/Site2SiteClientProtocol.cpp @@ -25,6 +25,7 @@ #include #include #include +#include "io/CRCStream.h" #include "Site2SitePeer.h" #include "Site2SiteClientProtocol.h" @@ -32,15 +33,15 @@ bool Site2SiteClientProtocol::establish() { if (_peerState != IDLE) { - _logger->log_error("Site2Site peer state is not idle while try to establish"); + logger_->log_error("Site2Site peer state is not idle while try to establish"); return false; } - bool ret = _peer->Open(); + bool ret = peer_->Open(); if (!ret) { - _logger->log_error("Site2Site peer socket open failed"); + logger_->log_error("Site2Site peer socket open failed"); return false; } @@ -49,14 +50,14 @@ bool Site2SiteClientProtocol::establish() if (!ret) { - _logger->log_error("Site2Site Protocol Version Negotiation failed"); + logger_->log_error("Site2Site Protocol Version Negotiation failed"); /* - _peer->yield(); + peer_->yield(); tearDown(); */ return false; } - _logger->log_info("Site2Site socket established"); + logger_->log_info("Site2Site socket established"); _peerState = ESTABLISHED; return true; @@ -67,51 +68,55 @@ bool Site2SiteClientProtocol::initiateResourceNegotiation() // Negotiate the version if (_peerState != IDLE) { - _logger->log_error("Site2Site peer state is not idle while initiateResourceNegotiation"); + logger_->log_error("Site2Site peer state is not idle while initiateResourceNegotiation"); return false; } - _logger->log_info("Negotiate protocol version with destination port %s current version %d", _portIdStr.c_str(), _currentVersion); + logger_->log_info("Negotiate protocol version with destination port %s current version %d", _portIdStr.c_str(), _currentVersion); - int ret = _peer->writeUTF(this->getResourceName()); + int ret = peer_->writeUTF(this->getResourceName()); + logger_->log_info("result of writing resource name is %i",ret); if (ret <= 0) { + logger_->log_debug("result of writing resource name is %i",ret); // tearDown(); return false; } - ret = _peer->write(_currentVersion); + ret = peer_->write(_currentVersion); if (ret <= 0) { + logger_->log_info("result of writing version is %i",ret); // tearDown(); return false; } uint8_t statusCode; - ret = _peer->read(statusCode); + ret = peer_->read(statusCode); if (ret <= 0) { + logger_->log_info("result of writing version status code %i",ret); // tearDown(); return false; } - + logger_->log_info("status code is %i",statusCode); switch (statusCode) { case RESOURCE_OK: - _logger->log_info("Site2Site Protocol Negotiate protocol version OK"); + logger_->log_info("Site2Site Protocol Negotiate protocol version OK"); return true; case DIFFERENT_RESOURCE_VERSION: uint32_t serverVersion; - ret = _peer->read(serverVersion); + ret = peer_->read(serverVersion); if (ret <= 0) { // tearDown(); return false; } - _logger->log_info("Site2Site Server Response asked for a different protocol version %d", serverVersion); + logger_->log_info("Site2Site Server Response asked for a different protocol version %d", serverVersion); for (unsigned int i = (_currentVersionIndex + 1); i < sizeof(_supportedVersion)/sizeof(uint32_t); i++) { if (serverVersion >= _supportedVersion[i]) @@ -125,12 +130,12 @@ bool Site2SiteClientProtocol::initiateResourceNegotiation() // tearDown(); return false; case NEGOTIATED_ABORT: - _logger->log_info("Site2Site Negotiate protocol response ABORT"); + logger_->log_info("Site2Site Negotiate protocol response ABORT"); ret = -1; // tearDown(); return false; default: - _logger->log_info("Negotiate protocol response unknown code %d", statusCode); + logger_->log_info("Negotiate protocol response unknown code %d", statusCode); return true; } @@ -142,30 +147,32 @@ bool Site2SiteClientProtocol::initiateCodecResourceNegotiation() // Negotiate the version if (_peerState != HANDSHAKED) { - _logger->log_error("Site2Site peer state is not handshaked while initiateCodecResourceNegotiation"); + logger_->log_error("Site2Site peer state is not handshaked while initiateCodecResourceNegotiation"); return false; } - _logger->log_info("Negotiate Codec version with destination port %s current version %d", _portIdStr.c_str(), _currentCodecVersion); + logger_->log_info("Negotiate Codec version with destination port %s current version %d", _portIdStr.c_str(), _currentCodecVersion); - int ret = _peer->writeUTF(this->getCodecResourceName()); + int ret = peer_->writeUTF(this->getCodecResourceName()); if (ret <= 0) { + logger_->log_debug("result of getCodecResourceName is %i",ret); // tearDown(); return false; } - ret = _peer->write(_currentCodecVersion); + ret = peer_->write(_currentCodecVersion); if (ret <= 0) { + logger_->log_debug("result of _currentCodecVersion is %i",ret); // tearDown(); return false; } uint8_t statusCode; - ret = _peer->read(statusCode); + ret = peer_->read(statusCode); if (ret <= 0) { @@ -176,17 +183,17 @@ bool Site2SiteClientProtocol::initiateCodecResourceNegotiation() switch (statusCode) { case RESOURCE_OK: - _logger->log_info("Site2Site Codec Negotiate version OK"); + logger_->log_info("Site2Site Codec Negotiate version OK"); return true; case DIFFERENT_RESOURCE_VERSION: uint32_t serverVersion; - ret = _peer->read(serverVersion); + ret = peer_->read(serverVersion); if (ret <= 0) { // tearDown(); return false; } - _logger->log_info("Site2Site Server Response asked for a different codec version %d", serverVersion); + logger_->log_info("Site2Site Server Response asked for a different codec version %d", serverVersion); for (unsigned int i = (_currentCodecVersionIndex + 1); i < sizeof(_supportedCodecVersion)/sizeof(uint32_t); i++) { if (serverVersion >= _supportedCodecVersion[i]) @@ -200,12 +207,12 @@ bool Site2SiteClientProtocol::initiateCodecResourceNegotiation() // tearDown(); return false; case NEGOTIATED_ABORT: - _logger->log_info("Site2Site Codec Negotiate response ABORT"); + logger_->log_info("Site2Site Codec Negotiate response ABORT"); ret = -1; // tearDown(); return false; default: - _logger->log_info("Negotiate Codec response unknown code %d", statusCode); + logger_->log_info("Negotiate Codec response unknown code %d", statusCode); return true; } @@ -216,10 +223,10 @@ bool Site2SiteClientProtocol::handShake() { if (_peerState != ESTABLISHED) { - _logger->log_error("Site2Site peer state is not established while handshake"); + logger_->log_error("Site2Site peer state is not established while handshake"); return false; } - _logger->log_info("Site2Site Protocol Perform hand shake with destination port %s", _portIdStr.c_str()); + logger_->log_info("Site2Site Protocol Perform hand shake with destination port %s", _portIdStr.c_str()); uuid_t uuid; // Generate the global UUID for the com identify uuid_generate(uuid); @@ -227,7 +234,7 @@ bool Site2SiteClientProtocol::handShake() uuid_unparse_lower(uuid, uuidStr); _commsIdentifier = uuidStr; - int ret = _peer->writeUTF(_commsIdentifier); + int ret = peer_->writeUTF(_commsIdentifier); if (ret <= 0) { @@ -251,7 +258,7 @@ bool Site2SiteClientProtocol::handShake() if (_currentVersion >= 3) { - ret = _peer->writeUTF(_peer->getURL()); + ret = peer_->writeUTF(peer_->getURL()); if (ret <= 0) { // tearDown(); @@ -260,7 +267,7 @@ bool Site2SiteClientProtocol::handShake() } uint32_t size = properties.size(); - ret = _peer->write(size); + ret = peer_->write(size); if (ret <= 0) { // tearDown(); @@ -270,19 +277,19 @@ bool Site2SiteClientProtocol::handShake() std::map::iterator it; for (it = properties.begin(); it!= properties.end(); it++) { - ret = _peer->writeUTF(it->first); + ret = peer_->writeUTF(it->first); if (ret <= 0) { // tearDown(); return false; } - ret = _peer->writeUTF(it->second); + ret = peer_->writeUTF(it->second); if (ret <= 0) { // tearDown(); return false; } - _logger->log_info("Site2Site Protocol Send handshake properties %s %s", it->first.c_str(), it->second.c_str()); + logger_->log_info("Site2Site Protocol Send handshake properties %s %s", it->first.c_str(), it->second.c_str()); } RespondCode code; @@ -299,23 +306,23 @@ bool Site2SiteClientProtocol::handShake() switch (code) { case PROPERTIES_OK: - _logger->log_info("Site2Site HandShake Completed"); + logger_->log_info("Site2Site HandShake Completed"); _peerState = HANDSHAKED; return true; case PORT_NOT_IN_VALID_STATE: case UNKNOWN_PORT: case PORTS_DESTINATION_FULL: - _logger->log_error("Site2Site HandShake Failed because destination port is either invalid or full"); + logger_->log_error("Site2Site HandShake Failed because destination port is either invalid or full"); ret = -1; /* - _peer->yield(); + peer_->yield(); tearDown(); */ return false; default: - _logger->log_info("HandShake Failed because of unknown respond code %d", code); + logger_->log_info("HandShake Failed because of unknown respond code %d", code); ret = -1; /* - _peer->yield(); + peer_->yield(); tearDown(); */ return false; } @@ -327,7 +334,7 @@ void Site2SiteClientProtocol::tearDown() { if (_peerState >= ESTABLISHED) { - _logger->log_info("Site2Site Protocol tearDown"); + logger_->log_info("Site2Site Protocol tearDown"); // need to write shutdown request writeRequestType(SHUTDOWN); } @@ -338,7 +345,7 @@ void Site2SiteClientProtocol::tearDown() delete it->second; } _transactionMap.clear(); - _peer->Close(); + peer_->Close(); _peerState = IDLE; } @@ -347,14 +354,14 @@ int Site2SiteClientProtocol::writeRequestType(RequestType type) if (type >= MAX_REQUEST_TYPE) return -1; - return _peer->writeUTF(RequestTypeStr[type]); + return peer_->writeUTF(RequestTypeStr[type]); } int Site2SiteClientProtocol::readRequestType(RequestType &type) { std::string requestTypeStr; - int ret = _peer->readUTF(requestTypeStr); + int ret = peer_->readUTF(requestTypeStr); if (ret <= 0) return ret; @@ -375,21 +382,21 @@ int Site2SiteClientProtocol::readRespond(RespondCode &code, std::string &message { uint8_t firstByte; - int ret = _peer->read(firstByte); + int ret = peer_->read(firstByte); if (ret <= 0 || firstByte != CODE_SEQUENCE_VALUE_1) return -1; uint8_t secondByte; - ret = _peer->read(secondByte); + ret = peer_->read(secondByte); if (ret <= 0 || secondByte != CODE_SEQUENCE_VALUE_2) return -1; uint8_t thirdByte; - ret = _peer->read(thirdByte); + ret = peer_->read(thirdByte); if (ret <= 0) return ret; @@ -405,7 +412,7 @@ int Site2SiteClientProtocol::readRespond(RespondCode &code, std::string &message } if (resCode->hasDescription) { - ret = _peer->readUTF(message); + ret = peer_->readUTF(message); if (ret <= 0) return -1; } @@ -427,14 +434,14 @@ int Site2SiteClientProtocol::writeRespond(RespondCode code, std::string message) codeSeq[1] = CODE_SEQUENCE_VALUE_2; codeSeq[2] = (uint8_t) code; - int ret = _peer->write(codeSeq, 3); + int ret = peer_->write(codeSeq, 3); if (ret != 3) return -1; if (resCode->hasDescription) { - ret = _peer->writeUTF(message); + ret = peer_->writeUTF(message); if (ret > 0) return (3 + ret); else @@ -448,11 +455,11 @@ bool Site2SiteClientProtocol::negotiateCodec() { if (_peerState != HANDSHAKED) { - _logger->log_error("Site2Site peer state is not handshaked while negotiate codec"); + logger_->log_error("Site2Site peer state is not handshaked while negotiate codec"); return false; } - _logger->log_info("Site2Site Protocol Negotiate Codec with destination port %s", _portIdStr.c_str()); + logger_->log_info("Site2Site Protocol Negotiate Codec with destination port %s", _portIdStr.c_str()); int status = this->writeRequestType(NEGOTIATE_FLOWFILE_CODEC); @@ -467,14 +474,14 @@ bool Site2SiteClientProtocol::negotiateCodec() if (!ret) { - _logger->log_error("Site2Site Codec Version Negotiation failed"); + logger_->log_error("Site2Site Codec Version Negotiation failed"); /* - _peer->yield(); + peer_->yield(); tearDown(); */ return false; } - _logger->log_info("Site2Site Codec Completed and move to READY state for data transfer"); + logger_->log_info("Site2Site Codec Completed and move to READY state for data transfer"); _peerState = READY; return true; @@ -489,12 +496,12 @@ bool Site2SiteClientProtocol::bootstrap() if (establish() && handShake() && negotiateCodec()) { - _logger->log_info("Site2Site Ready For data transaction"); + logger_->log_info("Site2Site Ready For data transaction"); return true; } else { - _peer->yield(); + peer_->yield(); tearDown(); return false; } @@ -537,28 +544,29 @@ Transaction* Site2SiteClientProtocol::createTransaction(std::string &transaction return NULL; } + CRCStream crcstream(peer_); switch (code) { case MORE_DATA: dataAvailable = true; - _logger->log_info("Site2Site peer indicates that data is available"); - transaction = new Transaction(direction); + logger_->log_info("Site2Site peer indicates that data is available"); + transaction = new Transaction(direction,crcstream); _transactionMap[transaction->getUUIDStr()] = transaction; transactionID = transaction->getUUIDStr(); transaction->setDataAvailable(dataAvailable); - _logger->log_info("Site2Site create transaction %s", transaction->getUUIDStr().c_str()); + logger_->log_info("Site2Site create transaction %s", transaction->getUUIDStr().c_str()); return transaction; case NO_MORE_DATA: dataAvailable = false; - _logger->log_info("Site2Site peer indicates that no data is available"); - transaction = new Transaction(direction); + logger_->log_info("Site2Site peer indicates that no data is available"); + transaction = new Transaction(direction,crcstream); _transactionMap[transaction->getUUIDStr()] = transaction; transactionID = transaction->getUUIDStr(); transaction->setDataAvailable(dataAvailable); - _logger->log_info("Site2Site create transaction %s", transaction->getUUIDStr().c_str()); + logger_->log_info("Site2Site create transaction %s", transaction->getUUIDStr().c_str()); return transaction; default: - _logger->log_info("Site2Site got unexpected response %d when asking for data", code); + logger_->log_info("Site2Site got unexpected response %d when asking for data", code); // tearDown(); return NULL; } @@ -574,10 +582,11 @@ Transaction* Site2SiteClientProtocol::createTransaction(std::string &transaction } else { - transaction = new Transaction(direction); + CRCStream crcstream(peer_); + transaction = new Transaction(direction,crcstream); _transactionMap[transaction->getUUIDStr()] = transaction; transactionID = transaction->getUUIDStr(); - _logger->log_info("Site2Site create transaction %s", transaction->getUUIDStr().c_str()); + logger_->log_info("Site2Site create transaction %s", transaction->getUUIDStr().c_str()); return transaction; } } @@ -611,13 +620,13 @@ bool Site2SiteClientProtocol::receive(std::string transactionID, DataPacket *pac if (transaction->getState() != TRANSACTION_STARTED && transaction->getState() != DATA_EXCHANGED) { - _logger->log_info("Site2Site transaction %s is not at started or exchanged state", transactionID.c_str()); + logger_->log_info("Site2Site transaction %s is not at started or exchanged state", transactionID.c_str()); return false; } if (transaction->getDirection() != RECEIVE) { - _logger->log_info("Site2Site transaction %s direction is wrong", transactionID.c_str()); + logger_->log_info("Site2Site transaction %s direction is wrong", transactionID.c_str()); return false; } @@ -641,17 +650,17 @@ bool Site2SiteClientProtocol::receive(std::string transactionID, DataPacket *pac } if (code == CONTINUE_TRANSACTION) { - _logger->log_info("Site2Site transaction %s peer indicate continue transaction", transactionID.c_str()); + logger_->log_info("Site2Site transaction %s peer indicate continue transaction", transactionID.c_str()); transaction->_dataAvailable = true; } else if (code == FINISH_TRANSACTION) { - _logger->log_info("Site2Site transaction %s peer indicate finish transaction", transactionID.c_str()); + logger_->log_info("Site2Site transaction %s peer indicate finish transaction", transactionID.c_str()); transaction->_dataAvailable = false; } else { - _logger->log_info("Site2Site transaction %s peer indicate wrong respond code %d", transactionID.c_str(), code); + logger_->log_info("Site2Site transaction %s peer indicate wrong respond code %d", transactionID.c_str(), code); return false; } } @@ -664,7 +673,7 @@ bool Site2SiteClientProtocol::receive(std::string transactionID, DataPacket *pac // start to read the packet uint32_t numAttributes; - ret = _peer->read(numAttributes, &transaction->_crc); + ret = transaction->getStream().read(numAttributes); if (ret <= 0 || numAttributes > MAX_NUM_ATTRIBUTES) { return false; @@ -675,22 +684,22 @@ bool Site2SiteClientProtocol::receive(std::string transactionID, DataPacket *pac { std::string key; std::string value; - ret = _peer->readUTF(key, true, &transaction->_crc); + ret = transaction->getStream().readUTF(key,true); if (ret <= 0) { return false; } - ret = _peer->readUTF(value, true, &transaction->_crc); + ret = transaction->getStream().readUTF(value,true); if (ret <= 0) { return false; } packet->_attributes[key] = value; - _logger->log_info("Site2Site transaction %s receives attribute key %s value %s", transactionID.c_str(), key.c_str(), value.c_str()); + logger_->log_info("Site2Site transaction %s receives attribute key %s value %s", transactionID.c_str(), key.c_str(), value.c_str()); } uint64_t len; - ret = _peer->read(len, &transaction->_crc); + ret = transaction->getStream().read(len); if (ret <= 0) { return false; @@ -700,7 +709,7 @@ bool Site2SiteClientProtocol::receive(std::string transactionID, DataPacket *pac transaction->_transfers++; transaction->_state = DATA_EXCHANGED; transaction->_bytes += len; - _logger->log_info("Site2Site transaction %s receives flow record %d, total length %d", transactionID.c_str(), + logger_->log_info("Site2Site transaction %s receives flow record %d, total length %d", transactionID.c_str(), transaction->_transfers, transaction->_bytes); return true; @@ -734,13 +743,13 @@ bool Site2SiteClientProtocol::send(std::string transactionID, DataPacket *packet if (transaction->getState() != TRANSACTION_STARTED && transaction->getState() != DATA_EXCHANGED) { - _logger->log_info("Site2Site transaction %s is not at started or exchanged state", transactionID.c_str()); + logger_->log_info("Site2Site transaction %s is not at started or exchanged state", transactionID.c_str()); return false; } if (transaction->getDirection() != SEND) { - _logger->log_info("Site2Site transaction %s direction is wrong", transactionID.c_str()); + logger_->log_info("Site2Site transaction %s direction is wrong", transactionID.c_str()); return false; } @@ -755,7 +764,7 @@ bool Site2SiteClientProtocol::send(std::string transactionID, DataPacket *packet // start to read the packet uint32_t numAttributes = packet->_attributes.size(); - ret = _peer->write(numAttributes, &transaction->_crc); + ret = transaction->getStream().write(numAttributes); if (ret != 4) { return false; @@ -764,22 +773,23 @@ bool Site2SiteClientProtocol::send(std::string transactionID, DataPacket *packet std::map::iterator itAttribute; for (itAttribute = packet->_attributes.begin(); itAttribute!= packet->_attributes.end(); itAttribute++) { - ret = _peer->writeUTF(itAttribute->first, true, &transaction->_crc); + ret = transaction->getStream().writeUTF(itAttribute->first, true); + if (ret <= 0) { return false; } - ret = _peer->writeUTF(itAttribute->second, true, &transaction->_crc); + ret = transaction->getStream().writeUTF(itAttribute->second, true); if (ret <= 0) { return false; } - _logger->log_info("Site2Site transaction %s send attribute key %s value %s", transactionID.c_str(), + logger_->log_info("Site2Site transaction %s send attribute key %s value %s", transactionID.c_str(), itAttribute->first.c_str(), itAttribute->second.c_str()); } uint64_t len = flowFile->getSize() ; - ret = _peer->write(len, &transaction->_crc); + ret = transaction->getStream().write(len); if (ret != 8) { return false; @@ -798,7 +808,7 @@ bool Site2SiteClientProtocol::send(std::string transactionID, DataPacket *packet transaction->_transfers++; transaction->_state = DATA_EXCHANGED; transaction->_bytes += len; - _logger->log_info("Site2Site transaction %s send flow record %d, total length %d", transactionID.c_str(), + logger_->log_info("Site2Site transaction %s send flow record %d, total length %d", transactionID.c_str(), transaction->_transfers, transaction->_bytes); return true; @@ -881,8 +891,8 @@ void Site2SiteClientProtocol::receiveFlowFiles(ProcessContext *context, ProcessS } Relationship relation; // undefined relationship uint64_t endTime = getTimeMillis(); - std::string transitUri = _peer->getURL() + "/" + sourceIdentifier; - std::string details = "urn:nifi:" + sourceIdentifier + "Remote Host=" + _peer->getHostName(); + std::string transitUri = peer_->getURL() + "/" + sourceIdentifier; + std::string details = "urn:nifi:" + sourceIdentifier + "Remote Host=" + peer_->getHostName(); session->getProvenanceReporter()->receive(flowFile, transitUri, sourceIdentifier, details, endTime - startTime); session->transfer(flowFile, relation); // receive the transfer for the flow record @@ -900,7 +910,7 @@ void Site2SiteClientProtocol::receiveFlowFiles(ProcessContext *context, ProcessS throw Exception(SITE2SITE_EXCEPTION, "Complete Transaction Failed"); return; } - _logger->log_info("Site2Site transaction %s successfully receive flow record %d, content bytes %d", + logger_->log_info("Site2Site transaction %s successfully receive flow record %d, content bytes %d", transactionID.c_str(), transfers, bytes); // we yield the receive if we did not get anything if (transfers == 0) @@ -912,7 +922,7 @@ void Site2SiteClientProtocol::receiveFlowFiles(ProcessContext *context, ProcessS deleteTransaction(transactionID); context->yield(); tearDown(); - _logger->log_debug("Caught Exception %s", exception.what()); + logger_->log_debug("Caught Exception %s", exception.what()); throw; } catch (...) @@ -921,7 +931,7 @@ void Site2SiteClientProtocol::receiveFlowFiles(ProcessContext *context, ProcessS deleteTransaction(transactionID); context->yield(); tearDown(); - _logger->log_debug("Caught Exception during Site2SiteClientProtocol::receiveFlowFiles"); + logger_->log_debug("Caught Exception during Site2SiteClientProtocol::receiveFlowFiles"); throw; } @@ -979,7 +989,7 @@ bool Site2SiteClientProtocol::confirm(std::string transactionID) // time window involved in the entire transaction, it is reduced to a simple round-trip conversation. long crcValue = transaction->getCRC(); std::string crc = std::to_string(crcValue); - _logger->log_info("Site2Site Send confirm with CRC %d to transaction %s", transaction->getCRC(), + logger_->log_info("Site2Site Send confirm with CRC %d to transaction %s", transaction->getCRC(), transactionID.c_str()); ret = writeRespond(CONFIRM_TRANSACTION, crc); if (ret <= 0) @@ -992,13 +1002,13 @@ bool Site2SiteClientProtocol::confirm(std::string transactionID) if (code == CONFIRM_TRANSACTION) { - _logger->log_info("Site2Site transaction %s peer confirm transaction", transactionID.c_str()); + logger_->log_info("Site2Site transaction %s peer confirm transaction", transactionID.c_str()); transaction->_state = TRANSACTION_CONFIRMED; return true; } else if (code == BAD_CHECKSUM) { - _logger->log_info("Site2Site transaction %s peer indicate bad checksum", transactionID.c_str()); + logger_->log_info("Site2Site transaction %s peer indicate bad checksum", transactionID.c_str()); /* transaction->_state = TRANSACTION_CONFIRMED; return true; */ @@ -1006,14 +1016,14 @@ bool Site2SiteClientProtocol::confirm(std::string transactionID) } else { - _logger->log_info("Site2Site transaction %s peer unknown respond code %d", + logger_->log_info("Site2Site transaction %s peer unknown respond code %d", transactionID.c_str(), code); return false; } } else { - _logger->log_info("Site2Site Send FINISH TRANSACTION for transaction %s", + logger_->log_info("Site2Site Send FINISH TRANSACTION for transaction %s", transactionID.c_str()); ret = writeRespond(FINISH_TRANSACTION, "FINISH_TRANSACTION"); if (ret <= 0) @@ -1027,14 +1037,14 @@ bool Site2SiteClientProtocol::confirm(std::string transactionID) // we've sent a FINISH_TRANSACTION. Now we'll wait for the peer to send a 'Confirm Transaction' response if (code == CONFIRM_TRANSACTION) { - _logger->log_info("Site2Site transaction %s peer confirm transaction with CRC %s", transactionID.c_str(), message.c_str()); + logger_->log_info("Site2Site transaction %s peer confirm transaction with CRC %s", transactionID.c_str(), message.c_str()); if (this->_currentVersion > 3) { long crcValue = transaction->getCRC(); std::string crc = std::to_string(crcValue); if (message == crc) { - _logger->log_info("Site2Site transaction %s CRC matched", transactionID.c_str()); + logger_->log_info("Site2Site transaction %s CRC matched", transactionID.c_str()); ret = writeRespond(CONFIRM_TRANSACTION, "CONFIRM_TRANSACTION"); if (ret <= 0) return false; @@ -1043,7 +1053,7 @@ bool Site2SiteClientProtocol::confirm(std::string transactionID) } else { - _logger->log_info("Site2Site transaction %s CRC not matched %s", transactionID.c_str(), crc.c_str()); + logger_->log_info("Site2Site transaction %s CRC not matched %s", transactionID.c_str(), crc.c_str()); ret = writeRespond(BAD_CHECKSUM, "BAD_CHECKSUM"); /* ret = writeRespond(CONFIRM_TRANSACTION, "CONFIRM_TRANSACTION"); @@ -1062,7 +1072,7 @@ bool Site2SiteClientProtocol::confirm(std::string transactionID) } else { - _logger->log_info("Site2Site transaction %s peer unknown respond code %d", + logger_->log_info("Site2Site transaction %s peer unknown respond code %d", transactionID.c_str(), code); return false; } @@ -1118,7 +1128,7 @@ void Site2SiteClientProtocol::deleteTransaction(std::string transactionID) transaction = it->second; } - _logger->log_info("Site2Site delete transaction %s", transaction->getUUIDStr().c_str()); + logger_->log_info("Site2Site delete transaction %s", transaction->getUUIDStr().c_str()); delete transaction; _transactionMap.erase(transactionID); } @@ -1184,7 +1194,7 @@ bool Site2SiteClientProtocol::complete(std::string transactionID) } else { - _logger->log_info("Site2Site transaction %s send finished", transactionID.c_str()); + logger_->log_info("Site2Site transaction %s send finished", transactionID.c_str()); ret = this->writeRespond(TRANSACTION_FINISHED, "Finished"); if (ret <= 0) return false; @@ -1208,13 +1218,13 @@ bool Site2SiteClientProtocol::complete(std::string transactionID) if (code == TRANSACTION_FINISHED) { - _logger->log_info("Site2Site transaction %s peer finished transaction", transactionID.c_str()); + logger_->log_info("Site2Site transaction %s peer finished transaction", transactionID.c_str()); transaction->_state = TRANSACTION_COMPLETED; return true; } else { - _logger->log_info("Site2Site transaction %s peer unknown respond code %d", + logger_->log_info("Site2Site transaction %s peer unknown respond code %d", transactionID.c_str(), code); return false; } @@ -1269,11 +1279,11 @@ void Site2SiteClientProtocol::transferFlowFiles(ProcessContext *context, Process throw Exception(SITE2SITE_EXCEPTION, "Send Failed"); return; } - _logger->log_info("Site2Site transaction %s send flow record %s", + logger_->log_info("Site2Site transaction %s send flow record %s", transactionID.c_str(), flow->getUUIDStr().c_str()); uint64_t endTime = getTimeMillis(); - std::string transitUri = _peer->getURL() + "/" + flow->getUUIDStr(); - std::string details = "urn:nifi:" + flow->getUUIDStr() + "Remote Host=" + _peer->getHostName(); + std::string transitUri = peer_->getURL() + "/" + flow->getUUIDStr(); + std::string details = "urn:nifi:" + flow->getUUIDStr() + "Remote Host=" + peer_->getHostName(); session->getProvenanceReporter()->send(flow, transitUri, details, endTime - startTime, false); session->remove(flow); @@ -1298,7 +1308,7 @@ void Site2SiteClientProtocol::transferFlowFiles(ProcessContext *context, Process throw Exception(SITE2SITE_EXCEPTION, "Complete Failed"); return; } - _logger->log_info("Site2Site transaction %s successfully send flow record %d, content bytes %d", + logger_->log_info("Site2Site transaction %s successfully send flow record %d, content bytes %d", transactionID.c_str(), transaction->_transfers, transaction->_bytes); } catch (std::exception &exception) @@ -1307,7 +1317,7 @@ void Site2SiteClientProtocol::transferFlowFiles(ProcessContext *context, Process deleteTransaction(transactionID); context->yield(); tearDown(); - _logger->log_debug("Caught Exception %s", exception.what()); + logger_->log_debug("Caught Exception %s", exception.what()); throw; } catch (...) @@ -1316,7 +1326,7 @@ void Site2SiteClientProtocol::transferFlowFiles(ProcessContext *context, Process deleteTransaction(transactionID); context->yield(); tearDown(); - _logger->log_debug("Caught Exception during Site2SiteClientProtocol::transferFlowFiles"); + logger_->log_debug("Caught Exception during Site2SiteClientProtocol::transferFlowFiles"); throw; } diff --git a/libminifi/src/Site2SitePeer.cpp b/libminifi/src/Site2SitePeer.cpp index 3d6166bdd0..ae3cc2da93 100644 --- a/libminifi/src/Site2SitePeer.cpp +++ b/libminifi/src/Site2SitePeer.cpp @@ -23,472 +23,33 @@ #include #include #include +#include #include #include +#include "io/ClientSocket.h" +#include "io/validation.h" #include "Site2SitePeer.h" #include "FlowController.h" -//! CRC tables -std::atomic CRC32::tableInit(false); -unsigned int CRC32::table[256]; +bool Site2SitePeer::Open() { -bool Site2SitePeer::Open() -{ - in_addr_t addr; - int sock = 0; - struct hostent *h; - const char *host; - uint16_t port; - - host = this->_host.c_str(); - port = this->_port; - - if (strlen(host) == 0) - return false; - -#ifdef __MACH__ - h = gethostbyname(host); -#else - char buf[1024]; - struct hostent he; - int hh_errno; - gethostbyname_r(host, &he, buf, sizeof(buf), &h, &hh_errno); -#endif - memcpy((char *) &addr, h->h_addr_list[0], h->h_length); - sock = socket(AF_INET, SOCK_STREAM, 0); - if (sock < 0) - { - _logger->log_error("Could not create socket to hostName %s", host); - this->yield(); - return false; - } - -#ifndef __MACH__ - int opt = 1; - bool nagle_off = true; - - if (nagle_off) - { - if (setsockopt(sock, SOL_TCP, TCP_NODELAY, (void *)&opt, sizeof(opt)) < 0) - { - _logger->log_error("setsockopt() TCP_NODELAY failed"); - close(sock); - this->yield(); - return false; - } - if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, - (char *)&opt, sizeof(opt)) < 0) - { - _logger->log_error("setsockopt() SO_REUSEADDR failed"); - close(sock); - this->yield(); - return false; - } - } - - int sndsize = 256*1024; - if (setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (char *)&sndsize, (int)sizeof(sndsize)) < 0) - { - _logger->log_error("setsockopt() SO_SNDBUF failed"); - close(sock); - this->yield(); - return false; - } - int rcvsize = 256*1024; - if (setsockopt(sock, SOL_SOCKET, SO_RCVBUF, (char *)&rcvsize, (int)sizeof(rcvsize)) < 0) - { - _logger->log_error("setsockopt() SO_RCVBUF failed"); - close(sock); - this->yield(); + if (IsNullOrEmpty (host_)) return false; - } -#endif - - struct sockaddr_in sa; - socklen_t socklen; - int status; - memset(&sa, 0, sizeof(sa)); - sa.sin_family = AF_INET; - sa.sin_addr.s_addr = htonl(INADDR_ANY); - sa.sin_port = htons(0); - socklen = sizeof(sa); - if (bind(sock, (struct sockaddr *)&sa, socklen) < 0) - { - _logger->log_error("socket bind failed"); - close(sock); - this->yield(); + if (stream_->initialize() < 0) return false; - } - - memset(&sa, 0, sizeof(sa)); - sa.sin_family = AF_INET; - sa.sin_addr.s_addr = addr; - sa.sin_port = htons(port); - socklen = sizeof(sa); - status = connect(sock, (struct sockaddr *)&sa, socklen); + uint16_t data_size = sizeof MAGIC_BYTES; - if (status < 0) - { - _logger->log_error("socket connect failed to %s %d", host, port); - close(sock); - this->yield(); - return false; - } - - // OpenSSL init - SSL_CTX *ctx = FlowControllerFactory::getFlowController()->getSSLContext(); - if (ctx) - { - // we have s2s secure config - this->_ssl = SSL_new(ctx); - SSL_set_fd(_ssl, sock); - if (SSL_connect(_ssl) == -1) - { - _logger->log_error("SSL socket connect failed to %s %d", host, port); - SSL_free(_ssl); - _ssl = NULL; - close(sock); - this->yield(); - return false; - } - else - { - _logger->log_info("SSL socket connect success to %s %d", host, port); - } - } - - _logger->log_info("Site2Site Peer socket %d connect to server %s port %d success", sock, host, port); - - _socket = sock; - - status = sendData((uint8_t *) MAGIC_BYTES, sizeof(MAGIC_BYTES)); - - if (status <= 0) - { - Close(); + if (stream_->writeData((uint8_t *) MAGIC_BYTES, data_size) != data_size) { return false; } return true; } -void Site2SitePeer::Close() -{ - if (_socket) - { - if (_ssl) - { - SSL_free(_ssl); - _ssl = NULL; - } - _logger->log_info("Site2Site Peer socket %d close", _socket); - close(_socket); - _socket = 0; - } +void Site2SitePeer::Close() { + if (stream_ != nullptr) + stream_->closeStream(); } -int Site2SitePeer::sendData(uint8_t *buf, int buflen, CRC32 *crc) -{ - int ret = 0, bytes = 0; - - if (_socket <= 0) - { - // this->yield(); - return -1; - } - - while (bytes < buflen) - { - if (!_ssl) - ret = send(_socket, buf+bytes, buflen-bytes, 0); - else - ret = SSL_write(_ssl, buf+bytes, buflen-bytes); - //check for errors - if (ret < 0) - { - _logger->log_error("Site2Site Peer socket %d send failed %s", _socket, strerror(errno)); - Close(); - // this->yield(); - return ret; - } - bytes+=ret; - } - - if (crc) - crc->update(buf, buflen); - - return bytes; -} - -int Site2SitePeer::Select(int msec) -{ - fd_set fds; - struct timeval tv; - int retval; - int fd = _socket; - - FD_ZERO(&fds); - FD_SET(fd, &fds); - - tv.tv_sec = msec/1000; - tv.tv_usec = (msec % 1000) * 1000; - - if (_ssl && SSL_pending(_ssl)) - return 1; - - if (msec > 0) - retval = select(fd+1, &fds, NULL, NULL, &tv); - else - retval = select(fd+1, &fds, NULL, NULL, NULL); - - if (retval <= 0) - return retval; - if (FD_ISSET(fd, &fds)) - return retval; - else - return 0; -} - -int Site2SitePeer::readData(uint8_t *buf, int buflen, CRC32 *crc) -{ - int sendSize = buflen; - uint8_t *start = buf; - - if (_socket <= 0) - { - // this->yield(); - return -1; - } - - while (buflen) - { - int status; - status = Select((int) _timeOut); - if (status <= 0) - { - Close(); - return status; - } - if (!_ssl) - { - status = recv(_socket, buf, buflen, 0); - if (status <= 0) - { - Close(); - return status; - } - } - else - { - // for SSL, wait for the TLS IO is completed - int sslStatus; - do { - status = SSL_read(_ssl, buf, buflen); - sslStatus = SSL_get_error(_ssl, status); - } - while (status < 0 && sslStatus == SSL_ERROR_WANT_READ); - if (status <= 0) - { - Close(); - return status; - } - } - buflen -= status; - buf += status; - } - - if (crc) - crc->update(start, sendSize); - - return sendSize; -} - -int Site2SitePeer::writeUTF(std::string str, bool widen, CRC32 *crc) -{ - int strlen = str.length(); - int utflen = 0; - int c, count = 0; - - /* use charAt instead of copying String to char array */ - for (int i = 0; i < strlen; i++) { - c = str.at(i); - if ((c >= 0x0001) && (c <= 0x007F)) { - utflen++; - } else if (c > 0x07FF) { - utflen += 3; - } else { - utflen += 2; - } - } - - if (utflen > 65535) - return -1; - - uint8_t *bytearr = NULL; - if (!widen) - { - bytearr = new uint8_t[utflen+2]; - bytearr[count++] = (uint8_t) ((utflen >> 8) & 0xFF); - bytearr[count++] = (uint8_t) ((utflen >> 0) & 0xFF); - } - else - { - bytearr = new uint8_t[utflen+4]; - bytearr[count++] = (uint8_t) ((utflen >> 24) & 0xFF); - bytearr[count++] = (uint8_t) ((utflen >> 16) & 0xFF); - bytearr[count++] = (uint8_t) ((utflen >> 8) & 0xFF); - bytearr[count++] = (uint8_t) ((utflen >> 0) & 0xFF); - } - - int i=0; - for (i=0; i= 0x0001) && (c <= 0x007F))) break; - bytearr[count++] = (uint8_t) c; - } - - for (;i < strlen; i++){ - c = str.at(i); - if ((c >= 0x0001) && (c <= 0x007F)) { - bytearr[count++] = (uint8_t) c; - } else if (c > 0x07FF) { - bytearr[count++] = (uint8_t) (0xE0 | ((c >> 12) & 0x0F)); - bytearr[count++] = (uint8_t) (0x80 | ((c >> 6) & 0x3F)); - bytearr[count++] = (uint8_t) (0x80 | ((c >> 0) & 0x3F)); - } else { - bytearr[count++] = (uint8_t) (0xC0 | ((c >> 6) & 0x1F)); - bytearr[count++] = (uint8_t) (0x80 | ((c >> 0) & 0x3F)); - } - } - int ret; - if (!widen) - { - ret = sendData(bytearr, utflen+2, crc); - } - else - { - ret = sendData(bytearr, utflen+4, crc); - } - delete[] bytearr; - return ret; -} - -int Site2SitePeer::readUTF(std::string &str, bool widen, CRC32 *crc) -{ - uint16_t utflen; - int ret; - - if (!widen) - { - ret = read(utflen, crc); - if (ret <= 0) - return ret; - } - else - { - uint32_t len; - ret = read(len, crc); - if (ret <= 0) - return ret; - utflen = len; - } - - uint8_t *bytearr = NULL; - char *chararr = NULL; - bytearr = new uint8_t[utflen]; - chararr = new char[utflen]; - memset(chararr, 0, utflen); - - int c, char2, char3; - int count = 0; - int chararr_count=0; - - ret = read(bytearr, utflen, crc); - if (ret <= 0) - { - delete[] bytearr; - delete[] chararr; - if (ret == 0) - { - if (!widen) - return (2 + utflen); - else - return (4 + utflen); - } - else - return ret; - } - - while (count < utflen) { - c = (int) bytearr[count] & 0xff; - if (c > 127) break; - count++; - chararr[chararr_count++]=(char)c; - } - - while (count < utflen) { - c = (int) bytearr[count] & 0xff; - switch (c >> 4) { - case 0: case 1: case 2: case 3: case 4: case 5: case 6: case 7: - /* 0xxxxxxx*/ - count++; - chararr[chararr_count++]=(char)c; - break; - case 12: case 13: - /* 110x xxxx 10xx xxxx*/ - count += 2; - if (count > utflen) - { - delete[] bytearr; - delete[] chararr; - return -1; - } - char2 = (int) bytearr[count-1]; - if ((char2 & 0xC0) != 0x80) - { - delete[] bytearr; - delete[] chararr; - return -1; - } - chararr[chararr_count++]=(char)(((c & 0x1F) << 6) | - (char2 & 0x3F)); - break; - case 14: - /* 1110 xxxx 10xx xxxx 10xx xxxx */ - count += 3; - if (count > utflen) - { - delete[] bytearr; - delete[] chararr; - return -1; - } - char2 = (int) bytearr[count-2]; - char3 = (int) bytearr[count-1]; - if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) - { - delete[] bytearr; - delete[] chararr; - return -1; - } - chararr[chararr_count++]=(char)(((c & 0x0F) << 12) | - ((char2 & 0x3F) << 6) | - ((char3 & 0x3F) << 0)); - break; - default: - delete[] bytearr; - delete[] chararr; - return -1; - } - } - // The number of chars produced may be less than utflen - std::string value(chararr, chararr_count); - str = value; - delete[] bytearr; - delete[] chararr; - if (!widen) - return (2 + utflen); - else - return (4 + utflen); -} diff --git a/libminifi/src/TailFile.cpp b/libminifi/src/TailFile.cpp index 0b389c78b6..36b5e53925 100644 --- a/libminifi/src/TailFile.cpp +++ b/libminifi/src/TailFile.cpp @@ -33,7 +33,8 @@ #include #include -#include "TimeUtil.h" +#include "utils/TimeUtil.h" +#include "utils/StringUtils.h" #include "TailFile.h" #include "ProcessContext.h" #include "ProcessSession.h" @@ -59,16 +60,12 @@ void TailFile::initialize() std::string TailFile::trimLeft(const std::string& s) { - const char *WHITESPACE = " \n\r\t"; - size_t startpos = s.find_first_not_of(WHITESPACE); - return (startpos == std::string::npos) ? "" : s.substr(startpos); + return StringUtils::trimLeft(s); } std::string TailFile::trimRight(const std::string& s) { - const char *WHITESPACE = " \n\r\t"; - size_t endpos = s.find_last_not_of(WHITESPACE); - return (endpos == std::string::npos) ? "" : s.substr(0, endpos+1); + return StringUtils::trimRight(s); } void TailFile::parseStateFileLine(char *buf) @@ -120,7 +117,7 @@ void TailFile::recoverState() std::ifstream file(_stateFile.c_str(), std::ifstream::in); if (!file.good()) { - _logger->log_error("load state file failed %s", _stateFile.c_str()); + logger_->log_error("load state file failed %s", _stateFile.c_str()); return; } const unsigned int bufSize = 512; @@ -136,7 +133,7 @@ void TailFile::storeState() std::ofstream file(_stateFile.c_str()); if (!file.is_open()) { - _logger->log_error("store state file failed %s", _stateFile.c_str()); + logger_->log_error("store state file failed %s", _stateFile.c_str()); return; } file << "FILENAME=" << this->_currentTailFileName << "\n"; @@ -205,7 +202,7 @@ void TailFile::checkRollOver() if (it!=matchedFiles.end()) { TailMatchedFileItem nextItem = *it; - _logger->log_info("TailFile File Roll Over from %s to %s", _currentTailFileName.c_str(), nextItem.fileName.c_str()); + logger_->log_info("TailFile File Roll Over from %s to %s", _currentTailFileName.c_str(), nextItem.fileName.c_str()); _currentTailFileName = nextItem.fileName; _currentTailFilePosition = 0; storeState(); @@ -261,7 +258,7 @@ void TailFile::onTrigger(ProcessContext *context, ProcessSession *session) flowFile->addAttribute(ABSOLUTE_PATH, fullPath); session->import(fullPath, flowFile, true, this->_currentTailFilePosition); session->transfer(flowFile, Success); - _logger->log_info("TailFile %s for %d bytes", _currentTailFileName.c_str(), flowFile->getSize()); + logger_->log_info("TailFile %s for %d bytes", _currentTailFileName.c_str(), flowFile->getSize()); std::string logName = baseName + "." + std::to_string(_currentTailFilePosition) + "-" + std::to_string(_currentTailFilePosition + flowFile->getSize()) + "." + extension; flowFile->updateAttribute(FILENAME, logName); diff --git a/libminifi/src/ThreadedSchedulingAgent.cpp b/libminifi/src/ThreadedSchedulingAgent.cpp index f7f1dbbc70..2008fecede 100644 --- a/libminifi/src/ThreadedSchedulingAgent.cpp +++ b/libminifi/src/ThreadedSchedulingAgent.cpp @@ -33,30 +33,30 @@ void ThreadedSchedulingAgent::schedule(Processor *processor) _administrativeYieldDuration = 0; std::string yieldValue; - if (_configure->get(Configure::nifi_administrative_yield_duration, yieldValue)) + if (configure_->get(Configure::nifi_administrative_yield_duration, yieldValue)) { TimeUnit unit; if (Property::StringToTime(yieldValue, _administrativeYieldDuration, unit) && Property::ConvertTimeUnitToMS(_administrativeYieldDuration, unit, _administrativeYieldDuration)) { - _logger->log_debug("nifi_administrative_yield_duration: [%d] ms", _administrativeYieldDuration); + logger_->log_debug("nifi_administrative_yield_duration: [%d] ms", _administrativeYieldDuration); } } _boredYieldDuration = 0; - if (_configure->get(Configure::nifi_bored_yield_duration, yieldValue)) + if (configure_->get(Configure::nifi_bored_yield_duration, yieldValue)) { TimeUnit unit; if (Property::StringToTime(yieldValue, _boredYieldDuration, unit) && Property::ConvertTimeUnitToMS(_boredYieldDuration, unit, _boredYieldDuration)) { - _logger->log_debug("nifi_bored_yield_duration: [%d] ms", _boredYieldDuration); + logger_->log_debug("nifi_bored_yield_duration: [%d] ms", _boredYieldDuration); } } if (processor->getScheduledState() != RUNNING) { - _logger->log_info("Can not schedule threads for processor %s because it is not running", processor->getName().c_str()); + logger_->log_info("Can not schedule threads for processor %s because it is not running", processor->getName().c_str()); return; } @@ -64,7 +64,7 @@ void ThreadedSchedulingAgent::schedule(Processor *processor) _threads.find(processor->getUUIDStr()); if (it != _threads.end()) { - _logger->log_info("Can not schedule threads for processor %s because there are existing threads running"); + logger_->log_info("Can not schedule threads for processor %s because there are existing threads running"); return; } @@ -82,7 +82,7 @@ void ThreadedSchedulingAgent::schedule(Processor *processor) }); thread->detach(); threads.push_back(thread); - _logger->log_info("Scheduled thread %d running for process %s", thread->get_id(), + logger_->log_info("Scheduled thread %d running for process %s", thread->get_id(), processor->getName().c_str()); } _threads[processor->getUUIDStr().c_str()] = threads; @@ -93,14 +93,14 @@ void ThreadedSchedulingAgent::schedule(Processor *processor) void ThreadedSchedulingAgent::unschedule(Processor *processor) { std::lock_guard lock(_mtx); - - _logger->log_info("Shutting down threads for processor %s/%s", + + logger_->log_info("Shutting down threads for processor %s/%s", processor->getName().c_str(), processor->getUUIDStr().c_str()); if (processor->getScheduledState() != RUNNING) { - _logger->log_info("Cannot unschedule threads for processor %s because it is not running", processor->getName().c_str()); + logger_->log_info("Cannot unschedule threads for processor %s because it is not running", processor->getName().c_str()); return; } @@ -109,16 +109,16 @@ void ThreadedSchedulingAgent::unschedule(Processor *processor) if (it == _threads.end()) { - _logger->log_info("Cannot unschedule threads for processor %s because there are no existing threads running", processor->getName().c_str()); + logger_->log_info("Cannot unschedule threads for processor %s because there are no existing threads running", processor->getName().c_str()); return; } for (std::vector::iterator itThread = it->second.begin(); itThread != it->second.end(); ++itThread) { std::thread *thread = *itThread; - _logger->log_info("Scheduled thread %d deleted for process %s", thread->get_id(), + logger_->log_info("Scheduled thread %d deleted for process %s", thread->get_id(), processor->getName().c_str()); delete thread; } _threads.erase(processor->getUUIDStr()); processor->clearActiveTask(); -} \ No newline at end of file +} diff --git a/libminifi/src/io/BaseStream.cpp b/libminifi/src/io/BaseStream.cpp new file mode 100644 index 0000000000..cf3fe45504 --- /dev/null +++ b/libminifi/src/io/BaseStream.cpp @@ -0,0 +1,153 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "io/BaseStream.h" +#include "io/Serializable.h" + +/** + * write 4 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ +int BaseStream::write(uint32_t base_value, bool is_little_endian) { + return ::Serializable::write(base_value, (DataStream*) this, is_little_endian); +} + +/** + * write 2 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ +int BaseStream::write(uint16_t base_value, bool is_little_endian) { + return ::Serializable::write(base_value, (DataStream*) this, is_little_endian); +} + +/** + * write valueto stream + * @param value non encoded value + * @param len length of value + * @param strema output stream + * @return resulting write size + **/ +int BaseStream::write(uint8_t *value, int len) { + return ::Serializable::write(value, len, (DataStream*) this); +} + +/** + * write 8 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ +int BaseStream::write(uint64_t base_value, bool is_little_endian) { + return ::Serializable::write(base_value, (DataStream*) this, is_little_endian); +} + +/** + * write bool to stream + * @param value non encoded value + * @return resulting write size + **/ +int BaseStream::write(bool value) { + uint8_t v = value; + return ::Serializable::write(v); +} + +/** + * write UTF string to stream + * @param str string to write + * @return resulting write size + **/ +int BaseStream::writeUTF(std::string str, bool widen) { + return ::Serializable::writeUTF(str, (DataStream*) this, widen); +} + +/** + * reads a byte from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ +int BaseStream::read(uint8_t &value) { + return ::Serializable::read(value, (DataStream*) this); +} + +/** + * reads two bytes from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ +int BaseStream::read(uint16_t &base_value, bool is_little_endian) { + return ::Serializable::read(base_value, (DataStream*) this); +} + +/** + * reads a byte from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ +int BaseStream::read(char &value) { + return ::Serializable::read(value, (DataStream*) this); +} + +/** + * reads a byte array from the stream + * @param value reference in which will set the result + * @param len length to read + * @param stream stream from which we will read + * @return resulting read size + **/ +int BaseStream::read(uint8_t *value, int len) { + return ::Serializable::read(value, len, (DataStream*) this); +} + +/** + * reads four bytes from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ +int BaseStream::read(uint32_t &value, bool is_little_endian) { + return ::Serializable::read(value, (DataStream*) this, is_little_endian); +} + +/** + * reads eight byte from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ +int BaseStream::read(uint64_t &value, bool is_little_endian) { + return ::Serializable::read(value, (DataStream*) this, is_little_endian); +} + +/** + * read UTF from stream + * @param str reference string + * @param stream stream from which we will read + * @return resulting read size + **/ +int BaseStream::readUTF(std::string &str, bool widen) { + return ::Serializable::readUTF(str, (DataStream*) this, widen); +} diff --git a/libminifi/src/io/CRCStream.cpp b/libminifi/src/io/CRCStream.cpp new file mode 100644 index 0000000000..47b45b56d9 --- /dev/null +++ b/libminifi/src/io/CRCStream.cpp @@ -0,0 +1,24 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "io/CRCStream.h" + + + diff --git a/libminifi/src/io/ClientSocket.cpp b/libminifi/src/io/ClientSocket.cpp new file mode 100644 index 0000000000..fac5c9dce2 --- /dev/null +++ b/libminifi/src/io/ClientSocket.cpp @@ -0,0 +1,425 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "io/validation.h" +#include "io/ClientSocket.h" + + +std::string Socket::HOSTNAME = Socket::getMyHostName(0); + + +Socket::Socket(const std::string &hostname, const uint16_t port, + const uint16_t listeners = -1) : + requested_hostname_(hostname), port_(port), addr_info_(0), socket_file_descriptor_( + -1), socket_max_(0), listeners_(listeners), canonical_hostname_("") { + logger_ = Logger::getLogger(); + FD_ZERO(&total_list_); + FD_ZERO(&read_fds_); + +} + +Socket::Socket(const std::string &hostname, const uint16_t port) : + ::Socket(hostname, port, 0) { + +} + +Socket::Socket(const Socket &&other) : + requested_hostname_(std::move(other.requested_hostname_)), port_( + std::move(other.port_)), addr_info_(std::move(other.addr_info_)), socket_file_descriptor_( + other.socket_file_descriptor_), socket_max_( + other.socket_max_.load()), listeners_(other.listeners_), total_list_( + other.total_list_), read_fds_(other.read_fds_), canonical_hostname_( + std::move(other.canonical_hostname_)) { + logger_ = Logger::getLogger(); + +} + +Socket::~Socket() { + closeStream(); +} + +void Socket::closeStream() +{ + if (0 != addr_info_) { + freeaddrinfo(addr_info_); + addr_info_=0; + } + + if (socket_file_descriptor_ >= 0) + { + close(socket_file_descriptor_); + socket_file_descriptor_ = -1; + } +} + +int8_t Socket::createConnection(const addrinfo *p) { + if ((socket_file_descriptor_ = socket(p->ai_family, p->ai_socktype, + p->ai_protocol)) == -1) { + logger_->log_error("error while connecting to server socket"); + return -1; + } + + setSocketOptions(socket_file_descriptor_); + + if (listeners_ > 0) { + + struct sockaddr_in *sa_loc = (struct sockaddr_in*) p->ai_addr; + sa_loc->sin_family = AF_INET; + sa_loc->sin_port = htons(port_); + sa_loc->sin_addr.s_addr = htonl(INADDR_ANY); + + if (bind(socket_file_descriptor_, p->ai_addr, p->ai_addrlen) == -1) { + logger_->log_error("Could not bind to socket", strerror(errno)); + return -1; + } + } + { + if (listeners_ <=0 ) + { + struct sockaddr_in *sa_loc = (struct sockaddr_in*) p->ai_addr; + sa_loc->sin_family = AF_INET; + //sa_loc->sin_port = htons(port); + sa_loc->sin_port = htons(port_); + sa_loc->sin_addr.s_addr = htonl(INADDR_ANY); // inet_addr(requested_hostname.c_str()); + if (connect(socket_file_descriptor_, p->ai_addr, p->ai_addrlen) == -1) { + close(socket_file_descriptor_); + socket_file_descriptor_ = -1; + logger_->log_warn("Could not connect to socket, error:%s", strerror(errno)); + return -1; + + } + } + } + + // listen + if (listeners_ > 0) { + if (listen(socket_file_descriptor_, listeners_) == -1) { + logger_->log_warn("attempted connection, saw %s", strerror(errno)); + return -1; + } + + } + // add the listener to the total set + FD_SET(socket_file_descriptor_, &total_list_); + socket_max_ = socket_file_descriptor_; + return 0; +} + +short Socket::initialize() { + + struct sockaddr_in servAddr; + + addrinfo hints = { sizeof(addrinfo) }; + memset(&hints, 0, sizeof hints); // make sure the struct is empty + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_CANONNAME; + if (listeners_ > 0) + hints.ai_flags |= AI_PASSIVE; + + hints.ai_protocol = 0; /* any protocol */ + + int errcode = getaddrinfo(requested_hostname_.c_str(), 0, &hints, &addr_info_); + + if (errcode != 0) { + logger_->log_error("Saw error during getaddrinfo, error: %s",strerror(errno)); + return -1; + } + + socket_file_descriptor_ = -1; + + auto p = addr_info_; + for (; p != NULL; p = p->ai_next) { + if (IsNullOrEmpty(canonical_hostname_)) { + if (!IsNullOrEmpty(p) && !IsNullOrEmpty(p->ai_canonname)) + canonical_hostname_ = p->ai_canonname; + } + + //we've successfully connected + if (port_ > 0 && createConnection(p) >= 0) + { + return 0; + break; + } + } + + return -1; + +} + +short Socket::select_descriptor(const uint16_t msec) { + struct timeval tv; + int retval; + + read_fds_ = total_list_; + + tv.tv_sec = msec / 1000; + tv.tv_usec = (msec % 1000) * 1000; + + std::lock_guard guard(selection_mutex_); + + if (msec > 0) + retval = select(socket_max_ + 1, &read_fds_, NULL, NULL, &tv); + else + retval = select(socket_max_ + 1, &read_fds_, NULL, NULL, NULL); + + if (retval < 0) { + logger_->log_error("Saw error during selection, error:%i %s",retval,strerror(errno)); + return retval; + } + + for (int i = 0; i <= socket_max_; i++) { + if (FD_ISSET(i, &read_fds_)) { + + if (i == socket_file_descriptor_) { + if (listeners_ > 0) + { + struct sockaddr_storage remoteaddr; // client address + socklen_t addrlen = sizeof remoteaddr; + int newfd = accept(socket_file_descriptor_, + (struct sockaddr *) &remoteaddr, &addrlen); + FD_SET(newfd, &total_list_); // add to master set + if (newfd > socket_max_) { // keep track of the max + socket_max_ = newfd; + } + return newfd; + + + + } + else{ + return socket_file_descriptor_; + } + // we have a new connection + } else { + // data to be received on i + return i; + } + } + + } + + return -1; +} + +short Socket::setSocketOptions(const int sock) { + int opt = 1; + bool nagle_off = true; +#ifndef __MACH__ + if (nagle_off) { + if (setsockopt(sock, SOL_TCP, TCP_NODELAY, (void *) &opt, sizeof(opt)) + < 0) { + logger_->log_error("setsockopt() TCP_NODELAY failed"); + close(sock); + return -1; + } + if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *) &opt, + sizeof(opt)) < 0) { + logger_->log_error("setsockopt() SO_REUSEADDR failed"); + close(sock); + return -1; + } + } + + int sndsize = 256 * 1024; + if (setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (char *) &sndsize, + (int) sizeof(sndsize)) < 0) { + logger_->log_error("setsockopt() SO_SNDBUF failed"); + close(sock); + return -1; + } + +#else + if (listeners_ > 0) { + // lose the pesky "address already in use" error message + if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *) &opt, + sizeof(opt)) < 0) { + logger_->log_error("setsockopt() SO_REUSEADDR failed"); + close(sock); + return -1; + } + } +#endif + return 0; +} + +std::string Socket::getHostname() const { + return canonical_hostname_; +} + +int Socket::writeData(std::vector &buf, int buflen) { + + if (buf.capacity() < buflen) + return -1; + return writeData((uint8_t*) &buf[0], buflen); +} + +// data stream overrides + +int Socket::writeData(uint8_t *value, int size) { + + int ret = 0, bytes = 0; + + while (bytes < size) { + + + + ret = send(socket_file_descriptor_, value + bytes, size - bytes, 0); + //check for errors + if (ret <= 0) { + close(socket_file_descriptor_); + logger_->log_error("Could not send to %d, error: %s",socket_file_descriptor_, strerror(errno)); + return ret; + } + bytes += ret; + + } + + if (ret) + logger_->log_debug("Send data size %d over socket %d", size, + socket_file_descriptor_); + + return bytes; + +} + +template +inline std::vector Socket::readBuffer(const T& t) { + std::vector buf; + buf.resize(sizeof t); + readData((uint8_t*) &buf[0], sizeof(t)); + return buf; +} + + +int Socket::write(uint64_t base_value, bool is_little_endian){ + + return Serializable::write(base_value,this,is_little_endian); +} + + +int Socket::write(uint32_t base_value, bool is_little_endian){ + return Serializable::write(base_value,this,is_little_endian); +} + +int Socket::write(uint16_t base_value, bool is_little_endian){ + return Serializable::write(base_value,this,is_little_endian); +} + + +int Socket::read(uint64_t &value, bool is_little_endian) { + + auto buf = readBuffer(value); + + if (is_little_endian) { + value = ((uint64_t) buf[0] << 56) | ((uint64_t) (buf[1] & 255) << 48) + | ((uint64_t) (buf[2] & 255) << 40) + | ((uint64_t) (buf[3] & 255) << 32) + | ((uint64_t) (buf[4] & 255) << 24) + | ((uint64_t) (buf[5] & 255) << 16) + | ((uint64_t) (buf[6] & 255) << 8) + | ((uint64_t) (buf[7] & 255) << 0); + } else { + value = ((uint64_t) buf[0] << 0) | ((uint64_t) (buf[1] & 255) << 8) + | ((uint64_t) (buf[2] & 255) << 16) + | ((uint64_t) (buf[3] & 255) << 24) + | ((uint64_t) (buf[4] & 255) << 32) + | ((uint64_t) (buf[5] & 255) << 40) + | ((uint64_t) (buf[6] & 255) << 48) + | ((uint64_t) (buf[7] & 255) << 56); + } + return sizeof(value); +} + +int Socket::read(uint32_t &value, bool is_little_endian) { + + auto buf = readBuffer(value); + + if (is_little_endian) { + value = (buf[0] << 24) | (buf[1] << 16) | (buf[2] << 8) | buf[3]; + } else { + value = buf[0] | buf[1] << 8 | buf[2] << 16 | buf[3] << 24; + + } + + return sizeof(value); +} + +int Socket::read(uint16_t &value, bool is_little_endian) { + + auto buf = readBuffer(value); + + if (is_little_endian) { + value = (buf[0] << 8) | buf[1]; + } else { + value = buf[0] | buf[1] << 8; + + } + return sizeof(value); +} + +int Socket::readData(std::vector &buf, int buflen) { + + if (buf.capacity() < buflen) + { + buf.resize(buflen); + } + return readData((uint8_t*) &buf[0], buflen); +} + +int Socket::readData(uint8_t *buf, int buflen) { + + int total_read = 0; + while (buflen) { + short fd = select_descriptor(1000); + if (fd < 0) { + + logger_->log_info("fd close %i",buflen); + close(socket_file_descriptor_); + return -1; + } + + int bytes_read = recv(fd, buf, buflen, 0); + if (bytes_read <= 0) { + if (bytes_read == 0) + logger_->log_info("Other side hung up on %d", fd); + else { + logger_->log_error("Could not recv on %d, error: %s",fd, strerror(errno)); + } + return -1; + } + + + buflen -= bytes_read; + buf += bytes_read; + total_read += bytes_read; + } + + return total_read; +} diff --git a/libminifi/src/io/DataStream.cpp b/libminifi/src/io/DataStream.cpp new file mode 100644 index 0000000000..3ff0a57383 --- /dev/null +++ b/libminifi/src/io/DataStream.cpp @@ -0,0 +1,128 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include "io/DataStream.h" + + + +int DataStream::writeData(uint8_t *value, int size) { + + std::copy(value,value+size,std::back_inserter(buffer)); + + return size; +} + +int DataStream::read(uint64_t &value, bool is_little_endian) { + if ((8 + readBuffer) > buffer.size()) { + // if read exceed + return -1; + } + uint8_t *buf = &buffer[readBuffer]; + + if (is_little_endian) { + value = ((uint64_t) buf[0] << 56) | ((uint64_t) (buf[1] & 255) << 48) + | ((uint64_t) (buf[2] & 255) << 40) + | ((uint64_t) (buf[3] & 255) << 32) + | ((uint64_t) (buf[4] & 255) << 24) + | ((uint64_t) (buf[5] & 255) << 16) + | ((uint64_t) (buf[6] & 255) << 8) + | ((uint64_t) (buf[7] & 255) << 0); + } else { + value = ((uint64_t) buf[0] << 0) | ((uint64_t) (buf[1] & 255) << 8) + | ((uint64_t) (buf[2] & 255) << 16) + | ((uint64_t) (buf[3] & 255) << 24) + | ((uint64_t) (buf[4] & 255) << 32) + | ((uint64_t) (buf[5] & 255) << 40) + | ((uint64_t) (buf[6] & 255) << 48) + | ((uint64_t) (buf[7] & 255) << 56); + } + readBuffer += 8; + return 8; +} + +int DataStream::read(uint32_t &value, bool is_little_endian) { + if ((4 + readBuffer) > buffer.size()) { + // if read exceed + return -1; + } + uint8_t *buf = &buffer[readBuffer]; + + if (is_little_endian) { + value = (buf[0] << 24) | (buf[1] << 16) | (buf[2] << 8) | buf[3]; + } else { + value = buf[0] | buf[1] << 8 | buf[2] << 16 | buf[3] << 24; + + } + readBuffer += 4; + return 4; +} + +int DataStream::read(uint16_t &value, bool is_little_endian) { + + if ((2 + readBuffer) > buffer.size()) { + // if read exceed + return -1; + } + uint8_t *buf = &buffer[readBuffer]; + + if (is_little_endian) { + value = (buf[0] << 8) | buf[1]; + } else { + value = buf[0] | buf[1] << 8; + + } + readBuffer += 2; + return 2; +} + +int DataStream::readData(std::vector &buf,int buflen) { + if ((buflen + readBuffer) > buffer.size()) { + // if read exceed + return -1; + } + + if (buf.capacity() < buflen) + buf.resize(buflen); + + buf.insert(buf.begin(),&buffer[readBuffer],&buffer[readBuffer+buflen]); + + readBuffer += buflen; + return buflen; +} + + +int DataStream::readData(uint8_t *buf,int buflen) { + if ((buflen + readBuffer) > buffer.size()) { + // if read exceed + return -1; + } + + std::copy(&buffer[readBuffer],&buffer[readBuffer+buflen],buf); + + readBuffer += buflen; + return buflen; +} diff --git a/libminifi/src/io/EndianCheck.cpp b/libminifi/src/io/EndianCheck.cpp new file mode 100644 index 0000000000..f1bc98b6e5 --- /dev/null +++ b/libminifi/src/io/EndianCheck.cpp @@ -0,0 +1,22 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "io/EndianCheck.h" + + +bool EndiannessCheck::IS_LITTLE = EndiannessCheck::is_little_endian(); diff --git a/libminifi/src/Serializable.cpp b/libminifi/src/io/Serializable.cpp similarity index 67% rename from libminifi/src/Serializable.cpp rename to libminifi/src/io/Serializable.cpp index 91330a0c75..8d67f15163 100644 --- a/libminifi/src/Serializable.cpp +++ b/libminifi/src/io/Serializable.cpp @@ -15,17 +15,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + #include -#include -#include +#include #include -#include #include -#include "Serializable.h" +#include "io/DataStream.h" +#include "io/Serializable.h" + #define htonll_r(x) ((((uint64_t)htonl(x)) << 32) + htonl((x) >> 32)) -bool EndiannessCheck::IS_LITTLE = EndiannessCheck::is_little_endian(); + #define IS_ASCII(c) __builtin_expect(!!((c >= 1) && (c <= 127)),1) @@ -78,7 +79,7 @@ int Serializable::write(bool value) { int Serializable::read(uint8_t &value,DataStream *stream) { uint8_t buf; - + int ret = stream->readData(&buf, 1); if (ret == 1) value = buf; @@ -100,17 +101,17 @@ int Serializable::read(uint8_t *value, int len,DataStream *stream) { int Serializable::read(uint16_t &value,DataStream *stream, bool is_little_endian) { - return stream->readShort(value, is_little_endian); + return stream->read(value, is_little_endian); } int Serializable::read(uint32_t &value,DataStream *stream, bool is_little_endian) { - return stream->readLong(value, is_little_endian); + return stream->read(value, is_little_endian); } int Serializable::read(uint64_t &value,DataStream *stream, bool is_little_endian) { - return stream->readLongLong(value, is_little_endian); + return stream->read(value, is_little_endian); } @@ -254,112 +255,9 @@ int Serializable::writeUTF(std::string str,DataStream *stream, bool widen) { } ret = stream->writeData(utf_to_write.data(), utflen); } else { - utflen += 4; + //utflen += 4; write(utflen,stream); ret = stream->writeData(utf_to_write.data(), utflen); } return ret; } - -int DataStream::writeData(uint8_t *value, int size) { - - /*if (buffer.size() + size < buffer.capacity()) - { - buffer.resize( buffer.size() + size ); - } - */ - std::copy(value,value+size,std::back_inserter(buffer)); - - return size; -} - -int DataStream::readLongLong(uint64_t &value, bool is_little_endian) { - if ((8 + readBuffer) > buffer.size()) { - // if read exceed - return -1; - } - uint8_t *buf = &buffer[readBuffer]; - - if (is_little_endian) { - value = ((uint64_t) buf[0] << 56) | ((uint64_t) (buf[1] & 255) << 48) - | ((uint64_t) (buf[2] & 255) << 40) - | ((uint64_t) (buf[3] & 255) << 32) - | ((uint64_t) (buf[4] & 255) << 24) - | ((uint64_t) (buf[5] & 255) << 16) - | ((uint64_t) (buf[6] & 255) << 8) - | ((uint64_t) (buf[7] & 255) << 0); - } else { - value = ((uint64_t) buf[0] << 0) | ((uint64_t) (buf[1] & 255) << 8) - | ((uint64_t) (buf[2] & 255) << 16) - | ((uint64_t) (buf[3] & 255) << 24) - | ((uint64_t) (buf[4] & 255) << 32) - | ((uint64_t) (buf[5] & 255) << 40) - | ((uint64_t) (buf[6] & 255) << 48) - | ((uint64_t) (buf[7] & 255) << 56); - } - readBuffer += 8; - return 8; -} - -int DataStream::readLong(uint32_t &value, bool is_little_endian) { - if ((4 + readBuffer) > buffer.size()) { - // if read exceed - return -1; - } - uint8_t *buf = &buffer[readBuffer]; - - if (is_little_endian) { - value = (buf[0] << 24) | (buf[1] << 16) | (buf[2] << 8) | buf[3]; - } else { - value = buf[0] | buf[1] << 8 | buf[2] << 16 | buf[3] << 24; - - } - readBuffer += 4; - return 4; -} - -int DataStream::readShort(uint16_t &value, bool is_little_endian) { - - if ((2 + readBuffer) > buffer.size()) { - // if read exceed - return -1; - } - uint8_t *buf = &buffer[readBuffer]; - - if (is_little_endian) { - value = (buf[0] << 8) | buf[1]; - } else { - value = buf[0] | buf[1] << 8; - - } - readBuffer += 2; - return 2; -} - -int DataStream::readData(std::vector &buf,int buflen) { - if ((buflen + readBuffer) > buffer.size()) { - // if read exceed - return -1; - } - - if (buf.capacity() < buflen) - buf.resize(buflen); - - buf.insert(buf.begin(),&buffer[readBuffer],&buffer[readBuffer+buflen]); - - readBuffer += buflen; - return buflen; -} - - -int DataStream::readData(uint8_t *buf,int buflen) { - if ((buflen + readBuffer) > buffer.size()) { - // if read exceed - return -1; - } - - std::copy(&buffer[readBuffer],&buffer[readBuffer+buflen],buf); - - readBuffer += buflen; - return buflen; -} diff --git a/libminifi/src/io/SocketFactory.cpp b/libminifi/src/io/SocketFactory.cpp new file mode 100644 index 0000000000..cbfdf965e3 --- /dev/null +++ b/libminifi/src/io/SocketFactory.cpp @@ -0,0 +1,24 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "io/SocketFactory.h" + +#include +#include + +std::atomic SocketFactory::context_instance_; +std::mutex SocketFactory::context_mutex_; diff --git a/libminifi/src/io/TLSSocket.cpp b/libminifi/src/io/TLSSocket.cpp new file mode 100644 index 0000000000..b38214e798 --- /dev/null +++ b/libminifi/src/io/TLSSocket.cpp @@ -0,0 +1,236 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Property.h" +#include "Configure.h" +#include "io/TLSSocket.h" +#include "utils/StringUtils.h" +#include +#include + + +std::atomic TLSContext::context_instance; +std::mutex TLSContext::context_mutex; + +TLSContext::TLSContext() : + error_value(0), ctx(0), logger(Logger::getLogger()), configuration( + Configure::getConfigure()) { + +} + +/** + * The memory barrier is defined by the singleton + */ +short TLSContext::initialize() { + if (ctx != 0) { + return error_value; + } + std::string clientAuthStr; + bool needClientCert = true; + if (!(configuration->get(Configure::nifi_security_need_ClientAuth, + clientAuthStr) + && StringUtils::StringToBool(clientAuthStr, needClientCert))) { + needClientCert = true; + } + + SSL_library_init(); + const SSL_METHOD *method; + + OpenSSL_add_all_algorithms(); + SSL_load_error_strings(); + method = TLSv1_2_client_method(); + ctx = SSL_CTX_new(method); + if (ctx == NULL) { + logger->log_error("Could not create SSL context, error: %s.", + std::strerror(errno)); + error_value = TLS_ERROR_CONTEXT; + return error_value; + } + if (needClientCert) { + std::string certificate; + std::string privatekey; + std::string passphrase; + std::string caCertificate; + + if (!(configuration->get(Configure::nifi_security_client_certificate, + certificate) + && configuration->get( + Configure::nifi_security_client_private_key, privatekey))) { + logger->log_error( + "Certificate and Private Key PEM file not configured, error: %s.", + std::strerror(errno)); + error_value = TLS_ERROR_PEM_MISSING; + return error_value; + } + // load certificates and private key in PEM format + if (SSL_CTX_use_certificate_file(ctx, certificate.c_str(), + SSL_FILETYPE_PEM) <= 0) { + logger->log_error("Could not create load certificate, error : %s", + std::strerror(errno)); + error_value = TLS_ERROR_CERT_MISSING; + return error_value; + + } + if (configuration->get(Configure::nifi_security_client_pass_phrase, + passphrase)) { + // if the private key has passphase + SSL_CTX_set_default_passwd_cb(ctx, pemPassWordCb); + } + + + int retp = SSL_CTX_use_PrivateKey_file(ctx, privatekey.c_str(), + SSL_FILETYPE_PEM); + if (retp != 1) { + logger->log_error("Could not create load private key,%i on %s error : %s", + retp,privatekey.c_str(),std::strerror(errno)); + error_value = TLS_ERROR_KEY_ERROR; + return error_value; + } + // verify private key + if (!SSL_CTX_check_private_key(ctx)) { + logger->log_error( + "Private key does not match the public certificate, error : %s", + std::strerror(errno)); + error_value = TLS_ERROR_KEY_ERROR; + return error_value; + } + // load CA certificates + if (configuration->get(Configure::nifi_security_client_ca_certificate, + caCertificate)) { + retp = SSL_CTX_load_verify_locations(ctx, caCertificate.c_str(), 0); + if (retp==0) { + logger->log_error( + "Can not load CA certificate, Exiting, error : %s", + std::strerror(errno)); + error_value = TLS_ERROR_CERT_ERROR; + return error_value; + } + } + + logger->log_info("Load/Verify Client Certificate OK."); + } + return 0; +} + +TLSSocket::~TLSSocket() +{ + if (ssl != 0) + SSL_free(ssl); +} +/** + * Constructor that accepts host name, port and listeners. With this + * contructor we will be creating a server socket + * @param hostname our host name + * @param port connecting port + * @param listeners number of listeners in the queue + */ +TLSSocket::TLSSocket(const std::string &hostname, const uint16_t port, + const uint16_t listeners) : + ::Socket(hostname, port, listeners), ssl(0) { +} + +TLSSocket::TLSSocket(const std::string &hostname, const uint16_t port) : + ::Socket(hostname, port, 0), ssl(0) { +} + +TLSSocket::TLSSocket(const TLSSocket &&d) : + ::Socket(std::move(d)), ssl(0) { +} + +short TLSSocket::initialize() { + TLSContext *context = TLSContext::getInstance(); + short ret = context->initialize(); + Socket::initialize(); + if (!ret) { + // we have s2s secure config + ssl = SSL_new(context->getContext()); + SSL_set_fd(ssl, socket_file_descriptor_); + if (SSL_connect(ssl) == -1) { + logger_->log_error("SSL socket connect failed to %s %d", + requested_hostname_.c_str(), port_); + SSL_free(ssl); + ssl = NULL; + close(socket_file_descriptor_); + return -1; + } else { + logger_->log_info("SSL socket connect success to %s %d", + requested_hostname_.c_str(), port_); + return 0; + } + } + return ret; +} + +short TLSSocket::select_descriptor(const uint16_t msec) { + if (ssl && SSL_pending(ssl)) + return 1; + return Socket::select_descriptor(msec); +} + +int TLSSocket::writeData(std::vector< uint8_t>& buf, int buflen) +{ + return Socket::writeData(buf,buflen); +} + +int TLSSocket::writeData(uint8_t *value, int size) { + if (IsNullOrEmpty(ssl)) + return -1; + // for SSL, wait for the TLS IO is completed + int bytes = 0; + int sent = 0; + while (bytes < size) { + + sent = SSL_write(ssl, value + bytes, size - bytes); + //check for errors + if (sent < 0) { + logger_->log_error("Site2Site Peer socket %d send failed %s", + socket_file_descriptor_, strerror(errno)); + return sent; + } + bytes += sent; + } + return size; +} + +int TLSSocket::readData(uint8_t *buf, int buflen) { + + if (IsNullOrEmpty(ssl)) + return -1; + int total_read = 0; + int status = 0; + while (buflen) { + short fd = select_descriptor(1000); + if (fd <= 0) { + + close(socket_file_descriptor_); + return -1; + } + + int sslStatus; + do { + status = SSL_read(ssl, buf, buflen); + sslStatus = SSL_get_error(ssl, status); + } while (status < 0 && sslStatus == SSL_ERROR_WANT_READ); + + buflen -= status; + buf += status; + total_read += status; + } + + return total_read; +} diff --git a/libminifi/test/unit/CRCTests.h b/libminifi/test/unit/CRCTests.h new file mode 100644 index 0000000000..3e6665e4f4 --- /dev/null +++ b/libminifi/test/unit/CRCTests.h @@ -0,0 +1,78 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "io/CRCStream.h" +#include "io/DataStream.h" +TEST_CASE("Test CRC1", "[testcrc1]") { + + BaseStream base; + CRCStream test(&base); + test.writeData((uint8_t*)"cow",3); + REQUIRE(2580823964 == test.getCRC()); + + +} + +TEST_CASE("Test CRC2", "[testcrc2]") { + + BaseStream base; + CRCStream test(&base); + std::string fox = "the quick brown fox jumped over the brown fox"; + std::vector charvect(fox.begin(), fox.end()); + test.writeData(charvect,charvect.size()); + REQUIRE(1922388889 == test.getCRC()); + + +} + + +TEST_CASE("Test CRC3", "[testcrc3]") { + + BaseStream base; + CRCStream test(&base); + uint64_t number=7; + test.write(number); + REQUIRE(1877464688 == test.getCRC()); + + +} + + +TEST_CASE("Test CRC4", "[testcrc4]") { + + BaseStream base; + CRCStream test(&base); + uint32_t number=7; + test.write(number); + REQUIRE(3163809701 == test.getCRC()); + + +} + +TEST_CASE("Test CRC5", "[testcrc5]") { + + BaseStream base; + CRCStream test(&base); + uint16_t number=7; + test.write(number); + REQUIRE(244876344 == test.getCRC()); + + +} \ No newline at end of file diff --git a/libminifi/test/unit/PropertyTests.h b/libminifi/test/unit/PropertyTests.h new file mode 100644 index 0000000000..2633de9449 --- /dev/null +++ b/libminifi/test/unit/PropertyTests.h @@ -0,0 +1,106 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBMINIFI_TEST_UNIT_PROPERTYTESTS_H_ +#define LIBMINIFI_TEST_UNIT_PROPERTYTESTS_H_ + +#include "utils/StringUtils.h" +#include "../TestBase.h" +#include "Property.h" + +TEST_CASE("Test Boolean Conversion", "[testboolConversion]") { + + bool b; + REQUIRE(true == StringUtils::StringToBool("true",b)); + REQUIRE(true == StringUtils::StringToBool("True",b)); + REQUIRE(true == StringUtils::StringToBool("TRue",b)); + REQUIRE(true == StringUtils::StringToBool("tRUE",b)); + + REQUIRE(false == StringUtils::StringToBool("FALSE",b)); + REQUIRE(false == StringUtils::StringToBool("FALLSEY",b)); + REQUIRE(false == StringUtils::StringToBool("FaLSE",b)); + REQUIRE(false == StringUtils::StringToBool("false",b)); + +} + +TEST_CASE("Test Trimmer Right", "[testTrims]") { + + std::string test = "a quick brown fox jumped over the road\t\n"; + + REQUIRE(test.c_str()[test.length() - 1] == '\n'); + REQUIRE(test.c_str()[test.length() - 2] == '\t'); + test = StringUtils::trimRight(test); + + REQUIRE(test.c_str()[test.length() - 1] == 'd'); + REQUIRE(test.c_str()[test.length() - 2] == 'a'); + + test = "a quick brown fox jumped over the road\v\t"; + + REQUIRE(test.c_str()[test.length() - 1] == '\t'); + REQUIRE(test.c_str()[test.length() - 2] == '\v'); + + test = StringUtils::trimRight(test); + + REQUIRE(test.c_str()[test.length() - 1] == 'd'); + REQUIRE(test.c_str()[test.length() - 2] == 'a'); + + test = "a quick brown fox jumped over the road \f"; + + REQUIRE(test.c_str()[test.length() - 1] == '\f'); + REQUIRE(test.c_str()[test.length() - 2] == ' '); + + test = StringUtils::trimRight(test); + + REQUIRE(test.c_str()[test.length() - 1] == 'd'); + +} + +TEST_CASE("Test Trimmer Left", "[testTrims]") { + + std::string test = "\t\na quick brown fox jumped over the road\t\n"; + + REQUIRE(test.c_str()[0] == '\t'); + REQUIRE(test.c_str()[1] == '\n'); + + test = StringUtils::trimLeft(test); + + REQUIRE(test.c_str()[0] == 'a'); + REQUIRE(test.c_str()[1] == ' '); + + test = "\v\ta quick brown fox jumped over the road\v\t"; + + REQUIRE(test.c_str()[0] == '\v'); + REQUIRE(test.c_str()[1] == '\t'); + + test = StringUtils::trimLeft(test); + + REQUIRE(test.c_str()[0] == 'a'); + REQUIRE(test.c_str()[1] == ' '); + + test = " \fa quick brown fox jumped over the road \f"; + + REQUIRE(test.c_str()[0] == ' '); + REQUIRE(test.c_str()[1] == '\f'); + + test = StringUtils::trimLeft(test); + + REQUIRE(test.c_str()[0] == 'a'); + REQUIRE(test.c_str()[1] == ' '); + +} + +#endif /* LIBMINIFI_TEST_UNIT_PROPERTYTESTS_H_ */ diff --git a/libminifi/test/unit/SerializationTests.h b/libminifi/test/unit/SerializationTests.h index ba329822cb..e444f4a33e 100644 --- a/libminifi/test/unit/SerializationTests.h +++ b/libminifi/test/unit/SerializationTests.h @@ -25,6 +25,7 @@ #include #include #include +#include #define FMT_DEFAULT fmt_lower @@ -33,7 +34,7 @@ TEST_CASE("TestSetPortId", "[S2S1]"){ - Site2SitePeer peer("fake_host",65433); + Site2SitePeer peer(std::unique_ptr(new DataStream()),"fake_host",65433); Site2SiteClientProtocol protocol(&peer); @@ -55,7 +56,7 @@ TEST_CASE("TestSetPortId", "[S2S1]"){ TEST_CASE("TestSetPortIdUppercase", "[S2S2]"){ - Site2SitePeer peer("fake_host",65433); + Site2SitePeer peer(std::unique_ptr(new DataStream()),"fake_host",65433); Site2SiteClientProtocol protocol(&peer); diff --git a/libminifi/test/unit/SocketTests.h b/libminifi/test/unit/SocketTests.h new file mode 100644 index 0000000000..ec7e9e3574 --- /dev/null +++ b/libminifi/test/unit/SocketTests.h @@ -0,0 +1,190 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBMINIFI_TEST_UNIT_REALTIMEDATACOLLECTOR_H_ +#define LIBMINIFI_TEST_UNIT_REALTIMEDATACOLLECTOR_H_ + +#include "../../include/io/ClientSocket.h" +TEST_CASE("TestSocket", "[TestSocket1]") { + + Socket socket("localhost",8183); + REQUIRE(-1 == socket.initialize() ); + REQUIRE("localhost" == socket.getHostname()); + socket.closeStream(); + +} + +TEST_CASE("TestSocketWriteTest1", "[TestSocket2]") { + + Socket socket(Socket::getMyHostName(),8183); + REQUIRE(-1 == socket.initialize() ); + + socket.writeData(0,0); + + std::vector buffer; + buffer.push_back('a'); + + REQUIRE(-1 == socket.writeData(buffer,1)); + + socket.closeStream(); + +} + +TEST_CASE("TestSocketWriteTest2", "[TestSocket3]") { + + std::vector buffer; + buffer.push_back('a'); + + Socket server(Socket::getMyHostName(),9183,1); + + REQUIRE(-1 != server.initialize() ); + + Socket client(Socket::getMyHostName(),9183); + + REQUIRE(-1 != client.initialize() ); + + REQUIRE( 1 == client.writeData(buffer,1) ); + + std::vector readBuffer; + readBuffer.resize(1); + + REQUIRE( 1== server.readData(readBuffer,1) ); + + REQUIRE( readBuffer == buffer ); + + server.closeStream(); + + + client.closeStream(); + +} + +TEST_CASE("TestGetHostName", "[TestSocket4]") { + + REQUIRE( Socket::getMyHostName().length() > 0 ); + + + +} + +TEST_CASE("TestWriteEndian64", "[TestSocket4]") { + + std::vector buffer; + buffer.push_back('a'); + + Socket server(Socket::getMyHostName(),9183,1); + + REQUIRE(-1 != server.initialize() ); + + Socket client(Socket::getMyHostName(),9183); + + REQUIRE(-1 != client.initialize() ); + + uint64_t negative_one = -1; + REQUIRE( 8 == client.write(negative_one) ); + + + uint64_t negative_two = 0; + REQUIRE( 8 == server.read(negative_two) ); + + REQUIRE( negative_two == negative_one ); + + + server.closeStream(); + + + client.closeStream(); + +} + +TEST_CASE("TestWriteEndian32", "[TestSocket5]") { + + std::vector buffer; + buffer.push_back('a'); + + Socket server(Socket::getMyHostName(),9183,1); + + REQUIRE(-1 != server.initialize() ); + + Socket client(Socket::getMyHostName(),9183); + + REQUIRE(-1 != client.initialize() ); + + { + uint32_t negative_one = -1; + REQUIRE( 4 == client.write(negative_one) ); + + + uint32_t negative_two = 0; + REQUIRE( 4 == server.read(negative_two) ); + + REQUIRE( negative_two == negative_one ); + } + + { + uint16_t negative_one = -1; + REQUIRE( 2 == client.write(negative_one) ); + + + uint16_t negative_two = 0; + REQUIRE( 2 == server.read(negative_two) ); + + REQUIRE( negative_two == negative_one ); + } + server.closeStream(); + + + client.closeStream(); + +} + + +TEST_CASE("TestSocketWriteTestAfterClose", "[TestSocket6]") { + + std::vector buffer; + buffer.push_back('a'); + + Socket server(Socket::getMyHostName(),9183,1); + + REQUIRE(-1 != server.initialize() ); + + Socket client(Socket::getMyHostName(),9183); + + REQUIRE(-1 != client.initialize() ); + + REQUIRE( 1 == client.writeData(buffer,1) ); + + std::vector readBuffer; + readBuffer.resize(1); + + REQUIRE( 1== server.readData(readBuffer,1) ); + + REQUIRE( readBuffer == buffer ); + + client.closeStream(); + + REQUIRE( -1 == client.writeData(buffer,1) ); + + server.closeStream(); + + + +} + + +#endif /* LIBMINIFI_TEST_UNIT_REALTIMEDATACOLLECTOR_H_ */ diff --git a/libminifi/test/unit/Tests.cpp b/libminifi/test/unit/Tests.cpp index bc90386a2f..1089a6c582 100644 --- a/libminifi/test/unit/Tests.cpp +++ b/libminifi/test/unit/Tests.cpp @@ -22,3 +22,7 @@ #include "ProvenanceTests.h" #include "ProcessorTests.h" #include "SerializationTests.h" +#include "PropertyTests.h" +#include "SocketTests.h" +#include "TimeUtilsTest.h" +#include "CRCTests.h" diff --git a/libminifi/test/unit/TimeUtilsTest.h b/libminifi/test/unit/TimeUtilsTest.h new file mode 100644 index 0000000000..a4b1c445f0 --- /dev/null +++ b/libminifi/test/unit/TimeUtilsTest.h @@ -0,0 +1,30 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBMINIFI_TEST_UNIT_TIMEUTILSTEST_H_ +#define LIBMINIFI_TEST_UNIT_TIMEUTILSTEST_H_ + +#include "utils/TimeUtil.h" + +TEST_CASE("Test time conversion", "[testtimeconversion]"){ + REQUIRE ( "2017-02-16 20:14:56.196" == getTimeStr(1487276096196,true) ); +} + + + +#endif /* LIBMINIFI_TEST_UNIT_TIMEUTILSTEST_H_ */ From fee67f257e7435b8623b7d54122de3857f4cf595 Mon Sep 17 00:00:00 2001 From: Marc Parisi Date: Wed, 22 Feb 2017 15:47:00 -0500 Subject: [PATCH 2/3] MINIFI-206: When testing TLS connectivity realized that we need to perform CRC on big endian encoded byte array. While this seems obvious, it was not clear at the time. --- libminifi/include/io/CRCStream.h | 44 ++++++++++++-------------------- libminifi/src/ListenHTTP.cpp | 2 +- libminifi/test/unit/CRCTests.h | 8 +++--- 3 files changed, 21 insertions(+), 33 deletions(-) diff --git a/libminifi/include/io/CRCStream.h b/libminifi/include/io/CRCStream.h index 7fdfb2ba1d..01b61995ae 100644 --- a/libminifi/include/io/CRCStream.h +++ b/libminifi/include/io/CRCStream.h @@ -126,7 +126,7 @@ class CRCStream: public BaseStream { virtual short initialize() { - child_stream->initialize(); + child_stream_->initialize(); reset(); return 0; } @@ -135,7 +135,7 @@ class CRCStream: public BaseStream { void updateCRC(uint8_t *buffer, uint32_t length); uint64_t getCRC() { - return crc; + return crc_; } void reset(); @@ -155,30 +155,21 @@ class CRCStream: public BaseStream { return buf; } - template - void updateCRCPrimitive(const K &t) { - uint8_t bytes[sizeof t]; - std::copy(static_cast(static_cast(&t)), - static_cast(static_cast(&t)) + sizeof t, - bytes); - updateCRC(bytes, sizeof t); - } - - uint64_t crc; - T *child_stream; + uint64_t crc_; + T *child_stream_; }; template CRCStream::CRCStream(T *other) : - child_stream(other) { - crc = crc32(0L, Z_NULL, 0); + child_stream_(other) { + crc_ = crc32(0L, Z_NULL, 0); } template CRCStream::CRCStream(CRCStream &&move) : - crc(std::move(move.crc)), child_stream(std::move(move.child_stream)) { + crc_(std::move(move.crc_)), child_stream_(std::move(move.child_stream_)) { } @@ -192,8 +183,8 @@ int CRCStream::readData(std::vector &buf, int buflen) { template int CRCStream::readData(uint8_t *buf, int buflen) { - int ret = child_stream->read(buf, buflen); - crc = crc32(crc, buf, buflen); + int ret = child_stream_->read(buf, buflen); + crc_ = crc32(crc_, buf, buflen); return ret; } @@ -208,55 +199,52 @@ int CRCStream::writeData(std::vector &buf, int buflen) { template int CRCStream::writeData(uint8_t *value, int size) { - int ret = child_stream->write(value, size); - crc = crc32(crc, value, size); + int ret = child_stream_->write(value, size); + crc_ = crc32(crc_, value, size); return ret; } template void CRCStream::reset() { - crc = crc32(0L, Z_NULL, 0); + crc_ = crc32(0L, Z_NULL, 0); } template void CRCStream::updateCRC(uint8_t *buffer, uint32_t length) { - crc = crc32(crc, buffer, length); + crc_ = crc32(crc_, buffer, length); } template int CRCStream::write(uint64_t base_value, bool is_little_endian){ - updateCRCPrimitive(base_value); const uint64_t value = is_little_endian == 1 ? htonll_r(base_value) : base_value; uint8_t bytes[sizeof value]; std::copy(static_cast(static_cast(&value)), static_cast(static_cast(&value)) + sizeof value, bytes); - return child_stream->write(bytes,sizeof value); + return writeData(bytes,sizeof value); } template int CRCStream::write(uint32_t base_value, bool is_little_endian){ - updateCRCPrimitive(base_value); const uint32_t value = is_little_endian ? htonl(base_value) : base_value; uint8_t bytes[sizeof value]; std::copy(static_cast(static_cast(&value)), static_cast(static_cast(&value)) + sizeof value, bytes); - return child_stream->write(bytes,sizeof value); + return writeData(bytes,sizeof value); } template int CRCStream::write(uint16_t base_value, bool is_little_endian){ - updateCRCPrimitive(base_value); const uint16_t value = is_little_endian == 1 ? htons(base_value) : base_value; uint8_t bytes[sizeof value]; std::copy(static_cast(static_cast(&value)), static_cast(static_cast(&value)) + sizeof value, bytes); - return child_stream->write(bytes,sizeof value); + return writeData(bytes,sizeof value); } diff --git a/libminifi/src/ListenHTTP.cpp b/libminifi/src/ListenHTTP.cpp index b86dad68da..89ce1d233f 100644 --- a/libminifi/src/ListenHTTP.cpp +++ b/libminifi/src/ListenHTTP.cpp @@ -28,7 +28,7 @@ #include "ListenHTTP.h" -#include "TimeUtil.h" +#include "utils/TimeUtil.h" #include "ProcessContext.h" #include "ProcessSession.h" #include "ProcessSessionFactory.h" diff --git a/libminifi/test/unit/CRCTests.h b/libminifi/test/unit/CRCTests.h index 3e6665e4f4..de96eee2a6 100644 --- a/libminifi/test/unit/CRCTests.h +++ b/libminifi/test/unit/CRCTests.h @@ -49,7 +49,7 @@ TEST_CASE("Test CRC3", "[testcrc3]") { CRCStream test(&base); uint64_t number=7; test.write(number); - REQUIRE(1877464688 == test.getCRC()); + REQUIRE(4215687882 == test.getCRC()); } @@ -61,7 +61,7 @@ TEST_CASE("Test CRC4", "[testcrc4]") { CRCStream test(&base); uint32_t number=7; test.write(number); - REQUIRE(3163809701 == test.getCRC()); + REQUIRE(3206564543 == test.getCRC()); } @@ -72,7 +72,7 @@ TEST_CASE("Test CRC5", "[testcrc5]") { CRCStream test(&base); uint16_t number=7; test.write(number); - REQUIRE(244876344 == test.getCRC()); + REQUIRE(3753740124 == test.getCRC()); -} \ No newline at end of file +} From cde15f0aec95abfd047238daf7be6ebb25481ae5 Mon Sep 17 00:00:00 2001 From: Marc Parisi Date: Thu, 23 Feb 2017 18:53:14 -0500 Subject: [PATCH 3/3] MINIFI-206: Don't recurse all source files per peer review --- libminifi/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libminifi/CMakeLists.txt b/libminifi/CMakeLists.txt index cada08b0b1..d80d63b44f 100644 --- a/libminifi/CMakeLists.txt +++ b/libminifi/CMakeLists.txt @@ -52,7 +52,7 @@ include_directories(../thirdparty/yaml-cpp-yaml-cpp-0.5.3/include) include_directories(../thirdparty/civetweb-1.9.1/include) include_directories(include) -file(GLOB_RECURSE SOURCES "src/*.cpp") +file(GLOB SOURCES "src/*.cpp" "src/io/*.cpp" "src/utils/*.cpp") file(GLOB SPD_SOURCES "../include/spdlog/*") # Workaround the limitations of having a