From 7c64c3b91a48febe6cf804d8c73fbc3974e5c700 Mon Sep 17 00:00:00 2001 From: Marc Parisi Date: Mon, 27 Feb 2017 14:02:58 -0500 Subject: [PATCH 1/4] MINIFI-217: First commit. Updates namespaces and removes use of raw pointers for user facing API. --- CMakeLists.txt | 5 + libminifi/CMakeLists.txt | 4 +- libminifi/include/AppendHostInfo.h | 67 - libminifi/include/BaseLogger.h | 207 -- libminifi/include/Connection.h | 329 ++- .../include/EventDrivenSchedulingAgent.h | 62 +- libminifi/include/Exception.h | 28 +- libminifi/include/ExecuteProcess.h | 112 - libminifi/include/FlowControlProtocol.h | 504 ++-- libminifi/include/FlowController.h | 510 ++-- libminifi/include/FlowFileRecord.h | 291 +- libminifi/include/FlowFileRepository.h | 204 -- libminifi/include/GenerateFlowFile.h | 87 - libminifi/include/GetFile.h | 117 - libminifi/include/ListenHTTP.h | 116 - libminifi/include/ListenSyslog.h | 209 -- libminifi/include/LogAppenders.h | 298 --- libminifi/include/LogAttribute.h | 128 - libminifi/include/Logger.h | 200 -- libminifi/include/ProcessContext.h | 113 - libminifi/include/ProcessGroup.h | 187 -- libminifi/include/ProcessSession.h | 125 - libminifi/include/Processor.h | 365 --- libminifi/include/Property.h | 259 -- libminifi/include/Provenance.h | 604 ----- libminifi/include/PutFile.h | 88 - libminifi/include/RealTimeDataCollector.h | 131 - libminifi/include/Relationship.h | 87 - libminifi/include/RemoteProcessorGroupPort.h | 128 +- libminifi/include/Repository.h | 318 --- libminifi/include/ResourceClaim.h | 44 +- libminifi/include/SchedulingAgent.h | 134 +- libminifi/include/Site2SiteClientProtocol.h | 1003 +++---- libminifi/include/Site2SitePeer.h | 453 ++-- libminifi/include/TailFile.h | 93 - libminifi/include/ThreadedSchedulingAgent.h | 74 +- .../include/TimerDrivenSchedulingAgent.h | 62 +- .../include/core/ConfigurableComponent.h | 104 + libminifi/include/core/Connectable.h | 165 ++ libminifi/include/core/FlowFile.h | 266 ++ libminifi/include/core/ProcessContext.h | 114 + libminifi/include/core/ProcessGroup.h | 189 ++ libminifi/include/core/ProcessSession.h | 163 ++ .../{ => core}/ProcessSessionFactory.h | 54 +- libminifi/include/core/Processor.h | 270 ++ libminifi/include/core/ProcessorNode.h | 246 ++ libminifi/include/core/Property.h | 264 ++ libminifi/include/core/Relationship.h | 96 + libminifi/include/core/Repository.h | 149 ++ libminifi/include/core/Scheduling.h | 64 + libminifi/include/core/core.h | 141 + libminifi/include/core/logging/BaseLogger.h | 224 ++ libminifi/include/core/logging/LogAppenders.h | 301 +++ libminifi/include/core/logging/Logger.h | 214 ++ .../core/repository/FlowFIleRepository.h | 116 + libminifi/include/io/BaseStream.h | 256 +- libminifi/include/io/CRCStream.h | 415 ++- libminifi/include/io/ClientSocket.h | 411 +-- libminifi/include/io/DataStream.h | 188 +- libminifi/include/io/EndianCheck.h | 44 +- libminifi/include/io/Serializable.h | 309 +-- libminifi/include/io/SocketFactory.h | 99 +- libminifi/include/io/TLSSocket.h | 223 +- libminifi/include/io/validation.h | 27 +- libminifi/include/processors/AppendHostInfo.h | 80 + libminifi/include/processors/ExecuteProcess.h | 125 + .../include/processors/GenerateFlowFile.h | 98 + libminifi/include/processors/GetFile.h | 129 + libminifi/include/processors/ListenHTTP.h | 126 + libminifi/include/processors/ListenSyslog.h | 216 ++ libminifi/include/processors/LogAttribute.h | 130 + libminifi/include/processors/PutFile.h | 101 + .../processors/RealTimeDataCollector.h | 145 + libminifi/include/processors/TailFile.h | 105 + .../include/{ => properties}/Configure.h | 67 +- libminifi/include/provenance/Provenance.h | 564 ++++ .../include/provenance/ProvenanceRepository.h | 161 ++ libminifi/include/utils/FailurePolicy.h | 13 + libminifi/include/utils/StringUtils.h | 15 +- libminifi/src/AppendHostInfo.cpp | 97 - libminifi/src/Configure.cpp | 33 +- libminifi/src/Connection.cpp | 303 +-- libminifi/src/EventDrivenSchedulingAgent.cpp | 55 +- libminifi/src/ExecuteProcess.cpp | 251 -- libminifi/src/FlowControlProtocol.cpp | 884 +++--- libminifi/src/FlowController.cpp | 664 ++--- libminifi/src/FlowFileRecord.cpp | 332 +-- libminifi/src/FlowFileRepository.cpp | 282 -- libminifi/src/GetFile.cpp | 329 --- libminifi/src/ListenHTTP.cpp | 395 --- libminifi/src/ListenSyslog.cpp | 343 --- libminifi/src/LogAttribute.cpp | 159 -- libminifi/src/ProcessGroup.cpp | 307 --- libminifi/src/ProcessSession.cpp | 790 ------ libminifi/src/Processor.cpp | 526 ---- libminifi/src/Provenance.cpp | 566 ---- libminifi/src/PutFile.cpp | 203 -- libminifi/src/RealTimeDataCollector.cpp | 481 ---- libminifi/src/RemoteProcessorGroupPort.cpp | 178 +- libminifi/src/Repository.cpp | 140 - libminifi/src/ResourceClaim.cpp | 47 +- libminifi/src/SchedulingAgent.cpp | 107 +- libminifi/src/Site2SiteClientProtocol.cpp | 2377 ++++++++--------- libminifi/src/Site2SitePeer.cpp | 12 + libminifi/src/TailFile.cpp | 269 -- libminifi/src/ThreadedSchedulingAgent.cpp | 216 +- libminifi/src/TimerDrivenSchedulingAgent.cpp | 50 +- libminifi/src/core/ConfigurableComponent.cpp | 143 + libminifi/src/core/Connectable.cpp | 174 ++ libminifi/src/core/Core.cpp | 51 + libminifi/src/core/ProcessGroup.cpp | 314 +++ libminifi/src/core/ProcessSession.cpp | 974 +++++++ .../src/{ => core}/ProcessSessionFactory.cpp | 18 +- libminifi/src/core/Processor.cpp | 272 ++ libminifi/src/core/ProcessorNode.cpp | 65 + libminifi/src/core/Property.cpp | 58 + libminifi/src/core/Record.cpp | 219 ++ libminifi/src/core/Repository.cpp | 65 + .../src/{ => core/logging}/BaseLogger.cpp | 166 +- .../src/{ => core/logging}/LogAppenders.cpp | 16 +- libminifi/src/{ => core/logging}/Logger.cpp | 14 +- .../core/repository/FlowFileRepository.cpp | 107 + libminifi/src/io/BaseStream.cpp | 39 +- libminifi/src/io/ClientSocket.cpp | 471 ++-- libminifi/src/io/DataStream.cpp | 12 + libminifi/src/io/EndianCheck.cpp | 13 +- libminifi/src/io/Serializable.cpp | 13 +- libminifi/src/io/SocketFactory.cpp | 13 + libminifi/src/io/TLSSocket.cpp | 30 +- libminifi/src/processors/AppendHostInfo.cpp | 124 + libminifi/src/processors/ExecuteProcess.cpp | 255 ++ .../src/{ => processors}/GenerateFlowFile.cpp | 46 +- libminifi/src/processors/GetFile.cpp | 340 +++ libminifi/src/processors/ListenHTTP.cpp | 380 +++ libminifi/src/processors/ListenSyslog.cpp | 331 +++ libminifi/src/processors/LogAttribute.cpp | 176 ++ libminifi/src/processors/PutFile.cpp | 213 ++ .../src/processors/RealTimeDataCollector.cpp | 480 ++++ libminifi/src/processors/TailFile.cpp | 271 ++ libminifi/src/provenance/Provenance.cpp | 578 ++++ .../src/provenance/ProvenanceRepository.cpp | 75 + libminifi/test/Server.cpp | 38 +- libminifi/test/TestBase.h | 81 +- libminifi/test/unit/CRCTests.cpp | 76 +- libminifi/test/unit/LoggerTests.cpp | 151 +- libminifi/test/unit/ProcessorTests.cpp | 295 +- libminifi/test/unit/PropertyTests.cpp | 31 +- libminifi/test/unit/ProvenanceTestHelper.h | 71 +- libminifi/test/unit/ProvenanceTests.cpp | 112 +- libminifi/test/unit/SerializationTests.cpp | 10 +- libminifi/test/unit/SocketTests.cpp | 160 +- libminifi/test/unit/TimeUtilsTest.cpp | 21 + main/MiNiFiMain.cpp | 208 +- 153 files changed, 17581 insertions(+), 15518 deletions(-) delete mode 100644 libminifi/include/AppendHostInfo.h delete mode 100644 libminifi/include/BaseLogger.h delete mode 100644 libminifi/include/ExecuteProcess.h delete mode 100644 libminifi/include/FlowFileRepository.h delete mode 100644 libminifi/include/GenerateFlowFile.h delete mode 100644 libminifi/include/GetFile.h delete mode 100644 libminifi/include/ListenHTTP.h delete mode 100644 libminifi/include/ListenSyslog.h delete mode 100644 libminifi/include/LogAppenders.h delete mode 100644 libminifi/include/LogAttribute.h delete mode 100644 libminifi/include/Logger.h delete mode 100644 libminifi/include/ProcessContext.h delete mode 100644 libminifi/include/ProcessGroup.h delete mode 100644 libminifi/include/ProcessSession.h delete mode 100644 libminifi/include/Processor.h delete mode 100644 libminifi/include/Property.h delete mode 100644 libminifi/include/Provenance.h delete mode 100644 libminifi/include/PutFile.h delete mode 100644 libminifi/include/RealTimeDataCollector.h delete mode 100644 libminifi/include/Relationship.h delete mode 100644 libminifi/include/Repository.h delete mode 100644 libminifi/include/TailFile.h create mode 100644 libminifi/include/core/ConfigurableComponent.h create mode 100644 libminifi/include/core/Connectable.h create mode 100644 libminifi/include/core/FlowFile.h create mode 100644 libminifi/include/core/ProcessContext.h create mode 100644 libminifi/include/core/ProcessGroup.h create mode 100644 libminifi/include/core/ProcessSession.h rename libminifi/include/{ => core}/ProcessSessionFactory.h (54%) create mode 100644 libminifi/include/core/Processor.h create mode 100644 libminifi/include/core/ProcessorNode.h create mode 100644 libminifi/include/core/Property.h create mode 100644 libminifi/include/core/Relationship.h create mode 100644 libminifi/include/core/Repository.h create mode 100644 libminifi/include/core/Scheduling.h create mode 100644 libminifi/include/core/core.h create mode 100644 libminifi/include/core/logging/BaseLogger.h create mode 100644 libminifi/include/core/logging/LogAppenders.h create mode 100644 libminifi/include/core/logging/Logger.h create mode 100644 libminifi/include/core/repository/FlowFIleRepository.h create mode 100644 libminifi/include/processors/AppendHostInfo.h create mode 100644 libminifi/include/processors/ExecuteProcess.h create mode 100644 libminifi/include/processors/GenerateFlowFile.h create mode 100644 libminifi/include/processors/GetFile.h create mode 100644 libminifi/include/processors/ListenHTTP.h create mode 100644 libminifi/include/processors/ListenSyslog.h create mode 100644 libminifi/include/processors/LogAttribute.h create mode 100644 libminifi/include/processors/PutFile.h create mode 100644 libminifi/include/processors/RealTimeDataCollector.h create mode 100644 libminifi/include/processors/TailFile.h rename libminifi/include/{ => properties}/Configure.h (71%) create mode 100644 libminifi/include/provenance/Provenance.h create mode 100644 libminifi/include/provenance/ProvenanceRepository.h delete mode 100644 libminifi/src/AppendHostInfo.cpp delete mode 100644 libminifi/src/ExecuteProcess.cpp delete mode 100644 libminifi/src/FlowFileRepository.cpp delete mode 100644 libminifi/src/GetFile.cpp delete mode 100644 libminifi/src/ListenHTTP.cpp delete mode 100644 libminifi/src/ListenSyslog.cpp delete mode 100644 libminifi/src/LogAttribute.cpp delete mode 100644 libminifi/src/ProcessGroup.cpp delete mode 100644 libminifi/src/ProcessSession.cpp delete mode 100644 libminifi/src/Processor.cpp delete mode 100644 libminifi/src/Provenance.cpp delete mode 100644 libminifi/src/PutFile.cpp delete mode 100644 libminifi/src/RealTimeDataCollector.cpp delete mode 100644 libminifi/src/Repository.cpp delete mode 100644 libminifi/src/TailFile.cpp create mode 100644 libminifi/src/core/ConfigurableComponent.cpp create mode 100644 libminifi/src/core/Connectable.cpp create mode 100644 libminifi/src/core/Core.cpp create mode 100644 libminifi/src/core/ProcessGroup.cpp create mode 100644 libminifi/src/core/ProcessSession.cpp rename libminifi/src/{ => core}/ProcessSessionFactory.cpp (74%) create mode 100644 libminifi/src/core/Processor.cpp create mode 100644 libminifi/src/core/ProcessorNode.cpp create mode 100644 libminifi/src/core/Property.cpp create mode 100644 libminifi/src/core/Record.cpp create mode 100644 libminifi/src/core/Repository.cpp rename libminifi/src/{ => core/logging}/BaseLogger.cpp (50%) rename libminifi/src/{ => core/logging}/LogAppenders.cpp (80%) rename libminifi/src/{ => core/logging}/Logger.cpp (77%) create mode 100644 libminifi/src/core/repository/FlowFileRepository.cpp create mode 100644 libminifi/src/processors/AppendHostInfo.cpp create mode 100644 libminifi/src/processors/ExecuteProcess.cpp rename libminifi/src/{ => processors}/GenerateFlowFile.cpp (67%) create mode 100644 libminifi/src/processors/GetFile.cpp create mode 100644 libminifi/src/processors/ListenHTTP.cpp create mode 100644 libminifi/src/processors/ListenSyslog.cpp create mode 100644 libminifi/src/processors/LogAttribute.cpp create mode 100644 libminifi/src/processors/PutFile.cpp create mode 100644 libminifi/src/processors/RealTimeDataCollector.cpp create mode 100644 libminifi/src/processors/TailFile.cpp create mode 100644 libminifi/src/provenance/Provenance.cpp create mode 100644 libminifi/src/provenance/ProvenanceRepository.cpp create mode 100644 libminifi/test/unit/TimeUtilsTest.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f3f7f82de6..62c0360dae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -128,6 +128,11 @@ enable_testing(test) target_include_directories(tests PRIVATE BEFORE ${LEVELDB_INCLUDE_DIRS}) target_include_directories(tests PRIVATE BEFORE "include") target_include_directories(tests PRIVATE BEFORE "libminifi/include/") + target_include_directories(tests PRIVATE BEFORE "libminifi/include/core") + target_include_directories(tests PRIVATE BEFORE "libminifi/include/io") + target_include_directories(tests PRIVATE BEFORE "libminifi/include/utils") + target_include_directories(tests PRIVATE BEFORE "libminifi/include/processors") + target_include_directories(tests PRIVATE BEFORE "libminifi/include/provenance") target_link_libraries(tests ${CMAKE_THREAD_LIBS_INIT} ${UUID_LIBRARIES} ${LEVELDB_LIBRARIES} ${OPENSSL_LIBRARIES} minifi yaml-cpp c-library civetweb-cpp) add_test(NAME LibMinifiTests COMMAND tests) diff --git a/libminifi/CMakeLists.txt b/libminifi/CMakeLists.txt index 69975ad81c..6a48483b86 100644 --- a/libminifi/CMakeLists.txt +++ b/libminifi/CMakeLists.txt @@ -48,7 +48,7 @@ include(CheckCXXCompilerFlag) CHECK_CXX_COMPILER_FLAG("-std=c++11" COMPILER_SUPPORTS_CXX11) CHECK_CXX_COMPILER_FLAG("-std=c++0x" COMPILER_SUPPORTS_CXX0X) if(COMPILER_SUPPORTS_CXX11) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -g") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") elseif(COMPILER_SUPPORTS_CXX0X) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x") else() @@ -60,7 +60,7 @@ include_directories(../thirdparty/yaml-cpp-yaml-cpp-0.5.3/include) include_directories(../thirdparty/civetweb-1.9.1/include) include_directories(include) -file(GLOB SOURCES "src/*.cpp" "src/io/*.cpp" "src/utils/*.cpp") +file(GLOB SOURCES "src/core/logging/*.cpp" "src/io/*.cpp" "src/core/*.cpp" "src/provenance/*.cpp" "src/processors/*.cpp" "src/*.cpp") file(GLOB SPD_SOURCES "../include/spdlog/*") # Workaround the limitations of having a diff --git a/libminifi/include/AppendHostInfo.h b/libminifi/include/AppendHostInfo.h deleted file mode 100644 index 8d9dd8f448..0000000000 --- a/libminifi/include/AppendHostInfo.h +++ /dev/null @@ -1,67 +0,0 @@ -/** - * @file AppendHostInfo.h - * AppendHostInfo class declaration - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __APPEND_HOSTINFO_H__ -#define __APPEND_HOSTINFO_H__ - -#include "FlowFileRecord.h" -#include "Processor.h" -#include "ProcessSession.h" - -//! AppendHostInfo Class -class AppendHostInfo : public Processor -{ -public: - //! Constructor - /*! - * Create a new processor - */ - AppendHostInfo(std::string name, uuid_t uuid = NULL) - : Processor(name, uuid) - { - logger_ = Logger::getLogger(); - } - //! Destructor - virtual ~AppendHostInfo() - { - } - //! Processor Name - static const std::string ProcessorName; - //! Supported Properties - static Property InterfaceName; - static Property HostAttribute; - static Property IPAttribute; - - //! Supported Relationships - static Relationship Success; - -public: - //! OnTrigger method, implemented by NiFi AppendHostInfo - virtual void onTrigger(ProcessContext *context, ProcessSession *session); - //! Initialize, over write by NiFi AppendHostInfo - virtual void initialize(void); - -protected: - -private: - //! Logger - std::shared_ptr logger_; -}; - -#endif diff --git a/libminifi/include/BaseLogger.h b/libminifi/include/BaseLogger.h deleted file mode 100644 index e9d2a02d89..0000000000 --- a/libminifi/include/BaseLogger.h +++ /dev/null @@ -1,207 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef LIBMINIFI_INCLUDE_BASELOGGER_H_ -#define LIBMINIFI_INCLUDE_BASELOGGER_H_ - -#include -#include -#include "spdlog/spdlog.h" -#include -//! 5M default log file size -#define DEFAULT_LOG_FILE_SIZE (5*1024*1024) -//! 3 log files rotation -#define DEFAULT_LOG_FILE_NUMBER 3 -#define LOG_NAME "minifi log" -#define LOG_FILE_NAME "minifi-app.log" - -/** - * Log level enumeration. - */ -typedef enum { - trace = 0, debug = 1, info = 2, warn = 3, err = 4, critical = 5, off = 6 -} LOG_LEVEL_E; - -#define LOG_BUFFER_SIZE 1024 -#define FILL_BUFFER char buffer[LOG_BUFFER_SIZE]; \ - va_list args; \ - va_start(args, format); \ - std::vsnprintf(buffer, LOG_BUFFER_SIZE,format, args); \ - va_end(args); - -/** - * Base class that represents a logger configuration. - */ -class BaseLogger { - -public: - static const char *nifi_log_level; - static const char *nifi_log_appender; - - /** - * Base Constructor - */ - BaseLogger() { - setLogLevel("info"); - logger_ = nullptr; - stderr_ = nullptr; - } - - /** - * Logger configuration constructorthat will set the base log level. - * @param config incoming configuration. - */ - BaseLogger(std::string log_level, std::shared_ptr logger) : logger_(logger) { - setLogLevel(log_level); - - } - - virtual ~BaseLogger() { - - } - - /** - * Move constructor that will atomically swap configuration - * shared pointers. - */ - BaseLogger(const BaseLogger &&other) : - configured_level_(other.configured_level_.load()) { - // must atomically exchange the pointers - logger_ = std::move(other.logger_); - set_error_logger(other.stderr_); - - } - - /** - * Returns the log level for this instance. - */ - virtual LOG_LEVEL_E getLogLevel() const { - return configured_level_; - } - - /** - * @brief Log error message - * @param format format string ('man printf' for syntax) - * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match - */ - virtual void log_error(const char * const format, ...); - /** - * @brief Log warn message - * @param format format string ('man printf' for syntax) - * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match - */ - virtual void log_warn(const char * const format, ...); - /** - * @brief Log info message - * @param format format string ('man printf' for syntax) - * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match - */ - virtual void log_info(const char * const format, ...); - /** - * @brief Log debug message - * @param format format string ('man printf' for syntax) - * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match - */ - virtual void log_debug(const char * const format, ...); - /** - * @brief Log trace message - * @param format format string ('man printf' for syntax) - * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match - */ - virtual void log_trace(const char * const format, ...); - - /** - * @brief Log error message - * @param format format string ('man printf' for syntax) - * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match - */ - virtual void log_str(LOG_LEVEL_E level,const std::string &buffer); - - /** - * Sets the log level for this instance based on the string - * @param level desired log leve. - * @param defaultLevel default level if we cannot match level. - */ - virtual void setLogLevel(const std::string &level, - LOG_LEVEL_E defaultLevel = info); - - /** - * Sets the log level atomic and sets it - * within logger if it can - * @param level desired log level. - */ - virtual void setLogLevel(LOG_LEVEL_E level) { - configured_level_ = level; - setLogLevel(); - } - - bool shouldLog(LOG_LEVEL_E level) - { - return level >= configured_level_.load(std::memory_order_relaxed); - } - - /** - * Move operator overload - */ - BaseLogger &operator=(const BaseLogger &&other) { - configured_level_ = (other.configured_level_.load()); - // must atomically exchange the pointers - logger_ = std::move(other.logger_); - set_error_logger(other.stderr_); - return *this; - } - -protected: - - - - /** - * Logger configuration constructorthat will set the base log level. - * @param config incoming configuration. - */ - BaseLogger(std::string log_level) : logger_(nullptr) { - setLogLevel(log_level); - } - - - void setLogger(std::shared_ptr logger) { - logger_ = logger; - } - - /** - * Since a thread may be using stderr and it can be null, - * we must atomically exchange the shared pointers. - * @param other other shared pointer. can be null ptr - */ - void set_error_logger(std::shared_ptr other); - - /** - * Sets the log level on the spdlogger if it is not null. - */ - void setLogLevel() { - if (logger_ != nullptr) - logger_->set_level( - (spdlog::level::level_enum) configured_level_.load()); - - } - - std::atomic configured_level_; - std::shared_ptr logger_; - std::shared_ptr stderr_; -}; - -#endif /* LIBMINIFI_INCLUDE_BASELOGGER_H_ */ diff --git a/libminifi/include/Connection.h b/libminifi/include/Connection.h index 5af0d2f5a9..1ea38b9a18 100644 --- a/libminifi/include/Connection.h +++ b/libminifi/include/Connection.h @@ -28,179 +28,170 @@ #include #include -#include "FlowFileRecord.h" -#include "Logger.h" -#include "Relationship.h" +#include "core/Connectable.h" +#include "core/logging/Logger.h" +#include "core/Relationship.h" +#include "core/Connectable.h" +#include "core/FlowFile.h" -//! Forwarder declaration -class Processor; +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +// Connection Class -//! Connection Class -class Connection -{ -public: - //! Constructor - /*! - * Create a new processor - */ - explicit Connection(std::string name, uuid_t uuid = NULL, uuid_t srcUUID = NULL, uuid_t destUUID = NULL); - //! Destructor - virtual ~Connection() {} - //! Set Connection Name - void setName(std::string name) { - _name = name; - } - //! Get Process Name - std::string getName(void) { - return (_name); - } - //! Set UUID - void setUUID(uuid_t uuid) { - uuid_copy(_uuid, uuid); - } - //! Set Source Processor UUID - void setSourceProcessorUUID(uuid_t uuid) { - uuid_copy(_srcUUID, uuid); - } - //! Set Destination Processor UUID - void setDestinationProcessorUUID(uuid_t uuid) { - uuid_copy(_destUUID, uuid); - } - //! Get Source Processor UUID - void getSourceProcessorUUID(uuid_t uuid) { - uuid_copy(uuid, _srcUUID); - } - //! Get Destination Processor UUID - void getDestinationProcessorUUID(uuid_t uuid) { - uuid_copy(uuid, _destUUID); - } - //! Get UUID - bool getUUID(uuid_t uuid) { - if (uuid) - { - uuid_copy(uuid, _uuid); - return true; - } - else - return false; - } - //! Get UUID Str - std::string getUUIDStr() { - return _uuidStr; - } - //! Set Connection Source Processor - void setSourceProcessor(Processor *source) { - _srcProcessor = source; - } - // ! Get Connection Source Processor - Processor *getSourceProcessor() { - return _srcProcessor; - } - //! Set Connection Destination Processor - void setDestinationProcessor(Processor *dest) { - _destProcessor = dest; - } - // ! Get Connection Destination Processor - Processor *getDestinationProcessor() { - return _destProcessor; - } - //! Set Connection relationship - void setRelationship(Relationship relationship) { - _relationship = relationship; - } - // ! Get Connection relationship - Relationship getRelationship() { - return _relationship; - } - //! Set Max Queue Size - void setMaxQueueSize(uint64_t size) - { - _maxQueueSize = size; - } - //! Get Max Queue Size - uint64_t getMaxQueueSize() - { - return _maxQueueSize; - } - //! Set Max Queue Data Size - void setMaxQueueDataSize(uint64_t size) - { - _maxQueueDataSize = size; - } - //! Get Max Queue Data Size - uint64_t getMaxQueueDataSize() - { - return _maxQueueDataSize; - } - //! Set Flow expiration duration in millisecond - void setFlowExpirationDuration(uint64_t duration) - { - _expiredDuration = duration; - } - //! Get Flow expiration duration in millisecond - uint64_t getFlowExpirationDuration() - { - return _expiredDuration; - } - //! Check whether the queue is empty - bool isEmpty(); - //! Check whether the queue is full to apply back pressure - bool isFull(); - //! Get queue size - uint64_t getQueueSize() { - std::lock_guard lock(_mtx); - return _queue.size(); - } - //! Get queue data size - uint64_t getQueueDataSize() - { - return _maxQueueDataSize; - } - //! Put the flow file into queue - void put(FlowFileRecord *flow); - //! Poll the flow file from queue, the expired flow file record also being returned - FlowFileRecord *poll(std::set &expiredFlowRecords); - //! Drain the flow records - void drain(); +class Connection : public core::Connectable, + public std::enable_shared_from_this { + public: + // Constructor + /* + * Create a new processor + */ + explicit Connection(std::shared_ptr flow_repository, + std::string name, uuid_t uuid = NULL, uuid_t srcUUID = + NULL, + uuid_t destUUID = NULL); + // Destructor + virtual ~Connection() { + } -protected: - //! A global unique identifier - uuid_t _uuid; - //! Source Processor UUID - uuid_t _srcUUID; - //! Destination Processor UUID - uuid_t _destUUID; - //! Connection Name - std::string _name; - //! Relationship for this connection - Relationship _relationship; - //! Source Processor (ProcessNode/Port) - Processor *_srcProcessor; - //! Destination Processor (ProcessNode/Port) - Processor *_destProcessor; - //! Max queue size to apply back pressure - std::atomic _maxQueueSize; - //! Max queue data size to apply back pressure - std::atomic _maxQueueDataSize; - //! Flow File Expiration Duration in= MilliSeconds - std::atomic _expiredDuration; - //! UUID string - std::string _uuidStr; + // Set Source Processor UUID + void setSourceUUID(uuid_t uuid) { + uuid_copy(src_uuid_, uuid); + } + // Set Destination Processor UUID + void setDestinationUUID(uuid_t uuid) { + uuid_copy(dest_uuid_, uuid); + } + // Get Source Processor UUID + void getSourceUUID(uuid_t uuid) { + uuid_copy(uuid, src_uuid_); + } + // Get Destination Processor UUID + void getDestinationUUID(uuid_t uuid) { + uuid_copy(uuid, dest_uuid_); + } -private: - //! Mutex for protection - std::mutex _mtx; - //! Queued data size - std::atomic _queuedDataSize; - //! Queue for the Flow File - std::queue _queue; - //! Logger - std::shared_ptr logger_; - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - Connection(const Connection &parent); - Connection &operator=(const Connection &parent); + // Set Connection Source Processor + void setSource(std::shared_ptr source) { + source_connectable_ = source; + } + // ! Get Connection Source Processor + std::shared_ptr getSource() { + return source_connectable_; + } + // Set Connection Destination Processor + void setDestination(std::shared_ptr dest) { + dest_connectable_ = dest; + } + // ! Get Connection Destination Processor + std::shared_ptr getDestination() { + return dest_connectable_; + } + // Set Connection relationship + void setRelationship(core::Relationship relationship) { + relationship_ = relationship; + } + // ! Get Connection relationship + core::Relationship getRelationship() { + return relationship_; + } + // Set Max Queue Size + void setMaxQueueSize(uint64_t size) { + max_queue_size_ = size; + } + // Get Max Queue Size + uint64_t getMaxQueueSize() { + return max_queue_size_; + } + // Set Max Queue Data Size + void setMaxQueueDataSize(uint64_t size) { + max_data_queue_size_ = size; + } + // Get Max Queue Data Size + uint64_t getMaxQueueDataSize() { + return max_data_queue_size_; + } + // Set Flow expiration duration in millisecond + void setFlowExpirationDuration(uint64_t duration) { + expired_duration_ = duration; + } + // Get Flow expiration duration in millisecond + uint64_t getFlowExpirationDuration() { + return expired_duration_; + } + // Check whether the queue is empty + bool isEmpty(); + // Check whether the queue is full to apply back pressure + bool isFull(); + // Get queue size + uint64_t getQueueSize() { + std::lock_guard lock(mutex_); + return queue_.size(); + } + // Get queue data size + uint64_t getQueueDataSize() { + return max_data_queue_size_; + } + // Put the flow file into queue + void put(std::shared_ptr flow); + // Poll the flow file from queue, the expired flow file record also being returned + std::shared_ptr poll( + std::set> &expiredFlowRecords); + // Drain the flow records + void drain(); -}; + void yield() { + + } + + bool isWorkAvailable() { + return !isEmpty(); + } + + bool isRunning() { + return true; + } + protected: + // Source Processor UUID + uuid_t src_uuid_; + // Destination Processor UUID + uuid_t dest_uuid_; + // Relationship for this connection + core::Relationship relationship_; + // Source Processor (ProcessNode/Port) + std::shared_ptr source_connectable_; + // Destination Processor (ProcessNode/Port) + std::shared_ptr dest_connectable_; + // Max queue size to apply back pressure + std::atomic max_queue_size_; + // Max queue data size to apply back pressure + std::atomic max_data_queue_size_; + // Flow File Expiration Duration in= MilliSeconds + std::atomic expired_duration_; + // flow file repository + std::shared_ptr flow_repository_; + + private: + // Mutex for protection + std::mutex mutex_; + // Queued data size + std::atomic queued_data_size_; + // Queue for the Flow File + std::queue> queue_; + // flow repository + std::shared_ptr flow_repository_; + // Logger + std::shared_ptr logger_; + // Prevent default copy constructor and assignment operation + // Only support pass by reference or pointer + Connection(const Connection &parent); + Connection &operator=(const Connection &parent); + +}; +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ #endif diff --git a/libminifi/include/EventDrivenSchedulingAgent.h b/libminifi/include/EventDrivenSchedulingAgent.h index b32f84f323..ab59725b93 100644 --- a/libminifi/include/EventDrivenSchedulingAgent.h +++ b/libminifi/include/EventDrivenSchedulingAgent.h @@ -20,36 +20,46 @@ #ifndef __EVENT_DRIVEN_SCHEDULING_AGENT_H__ #define __EVENT_DRIVEN_SCHEDULING_AGENT_H__ -#include "Logger.h" -#include "Processor.h" -#include "ProcessContext.h" +#include "core/logging/Logger.h" +#include "core/Processor.h" +#include "core/ProcessContext.h" +#include "core/ProcessSessionFactory.h" #include "ThreadedSchedulingAgent.h" -//! EventDrivenSchedulingAgent Class -class EventDrivenSchedulingAgent : public ThreadedSchedulingAgent -{ -public: - //! Constructor - /*! - * Create a new processor - */ - EventDrivenSchedulingAgent() - : ThreadedSchedulingAgent() - { - } - //! Destructor - virtual ~EventDrivenSchedulingAgent() - { - } - //! Run function for the thread - void run(Processor *processor, ProcessContext *processContext, ProcessSessionFactory *sessionFactory); +namespace org { +namespace apache { +namespace nifi { +namespace minifi { -private: - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - EventDrivenSchedulingAgent(const EventDrivenSchedulingAgent &parent); - EventDrivenSchedulingAgent &operator=(const EventDrivenSchedulingAgent &parent); +// EventDrivenSchedulingAgent Class +class EventDrivenSchedulingAgent : public ThreadedSchedulingAgent { + public: + // Constructor + /*! + * Create a new processor + */ + EventDrivenSchedulingAgent(std::shared_ptr repo) + : ThreadedSchedulingAgent(repo) { + } + // Destructor + virtual ~EventDrivenSchedulingAgent() { + } + // Run function for the thread + void run(std::shared_ptr processor, + core::ProcessContext *processContext, + core::ProcessSessionFactory *sessionFactory); + + private: + // Prevent default copy constructor and assignment operation + // Only support pass by reference or pointer + EventDrivenSchedulingAgent(const EventDrivenSchedulingAgent &parent); + EventDrivenSchedulingAgent &operator=( + const EventDrivenSchedulingAgent &parent); }; +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ #endif diff --git a/libminifi/include/Exception.h b/libminifi/include/Exception.h index 1e02fa5ea6..a0c70e6217 100644 --- a/libminifi/include/Exception.h +++ b/libminifi/include/Exception.h @@ -26,7 +26,13 @@ #include #include -//! ExceptionType + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { + +// ExceptionType enum ExceptionType { FILE_OPERATION_EXCEPTION = 0, @@ -39,7 +45,7 @@ enum ExceptionType MAX_EXCEPTION }; -//! Exception String +// Exception String static const char *ExceptionStr[MAX_EXCEPTION] = { "File Operation", @@ -51,7 +57,7 @@ static const char *ExceptionStr[MAX_EXCEPTION] = "General Operation" }; -//! Exception Type to String +// Exception Type to String inline const char *ExceptionTypeToString(ExceptionType type) { if (type < MAX_EXCEPTION) @@ -60,17 +66,17 @@ inline const char *ExceptionTypeToString(ExceptionType type) return NULL; } -//! Exception Class +// Exception Class class Exception : public std::exception { public: - //! Constructor + // Constructor /*! * Create a new flow record */ Exception(ExceptionType type, const char *errorMsg) : _type(type), _errorMsg(errorMsg) { } - //! Destructor + // Destructor virtual ~Exception() throw () {} virtual const char * what() const throw () { @@ -82,13 +88,17 @@ class Exception : public std::exception private: - //! Exception type + // Exception type ExceptionType _type; - //! Exception detailed information + // Exception detailed information std::string _errorMsg; - //! Hold the what result + // Hold the what result mutable std::string _whatStr; }; +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ #endif diff --git a/libminifi/include/ExecuteProcess.h b/libminifi/include/ExecuteProcess.h deleted file mode 100644 index 3ddd815155..0000000000 --- a/libminifi/include/ExecuteProcess.h +++ /dev/null @@ -1,112 +0,0 @@ -/** - * @file ExecuteProcess.h - * ExecuteProcess class declaration - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __EXECUTE_PROCESS_H__ -#define __EXECUTE_PROCESS_H__ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "FlowFileRecord.h" -#include "Processor.h" -#include "ProcessSession.h" - -//! ExecuteProcess Class -class ExecuteProcess : public Processor -{ -public: - //! Constructor - /*! - * Create a new processor - */ - ExecuteProcess(std::string name, uuid_t uuid = NULL) - : Processor(name, uuid) - { - logger_ = Logger::getLogger(); - _redirectErrorStream = false; - _batchDuration = 0; - _workingDir = "."; - _processRunning = false; - _pid = 0; - } - //! Destructor - virtual ~ExecuteProcess() - { - if (_processRunning && _pid > 0) - kill(_pid, SIGTERM); - } - //! Processor Name - static const std::string ProcessorName; - //! Supported Properties - static Property Command; - static Property CommandArguments; - static Property WorkingDir; - static Property BatchDuration; - static Property RedirectErrorStream; - //! Supported Relationships - static Relationship Success; - - //! Nest Callback Class for write stream - class WriteCallback : public OutputStreamCallback - { - public: - WriteCallback(char *data, uint64_t size) - : _data(data), _dataSize(size) {} - char *_data; - uint64_t _dataSize; - void process(std::ofstream *stream) { - if (_data && _dataSize > 0) - stream->write(_data, _dataSize); - } - }; - -public: - //! OnTrigger method, implemented by NiFi ExecuteProcess - virtual void onTrigger(ProcessContext *context, ProcessSession *session); - //! Initialize, over write by NiFi ExecuteProcess - virtual void initialize(void); - -protected: - -private: - //! Logger - std::shared_ptr logger_; - //! Property - std::string _command; - std::string _commandArgument; - std::string _workingDir; - int64_t _batchDuration; - bool _redirectErrorStream; - //! Full command - std::string _fullCommand; - //! whether the process is running - bool _processRunning; - int _pipefd[2]; - pid_t _pid; -}; - -#endif diff --git a/libminifi/include/FlowControlProtocol.h b/libminifi/include/FlowControlProtocol.h index ffc454a495..664f612f1b 100644 --- a/libminifi/include/FlowControlProtocol.h +++ b/libminifi/include/FlowControlProtocol.h @@ -32,310 +32,288 @@ #include #include #include + +#include "core/Property.h" #include "Configure.h" -#include "Logger.h" -#include "Property.h" +#include "core/logging/Logger.h" + -//! Forwarder declaration +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +// Forwarder declaration class FlowController; #define DEFAULT_NIFI_SERVER_PORT 9000 #define DEFAULT_REPORT_INTERVAL 1000 // 1 sec #define MAX_READ_TIMEOUT 30000 // 30 seconds -//! FlowControl Protocol Msg Type +// FlowControl Protocol Msg Type typedef enum { - REGISTER_REQ, // Device Register Request from device to server which contain device serial number, current running flow YAML version - REGISTER_RESP, // Device Register Respond from server to device, may contain new flow.YAML from server ask device to apply and also device report interval - REPORT_REQ, // Period Device Report from device to server which contain device serial number, current running flow YAML name/version and other period report info - REPORT_RESP, // Report Respond from server to device, may ask device to update flow YAML or processor property - MAX_FLOW_CONTROL_MSG_TYPE + REGISTER_REQ, // Device Register Request from device to server which contain device serial number, current running flow YAML version + REGISTER_RESP, // Device Register Respond from server to device, may contain new flow.YAML from server ask device to apply and also device report interval + REPORT_REQ, // Period Device Report from device to server which contain device serial number, current running flow YAML name/version and other period report info + REPORT_RESP, // Report Respond from server to device, may ask device to update flow YAML or processor property + MAX_FLOW_CONTROL_MSG_TYPE } FlowControlMsgType; -//! FlowControl Protocol Msg Type String -static const char *FlowControlMsgTypeStr[MAX_FLOW_CONTROL_MSG_TYPE] = -{ - "REGISTER_REQ", - "REGISTER_RESP", - "REPORT_REQ", - "REPORT_RESP" -}; +// FlowControl Protocol Msg Type String +static const char *FlowControlMsgTypeStr[MAX_FLOW_CONTROL_MSG_TYPE] = { + "REGISTER_REQ", "REGISTER_RESP", "REPORT_REQ", "REPORT_RESP" }; -//! Flow Control Msg Type to String -inline const char *FlowControlMsgTypeToStr(FlowControlMsgType type) -{ - if (type < MAX_FLOW_CONTROL_MSG_TYPE) - return FlowControlMsgTypeStr[type]; - else - return NULL; +// Flow Control Msg Type to String +inline const char *FlowControlMsgTypeToStr(FlowControlMsgType type) { + if (type < MAX_FLOW_CONTROL_MSG_TYPE) + return FlowControlMsgTypeStr[type]; + else + return NULL; } -//! FlowControll Protocol Msg ID (Some Messages are fix length, Some are variable length (TLV) +// FlowControll Protocol Msg ID (Some Messages are fix length, Some are variable length (TLV) typedef enum { - //Fix length 8 bytes: client to server in register request, required field - FLOW_SERIAL_NUMBER, - // Flow YAML name TLV: client to server in register request and report request, required field - FLOW_YML_NAME, - // Flow YAML content, TLV: server to client in register respond, option field in case server want to ask client to load YAML from server - FLOW_YML_CONTENT, - // Fix length, 4 bytes Report interval in msec: server to client in register respond, option field - REPORT_INTERVAL, - // Processor Name TLV: server to client in report respond, option field in case server want to ask client to update processor property - PROCESSOR_NAME, - // Processor Property Name TLV: server to client in report respond, option field in case server want to ask client to update processor property - PROPERTY_NAME, - // Processor Property Value TLV: server to client in report respond, option field in case server want to ask client to update processor property - PROPERTY_VALUE, - // Report Blob TLV: client to server in report request, option field in case client want to pickyback the report blob in report request to server - REPORT_BLOB, - MAX_FLOW_MSG_ID + //Fix length 8 bytes: client to server in register request, required field + FLOW_SERIAL_NUMBER, + // Flow YAML name TLV: client to server in register request and report request, required field + FLOW_YML_NAME, + // Flow YAML content, TLV: server to client in register respond, option field in case server want to ask client to load YAML from server + FLOW_YML_CONTENT, + // Fix length, 4 bytes Report interval in msec: server to client in register respond, option field + REPORT_INTERVAL, + // Processor Name TLV: server to client in report respond, option field in case server want to ask client to update processor property + PROCESSOR_NAME, + // Processor Property Name TLV: server to client in report respond, option field in case server want to ask client to update processor property + PROPERTY_NAME, + // Processor Property Value TLV: server to client in report respond, option field in case server want to ask client to update processor property + PROPERTY_VALUE, + // Report Blob TLV: client to server in report request, option field in case client want to pickyback the report blob in report request to server + REPORT_BLOB, + MAX_FLOW_MSG_ID } FlowControlMsgID; -//! FlowControl Protocol Msg ID String -static const char *FlowControlMsgIDStr[MAX_FLOW_MSG_ID] = -{ - "FLOW_SERIAL_NUMBER", - "FLOW_YAML_NAME", - "FLOW_YAML_CONTENT", - "REPORT_INTERVAL", - "PROCESSOR_NAME" - "PROPERTY_NAME", - "PROPERTY_VALUE", - "REPORT_BLOB" -}; +// FlowControl Protocol Msg ID String +static const char *FlowControlMsgIDStr[MAX_FLOW_MSG_ID] = { + "FLOW_SERIAL_NUMBER", "FLOW_YAML_NAME", "FLOW_YAML_CONTENT", + "REPORT_INTERVAL", "PROCESSOR_NAME" + "PROPERTY_NAME", "PROPERTY_VALUE", "REPORT_BLOB" }; #define TYPE_HDR_LEN 4 // Fix Hdr Type #define TLV_HDR_LEN 8 // Type 4 bytes and Len 4 bytes -//! FlowControl Protocol Msg Len -inline int FlowControlMsgIDEncodingLen(FlowControlMsgID id, int payLoadLen) -{ - if (id == FLOW_SERIAL_NUMBER) - return (TYPE_HDR_LEN + 8); - else if (id == REPORT_INTERVAL) - return (TYPE_HDR_LEN + 4); - else if (id < MAX_FLOW_MSG_ID) - return (TLV_HDR_LEN + payLoadLen); - else - return -1; +// FlowControl Protocol Msg Len +inline int FlowControlMsgIDEncodingLen(FlowControlMsgID id, int payLoadLen) { + if (id == FLOW_SERIAL_NUMBER) + return (TYPE_HDR_LEN + 8); + else if (id == REPORT_INTERVAL) + return (TYPE_HDR_LEN + 4); + else if (id < MAX_FLOW_MSG_ID) + return (TLV_HDR_LEN + payLoadLen); + else + return -1; } -//! Flow Control Msg Id to String -inline const char *FlowControlMsgIdToStr(FlowControlMsgID id) -{ - if (id < MAX_FLOW_MSG_ID) - return FlowControlMsgIDStr[id]; - else - return NULL; +// Flow Control Msg Id to String +inline const char *FlowControlMsgIdToStr(FlowControlMsgID id) { + if (id < MAX_FLOW_MSG_ID) + return FlowControlMsgIDStr[id]; + else + return NULL; } -//! Flow Control Respond status code +// Flow Control Respond status code typedef enum { - RESP_SUCCESS, - RESP_TRIGGER_REGISTER, // Server respond to client report to re trigger register - RESP_START_FLOW_CONTROLLER, // Server respond to client to start flow controller - RESP_STOP_FLOW_CONTROLLER, // Server respond to client to stop flow controller - RESP_FAILURE, - MAX_RESP_CODE + RESP_SUCCESS, + RESP_TRIGGER_REGISTER, // Server respond to client report to re trigger register + RESP_START_FLOW_CONTROLLER, // Server respond to client to start flow controller + RESP_STOP_FLOW_CONTROLLER, // Server respond to client to stop flow controller + RESP_FAILURE, + MAX_RESP_CODE } FlowControlRespCode; -//! FlowControl Resp Code str -static const char *FlowControlRespCodeStr[MAX_RESP_CODE] = -{ - "RESP_SUCCESS", - "RESP_TRIGGER_REGISTER", - "RESP_START_FLOW_CONTROLLER", - "RESP_STOP_FLOW_CONTROLLER", - "RESP_FAILURE" -}; +// FlowControl Resp Code str +static const char *FlowControlRespCodeStr[MAX_RESP_CODE] = { "RESP_SUCCESS", + "RESP_TRIGGER_REGISTER", "RESP_START_FLOW_CONTROLLER", + "RESP_STOP_FLOW_CONTROLLER", "RESP_FAILURE" }; -//! Flow Control Resp Code to String -inline const char *FlowControlRespCodeToStr(FlowControlRespCode code) -{ - if (code < MAX_RESP_CODE) - return FlowControlRespCodeStr[code]; - else - return NULL; +// Flow Control Resp Code to String +inline const char *FlowControlRespCodeToStr(FlowControlRespCode code) { + if (code < MAX_RESP_CODE) + return FlowControlRespCodeStr[code]; + else + return NULL; } -//! Common FlowControlProtocol Header +// Common FlowControlProtocol Header typedef struct { - uint32_t msgType; //! Msg Type - uint32_t seqNumber; //! Seq Number to match Req with Resp - uint32_t status; //! Resp Code, see FlowControlRespCode - uint32_t payloadLen; //! Msg Payload length + uint32_t msgType; // Msg Type + uint32_t seqNumber; // Seq Number to match Req with Resp + uint32_t status; // Resp Code, see FlowControlRespCode + uint32_t payloadLen; // Msg Payload length } FlowControlProtocolHeader; -//! FlowControlProtocol Class -class FlowControlProtocol -{ -public: - //! Constructor - /*! - * Create a new control protocol - */ - FlowControlProtocol(FlowController *controller) { - _controller = controller; - logger_ = Logger::getLogger(); - configure_ = Configure::getConfigure(); - _socket = 0; - _serverName = "localhost"; - _serverPort = DEFAULT_NIFI_SERVER_PORT; - _registered = false; - _seqNumber = 0; - _reportBlob = NULL; - _reportBlobLen = 0; - _reportInterval = DEFAULT_REPORT_INTERVAL; - _running = false; +// FlowControlProtocol Class +class FlowControlProtocol { + public: + // Constructor + /*! + * Create a new control protocol + */ + FlowControlProtocol(FlowController *controller) { + _controller = controller; + logger_ = logging::Logger::getLogger(); + configure_ = Configure::getConfigure(); + _socket = 0; + _serverName = "localhost"; + _serverPort = DEFAULT_NIFI_SERVER_PORT; + _registered = false; + _seqNumber = 0; + _reportBlob = NULL; + _reportBlobLen = 0; + _reportInterval = DEFAULT_REPORT_INTERVAL; + running_ = false; - std::string value; + std::string value; - if (configure_->get(Configure::nifi_server_name, value)) - { - _serverName = value; - logger_->log_info("NiFi Server Name %s", _serverName.c_str()); - } - if (configure_->get(Configure::nifi_server_port, value) && Property::StringToInt(value, _serverPort)) - { - logger_->log_info("NiFi Server Port: [%d]", _serverPort); - } - if (configure_->get(Configure::nifi_server_report_interval, value)) - { - TimeUnit unit; - if (Property::StringToTime(value, _reportInterval, unit) && - Property::ConvertTimeUnitToMS(_reportInterval, unit, _reportInterval)) - { - logger_->log_info("NiFi server report interval: [%d] ms", _reportInterval); - } - } - else - _reportInterval = 0; - } - //! Destructor - virtual ~FlowControlProtocol() - { - stop(); - if (_socket) - close(_socket); - if (_reportBlob) - delete [] _reportBlob; - if (this->_thread) - delete this->_thread; - } + if (configure_->get(Configure::nifi_server_name, value)) { + _serverName = value; + logger_->log_info("NiFi Server Name %s", _serverName.c_str()); + } + if (configure_->get(Configure::nifi_server_port, value) + && core::Property::StringToInt( + value, _serverPort)) { + logger_->log_info("NiFi Server Port: [%d]", _serverPort); + } + if (configure_->get(Configure::nifi_server_report_interval, value)) { + core::TimeUnit unit; + if (core::Property::StringToTime( + value, _reportInterval, unit) + && core::Property::ConvertTimeUnitToMS( + _reportInterval, unit, _reportInterval)) { + logger_->log_info("NiFi server report interval: [%d] ms", + _reportInterval); + } + } else + _reportInterval = 0; + } + // Destructor + virtual ~FlowControlProtocol() { + stop(); + if (_socket) + close(_socket); + if (_reportBlob) + delete[] _reportBlob; + if (this->_thread) + delete this->_thread; + } -public: + public: - //! SendRegisterRequest and Process Register Respond, return 0 for success - int sendRegisterReq(); - //! SendReportReq and Process Report Respond, return 0 for success - int sendReportReq(); - //! Start the flow control protocol - void start(); - //! Stop the flow control protocol - void stop(); - //! Set Report BLOB for periodically report - void setReportBlob(char *blob, int len) - { - std::lock_guard lock(_mtx); - if (_reportBlob && _reportBlobLen >= len) - { - memcpy(_reportBlob, blob, len); - _reportBlobLen = len; - } - else - { - if (_reportBlob) - delete[] _reportBlob; - _reportBlob = new char[len]; - _reportBlobLen = len; - } - } - //! Run function for the thread - static void run(FlowControlProtocol *protocol); - //! set 8 bytes SerialNumber - void setSerialNumber(uint8_t *number) - { - memcpy(_serialNumber, number, 8); - } + // SendRegisterRequest and Process Register Respond, return 0 for success + int sendRegisterReq(); + // SendReportReq and Process Report Respond, return 0 for success + int sendReportReq(); + // Start the flow control protocol + void start(); + // Stop the flow control protocol + void stop(); + // Set Report BLOB for periodically report + void setReportBlob(char *blob, int len) { + std::lock_guard lock(mutex_); + if (_reportBlob && _reportBlobLen >= len) { + memcpy(_reportBlob, blob, len); + _reportBlobLen = len; + } else { + if (_reportBlob) + delete[] _reportBlob; + _reportBlob = new char[len]; + _reportBlobLen = len; + } + } + // Run function for the thread + static void run(FlowControlProtocol *protocol); + // set 8 bytes SerialNumber + void setSerialNumber(uint8_t *number) { + memcpy(_serialNumber, number, 8); + } -protected: + protected: -private: - //! Connect to the socket, return sock descriptor if success, 0 for failure - int connectServer(const char *host, uint16_t port); - //! Send Data via the socket, return -1 for failure - int sendData(uint8_t *buf, int buflen); - //! Read length into buf, return -1 for failure and 0 for EOF - int readData(uint8_t *buf, int buflen); - //! Select on the socket - int selectClient(int msec); - //! Read the header - int readHdr(FlowControlProtocolHeader *hdr); - //! encode uint32_t - uint8_t *encode(uint8_t *buf, uint32_t value) - { - *buf++ = (value & 0xFF000000) >> 24; - *buf++ = (value & 0x00FF0000) >> 16; - *buf++ = (value & 0x0000FF00) >> 8; - *buf++ = (value & 0x000000FF); - return buf; - } - //! encode uint32_t - uint8_t *decode(uint8_t *buf, uint32_t &value) - { - value = ((buf[0]<<24)|(buf[1]<<16)|(buf[2]<<8)|(buf[3])); - return (buf + 4); - } - //! encode byte array - uint8_t *encode(uint8_t *buf, uint8_t *bufArray, int size) - { - memcpy(buf, bufArray, size); - buf += size; - return buf; - } - //! encode std::string - uint8_t *encode(uint8_t *buf, std::string value) - { - // add the \0 for size - buf = encode(buf, value.size()+1); - buf = encode(buf, (uint8_t *) value.c_str(), value.size()+1); - return buf; - } - //! Mutex for protection - std::mutex _mtx; - //! Logger - std::shared_ptr logger_; - //! Configure - Configure *configure_ = NULL; - //! NiFi server Name - std::string _serverName; - //! NiFi server port - int64_t _serverPort; - //! Serial Number - uint8_t _serialNumber[8]; - //! socket to server - int _socket; - //! report interal in msec - int64_t _reportInterval; - //! whether it was registered to the NiFi server - bool _registered; - //! seq number - uint32_t _seqNumber; - //! FlowController - FlowController *_controller = NULL; - //! report Blob - char *_reportBlob; - //! report Blob len; - int _reportBlobLen; - //! thread - std::thread *_thread = NULL; - //! whether it is running - bool _running; - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - FlowControlProtocol(const FlowControlProtocol &parent); - FlowControlProtocol &operator=(const FlowControlProtocol &parent); + private: + // Connect to the socket, return sock descriptor if success, 0 for failure + int connectServer(const char *host, uint16_t port); + // Send Data via the socket, return -1 for failure + int sendData(uint8_t *buf, int buflen); + // Read length into buf, return -1 for failure and 0 for EOF + int readData(uint8_t *buf, int buflen); + // Select on the socket + int selectClient(int msec); + // Read the header + int readHdr(FlowControlProtocolHeader *hdr); + // encode uint32_t + uint8_t *encode(uint8_t *buf, uint32_t value) { + *buf++ = (value & 0xFF000000) >> 24; + *buf++ = (value & 0x00FF0000) >> 16; + *buf++ = (value & 0x0000FF00) >> 8; + *buf++ = (value & 0x000000FF); + return buf; + } + // encode uint32_t + uint8_t *decode(uint8_t *buf, uint32_t &value) { + value = ((buf[0] << 24) | (buf[1] << 16) | (buf[2] << 8) | (buf[3])); + return (buf + 4); + } + // encode byte array + uint8_t *encode(uint8_t *buf, uint8_t *bufArray, int size) { + memcpy(buf, bufArray, size); + buf += size; + return buf; + } + // encode std::string + uint8_t *encode(uint8_t *buf, std::string value) { + // add the \0 for size + buf = encode(buf, value.size() + 1); + buf = encode(buf, (uint8_t *) value.c_str(), value.size() + 1); + return buf; + } + // Mutex for protection + std::mutex mutex_; + // Logger + std::shared_ptr logger_; + // Configure + Configure *configure_ = NULL; + // NiFi server Name + std::string _serverName; + // NiFi server port + int64_t _serverPort; + // Serial Number + uint8_t _serialNumber[8]; + // socket to server + int _socket; + // report interal in msec + int64_t _reportInterval; + // whether it was registered to the NiFi server + bool _registered; + // seq number + uint32_t _seqNumber; + // FlowController + FlowController *_controller = NULL; + // report Blob + char *_reportBlob; + // report Blob len; + int _reportBlobLen; + // thread + std::thread *_thread = NULL; + // whether it is running + bool running_; + // Prevent default copy constructor and assignment operation + // Only support pass by reference or pointer + FlowControlProtocol(const FlowControlProtocol &parent); + FlowControlProtocol &operator=(const FlowControlProtocol &parent); }; +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ #endif diff --git a/libminifi/include/FlowController.h b/libminifi/include/FlowController.h index 9db13ff1ae..4144409739 100644 --- a/libminifi/include/FlowController.h +++ b/libminifi/include/FlowController.h @@ -28,324 +28,280 @@ #include #include #include -#ifdef YAML_SUPPORT -#include "yaml-cpp/yaml.h" -#endif #include "Configure.h" -#include "Property.h" -#include "Relationship.h" +#include "core/Relationship.h" #include "FlowFileRecord.h" #include "Connection.h" -#include "Processor.h" -#include "ProcessContext.h" -#include "ProcessSession.h" -#include "ProcessGroup.h" -#include "GenerateFlowFile.h" -#include "LogAttribute.h" -#include "RealTimeDataCollector.h" +#include "core/Processor.h" +#include "core/ProcessContext.h" +#include "core/ProcessSession.h" +#include "core/ProcessGroup.h" +#include "processors/GenerateFlowFile.h" +#include "processors/LogAttribute.h" +#include "processors/RealTimeDataCollector.h" #include "TimerDrivenSchedulingAgent.h" #include "EventDrivenSchedulingAgent.h" #include "FlowControlProtocol.h" #include "RemoteProcessorGroupPort.h" -#include "Provenance.h" -#include "FlowFileRepository.h" -#include "GetFile.h" -#include "PutFile.h" -#include "TailFile.h" -#include "ListenSyslog.h" -#include "ListenHTTP.h" -#include "ExecuteProcess.h" -#include "AppendHostInfo.h" -// OpenSSL related -#ifdef OPENSSL_SUPPORT -#include -#include -#endif +#include "provenance/Provenance.h" +#include "processors/GetFile.h" +#include "processors/PutFile.h" +#include "processors/TailFile.h" +#include "processors/ListenSyslog.h" +#include "processors/ListenHTTP.h" +#include "processors/ExecuteProcess.h" +#include "processors/AppendHostInfo.h" +#include "core/Property.h" -//! Default NiFi Root Group Name +namespace org { +namespace apache { +namespace nifi { +namespace minifi { + +// Default NiFi Root Group Name #define DEFAULT_ROOT_GROUP_NAME "" #define DEFAULT_FLOW_YAML_FILE_NAME "conf/flow.yml" #define CONFIG_YAML_PROCESSORS_KEY "Processors" struct ProcessorConfig { - std::string id; - std::string name; - std::string javaClass; - std::string maxConcurrentTasks; - std::string schedulingStrategy; - std::string schedulingPeriod; - std::string penalizationPeriod; - std::string yieldPeriod; - std::string runDurationNanos; - std::vector autoTerminatedRelationships; - std::vector properties; + std::string id; + std::string name; + std::string javaClass; + std::string maxConcurrentTasks; + std::string schedulingStrategy; + std::string schedulingPeriod; + std::string penalizationPeriod; + std::string yieldPeriod; + std::string runDurationNanos; + std::vector autoTerminatedRelationships; + std::vector properties; }; /** * Flow Controller class. Generally used by FlowController factory * as a singleton. */ -class FlowController { -public: - static const int DEFAULT_MAX_TIMER_DRIVEN_THREAD = 10; - static const int DEFAULT_MAX_EVENT_DRIVEN_THREAD = 5; +class FlowController : public core::CoreComponent { + public: + static const int DEFAULT_MAX_TIMER_DRIVEN_THREAD = 10; + static const int DEFAULT_MAX_EVENT_DRIVEN_THREAD = 5; + // Destructor + virtual ~FlowController() { + } - //! Destructor - virtual ~FlowController(){ - } - //! Set FlowController Name - virtual void setName(std::string name) { - _name = name; - } - //! Get Flow Controller Name - virtual std::string getName(void) { - return (_name); - } - //! Set UUID - virtual void setUUID(uuid_t uuid) { - uuid_copy(_uuid, uuid); - } - //! Get UUID - virtual bool getUUID(uuid_t uuid) { - if (uuid) { - uuid_copy(uuid, _uuid); - return true; - } else - return false; - } - //! Set MAX TimerDrivenThreads - virtual void setMaxTimerDrivenThreads(int number) { - _maxTimerDrivenThreads = number; - } - //! Get MAX TimerDrivenThreads - virtual int getMaxTimerDrivenThreads() { - return _maxTimerDrivenThreads; - } - //! Set MAX EventDrivenThreads - virtual void setMaxEventDrivenThreads(int number) { - _maxEventDrivenThreads = number; - } - //! Get MAX EventDrivenThreads - virtual int getMaxEventDrivenThreads() { - return _maxEventDrivenThreads; - } - //! Get the provenance repository - virtual ProvenanceRepository *getProvenanceRepository() { - return this->_provenanceRepo; - } - //! Get the flowfile repository - virtual FlowFileRepository *getFlowFileRepository() { - return this->_flowfileRepo; - } - //! Load flow xml from disk, after that, create the root process group and its children, initialize the flows - virtual void load() = 0; + // Set MAX TimerDrivenThreads + virtual void setMaxTimerDrivenThreads(int number) { + max_timer_driven_threads_ = number; + } + // Get MAX TimerDrivenThreads + virtual int getMaxTimerDrivenThreads() { + return max_timer_driven_threads_; + } + // Set MAX EventDrivenThreads + virtual void setMaxEventDrivenThreads(int number) { + max_event_driven_threads_ = number; + } + // Get MAX EventDrivenThreads + virtual int getMaxEventDrivenThreads() { + return max_event_driven_threads_; + } + // Get the provenance repository + virtual std::shared_ptr getProvenanceRepository() { + return this->provenance_repo_; + } - //! Whether the Flow Controller is start running - virtual bool isRunning() { - return _running.load(); - } - //! Whether the Flow Controller has already been initialized (loaded flow XML) - virtual bool isInitialized() { - return _initialized.load(); - } - //! Start to run the Flow Controller which internally start the root process group and all its children - virtual bool start() = 0; - //! Unload the current flow YAML, clean the root process group and all its children - virtual void stop(bool force) = 0; - //! Asynchronous function trigger unloading and wait for a period of time - virtual void waitUnload(const uint64_t timeToWaitMs) = 0; - //! Unload the current flow xml, clean the root process group and all its children - virtual void unload() = 0; - //! Load new xml - virtual void reload(std::string yamlFile) = 0; - //! update property value - void updatePropertyValue(std::string processorName, - std::string propertyName, std::string propertyValue) { - if (_root) - _root->updatePropertyValue(processorName, propertyName, - propertyValue); - } + // Get the flowfile repository + virtual std::shared_ptr getFlowFileRepository() { + return this->flow_file_repo_; + } - //! Create Processor (Node/Input/Output Port) based on the name - virtual Processor *createProcessor(std::string name, uuid_t uuid) = 0; - //! Create Root Processor Group - virtual ProcessGroup *createRootProcessGroup(std::string name, uuid_t uuid) = 0; - //! Create Remote Processor Group - virtual ProcessGroup *createRemoteProcessGroup(std::string name, - uuid_t uuid) = 0; - //! Create Connection - virtual Connection *createConnection(std::string name, uuid_t uuid) = 0; - //! set 8 bytes SerialNumber - virtual void setSerialNumber(uint8_t *number) { - _protocol->setSerialNumber(number); - } + // Load flow xml from disk, after that, create the root process group and its children, initialize the flows + virtual void load() = 0; -protected: - - //! A global unique identifier - uuid_t _uuid; - //! FlowController Name - std::string _name; - //! Configuration File Name - std::string _configurationFileName; - //! NiFi property File Name - std::string _propertiesFileName; - //! Root Process Group - ProcessGroup *_root; - //! MAX Timer Driven Threads - int _maxTimerDrivenThreads; - //! MAX Event Driven Threads - int _maxEventDrivenThreads; - //! Config - //! FlowFile Repo - //! Whether it is running - std::atomic _running; - //! Whether it has already been initialized (load the flow XML already) - std::atomic _initialized; - //! Provenance Repo - ProvenanceRepository *_provenanceRepo; - //! FlowFile Repo - FlowFileRepository *_flowfileRepo; - //! Flow Engines - //! Flow Timer Scheduler - TimerDrivenSchedulingAgent _timerScheduler; - //! Flow Event Scheduler - EventDrivenSchedulingAgent _eventScheduler; - //! Controller Service - //! Config - //! Site to Site Server Listener - //! Heart Beat - //! FlowControl Protocol - FlowControlProtocol *_protocol; - + // Whether the Flow Controller is start running + virtual bool isRunning() { + return running_.load(); + } + // Whether the Flow Controller has already been initialized (loaded flow XML) + virtual bool isInitialized() { + return initialized_.load(); + } + // Start to run the Flow Controller which internally start the root process group and all its children + virtual bool start() = 0; + // Unload the current flow YAML, clean the root process group and all its children + virtual void stop(bool force) = 0; + // Asynchronous function trigger unloading and wait for a period of time + virtual void waitUnload(const uint64_t timeToWaitMs) = 0; + // Unload the current flow xml, clean the root process group and all its children + virtual void unload() = 0; + // Load new xml + virtual void reload(std::string yamlFile) = 0; + // update property value + void updatePropertyValue(std::string processorName, std::string propertyName, + std::string propertyValue) { + if (root_) + root_->updatePropertyValue(processorName, propertyName, propertyValue); + } - FlowController() : - _root(0), _maxTimerDrivenThreads(0), _maxEventDrivenThreads(0), _running( - false), _initialized(false), _provenanceRepo(0), _flowfileRepo(0), _protocol( - 0), logger_(Logger::getLogger()){ - } + // Create Processor (Node/Input/Output Port) based on the name + virtual std::shared_ptr createProcessor(std::string name, + uuid_t uuid) = 0; + // Create Root Processor Group + virtual core::ProcessGroup *createRootProcessGroup(std::string name, + uuid_t uuid) = 0; + // Create Remote Processor Group + virtual core::ProcessGroup *createRemoteProcessGroup(std::string name, + uuid_t uuid) = 0; + // Create Connection + virtual std::shared_ptr createConnection(std::string name, + uuid_t uuid) = 0; + // set 8 bytes SerialNumber + virtual void setSerialNumber(uint8_t *number) { + protocol_->setSerialNumber(number); + } -private: + protected: - //! Logger - std::shared_ptr logger_; + // Configuration File Name + std::string configuration_file_name_; + // NiFi property File Name + std::string properties_file_name_; + // Root Process Group + core::ProcessGroup *root_; + // MAX Timer Driven Threads + int max_timer_driven_threads_; + // MAX Event Driven Threads + int max_event_driven_threads_; + // Config + // FlowFile Repo + // Whether it is running + std::atomic running_; + // conifiguration filename + std::string configuration_filename_; + // Whether it has already been initialized (load the flow XML already) + std::atomic initialized_; + // Provenance Repo + std::shared_ptr provenance_repo_; -}; + // FlowFile Repo + std::shared_ptr flow_file_repo_; -/** - * Flow Controller implementation that defines the typical flow. - * of events. - */ -class FlowControllerImpl: public FlowController { -public: + // Flow Engines + // Flow Timer Scheduler + TimerDrivenSchedulingAgent _timerScheduler; + // Flow Event Scheduler + EventDrivenSchedulingAgent _eventScheduler; + // Controller Service + // Config + // Site to Site Server Listener + // Heart Beat + // FlowControl Protocol + FlowControlProtocol *protocol_; + + FlowController(std::shared_ptr provenance_repo, + std::shared_ptr flow_file_repo) + : root_(0), + max_timer_driven_threads_(0), + max_event_driven_threads_(0), + running_(false), + initialized_(false), + provenance_repo_(provenance_repo), + flow_file_repo_(flow_file_repo), + protocol_(0), + _timerScheduler(provenance_repo_), + _eventScheduler(provenance_repo_), + logger_(logging::Logger::getLogger()) { + } - //! Destructor - virtual ~FlowControllerImpl(); + private: - //! Life Cycle related function - //! Load flow xml from disk, after that, create the root process group and its children, initialize the flows - void load(); - //! Start to run the Flow Controller which internally start the root process group and all its children - bool start(); - //! Stop to run the Flow Controller which internally stop the root process group and all its children - void stop(bool force); - //! Asynchronous function trigger unloading and wait for a period of time - void waitUnload(const uint64_t timeToWaitMs); - //! Unload the current flow xml, clean the root process group and all its children - void unload(); - //! Load new xml - void reload(std::string yamlFile); - //! Load Flow File from persistent Flow Repo - void loadFlowRepo(); - //! update property value - void updatePropertyValue(std::string processorName, - std::string propertyName, std::string propertyValue) { - if (_root) - _root->updatePropertyValue(processorName, propertyName, - propertyValue); - } + // Logger + std::shared_ptr logger_; - //! Create Processor (Node/Input/Output Port) based on the name - Processor *createProcessor(std::string name, uuid_t uuid); - //! Create Root Processor Group - ProcessGroup *createRootProcessGroup(std::string name, uuid_t uuid); - //! Create Remote Processor Group - ProcessGroup *createRemoteProcessGroup(std::string name, uuid_t uuid); - //! Create Connection - Connection *createConnection(std::string name, uuid_t uuid); +}; - //! Constructor - /*! - * Create a new Flow Controller - */ - FlowControllerImpl(std::string name = DEFAULT_ROOT_GROUP_NAME); +class FlowControllerImpl : public FlowController { + public: - - - - friend class FlowControlFactory; + // Destructor + virtual ~FlowControllerImpl(); -private: - - + // Life Cycle related function + // Load flow xml from disk, after that, create the root process group and its children, initialize the flows + void load(); + // Start to run the Flow Controller which internally start the root process group and all its children + bool start(); + // Stop to run the Flow Controller which internally stop the root process group and all its children + void stop(bool force); + // Asynchronous function trigger unloading and wait for a period of time + void waitUnload(const uint64_t timeToWaitMs); + // Unload the current flow xml, clean the root process group and all its children + void unload(); + // Load Flow File from persistent Flow Repo + void loadFlowRepo(); + // Load new xml + void reload(std::string yamlFile); + // update property value + void updatePropertyValue(std::string processorName, std::string propertyName, + std::string propertyValue) { + if (root_) + root_->updatePropertyValue(processorName, propertyName, propertyValue); + } - //! Mutex for protection - std::mutex _mtx; - //! Logger - std::shared_ptr logger_; - Configure *configure_; + // Create Processor (Node/Input/Output Port) based on the name + std::shared_ptr createProcessor(std::string name, + uuid_t uuid); + // Create Root Processor Group + core::ProcessGroup *createRootProcessGroup(std::string name, uuid_t uuid); + // Create Remote Processor Group + core::ProcessGroup *createRemoteProcessGroup(std::string name, uuid_t uuid); + // Create Connection + std::shared_ptr createConnection(std::string name, uuid_t uuid); -#ifdef YAML_SUPPORT - //! Process Processor Node YAML - void parseProcessorNodeYaml(YAML::Node processorNode, ProcessGroup *parent); - //! Process Port YAML - void parsePortYaml(YAML::Node *portNode, ProcessGroup *parent, - TransferDirection direction); - //! Process Root Processor Group YAML - void parseRootProcessGroupYaml(YAML::Node rootNode); - //! Process Property YAML - void parseProcessorPropertyYaml(YAML::Node *doc, YAML::Node *node, - Processor *processor); - //! Process connection YAML - void parseConnectionYaml(YAML::Node *node, ProcessGroup *parent); - //! Process Remote Process Group YAML - void parseRemoteProcessGroupYaml(YAML::Node *node, ProcessGroup *parent); - //! Parse Properties Node YAML for a processor - void parsePropertiesNodeYaml(YAML::Node *propertiesNode, - Processor *processor); -#endif + // Constructor + /*! + * Create a new Flow Controller + */ + FlowControllerImpl(std::shared_ptr repo, + std::string name = DEFAULT_ROOT_GROUP_NAME); - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - FlowControllerImpl(const FlowController &parent); - FlowControllerImpl &operator=(const FlowController &parent); + // Prevent default copy constructor and assignment operation + // Only support pass by reference or pointer + FlowControllerImpl(const FlowController &parent) = delete; + FlowControllerImpl &operator=(const FlowController &parent) = delete; -}; + private: -/** - * Flow Controller factory that creates flow controllers or gets the - * assigned instance. - */ -class FlowControllerFactory { -public: - //! Get the singleton flow controller - static FlowController * getFlowController(FlowController *instance = 0) { - if (!_flowController) { - if (NULL == instance) - _flowController = createFlowController(); - else - _flowController = instance; - } - return _flowController; - } + std::mutex mutex_; + + // configuration object + Configure *configure_; + // Process Processor Node YAML + void parseProcessorNodeYaml(YAML::Node processorNode, + core::ProcessGroup *parent); + // Process Port YAML + void parsePortYaml(YAML::Node *portNode, core::ProcessGroup *parent, + TransferDirection direction); + // Process Root Processor Group YAML + void parseRootProcessGroupYaml(YAML::Node rootNode); + // Process Property YAML + void parseProcessorPropertyYaml(YAML::Node *doc, YAML::Node *node, + std::shared_ptr processor); + // Process connection YAML + void parseConnectionYaml(YAML::Node *node, core::ProcessGroup *parent); + // Process Remote Process Group YAML + void parseRemoteProcessGroupYaml(YAML::Node *node, + core::ProcessGroup *parent); + // Parse Properties Node YAML for a processor + void parsePropertiesNodeYaml(YAML::Node *propertiesNode, + std::shared_ptr processor); - //! Get the singleton flow controller - static FlowController * createFlowController() { - return dynamic_cast(new FlowControllerImpl()); - } -private: - static FlowController *_flowController; }; +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + #endif diff --git a/libminifi/include/FlowFileRecord.h b/libminifi/include/FlowFileRecord.h index ded0623413..f479787d30 100644 --- a/libminifi/include/FlowFileRecord.h +++ b/libminifi/include/FlowFileRecord.h @@ -31,215 +31,116 @@ #include #include +#include "core/FlowFile.h" #include "utils/TimeUtil.h" -#include "Logger.h" +#include "core/logging/Logger.h" #include "ResourceClaim.h" +#include "Connection.h" -class ProcessSession; -class Connection; -class FlowFileEventRecord; +namespace org { +namespace apache { +namespace nifi { +namespace minifi { #define DEFAULT_FLOWFILE_PATH "." -//! FlowFile Attribute -enum FlowAttribute -{ - //! The flowfile's path indicates the relative directory to which a FlowFile belongs and does not contain the filename - PATH = 0, - //! The flowfile's absolute path indicates the absolute directory to which a FlowFile belongs and does not contain the filename - ABSOLUTE_PATH, - //! The filename of the FlowFile. The filename should not contain any directory structure. - FILENAME, - //! A unique UUID assigned to this FlowFile. - UUID, - //! A numeric value indicating the FlowFile priority - priority, - //! The MIME Type of this FlowFile - MIME_TYPE, - //! Specifies the reason that a FlowFile is being discarded - DISCARD_REASON, - //! Indicates an identifier other than the FlowFile's UUID that is known to refer to this FlowFile. - ALTERNATE_IDENTIFIER, - MAX_FLOW_ATTRIBUTES +// FlowFile Attribute +enum FlowAttribute { + // The flowfile's path indicates the relative directory to which a FlowFile belongs and does not contain the filename + PATH = 0, + // The flowfile's absolute path indicates the absolute directory to which a FlowFile belongs and does not contain the filename + ABSOLUTE_PATH, + // The filename of the FlowFile. The filename should not contain any directory structure. + FILENAME, + // A unique UUID assigned to this FlowFile. + UUID, + // A numeric value indicating the FlowFile priority + priority, + // The MIME Type of this FlowFile + MIME_TYPE, + // Specifies the reason that a FlowFile is being discarded + DISCARD_REASON, + // Indicates an identifier other than the FlowFile's UUID that is known to refer to this FlowFile. + ALTERNATE_IDENTIFIER, + MAX_FLOW_ATTRIBUTES }; -//! FlowFile Attribute Key -static const char *FlowAttributeKeyArray[MAX_FLOW_ATTRIBUTES] = -{ - "path", - "absolute.path", - "filename", - "uuid", - "priority", - "mime.type", - "discard.reason", - "alternate.identifier" -}; - -//! FlowFile Attribute Enum to Key -inline const char *FlowAttributeKey(FlowAttribute attribute) -{ - if (attribute < MAX_FLOW_ATTRIBUTES) - return FlowAttributeKeyArray[attribute]; - else - return NULL; +// FlowFile Attribute Key +static const char *FlowAttributeKeyArray[MAX_FLOW_ATTRIBUTES] = { "path", + "absolute.path", "filename", "uuid", "priority", "mime.type", + "discard.reason", "alternate.identifier" }; + +// FlowFile Attribute Enum to Key +inline const char *FlowAttributeKey(FlowAttribute attribute) { + if (attribute < MAX_FLOW_ATTRIBUTES) + return FlowAttributeKeyArray[attribute]; + else + return NULL; } -//! FlowFile IO Callback functions for input and output -//! throw exception for error -class InputStreamCallback -{ -public: - virtual void process(std::ifstream *stream) = 0; +// FlowFile IO Callback functions for input and output +// throw exception for error +class InputStreamCallback { + public: + virtual void process(std::ifstream *stream) = 0; }; -class OutputStreamCallback -{ -public: - virtual void process(std::ofstream *stream) = 0; +class OutputStreamCallback { + public: + virtual void process(std::ofstream *stream) = 0; }; - -//! FlowFile Record Class -class FlowFileRecord -{ - friend class ProcessSession; -public: - //! Constructor - /*! - * Create a new flow record - */ - explicit FlowFileRecord(std::map attributes, ResourceClaim *claim = NULL); - /*! - * Create a new flow record from repo flow event - */ - explicit FlowFileRecord(FlowFileEventRecord *event); - //! Destructor - virtual ~FlowFileRecord(); - //! addAttribute key is enum - bool addAttribute(FlowAttribute key, std::string value); - //! addAttribute key is string - bool addAttribute(std::string key, std::string value); - //! removeAttribute key is enum - bool removeAttribute(FlowAttribute key); - //! removeAttribute key is string - bool removeAttribute(std::string key); - //! updateAttribute key is enum - bool updateAttribute(FlowAttribute key, std::string value); - //! updateAttribute key is string - bool updateAttribute(std::string key, std::string value); - //! getAttribute key is enum - bool getAttribute(FlowAttribute key, std::string &value); - //! getAttribute key is string - bool getAttribute(std::string key, std::string &value); - //! setAttribute, if attribute already there, update it, else, add it - void setAttribute(std::string key, std::string value) { - _attributes[key] = value; - } - //! Get the UUID as string - std::string getUUIDStr() { - return _uuidStr; - } - //! Get Attributes - std::map getAttributes() { - return _attributes; - } - //! Check whether it is still being penalized - bool isPenalized() { - return (_penaltyExpirationMs > 0 ? _penaltyExpirationMs > getTimeMillis() : false); - } - //! Get Size - uint64_t getSize() { - return _size; - } - // ! Get Offset - uint64_t getOffset() { - return _offset; - } - // ! Get Entry Date - uint64_t getEntryDate() { - return _entryDate; - } - // ! Get Lineage Start Date - uint64_t getlineageStartDate() { - return _lineageStartDate; - } - // ! Set Original connection - void setOriginalConnection (Connection *connection) { - _orginalConnection = connection; - } - //! Get Original connection - Connection * getOriginalConnection() { - return _orginalConnection; - } - //! Get Resource Claim - ResourceClaim *getResourceClaim() { - return _claim; - } - //! Get lineageIdentifiers - std::set getlineageIdentifiers() - { - return _lineageIdentifiers; - } - //! Check whether it is stored to DB already - bool isStoredToRepository() - { - return _isStoredToRepo; - } - void setStoredToRepository(bool value) - { - _isStoredToRepo = value; - } - -protected: - - //! Date at which the flow file entered the flow - uint64_t _entryDate; - //! Date at which the origin of this flow file entered the flow - uint64_t _lineageStartDate; - //! Date at which the flow file was queued - uint64_t _lastQueueDate; - //! Size in bytes of the data corresponding to this flow file - uint64_t _size; - //! A global unique identifier - uuid_t _uuid; - //! A local unique identifier - uint64_t _id; - //! Offset to the content - uint64_t _offset; - //! Penalty expiration - uint64_t _penaltyExpirationMs; - //! Attributes key/values pairs for the flow record - std::map _attributes; - //! Pointer to the associated content resource claim - ResourceClaim *_claim; - //! UUID string - std::string _uuidStr; - //! UUID string for all parents - std::set _lineageIdentifiers; - //! whether it is stored to DB - bool _isStoredToRepo; - //! duplicate the original flow file - void duplicate(FlowFileRecord *original); - -private: - - //! Local flow sequence ID - static std::atomic _localFlowSeqNumber; - //! Mark for deletion - bool _markedDelete; - //! Connection queue that this flow file will be transfer or current in - Connection *_connection; - //! Orginal connection queue that this flow file was dequeued from - Connection *_orginalConnection; - //! Logger - std::shared_ptr logger_; - //! Snapshot flow record for session rollback - bool _snapshot; - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - FlowFileRecord(const FlowFileRecord &parent); - FlowFileRecord &operator=(const FlowFileRecord &parent); +class FlowFileRecord : public core::FlowFile { + public: + // Constructor + /*! + * Create a new flow record + */ + explicit FlowFileRecord(std::shared_ptr flow_repository, + std::map attributes, + std::shared_ptr claim = nullptr); + + explicit FlowFileRecord(std::shared_ptr flow_repository,std::shared_ptr event); + // Destructor + virtual ~FlowFileRecord(); + // addAttribute key is enum + bool addKeyedAttribute(FlowAttribute key, std::string value); + // removeAttribute key is enum + bool removeKeyedAttribute(FlowAttribute key); + // updateAttribute key is enum + bool updateKeyedAttribute(FlowAttribute key, std::string value); + // getAttribute key is enum + bool getKeyedAttribute(FlowAttribute key, std::string &value); + + void setSnapShot(bool snapshot) { + snapshot_ = snapshot; + } + + FlowFileRecord &operator=(const FlowFileRecord &) = delete; + + FlowFileRecord(const FlowFileRecord &parent) = delete; + + protected: + + private: + + // Local flow sequence ID + static std::atomic local_flow_seq_number_; + + // repository reference. + std::shared_ptr flow_repository_; + + // Logger + std::shared_ptr logger_; + // Snapshot flow record for session rollback + bool snapshot_; + // Prevent default copy constructor and assignment operation + // Only support pass by reference or pointer }; +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + #endif diff --git a/libminifi/include/FlowFileRepository.h b/libminifi/include/FlowFileRepository.h deleted file mode 100644 index 50d2c418b3..0000000000 --- a/libminifi/include/FlowFileRepository.h +++ /dev/null @@ -1,204 +0,0 @@ -/** - * @file FlowFileRepository - * Flow file repository class declaration - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __FLOWFILE_REPOSITORY_H__ -#define __FLOWFILE_REPOSITORY_H__ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "Configure.h" -#include "Connection.h" -#include "FlowFileRecord.h" -#include "Logger.h" -#include "Property.h" -#include "ResourceClaim.h" -#include "io/Serializable.h" -#include "utils/TimeUtil.h" -#include "Repository.h" - -#define FLOWFILE_REPOSITORY_DIRECTORY "./flowfile_repository" -#define MAX_FLOWFILE_REPOSITORY_STORAGE_SIZE (10*1024*1024) // 10M -#define MAX_FLOWFILE_REPOSITORY_ENTRY_LIFE_TIME (600000) // 10 minute -#define FLOWFILE_REPOSITORY_PURGE_PERIOD (2500) // 2500 msec - -//! FlowFile Repository -class FlowFileRepository : public Repository -{ -public: - //! Constructor - /*! - * Create a new provenance repository - */ - FlowFileRepository() - : Repository(Repository::FLOWFILE, FLOWFILE_REPOSITORY_DIRECTORY, - MAX_FLOWFILE_REPOSITORY_ENTRY_LIFE_TIME, MAX_FLOWFILE_REPOSITORY_STORAGE_SIZE, FLOWFILE_REPOSITORY_PURGE_PERIOD) - { - } - //! Destructor - virtual ~FlowFileRepository() { - } - //! Load Repo to Connections - void loadFlowFileToConnections(std::map *connectionMap); - -protected: - -private: - - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - FlowFileRepository(const FlowFileRepository &parent); - FlowFileRepository &operator=(const FlowFileRepository &parent); -}; - -//! FlowFile Event Record -class FlowFileEventRecord : protected Serializable -{ -public: - //! Constructor - /*! - * Create a new provenance event record - */ - FlowFileEventRecord() - : _entryDate(0), _lineageStartDate(0), _size(0), _offset(0) - { - _eventTime = getTimeMillis(); - logger_ = Logger::getLogger(); - } - - //! Destructor - virtual ~FlowFileEventRecord() { - } - //! Get Attributes - std::map getAttributes() { - return _attributes; - } - //! Get Size - uint64_t getFileSize() { - return _size; - } - // ! Get Offset - uint64_t getFileOffset() { - return _offset; - } - // ! Get Entry Date - uint64_t getFlowFileEntryDate() { - return _entryDate; - } - // ! Get Lineage Start Date - uint64_t getlineageStartDate() { - return _lineageStartDate; - } - // ! Get Event Time - uint64_t getEventTime() { - return _eventTime; - } - //! Get FlowFileUuid - std::string getFlowFileUuid() - { - return _uuid; - } - //! Get ConnectionUuid - std::string getConnectionUuid() - { - return _uuidConnection; - } - //! Get content full path - std::string getContentFullPath() - { - return _contentFullPath; - } - //! Get LineageIdentifiers - std::set getLineageIdentifiers() - { - return _lineageIdentifiers; - } - //! fromFlowFile - void fromFlowFile(FlowFileRecord *flow, std::string uuidConnection) - { - _entryDate = flow->getEntryDate(); - _lineageStartDate = flow->getlineageStartDate(); - _lineageIdentifiers = flow->getlineageIdentifiers(); - _uuid = flow->getUUIDStr(); - _attributes = flow->getAttributes(); - _size = flow->getSize(); - _offset = flow->getOffset(); - _uuidConnection = uuidConnection; - if (flow->getResourceClaim()) - { - _contentFullPath = flow->getResourceClaim()->getContentFullPath(); - } - } - //! Serialize and Persistent to the repository - bool Serialize(FlowFileRepository *repo); - //! DeSerialize - bool DeSerialize(const uint8_t *buffer, const int bufferSize); - //! DeSerialize - bool DeSerialize(DataStream &stream) - { - return DeSerialize(stream.getBuffer(),stream.getSize()); - } - //! DeSerialize - bool DeSerialize(FlowFileRepository *repo, std::string key); - -protected: - - //! Date at which the event was created - uint64_t _eventTime; - //! Date at which the flow file entered the flow - uint64_t _entryDate; - //! Date at which the origin of this flow file entered the flow - uint64_t _lineageStartDate; - //! Size in bytes of the data corresponding to this flow file - uint64_t _size; - //! flow uuid - std::string _uuid; - //! connection uuid - std::string _uuidConnection; - //! Offset to the content - uint64_t _offset; - //! Full path to the content - std::string _contentFullPath; - //! Attributes key/values pairs for the flow record - std::map _attributes; - //! UUID string for all parents - std::set _lineageIdentifiers; - -private: - - //! Logger - std::shared_ptr logger_; - - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - FlowFileEventRecord(const FlowFileEventRecord &parent); - FlowFileEventRecord &operator=(const FlowFileEventRecord &parent); - -}; - -#endif diff --git a/libminifi/include/GenerateFlowFile.h b/libminifi/include/GenerateFlowFile.h deleted file mode 100644 index 27aa43b57a..0000000000 --- a/libminifi/include/GenerateFlowFile.h +++ /dev/null @@ -1,87 +0,0 @@ -/** - * @file GenerateFlowFile.h - * GenerateFlowFile class declaration - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __GENERATE_FLOW_FILE_H__ -#define __GENERATE_FLOW_FILE_H__ - -#include "FlowFileRecord.h" -#include "Processor.h" -#include "ProcessSession.h" - -//! GenerateFlowFile Class -class GenerateFlowFile : public Processor -{ -public: - //! Constructor - /*! - * Create a new processor - */ - GenerateFlowFile(std::string name, uuid_t uuid = NULL) - : Processor(name, uuid) - { - _data = NULL; - _dataSize = 0; - } - //! Destructor - virtual ~GenerateFlowFile() - { - if (_data) - delete[] _data; - } - //! Processor Name - static const std::string ProcessorName; - //! Supported Properties - static Property FileSize; - static Property BatchSize; - static Property DataFormat; - static Property UniqueFlowFiles; - static const char *DATA_FORMAT_BINARY; - static const char *DATA_FORMAT_TEXT; - //! Supported Relationships - static Relationship Success; - //! Nest Callback Class for write stream - class WriteCallback : public OutputStreamCallback - { - public: - WriteCallback(char *data, uint64_t size) - : _data(data), _dataSize(size) {} - char *_data; - uint64_t _dataSize; - void process(std::ofstream *stream) { - if (_data && _dataSize > 0) - stream->write(_data, _dataSize); - } - }; - -public: - //! OnTrigger method, implemented by NiFi GenerateFlowFile - virtual void onTrigger(ProcessContext *context, ProcessSession *session); - //! Initialize, over write by NiFi GenerateFlowFile - virtual void initialize(void); - -protected: - -private: - //! Generated data - char * _data; - //! Size of the generate data - uint64_t _dataSize; -}; - -#endif diff --git a/libminifi/include/GetFile.h b/libminifi/include/GetFile.h deleted file mode 100644 index 8f8068c8d0..0000000000 --- a/libminifi/include/GetFile.h +++ /dev/null @@ -1,117 +0,0 @@ -/** - * @file GetFile.h - * GetFile class declaration - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __GET_FILE_H__ -#define __GET_FILE_H__ - -#include "FlowFileRecord.h" -#include "Processor.h" -#include "ProcessSession.h" - -//! GetFile Class -class GetFile : public Processor -{ -public: - //! Constructor - /*! - * Create a new processor - */ - GetFile(std::string name, uuid_t uuid = NULL) - : Processor(name, uuid) - { - logger_ = Logger::getLogger(); - _directory = "."; - _recursive = true; - _keepSourceFile = false; - _minAge = 0; - _maxAge = 0; - _minSize = 0; - _maxSize = 0; - _ignoreHiddenFile = true; - _pollInterval = 0; - _batchSize = 10; - _lastDirectoryListingTime = getTimeMillis(); - _fileFilter = "[^\\.].*"; - } - //! Destructor - virtual ~GetFile() - { - } - //! Processor Name - static const std::string ProcessorName; - //! Supported Properties - static Property Directory; - static Property Recurse; - static Property KeepSourceFile; - static Property MinAge; - static Property MaxAge; - static Property MinSize; - static Property MaxSize; - static Property IgnoreHiddenFile; - static Property PollInterval; - static Property BatchSize; - static Property FileFilter; - //! Supported Relationships - static Relationship Success; - -public: - //! OnTrigger method, implemented by NiFi GetFile - virtual void onTrigger(ProcessContext *context, ProcessSession *session); - //! Initialize, over write by NiFi GetFile - virtual void initialize(void); - //! perform directory listing - void performListing(std::string dir); - -protected: - -private: - //! Logger - std::shared_ptr logger_; - //! Queue for store directory list - std::queue _dirList; - //! Get Listing size - uint64_t getListingSize() { - std::lock_guard lock(_mtx); - return _dirList.size(); - } - //! Whether the directory listing is empty - bool isListingEmpty(); - //! Put full path file name into directory listing - void putListing(std::string fileName); - //! Poll directory listing for files - void pollListing(std::queue &list, int maxSize); - //! Check whether file can be added to the directory listing - bool acceptFile(std::string fullName, std::string name); - //! Mutex for protection of the directory listing - std::mutex _mtx; - std::string _directory; - bool _recursive; - bool _keepSourceFile; - int64_t _minAge; - int64_t _maxAge; - int64_t _minSize; - int64_t _maxSize; - bool _ignoreHiddenFile; - int64_t _pollInterval; - int64_t _batchSize; - uint64_t _lastDirectoryListingTime; - std::string _fileFilter; -}; - -#endif diff --git a/libminifi/include/ListenHTTP.h b/libminifi/include/ListenHTTP.h deleted file mode 100644 index 5a467f2341..0000000000 --- a/libminifi/include/ListenHTTP.h +++ /dev/null @@ -1,116 +0,0 @@ -/** - * @file ListenHTTP.h - * ListenHTTP class declaration - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __LISTEN_HTTP_H__ -#define __LISTEN_HTTP_H__ - -#include -#include - -#include - -#include "FlowFileRecord.h" -#include "Processor.h" -#include "ProcessSession.h" - - -//! ListenHTTP Class -class ListenHTTP : public Processor -{ -public: - - //! Constructor - /*! - * Create a new processor - */ - ListenHTTP(std::string name, uuid_t uuid = NULL) - : Processor(name, uuid) - { - _logger = Logger::getLogger(); - } - //! Destructor - ~ListenHTTP() - { - } - //! Processor Name - static const std::string ProcessorName; - //! Supported Properties - static Property BasePath; - static Property Port; - static Property AuthorizedDNPattern; - static Property SSLCertificate; - static Property SSLCertificateAuthority; - static Property SSLVerifyPeer; - static Property SSLMinimumVersion; - static Property HeadersAsAttributesRegex; - //! Supported Relationships - static Relationship Success; - - void onTrigger(ProcessContext *context, ProcessSession *session); - void initialize(); - void onSchedule(ProcessContext *context, ProcessSessionFactory *sessionFactory); - - //! HTTP request handler - class Handler : public CivetHandler - { - public: - Handler(ProcessContext *context, - ProcessSessionFactory *sessionFactory, - std::string &&authDNPattern, - std::string &&headersAsAttributesPattern); - bool handlePost(CivetServer *server, struct mg_connection *conn); - - private: - //! Send HTTP 500 error response to client - void sendErrorResponse(struct mg_connection *conn); - //! Logger - std::shared_ptr _logger; - - std::regex _authDNRegex; - std::regex _headersAsAttributesRegex; - ProcessContext *_processContext; - ProcessSessionFactory *_processSessionFactory; - }; - - //! Write callback for transferring data from HTTP request to content repo - class WriteCallback : public OutputStreamCallback - { - public: - WriteCallback(struct mg_connection *conn, const struct mg_request_info *reqInfo); - void process(std::ofstream *stream); - - private: - //! Logger - std::shared_ptr _logger; - - struct mg_connection *_conn; - const struct mg_request_info *_reqInfo; - }; - -protected: - -private: - //! Logger - std::shared_ptr _logger; - - std::unique_ptr _server; - std::unique_ptr _handler; -}; - -#endif diff --git a/libminifi/include/ListenSyslog.h b/libminifi/include/ListenSyslog.h deleted file mode 100644 index 339dbc1efc..0000000000 --- a/libminifi/include/ListenSyslog.h +++ /dev/null @@ -1,209 +0,0 @@ -/** - * @file ListenSyslog.h - * ListenSyslog class declaration - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __LISTEN_SYSLOG_H__ -#define __LISTEN_SYSLOG_H__ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "FlowFileRecord.h" -#include "Processor.h" -#include "ProcessSession.h" - -//! SyslogEvent -typedef struct { - uint8_t *payload; - uint64_t len; -} SysLogEvent; - -//! ListenSyslog Class -class ListenSyslog : public Processor -{ -public: - //! Constructor - /*! - * Create a new processor - */ - ListenSyslog(std::string name, uuid_t uuid = NULL) - : Processor(name, uuid) - { - logger_ = Logger::getLogger(); - _eventQueueByteSize = 0; - _serverSocket = 0; - _recvBufSize = 65507; - _maxSocketBufSize = 1024*1024; - _maxConnections = 2; - _maxBatchSize = 1; - _messageDelimiter = "\n"; - _protocol = "UDP"; - _port = 514; - _parseMessages = false; - _serverSocket = 0; - _maxFds = 0; - FD_ZERO(&_readfds); - _thread = NULL; - _resetServerSocket = false; - _serverTheadRunning = false; - } - //! Destructor - virtual ~ListenSyslog() - { - _serverTheadRunning = false; - if (this->_thread) - delete this->_thread; - // need to reset the socket - std::vector::iterator it; - for (it = _clientSockets.begin(); it != _clientSockets.end(); ++it) - { - int clientSocket = *it; - close(clientSocket); - } - _clientSockets.clear(); - if (_serverSocket > 0) - { - logger_->log_info("ListenSysLog Server socket %d close", _serverSocket); - close(_serverSocket); - _serverSocket = 0; - } - } - //! Processor Name - static const std::string ProcessorName; - //! Supported Properties - static Property RecvBufSize; - static Property MaxSocketBufSize; - static Property MaxConnections; - static Property MaxBatchSize; - static Property MessageDelimiter; - static Property ParseMessages; - static Property Protocol; - static Property Port; - //! Supported Relationships - static Relationship Success; - static Relationship Invalid; - //! Nest Callback Class for write stream - class WriteCallback : public OutputStreamCallback - { - public: - WriteCallback(char *data, uint64_t size) - : _data(data), _dataSize(size) {} - char *_data; - uint64_t _dataSize; - void process(std::ofstream *stream) { - if (_data && _dataSize > 0) - stream->write(_data, _dataSize); - } - }; - -public: - //! OnTrigger method, implemented by NiFi ListenSyslog - virtual void onTrigger(ProcessContext *context, ProcessSession *session); - //! Initialize, over write by NiFi ListenSyslog - virtual void initialize(void); - -protected: - -private: - //! Logger - std::shared_ptr logger_; - //! Run function for the thread - static void run(ListenSyslog *process); - //! Run Thread - void runThread(); - //! Queue for store syslog event - std::queue _eventQueue; - //! Size of Event queue in bytes - uint64_t _eventQueueByteSize; - //! Get event queue size - uint64_t getEventQueueSize() { - std::lock_guard lock(_mtx); - return _eventQueue.size(); - } - //! Get event queue byte size - uint64_t getEventQueueByteSize() { - std::lock_guard lock(_mtx); - return _eventQueueByteSize; - } - //! Whether the event queue is empty - bool isEventQueueEmpty() - { - std::lock_guard lock(_mtx); - return _eventQueue.empty(); - } - //! Put event into directory listing - void putEvent(uint8_t *payload, uint64_t len) - { - std::lock_guard lock(_mtx); - SysLogEvent event; - event.payload = payload; - event.len = len; - _eventQueue.push(event); - _eventQueueByteSize += len; - } - //! Read \n terminated line from TCP socket - int readline( int fd, char *bufptr, size_t len ); - //! start server socket and handling client socket - void startSocketThread(); - //! Poll event - void pollEvent(std::queue &list, int maxSize) - { - std::lock_guard lock(_mtx); - - while (!_eventQueue.empty() && (maxSize == 0 || list.size() < maxSize)) - { - SysLogEvent event = _eventQueue.front(); - _eventQueue.pop(); - _eventQueueByteSize -= event.len; - list.push(event); - } - return; - } - //! Mutex for protection of the directory listing - std::mutex _mtx; - int64_t _recvBufSize; - int64_t _maxSocketBufSize; - int64_t _maxConnections; - int64_t _maxBatchSize; - std::string _messageDelimiter; - std::string _protocol; - int64_t _port; - bool _parseMessages; - int _serverSocket; - std::vector _clientSockets; - int _maxFds; - fd_set _readfds; - //! thread - std::thread *_thread; - //! whether to reset the server socket - bool _resetServerSocket; - bool _serverTheadRunning; - //! buffer for read socket - uint8_t _buffer[2048]; -}; - -#endif diff --git a/libminifi/include/LogAppenders.h b/libminifi/include/LogAppenders.h deleted file mode 100644 index ef28bb8e8e..0000000000 --- a/libminifi/include/LogAppenders.h +++ /dev/null @@ -1,298 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef LIBMINIFI_INCLUDE_LOGAPPENDERS_H_ -#define LIBMINIFI_INCLUDE_LOGAPPENDERS_H_ - -#include "BaseLogger.h" -#include "spdlog/sinks/null_sink.h" -#include "spdlog/sinks/ostream_sink.h" -#include -#include "Configure.h" - -template -static std::string getUniqueName() { - std::string name = LOG_NAME; - name += " -- "; - name += abi::__cxa_demangle(typeid(T).name(), 0, 0, 0); - spdlog::drop(name); - return name; -} - -/** - * Null appender sets a null sink, thereby performing no logging. - */ -class NullAppender: public BaseLogger { -public: - /** - * Base constructor that creates the null sink. - */ - explicit NullAppender() : - ::BaseLogger("off") { - auto null_sink = std::make_shared(); - std::string unique_name = getUniqueName(); - logger_ = std::make_shared(unique_name, null_sink); - configured_level_ = off; - setLogLevel(); - } - - /** - * Move constructor for the null appender. - */ - explicit NullAppender(const NullAppender &&other) : - ::BaseLogger(std::move(other)) { - - } - -}; - -/** - * Basic output stream configuration that uses a supplied ostream - * - * Design : extends LoggerConfiguration using the logger and log level - * encapsulated within the base configuration class. - */ -class OutputStreamAppender: public BaseLogger { - -public: - - static const char *nifi_log_output_stream_error_stderr; - - /** - * Output stream move constructor. - */ - explicit OutputStreamAppender(const OutputStreamAppender &&other) : - ::BaseLogger(std::move(other)) { - - } - - /** - * Base constructor. Creates a ostream sink. - * @param stream incoming stream reference. - * @param config configuration. - */ - explicit OutputStreamAppender(Configure *config) : - ::BaseLogger("info") { - auto ostream_sink = std::make_shared( - std::cout); - - std::string unique_name = getUniqueName(); - logger_ = std::make_shared(unique_name, ostream_sink); - - std::string use_std_err; - - if (NULL != config - && config->get(nifi_log_output_stream_error_stderr, - use_std_err)) { - - std::transform(use_std_err.begin(), use_std_err.end(), - use_std_err.begin(), ::tolower); - - if (use_std_err == "true") { - std::string err_unique_name = - getUniqueName(); - auto error_ostream_sink = std::make_shared< - spdlog::sinks::ostream_sink_mt>(std::cerr); - stderr_ = std::make_shared(err_unique_name, - error_ostream_sink); - } - } else { - stderr_ = nullptr; - } - - std::string log_level; - if (NULL != config - && config->get(BaseLogger::nifi_log_level, log_level)) { - setLogLevel(log_level); - } else{ - setLogLevel("info"); - } - - } - - /** - * Base constructor. Creates a ostream sink. - * @param stream incoming stream reference. - * @param config configuration. - */ - OutputStreamAppender(std::ostream &stream, Configure *config) : - ::BaseLogger("info") { - auto ostream_sink = std::make_shared( - stream); - std::string unique_name = getUniqueName(); - logger_ = std::make_shared(unique_name, ostream_sink); - - stderr_ = nullptr; - - std::string log_level; - if (NULL != config - && config->get(BaseLogger::nifi_log_level, log_level)) { - setLogLevel(log_level); - } else { - setLogLevel("info"); - } - - } - -protected: - -}; - -/** - * Rolling configuration - * Design : extends LoggerConfiguration using the logger and log level - * encapsulated within the base configuration class. - */ -class RollingAppender: public BaseLogger { -public: - static const char *nifi_log_rolling_apender_file; - static const char *nifi_log_rolling_appender_max_files; - static const char *nifi_log_rolling_appender_max_file_size; - - /** - * RollingAppenderConfiguration move constructor. - */ - explicit RollingAppender(const RollingAppender&& other) : - ::BaseLogger(std::move(other)), max_files_( - std::move(other.max_files_)), file_name_( - std::move(other.file_name_)), max_file_size_( - std::move(other.max_file_size_)) { - } - /** - * Base Constructor. - * @param config pointer to the configuration for this instance. - */ - explicit RollingAppender(Configure * config = 0) : - ::BaseLogger("info") { - std::string file_name = ""; - if (NULL != config - && config->get(nifi_log_rolling_apender_file, file_name)) { - file_name_ = file_name; - } else{ - file_name_ = LOG_FILE_NAME; - } - - std::string max_files = ""; - if (NULL != config - && config->get(nifi_log_rolling_appender_max_files, - max_files)) { - try { - max_files_ = std::stoi(max_files); - } catch (const std::invalid_argument &ia) { - max_files_ = DEFAULT_LOG_FILE_NUMBER; - } catch (const std::out_of_range &oor) { - max_files_ = DEFAULT_LOG_FILE_NUMBER; - } - } else { - max_files_ = DEFAULT_LOG_FILE_NUMBER; - } - - std::string max_file_size = ""; - if (NULL != config - && config->get(nifi_log_rolling_appender_max_file_size, - max_file_size)) { - try { - max_file_size_ = std::stoi(max_file_size); - } catch (const std::invalid_argument &ia) { - max_file_size_ = DEFAULT_LOG_FILE_SIZE; - } catch (const std::out_of_range &oor) { - max_file_size_ = DEFAULT_LOG_FILE_SIZE; - } - } else { - max_file_size_ = DEFAULT_LOG_FILE_SIZE; - } - - std::string unique_name = getUniqueName(); - logger_ = spdlog::rotating_logger_mt(unique_name, file_name_, - max_file_size_, max_files_); - - std::string log_level; - if (NULL != config - && config->get(BaseLogger::nifi_log_level, log_level)) { - setLogLevel(log_level); - } - } - - /** - * To maintain current functionality we will flush on write. - */ - void log_str(LOG_LEVEL_E level, const std::string &buffer) { - ::BaseLogger::log_str(level, buffer); - logger_->flush(); - } - -protected: - - /** - * file name. - */ - std::string file_name_; - /** - * maximum number of files to keep in the rotation. - */ - size_t max_files_; - /** - * Maximum file size per rotated file. - */ - size_t max_file_size_; - -}; - -class LogInstance { -public: - /** - * Returns a logger configuration based on - * the configuration within this instance. - * @param config configuration for this instance. - */ - static std::unique_ptr getConfiguredLogger(Configure *config) { - std::string appender = ""; - - if (config->get(BaseLogger::nifi_log_appender, appender)) { - std::transform(appender.begin(), appender.end(), appender.begin(), - ::tolower); - - if ("nullappender" == appender || "null appender" == appender - || "null" == appender) { - - return std::move( - std::unique_ptr(new NullAppender())); - - } else if ("rollingappender" == appender - || "rolling appender" == appender - || "rolling" == appender) { - - return std::move( - std::unique_ptr( - new RollingAppender(config))); - - } else if ("outputstream" == appender - || "outputstreamappender" == appender - || "outputstream appender" == appender) { - - return std::move( - std::unique_ptr( - new OutputStreamAppender(config))); - - } - } - return nullptr; - - } -}; - -#endif /* LIBMINIFI_INCLUDE_LOGAPPENDERS_H_ */ diff --git a/libminifi/include/LogAttribute.h b/libminifi/include/LogAttribute.h deleted file mode 100644 index 429a594bdb..0000000000 --- a/libminifi/include/LogAttribute.h +++ /dev/null @@ -1,128 +0,0 @@ -/** - * @file LogAttribute.h - * LogAttribute class declaration - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __LOG_ATTRIBUTE_H__ -#define __LOG_ATTRIBUTE_H__ - -#include "FlowFileRecord.h" -#include "Processor.h" -#include "ProcessSession.h" - -//! LogAttribute Class -class LogAttribute : public Processor -{ -public: - //! Constructor - /*! - * Create a new processor - */ - LogAttribute(std::string name, uuid_t uuid = NULL) - : Processor(name, uuid) - { - logger_ = Logger::getLogger(); - } - //! Destructor - virtual ~LogAttribute() - { - } - //! Processor Name - static const std::string ProcessorName; - //! Supported Properties - static Property LogLevel; - static Property AttributesToLog; - static Property AttributesToIgnore; - static Property LogPayload; - static Property LogPrefix; - //! Supported Relationships - static Relationship Success; - enum LogAttrLevel { - LogAttrLevelTrace, LogAttrLevelDebug, LogAttrLevelInfo, LogAttrLevelWarn, LogAttrLevelError - }; - //! Convert log level from string to enum - bool logLevelStringToEnum(std::string logStr, LogAttrLevel &level) - { - if (logStr == "trace") - { - level = LogAttrLevelTrace; - return true; - } - else if (logStr == "debug") - { - level = LogAttrLevelDebug; - return true; - } - else if (logStr == "info") - { - level = LogAttrLevelInfo; - return true; - } - else if (logStr == "warn") - { - level = LogAttrLevelWarn; - return true; - } - else if (logStr == "error") - { - level = LogAttrLevelError; - return true; - } - else - return false; - } - //! Nest Callback Class for read stream - class ReadCallback : public InputStreamCallback - { - public: - ReadCallback(uint64_t size) - { - _bufferSize = size; - _buffer = new char[_bufferSize]; - } - ~ReadCallback() - { - if (_buffer) - delete[] _buffer; - } - void process(std::ifstream *stream) { - - stream->read(_buffer, _bufferSize); - if (!stream) - _readSize = stream->gcount(); - else - _readSize = _bufferSize; - } - char *_buffer; - uint64_t _bufferSize; - uint64_t _readSize; - }; - -public: - //! OnTrigger method, implemented by NiFi LogAttribute - virtual void onTrigger(ProcessContext *context, ProcessSession *session); - //! Initialize, over write by NiFi LogAttribute - virtual void initialize(void); - -protected: - -private: - //! Logger - std::shared_ptr logger_; -}; - -#endif diff --git a/libminifi/include/Logger.h b/libminifi/include/Logger.h deleted file mode 100644 index 81d444680d..0000000000 --- a/libminifi/include/Logger.h +++ /dev/null @@ -1,200 +0,0 @@ -/** - * @file Logger.h - * Logger class declaration - * This is a C++ wrapper for spdlog, a lightweight C++ logging library - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __LOGGER_H__ -#define __LOGGER_H__ - -#include -#include -#include -#include -#include -#include -#include - -#include "BaseLogger.h" -#include "spdlog/spdlog.h" - -/** - * Logger class - * Design: Extends BaseLogger, leaving this class to be the facade to the underlying - * logging mechanism. Is a facade to BaseLogger's underlying log stream. This allows - * the underlying implementation to be replaced real time. - */ -class Logger: public BaseLogger { -protected: - struct singleton; -public: - - /** - * Returns a shared pointer to the logger instance. - * Note that while there is no synchronization this is expected - * to be called and initialized first - * @returns shared pointer to the base logger. - */ - static std::shared_ptr getLogger() { - - if (singleton_logger_ == nullptr) - singleton_logger_ = std::make_shared(singleton { 0 }); - return singleton_logger_; - } - - /** - * Returns the log level for this instance. - */ - LOG_LEVEL_E getLogLevel() const { - return current_logger_.load()->getLogLevel(); - } - - /** - * Sets the log level atomic and sets it - * within logger if it can - * @param level desired log level. - */ - void setLogLevel(LOG_LEVEL_E level) { - current_logger_.load()->setLogLevel(level); - } - - /** - * Sets the log level for this instance based on the string - * @param level desired log leve. - * @param defaultLevel default level if we cannot match level. - */ - void setLogLevel(const std::string &level, - LOG_LEVEL_E defaultLevel = info) { - current_logger_.load()->setLogLevel(level, info); - } - - void updateLogger(std::unique_ptr logger) { - - if (logger == nullptr ) - return; - current_logger_.store(logger.release()); - } - - /** - * @brief Log error message - * @param format format string ('man printf' for syntax) - * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match - */ - void log_error(const char * const format, ...) { - if (!current_logger_.load()->shouldLog(err)) - return; - FILL_BUFFER - current_logger_.load()->log_str(err, buffer); - } - /** - * @brief Log warn message - * @param format format string ('man printf' for syntax) - * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match - */ - void log_warn(const char * const format, ...) { - if (!current_logger_.load()->shouldLog(warn)) - return; - FILL_BUFFER - current_logger_.load()->log_str(warn, buffer); - } - /** - * @brief Log info message - * @param format format string ('man printf' for syntax) - * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match - */ - void log_info(const char * const format, ...) { - if (!current_logger_.load()->shouldLog(info)) - return; - FILL_BUFFER - current_logger_.load()->log_str(info, buffer); - } - /** - * @brief Log debug message - * @param format format string ('man printf' for syntax) - * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match - */ - void log_debug(const char * const format, ...) { - - if (!current_logger_.load()->shouldLog(debug)) - return; - FILL_BUFFER - current_logger_.load()->log_str(debug, buffer); - } - /** - * @brief Log trace message - * @param format format string ('man printf' for syntax) - * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match - */ - void log_trace(const char * const format, ...) { - - if (!current_logger_.load()->shouldLog(trace)) - return; - FILL_BUFFER - current_logger_.load()->log_str(trace, buffer); - } - - /** - * @brief Log message - * @param format format string ('man printf' for syntax) - * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match - */ - virtual void log_str(LOG_LEVEL_E level, const std::string &buffer) { - current_logger_.load()->log_str(level, buffer); - } - - //! Destructor - ~Logger() { - } - - explicit Logger(const singleton &a) { - - /** - * flush on info to maintain current functionality - */ - std::shared_ptr defaultsink = spdlog::rotating_logger_mt(LOG_NAME, - LOG_FILE_NAME, - DEFAULT_LOG_FILE_SIZE, DEFAULT_LOG_FILE_NUMBER); - defaultsink->flush_on(spdlog::level::level_enum::info); - - std::unique_ptr new_logger_ = std::unique_ptr( - new BaseLogger("info", defaultsink)); - - new_logger_->setLogLevel(info); - current_logger_.store(new_logger_.release()); - } - - Logger(const Logger &parent) = delete; - Logger &operator=(const Logger &parent) = delete; - -protected: - - /** - * Allows for a null constructor above so that we can have a public constructor that - * effectively limits us to being a singleton by having a protected argument in the constructor - */ - struct singleton { - explicit singleton(int) { - } - }; - - std::atomic current_logger_; - -//! Singleton logger instance - static std::shared_ptr singleton_logger_; -}; - -#endif diff --git a/libminifi/include/ProcessContext.h b/libminifi/include/ProcessContext.h deleted file mode 100644 index 585e1879ce..0000000000 --- a/libminifi/include/ProcessContext.h +++ /dev/null @@ -1,113 +0,0 @@ -/** - * @file ProcessContext.h - * ProcessContext class declaration - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __PROCESS_CONTEXT_H__ -#define __PROCESS_CONTEXT_H__ - -#include -#include -#include -#include -#include -#include -#include - -#include "Logger.h" -#include "Processor.h" - -//! ProcessContext Class -class ProcessContext -{ -public: - //! Constructor - /*! - * Create a new process context associated with the processor/controller service/state manager - */ - ProcessContext(Processor *processor = NULL) : _processor(processor) { - logger_ = Logger::getLogger(); - } - //! Destructor - virtual ~ProcessContext() {} - //! Get Processor associated with the Process Context - Processor *getProcessor() { - return _processor; - } - bool getProperty(std::string name, std::string &value) { - if (_processor) - return _processor->getProperty(name, value); - else - return false; - } - //! Sets the property value using the property's string name - bool setProperty(std::string name, std::string value) - { - if (_processor) - return _processor->setProperty(name, value); - else - return false; - } - //! Sets the property value using the Property object - bool setProperty(Property prop, std::string value) { - if (_processor) - return _processor->setProperty(prop, value); - else - return false; - } - //! Whether the relationship is supported - bool isSupportedRelationship(Relationship relationship) { - if (_processor) - return _processor->isSupportedRelationship(relationship); - else - return false; - } - //! Check whether the relationship is auto terminated - bool isAutoTerminated(Relationship relationship) { - if (_processor) - return _processor->isAutoTerminated(relationship); - else - return false; - } - //! Get ProcessContext Maximum Concurrent Tasks - uint8_t getMaxConcurrentTasks(void) { - if (_processor) - return _processor->getMaxConcurrentTasks(); - else - return 0; - } - //! Yield based on the yield period - void yield() { - if (_processor) - _processor->yield(); - } - - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - ProcessContext(const ProcessContext &parent) = delete; - ProcessContext &operator=(const ProcessContext &parent) = delete; - -private: - - //! Processor - Processor *_processor; - //! Logger - std::shared_ptr logger_; - -}; - -#endif diff --git a/libminifi/include/ProcessGroup.h b/libminifi/include/ProcessGroup.h deleted file mode 100644 index dfec6c5f30..0000000000 --- a/libminifi/include/ProcessGroup.h +++ /dev/null @@ -1,187 +0,0 @@ -/** - * @file ProcessGroup.h - * ProcessGroup class declaration - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __PROCESS_GROUP_H__ -#define __PROCESS_GROUP_H__ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "Processor.h" -#include "Exception.h" -#include "TimerDrivenSchedulingAgent.h" -#include "EventDrivenSchedulingAgent.h" -#include "Logger.h" - -//! Process Group Type -enum ProcessGroupType -{ - ROOT_PROCESS_GROUP = 0, - REMOTE_PROCESS_GROUP, - MAX_PROCESS_GROUP_TYPE -}; - -//! ProcessGroup Class -class ProcessGroup -{ -public: - //! Constructor - /*! - * Create a new process group - */ - ProcessGroup(ProcessGroupType type, std::string name, uuid_t uuid = NULL, ProcessGroup *parent = NULL); - //! Destructor - virtual ~ProcessGroup(); - //! Set Processor Name - void setName(std::string name) { - name_ = name; - } - //! Get Process Name - std::string getName(void) { - return (name_); - } - //! Set URL - void setURL(std::string url) { - url_ = url; - } - //! Get URL - std::string getURL(void) { - return (url_); - } - //! SetTransmitting - void setTransmitting(bool val) - { - transmitting_ = val; - } - //! Get Transmitting - bool getTransmitting() - { - return transmitting_; - } - //! setTimeOut - void setTimeOut(uint64_t time) - { - timeOut_ = time; - } - uint64_t getTimeOut() - { - return timeOut_; - } - //! Set Processor yield period in MilliSecond - void setYieldPeriodMsec(uint64_t period) { - yield_period_msec_ = period; - } - //! Get Processor yield period in MilliSecond - uint64_t getYieldPeriodMsec(void) { - return(yield_period_msec_); - } - //! Set UUID - void setUUID(uuid_t uuid) { - uuid_copy(uuid_, uuid); - } - //! Get UUID - bool getUUID(uuid_t uuid) { - if (uuid) - { - uuid_copy(uuid, uuid_); - return true; - } - else - return false; - } - //! Start Processing - void startProcessing(TimerDrivenSchedulingAgent *timeScheduler, - EventDrivenSchedulingAgent *eventScheduler); - //! Stop Processing - void stopProcessing(TimerDrivenSchedulingAgent *timeScheduler, - EventDrivenSchedulingAgent *eventScheduler); - //! Whether it is root process group - bool isRootProcessGroup(); - //! set parent process group - void setParent(ProcessGroup *parent) { - std::lock_guard lock(mtx_); - parent_process_group_ = parent; - } - //! get parent process group - ProcessGroup *getParent(void) { - std::lock_guard lock(mtx_); - return parent_process_group_; - } - //! Add processor - void addProcessor(Processor *processor); - //! Remove processor - void removeProcessor(Processor *processor); - //! Add child processor group - void addProcessGroup(ProcessGroup *child); - //! Remove child processor group - void removeProcessGroup(ProcessGroup *child); - // ! Add connections - void addConnection(Connection *connection); - //! findProcessor based on UUID - Processor *findProcessor(uuid_t uuid); - //! findProcessor based on name - Processor *findProcessor(std::string processorName); - //! removeConnection - void removeConnection(Connection *connection); - //! update property value - void updatePropertyValue(std::string processorName, std::string propertyName, std::string propertyValue); - //! get connections under the process group - void getConnections(std::map *connectionMap); - -protected: - //! A global unique identifier - uuid_t uuid_; - //! Processor Group Name - std::string name_; - //! Process Group Type - ProcessGroupType type_; - //! Processors (ProcessNode) inside this process group which include Input/Output Port, Remote Process Group input/Output port - std::set processors_; - std::set child_process_groups_; - //! Connections between the processor inside the group; - std::set connections_; - //! Parent Process Group - ProcessGroup* parent_process_group_; - //! Yield Period in Milliseconds - std::atomic yield_period_msec_; - std::atomic timeOut_; - //! URL - std::string url_; - //! Transmitting - std::atomic transmitting_; - -private: - - //! Mutex for protection - std::mutex mtx_; - //! Logger - std::shared_ptr logger_; - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - ProcessGroup(const ProcessGroup &parent); - ProcessGroup &operator=(const ProcessGroup &parent); -}; - -#endif diff --git a/libminifi/include/ProcessSession.h b/libminifi/include/ProcessSession.h deleted file mode 100644 index 4e267581f2..0000000000 --- a/libminifi/include/ProcessSession.h +++ /dev/null @@ -1,125 +0,0 @@ -/** - * @file ProcessSession.h - * ProcessSession class declaration - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __PROCESS_SESSION_H__ -#define __PROCESS_SESSION_H__ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "Processor.h" -#include "ProcessContext.h" -#include "FlowFileRecord.h" -#include "Exception.h" -#include "Logger.h" -#include "Provenance.h" - -//! ProcessSession Class -class ProcessSession -{ -public: - //! Constructor - /*! - * Create a new process session - */ - ProcessSession(ProcessContext *processContext = NULL); - //! Destructor - virtual ~ProcessSession() { - if (_provenanceReport) - delete _provenanceReport; - } - //! Commit the session - void commit(); - //! Roll Back the session - void rollback(); - //! Get Provenance Report - ProvenanceReporter *getProvenanceReporter() - { - return _provenanceReport; - } - //! - //! Get the FlowFile from the highest priority queue - FlowFileRecord *get(); - //! Create a new UUID FlowFile with no content resource claim and without parent - FlowFileRecord *create(); - //! Create a new UUID FlowFile with no content resource claim and inherit all attributes from parent - FlowFileRecord *create(FlowFileRecord *parent); - //! Clone a new UUID FlowFile from parent both for content resource claim and attributes - FlowFileRecord *clone(FlowFileRecord *parent); - //! Clone a new UUID FlowFile from parent for attributes and sub set of parent content resource claim - FlowFileRecord *clone(FlowFileRecord *parent, long offset, long size); - //! Duplicate a FlowFile with the same UUID and all attributes and content resource claim for the roll back of the session - FlowFileRecord *duplicate(FlowFileRecord *orignal); - //! Transfer the FlowFile to the relationship - void transfer(FlowFileRecord *flow, Relationship relationship); - //! Put Attribute - void putAttribute(FlowFileRecord *flow, std::string key, std::string value); - //! Remove Attribute - void removeAttribute(FlowFileRecord *flow, std::string key); - //! Remove Flow File - void remove(FlowFileRecord *flow); - //! Execute the given read callback against the content - void read(FlowFileRecord *flow, InputStreamCallback *callback); - //! Execute the given write callback against the content - void write(FlowFileRecord *flow, OutputStreamCallback *callback); - //! Execute the given write/append callback against the content - void append(FlowFileRecord *flow, OutputStreamCallback *callback); - //! Penalize the flow - void penalize(FlowFileRecord *flow); - //! Import the existed file into the flow - void import(std::string source, FlowFileRecord *flow, bool keepSource = true, uint64_t offset = 0); - - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - ProcessSession(const ProcessSession &parent) = delete; - ProcessSession &operator=(const ProcessSession &parent) = delete; - -protected: - //! FlowFiles being modified by current process session - std::map _updatedFlowFiles; - //! Copy of the original FlowFiles being modified by current process session as above - std::map _originalFlowFiles; - //! FlowFiles being added by current process session - std::map _addedFlowFiles; - //! FlowFiles being deleted by current process session - std::map _deletedFlowFiles; - //! FlowFiles being transfered to the relationship - std::map _transferRelationship; - //! FlowFiles being cloned for multiple connections per relationship - std::map _clonedFlowFiles; - -private: - // Clone the flow file during transfer to multiple connections for a relationship - FlowFileRecord* cloneDuringTransfer(FlowFileRecord *parent); - //! ProcessContext - ProcessContext *_processContext; - //! Logger - std::shared_ptr logger_; - //! Provenance Report - ProvenanceReporter *_provenanceReport; - -}; - -#endif diff --git a/libminifi/include/Processor.h b/libminifi/include/Processor.h deleted file mode 100644 index 1634bc098a..0000000000 --- a/libminifi/include/Processor.h +++ /dev/null @@ -1,365 +0,0 @@ -/** - * @file Processor.h - * Processor class declaration - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __PROCESSOR_H__ -#define __PROCESSOR_H__ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "utils/TimeUtil.h" -#include "Property.h" -#include "Relationship.h" -#include "Connection.h" - -//! Forwarder declaration -class ProcessContext; -class ProcessSession; -class ProcessSessionFactory; - -//! Minimum scheduling period in Nano Second -#define MINIMUM_SCHEDULING_NANOS 30000 - -//! Default yield period in second -#define DEFAULT_YIELD_PERIOD_SECONDS 1 - -//! Default penalization period in second -#define DEFAULT_PENALIZATION_PERIOD_SECONDS 30 - -/*! - * Indicates the valid values for the state of a entity - * with respect to scheduling the entity to run. - */ -enum ScheduledState { - - /** - * Entity cannot be scheduled to run - */ - DISABLED, - /** - * Entity can be scheduled to run but currently is not - */ - STOPPED, - /** - * Entity is currently scheduled to run - */ - RUNNING -}; - -/*! - * Scheduling Strategy - */ -enum SchedulingStrategy { - //! Event driven - EVENT_DRIVEN, - //! Timer driven - TIMER_DRIVEN, - //! Cron Driven - CRON_DRIVEN -}; - -//! Processor Class -class Processor -{ - friend class ProcessContext; -public: - //! Constructor - /*! - * Create a new processor - */ - Processor(std::string name, uuid_t uuid = NULL); - //! Destructor - virtual ~Processor(); - //! Set Processor Name - void setName(std::string name) { - _name = name; - } - //! Get Process Name - std::string getName(void) { - return (_name); - } - //! Set UUID - void setUUID(uuid_t uuid) { - uuid_copy(_uuid, uuid); - char uuidStr[37]; - uuid_unparse_lower(_uuid, uuidStr); - _uuidStr = uuidStr; - } - //! Get UUID - bool getUUID(uuid_t uuid) { - if (uuid) - { - uuid_copy(uuid, _uuid); - return true; - } - else - { - return false; - } - } - //! Set the supported processor properties while the process is not running - bool setSupportedProperties(std::set properties); - //! Set the supported relationships while the process is not running - bool setSupportedRelationships(std::set relationships); - //! Get the supported property value by name - bool getProperty(std::string name, std::string &value); - //! Set the supported property value by name wile the process is not running - bool setProperty(std::string name, std::string value); - //! Set the supported property value by using the property itself. - bool setProperty(Property prop, std::string value); - //! Whether the relationship is supported - bool isSupportedRelationship(Relationship relationship); - //! Set the auto terminated relationships while the process is not running - bool setAutoTerminatedRelationships(std::set relationships); - //! Check whether the relationship is auto terminated - bool isAutoTerminated(Relationship relationship); - //! Check whether the processor is running - bool isRunning(); - //! Set Processor Scheduled State - void setScheduledState(ScheduledState state); - //! Get Processor Scheduled State - ScheduledState getScheduledState(void) { - return _state; - } - //! Set Processor Scheduling Strategy - void setSchedulingStrategy(SchedulingStrategy strategy) { - _strategy = strategy; - } - //! Get Processor Scheduling Strategy - SchedulingStrategy getSchedulingStrategy(void) { - return _strategy; - } - //! Set Processor Loss Tolerant - void setlossTolerant(bool lossTolerant) { - _lossTolerant = lossTolerant; - } - //! Get Processor Loss Tolerant - bool getlossTolerant(void) { - return _lossTolerant; - } - //! Set Processor Scheduling Period in Nano Second - void setSchedulingPeriodNano(uint64_t period) { - uint64_t minPeriod = MINIMUM_SCHEDULING_NANOS; - _schedulingPeriodNano = std::max(period, minPeriod); - } - //! Get Processor Scheduling Period in Nano Second - uint64_t getSchedulingPeriodNano(void) { - return _schedulingPeriodNano; - } - //! Set Processor Run Duration in Nano Second - void setRunDurationNano(uint64_t period) { - _runDurantionNano = period; - } - //! Get Processor Run Duration in Nano Second - uint64_t getRunDurationNano(void) { - return(_runDurantionNano); - } - //! Set Processor yield period in MilliSecond - void setYieldPeriodMsec(uint64_t period) { - _yieldPeriodMsec = period; - } - //! Get Processor yield period in MilliSecond - uint64_t getYieldPeriodMsec(void) { - return(_yieldPeriodMsec); - } - //! Set Processor penalization period in MilliSecond - void setPenalizationPeriodMsec(uint64_t period) { - _penalizationPeriodMsec = period; - } - //! Get Processor penalization period in MilliSecond - uint64_t getPenalizationPeriodMsec(void) { - return(_penalizationPeriodMsec); - } - //! Set Processor Maximum Concurrent Tasks - void setMaxConcurrentTasks(uint8_t tasks) { - _maxConcurrentTasks = tasks; - } - //! Get Processor Maximum Concurrent Tasks - uint8_t getMaxConcurrentTasks(void) { - return(_maxConcurrentTasks); - } - //! Set Trigger when empty - void setTriggerWhenEmpty(bool value) { - _triggerWhenEmpty = value; - } - //! Get Trigger when empty - bool getTriggerWhenEmpty(void) { - return(_triggerWhenEmpty); - } - //! Get Active Task Counts - uint8_t getActiveTasks(void) { - return(_activeTasks); - } - //! Increment Active Task Counts - void incrementActiveTasks(void) { - _activeTasks++; - } - //! decrement Active Task Counts - void decrementActiveTask(void) { - _activeTasks--; - } - void clearActiveTask(void) { - _activeTasks = 0; - } - //! Yield based on the yield period - void yield() - { - _yieldExpiration = (getTimeMillis() + _yieldPeriodMsec); - } - //! Yield based on the input time - void yield(uint64_t time) - { - _yieldExpiration = (getTimeMillis() + time); - } - //! whether need be to yield - bool isYield() - { - if (_yieldExpiration > 0) - return (_yieldExpiration >= getTimeMillis()); - else - return false; - } - // clear yield expiration - void clearYield() - { - _yieldExpiration = 0; - } - // get yield time - uint64_t getYieldTime() - { - uint64_t curTime = getTimeMillis(); - if (_yieldExpiration > curTime) - return (_yieldExpiration - curTime); - else - return 0;; - } - //! Whether flow file queued in incoming connection - bool flowFilesQueued(); - //! Whether flow file queue full in any of the outgoin connection - bool flowFilesOutGoingFull(); - //! Get incoming connections - std::set getIncomingConnections() { - return _incomingConnections; - } - //! Has Incoming Connection - bool hasIncomingConnections() { - return (_incomingConnections.size() > 0); - } - //! Get outgoing connections based on relationship name - std::set getOutGoingConnections(std::string relationship); - //! Add connection - bool addConnection(Connection *connection); - //! Remove connection - void removeConnection(Connection *connection); - //! Get the UUID as string - std::string getUUIDStr() { - return _uuidStr; - } - //! Get the Next RoundRobin incoming connection - Connection *getNextIncomingConnection(); - //! On Trigger - void onTrigger(ProcessContext *context, ProcessSessionFactory *sessionFactory); - //! Block until work is available on any input connection, or the given duration elapses - void waitForWork(uint64_t timeoutMs); - //! Notify this processor that work may be available - void notifyWork(); - -public: - //! OnTrigger method, implemented by NiFi Processor Designer - virtual void onTrigger(ProcessContext *context, ProcessSession *session) = 0; - //! Initialize, overridden by NiFi Process Designer - virtual void initialize() {} - //! Scheduled event hook, overridden by NiFi Process Designer - virtual void onSchedule(ProcessContext *context, ProcessSessionFactory *sessionFactory) {} - -protected: - - //! A global unique identifier - uuid_t _uuid; - //! Processor Name - std::string _name; - //! Supported properties - std::map _properties; - //! Supported relationships - std::map _relationships; - //! Autoterminated relationships - std::map _autoTerminatedRelationships; - //! Processor state - std::atomic _state; - //! Scheduling Strategy - std::atomic _strategy; - //! lossTolerant - std::atomic _lossTolerant; - //! SchedulePeriod in Nano Seconds - std::atomic _schedulingPeriodNano; - //! Run Duration in Nano Seconds - std::atomic _runDurantionNano; - //! Yield Period in Milliseconds - std::atomic _yieldPeriodMsec; - //! Penalization Period in MilliSecond - std::atomic _penalizationPeriodMsec; - //! Maximum Concurrent Tasks - std::atomic _maxConcurrentTasks; - //! Active Tasks - std::atomic _activeTasks; - //! Trigger the Processor even if the incoming connection is empty - std::atomic _triggerWhenEmpty; - //! Incoming connections - std::set _incomingConnections; - //! Outgoing connections map based on Relationship name - std::map> _outGoingConnections; - //! UUID string - std::string _uuidStr; - -private: - - //! Mutex for protection - std::mutex _mtx; - //! Yield Expiration - std::atomic _yieldExpiration; - //! Incoming connection Iterator - std::set::iterator _incomingConnectionsIter; - //! Condition for whether there is incoming work to do - std::atomic _hasWork; - //! Concurrent condition mutex for whether there is incoming work to do - std::mutex _workAvailableMtx; - //! Concurrent condition variable for whether there is incoming work to do - std::condition_variable _hasWorkCondition; - //! Check all incoming connections for work - bool isWorkAvailable(); - //! Logger - std::shared_ptr logger_; - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - Processor(const Processor &parent); - Processor &operator=(const Processor &parent); - -}; - -#endif diff --git a/libminifi/include/Property.h b/libminifi/include/Property.h deleted file mode 100644 index bf33b35761..0000000000 --- a/libminifi/include/Property.h +++ /dev/null @@ -1,259 +0,0 @@ -/** - * @file Property.h - * Processor Property class declaration - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __PROPERTY_H__ -#define __PROPERTY_H__ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -//! Time Unit -enum TimeUnit { - DAY, HOUR, MINUTE, SECOND, MILLISECOND, NANOSECOND -}; - -//! Property Class -class Property { - -public: - //! Constructor - /*! - * Create a new property - */ - Property(const std::string name, const std::string description, - const std::string value) : - _name(name), _description(description), _value(value) { - } - Property() { - } - //! Destructor - virtual ~Property() { - } - //! Get Name for the property - std::string getName() { - return _name; - } - //! Get Description for the property - std::string getDescription() { - return _description; - } - //! Get value for the property - std::string getValue() { - return _value; - } - //! Set value for the property - void setValue(std::string value) { - _value = value; - } - //! Compare - bool operator <(const Property & right) const { - return _name < right._name; - } - - //! Convert TimeUnit to MilliSecond - static bool ConvertTimeUnitToMS(int64_t input, TimeUnit unit, - int64_t &out) { - if (unit == MILLISECOND) { - out = input; - return true; - } else if (unit == SECOND) { - out = input * 1000; - return true; - } else if (unit == MINUTE) { - out = input * 60 * 1000; - return true; - } else if (unit == HOUR) { - out = input * 60 * 60 * 1000; - return true; - } else if (unit == DAY) { - out = 24 * 60 * 60 * 1000; - return true; - } else if (unit == NANOSECOND) { - out = input / 1000 / 1000; - return true; - } else { - return false; - } - } - //! Convert TimeUnit to NanoSecond - static bool ConvertTimeUnitToNS(int64_t input, TimeUnit unit, - int64_t &out) { - if (unit == MILLISECOND) { - out = input * 1000 * 1000; - return true; - } else if (unit == SECOND) { - out = input * 1000 * 1000 * 1000; - return true; - } else if (unit == MINUTE) { - out = input * 60 * 1000 * 1000 * 1000; - return true; - } else if (unit == HOUR) { - out = input * 60 * 60 * 1000 * 1000 * 1000; - return true; - } else if (unit == NANOSECOND) { - out = input; - return true; - } else { - return false; - } - } - //! Convert String - static bool StringToTime(std::string input, int64_t &output, - TimeUnit &timeunit) { - if (input.size() == 0) { - return false; - } - - const char *cvalue = input.c_str(); - char *pEnd; - long int ival = strtol(cvalue, &pEnd, 0); - - if (pEnd[0] == '\0') { - return false; - } - - while (*pEnd == ' ') { - // Skip the space - pEnd++; - } - - std::string unit(pEnd); - - if (unit == "sec" || unit == "s" || unit == "second" - || unit == "seconds" || unit == "secs") { - timeunit = SECOND; - output = ival; - return true; - } else if (unit == "min" || unit == "m" || unit == "mins" - || unit == "minute" || unit == "minutes") { - timeunit = MINUTE; - output = ival; - return true; - } else if (unit == "ns" || unit == "nano" || unit == "nanos" - || unit == "nanoseconds") { - timeunit = NANOSECOND; - output = ival; - return true; - } else if (unit == "ms" || unit == "milli" || unit == "millis" - || unit == "milliseconds") { - timeunit = MILLISECOND; - output = ival; - return true; - } else if (unit == "h" || unit == "hr" || unit == "hour" - || unit == "hrs" || unit == "hours") { - timeunit = HOUR; - output = ival; - return true; - } else if (unit == "d" || unit == "day" || unit == "days") { - timeunit = DAY; - output = ival; - return true; - } else - return false; - } - - //! Convert String to Integer - static bool StringToInt(std::string input, int64_t &output) { - if (input.size() == 0) { - return false; - } - - const char *cvalue = input.c_str(); - char *pEnd; - long int ival = strtol(cvalue, &pEnd, 0); - - if (pEnd[0] == '\0') { - output = ival; - return true; - } - - while (*pEnd == ' ') { - // Skip the space - pEnd++; - } - - char end0 = toupper(pEnd[0]); - if ((end0 == 'K') || (end0 == 'M') || (end0 == 'G') || (end0 == 'T') - || (end0 == 'P')) { - if (pEnd[1] == '\0') { - unsigned long int multiplier = 1000; - - if ((end0 != 'K')) { - multiplier *= 1000; - if (end0 != 'M') { - multiplier *= 1000; - if (end0 != 'G') { - multiplier *= 1000; - if (end0 != 'T') { - multiplier *= 1000; - } - } - } - } - output = ival * multiplier; - return true; - - } else if ((pEnd[1] == 'b' || pEnd[1] == 'B') - && (pEnd[2] == '\0')) { - - unsigned long int multiplier = 1024; - - if ((end0 != 'K')) { - multiplier *= 1024; - if (end0 != 'M') { - multiplier *= 1024; - if (end0 != 'G') { - multiplier *= 1024; - if (end0 != 'T') { - multiplier *= 1024; - } - } - } - } - output = ival * multiplier; - return true; - } - } - - return false; - } - -protected: - //! Name - std::string _name; - //! Description - std::string _description; - //! Value - std::string _value; - -private: - -}; - -#endif diff --git a/libminifi/include/Provenance.h b/libminifi/include/Provenance.h deleted file mode 100644 index 3ba9792c8e..0000000000 --- a/libminifi/include/Provenance.h +++ /dev/null @@ -1,604 +0,0 @@ -/** - * @file Provenance.h - * Flow file record class declaration - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __PROVENANCE_H__ -#define __PROVENANCE_H__ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "Configure.h" -#include "Connection.h" -#include "FlowFileRecord.h" -#include "Logger.h" -#include "Property.h" -#include "ResourceClaim.h" -#include "io/Serializable.h" -#include "utils/TimeUtil.h" -#include "Repository.h" - -class ProvenanceRepository; - -//! Provenance Event Record -class ProvenanceEventRecord : protected Serializable -{ -public: - enum ProvenanceEventType { - - /** - * A CREATE event is used when a FlowFile is generated from data that was - * not received from a remote system or external process - */ - CREATE, - - /** - * Indicates a provenance event for receiving data from an external process. This Event Type - * is expected to be the first event for a FlowFile. As such, a Processor that receives data - * from an external source and uses that data to replace the content of an existing FlowFile - * should use the {@link #FETCH} event type, rather than the RECEIVE event type. - */ - RECEIVE, - - /** - * Indicates that the contents of a FlowFile were overwritten using the contents of some - * external resource. This is similar to the {@link #RECEIVE} event but varies in that - * RECEIVE events are intended to be used as the event that introduces the FlowFile into - * the system, whereas FETCH is used to indicate that the contents of an existing FlowFile - * were overwritten. - */ - FETCH, - - /** - * Indicates a provenance event for sending data to an external process - */ - SEND, - - /** - * Indicates that the contents of a FlowFile were downloaded by a user or external entity. - */ - DOWNLOAD, - - /** - * Indicates a provenance event for the conclusion of an object's life for - * some reason other than object expiration - */ - DROP, - - /** - * Indicates a provenance event for the conclusion of an object's life due - * to the fact that the object could not be processed in a timely manner - */ - EXPIRE, - - /** - * FORK is used to indicate that one or more FlowFile was derived from a - * parent FlowFile. - */ - FORK, - - /** - * JOIN is used to indicate that a single FlowFile is derived from joining - * together multiple parent FlowFiles. - */ - JOIN, - - /** - * CLONE is used to indicate that a FlowFile is an exact duplicate of its - * parent FlowFile. - */ - CLONE, - - /** - * CONTENT_MODIFIED is used to indicate that a FlowFile's content was - * modified in some way. When using this Event Type, it is advisable to - * provide details about how the content is modified. - */ - CONTENT_MODIFIED, - - /** - * ATTRIBUTES_MODIFIED is used to indicate that a FlowFile's attributes were - * modified in some way. This event is not needed when another event is - * reported at the same time, as the other event will already contain all - * FlowFile attributes. - */ - ATTRIBUTES_MODIFIED, - - /** - * ROUTE is used to show that a FlowFile was routed to a specified - * {@link org.apache.nifi.processor.Relationship Relationship} and should provide - * information about why the FlowFile was routed to this relationship. - */ - ROUTE, - - /** - * Indicates a provenance event for adding additional information such as a - * new linkage to a new URI or UUID - */ - ADDINFO, - - /** - * Indicates a provenance event for replaying a FlowFile. The UUID of the - * event will indicate the UUID of the original FlowFile that is being - * replayed. The event will contain exactly one Parent UUID that is also the - * UUID of the FlowFile that is being replayed and exactly one Child UUID - * that is the UUID of the a newly created FlowFile that will be re-queued - * for processing. - */ - REPLAY - }; - friend class ProcessSession; -public: - //! Constructor - /*! - * Create a new provenance event record - */ - ProvenanceEventRecord(ProvenanceEventType event, std::string componentId, std::string componentType) { - _eventType = event; - _componentId = componentId; - _componentType = componentType; - _eventTime = getTimeMillis(); - char eventIdStr[37]; - // Generate the global UUID for th event - uuid_generate(_eventId); - uuid_unparse_lower(_eventId, eventIdStr); - _eventIdStr = eventIdStr; - logger_ = Logger::getLogger(); - } - - ProvenanceEventRecord() { - _eventTime = getTimeMillis(); - logger_ = Logger::getLogger(); - } - - //! Destructor - virtual ~ProvenanceEventRecord() { - } - //! Get the Event ID - std::string getEventId() { - return _eventIdStr; - } - //! Get Attributes - std::map getAttributes() { - return _attributes; - } - //! Get Size - uint64_t getFileSize() { - return _size; - } - // ! Get Offset - uint64_t getFileOffset() { - return _offset; - } - // ! Get Entry Date - uint64_t getFlowFileEntryDate() { - return _entryDate; - } - // ! Get Lineage Start Date - uint64_t getlineageStartDate() { - return _lineageStartDate; - } - // ! Get Event Time - uint64_t getEventTime() { - return _eventTime; - } - // ! Get Event Duration - uint64_t getEventDuration() { - return _eventDuration; - } - //! Set Event Duration - void setEventDuration(uint64_t duration) - { - _eventDuration = duration; - } - // ! Get Event Type - ProvenanceEventType getEventType() { - return _eventType; - } - //! Get Component ID - std::string getComponentId() - { - return _componentId; - } - //! Get Component Type - std::string getComponentType() - { - return _componentType; - } - //! Get FlowFileUuid - std::string getFlowFileUuid() - { - return _uuid; - } - //! Get content full path - std::string getContentFullPath() - { - return _contentFullPath; - } - //! Get LineageIdentifiers - std::set getLineageIdentifiers() - { - return _lineageIdentifiers; - } - //! Get Details - std::string getDetails() - { - return _details; - } - //! Set Details - void setDetails(std::string details) - { - _details = details; - } - //! Get TransitUri - std::string getTransitUri() - { - return _transitUri; - } - //! Set TransitUri - void setTransitUri(std::string uri) - { - _transitUri = uri; - } - //! Get SourceSystemFlowFileIdentifier - std::string getSourceSystemFlowFileIdentifier() - { - return _sourceSystemFlowFileIdentifier; - } - //! Set SourceSystemFlowFileIdentifier - void setSourceSystemFlowFileIdentifier(std::string identifier) - { - _sourceSystemFlowFileIdentifier = identifier; - } - //! Get Parent UUIDs - std::vector getParentUuids() - { - return _parentUuids; - } - //! Add Parent UUID - void addParentUuid(std::string uuid) - { - if (std::find(_parentUuids.begin(), _parentUuids.end(), uuid) != _parentUuids.end()) - return; - else - _parentUuids.push_back(uuid); - } - //! Add Parent Flow File - void addParentFlowFile(FlowFileRecord *flow) - { - addParentUuid(flow->getUUIDStr()); - return; - } - //! Remove Parent UUID - void removeParentUuid(std::string uuid) - { - _parentUuids.erase(std::remove(_parentUuids.begin(), _parentUuids.end(), uuid), _parentUuids.end()); - } - //! Remove Parent Flow File - void removeParentFlowFile(FlowFileRecord *flow) - { - removeParentUuid(flow->getUUIDStr()); - return; - } - //! Get Children UUIDs - std::vector getChildrenUuids() - { - return _childrenUuids; - } - //! Add Child UUID - void addChildUuid(std::string uuid) - { - if (std::find(_childrenUuids.begin(), _childrenUuids.end(), uuid) != _childrenUuids.end()) - return; - else - _childrenUuids.push_back(uuid); - } - //! Add Child Flow File - void addChildFlowFile(FlowFileRecord *flow) - { - addChildUuid(flow->getUUIDStr()); - return; - } - //! Remove Child UUID - void removeChildUuid(std::string uuid) - { - _childrenUuids.erase(std::remove(_childrenUuids.begin(), _childrenUuids.end(), uuid), _childrenUuids.end()); - } - //! Remove Child Flow File - void removeChildFlowFile(FlowFileRecord *flow) - { - removeChildUuid(flow->getUUIDStr()); - return; - } - //! Get AlternateIdentifierUri - std::string getAlternateIdentifierUri() - { - return _alternateIdentifierUri; - } - //! Set AlternateIdentifierUri - void setAlternateIdentifierUri(std::string uri) - { - _alternateIdentifierUri = uri; - } - //! Get Relationship - std::string getRelationship() - { - return _relationship; - } - //! Set Relationship - void setRelationship(std::string relation) - { - _relationship = relation; - } - //! Get sourceQueueIdentifier - std::string getSourceQueueIdentifier() - { - return _sourceQueueIdentifier; - } - //! Set sourceQueueIdentifier - void setSourceQueueIdentifier(std::string identifier) - { - _sourceQueueIdentifier = identifier; - } - //! fromFlowFile - void fromFlowFile(FlowFileRecord *flow) - { - _entryDate = flow->getEntryDate(); - _lineageStartDate = flow->getlineageStartDate(); - _lineageIdentifiers = flow->getlineageIdentifiers(); - _uuid = flow->getUUIDStr(); - _attributes = flow->getAttributes(); - _size = flow->getSize(); - _offset = flow->getOffset(); - if (flow->getOriginalConnection()) - _sourceQueueIdentifier = flow->getOriginalConnection()->getName(); - if (flow->getResourceClaim()) - { - _contentFullPath = flow->getResourceClaim()->getContentFullPath(); - } - } - //! Serialize and Persistent to the repository - bool Serialize(ProvenanceRepository *repo); - //! DeSerialize - bool DeSerialize(const uint8_t *buffer, const int bufferSize); - //! DeSerialize - bool DeSerialize(DataStream &stream) - { - return DeSerialize(stream.getBuffer(),stream.getSize()); - } - //! DeSerialize - bool DeSerialize(ProvenanceRepository *repo, std::string key); - -protected: - - //! Event type - ProvenanceEventType _eventType; - //! Date at which the event was created - uint64_t _eventTime; - //! Date at which the flow file entered the flow - uint64_t _entryDate; - //! Date at which the origin of this flow file entered the flow - uint64_t _lineageStartDate; - //! Event Duration - uint64_t _eventDuration; - //! Component ID - std::string _componentId; - //! Component Type - std::string _componentType; - //! Size in bytes of the data corresponding to this flow file - uint64_t _size; - //! flow uuid - std::string _uuid; - //! Offset to the content - uint64_t _offset; - //! Full path to the content - std::string _contentFullPath; - //! Attributes key/values pairs for the flow record - std::map _attributes; - //! provenance ID - uuid_t _eventId; - //! UUID string for all parents - std::set _lineageIdentifiers; - //! transitUri - std::string _transitUri; - //! sourceSystemFlowFileIdentifier - std::string _sourceSystemFlowFileIdentifier; - //! parent UUID - std::vector _parentUuids; - //! child UUID - std::vector _childrenUuids; - //! detail - std::string _details; - //! sourceQueueIdentifier - std::string _sourceQueueIdentifier; - //! event ID Str - std::string _eventIdStr; - //! relationship - std::string _relationship; - //! alternateIdentifierUri; - std::string _alternateIdentifierUri; - -private: - - //! Logger - std::shared_ptr logger_; - - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - ProvenanceEventRecord(const ProvenanceEventRecord &parent); - ProvenanceEventRecord &operator=(const ProvenanceEventRecord &parent); - -}; - -//! Provenance Reporter -class ProvenanceReporter -{ - friend class ProcessSession; -public: - //! Constructor - /*! - * Create a new provenance reporter associated with the process session - */ - ProvenanceReporter(std::string componentId, std::string componentType) { - logger_ = Logger::getLogger(); - _componentId = componentId; - _componentType = componentType; - } - - //! Destructor - virtual ~ProvenanceReporter() { - clear(); - } - //! Get events - std::set getEvents() - { - return _events; - } - //! Add event - void add(ProvenanceEventRecord *event) - { - _events.insert(event); - } - //! Remove event - void remove(ProvenanceEventRecord *event) - { - if (_events.find(event) != _events.end()) - { - _events.erase(event); - } - } - //! - //! clear - void clear() - { - for (auto it : _events) - { - delete it; - } - _events.clear(); - } - //! allocate - ProvenanceEventRecord *allocate(ProvenanceEventRecord::ProvenanceEventType eventType, FlowFileRecord *flow) - { - ProvenanceEventRecord *event = new ProvenanceEventRecord(eventType, _componentId, _componentType); - if (event) - event->fromFlowFile(flow); - - return event; - } - //! commit - void commit(); - //! create - void create(FlowFileRecord *flow, std::string detail); - //! route - void route(FlowFileRecord *flow, Relationship relation, std::string detail, uint64_t processingDuration); - //! modifyAttributes - void modifyAttributes(FlowFileRecord *flow, std::string detail); - //! modifyContent - void modifyContent(FlowFileRecord *flow, std::string detail, uint64_t processingDuration); - //! clone - void clone(FlowFileRecord *parent, FlowFileRecord *child); - //! join - void join(std::vector parents, FlowFileRecord *child, std::string detail, uint64_t processingDuration); - //! fork - void fork(std::vector child, FlowFileRecord *parent, std::string detail, uint64_t processingDuration); - //! expire - void expire(FlowFileRecord *flow, std::string detail); - //! drop - void drop(FlowFileRecord *flow, std::string reason); - //! send - void send(FlowFileRecord *flow, std::string transitUri, std::string detail, uint64_t processingDuration, bool force); - //! fetch - void fetch(FlowFileRecord *flow, std::string transitUri, std::string detail, uint64_t processingDuration); - //! receive - void receive(FlowFileRecord *flow, std::string transitUri, std::string sourceSystemFlowFileIdentifier, std::string detail, uint64_t processingDuration); - -protected: - - //! Component ID - std::string _componentId; - //! Component Type - std::string _componentType; - -private: - - //! Incoming connection Iterator - std::set _events; - //! Logger - std::shared_ptr logger_; - - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - ProvenanceReporter(const ProvenanceReporter &parent); - ProvenanceReporter &operator=(const ProvenanceReporter &parent); -}; - -#define PROVENANCE_DIRECTORY "./provenance_repository" -#define MAX_PROVENANCE_STORAGE_SIZE (10*1024*1024) // 10M -#define MAX_PROVENANCE_ENTRY_LIFE_TIME (60000) // 1 minute -#define PROVENANCE_PURGE_PERIOD (2500) // 2500 msec - -//! Provenance Repository -class ProvenanceRepository : public Repository -{ -public: - //! Constructor - /*! - * Create a new provenance repository - */ - ProvenanceRepository() - : Repository(Repository::PROVENANCE, PROVENANCE_DIRECTORY, - MAX_PROVENANCE_ENTRY_LIFE_TIME, MAX_PROVENANCE_STORAGE_SIZE, PROVENANCE_PURGE_PERIOD) - { - } - - //! Destructor - virtual ~ProvenanceRepository() { - } - - //! Persistent event - void registerEvent(ProvenanceEventRecord *event) - { - event->Serialize(this); - } - //! Remove event - void removeEvent(ProvenanceEventRecord *event) - { - Delete(event->getEventId()); - } - -protected: - -private: - - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - ProvenanceRepository(const ProvenanceRepository &parent); - ProvenanceRepository &operator=(const ProvenanceRepository &parent); -}; - -#endif diff --git a/libminifi/include/PutFile.h b/libminifi/include/PutFile.h deleted file mode 100644 index 015605ecf6..0000000000 --- a/libminifi/include/PutFile.h +++ /dev/null @@ -1,88 +0,0 @@ -/** - * @file PutFile.h - * PutFile class declaration - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __PUT_FILE_H__ -#define __PUT_FILE_H__ - -#include "FlowFileRecord.h" -#include "Processor.h" -#include "ProcessSession.h" - -//! PutFile Class -class PutFile : public Processor -{ -public: - - static const std::string CONFLICT_RESOLUTION_STRATEGY_REPLACE; - static const std::string CONFLICT_RESOLUTION_STRATEGY_IGNORE; - static const std::string CONFLICT_RESOLUTION_STRATEGY_FAIL; - - //! Constructor - /*! - * Create a new processor - */ - PutFile(std::string name, uuid_t uuid = NULL) - : Processor(name, uuid) - { - logger_ = Logger::getLogger(); - } - //! Destructor - virtual ~PutFile() - { - } - //! Processor Name - static const std::string ProcessorName; - //! Supported Properties - static Property Directory; - static Property ConflictResolution; - //! Supported Relationships - static Relationship Success; - static Relationship Failure; - - //! OnTrigger method, implemented by NiFi PutFile - virtual void onTrigger(ProcessContext *context, ProcessSession *session); - //! Initialize, over write by NiFi PutFile - virtual void initialize(void); - - class ReadCallback : public InputStreamCallback - { - public: - ReadCallback(const std::string &tmpFile, const std::string &destFile); - ~ReadCallback(); - virtual void process(std::ifstream *stream); - bool commit(); - - private: - std::shared_ptr logger_; - std::ofstream _tmpFileOs; - bool _writeSucceeded = false; - std::string _tmpFile; - std::string _destFile; - }; - -protected: - -private: - //! Logger - std::shared_ptr logger_; - - bool putFile(ProcessSession *session, FlowFileRecord *flowFile, const std::string &tmpFile, const std::string &destFile); -}; - -#endif diff --git a/libminifi/include/RealTimeDataCollector.h b/libminifi/include/RealTimeDataCollector.h deleted file mode 100644 index 3b6d05fe2b..0000000000 --- a/libminifi/include/RealTimeDataCollector.h +++ /dev/null @@ -1,131 +0,0 @@ -/** - * @file RealTimeDataCollector.h - * RealTimeDataCollector class declaration - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __REAL_TIME_DATA_COLLECTOR_H__ -#define __REAL_TIME_DATA_COLLECTOR_H__ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "FlowFileRecord.h" -#include "Processor.h" -#include "ProcessSession.h" - -//! RealTimeDataCollector Class -class RealTimeDataCollector : public Processor -{ -public: - //! Constructor - /*! - * Create a new processor - */ - RealTimeDataCollector(std::string name, uuid_t uuid = NULL) - : Processor(name, uuid) - { - _realTimeSocket = 0; - _batchSocket = 0; - logger_ = Logger::getLogger(); - _firstInvoking = false; - _realTimeAccumulated = 0; - _batchAcccumulated = 0; - _queuedDataSize = 0; - } - //! Destructor - virtual ~RealTimeDataCollector() - { - if (_realTimeSocket) - close(_realTimeSocket); - if (_batchSocket) - close(_batchSocket); - if (_fileStream.is_open()) - _fileStream.close(); - } - //! Processor Name - static const std::string ProcessorName; - //! Supported Properties - static Property REALTIMESERVERNAME; - static Property REALTIMESERVERPORT; - static Property BATCHSERVERNAME; - static Property BATCHSERVERPORT; - static Property FILENAME; - static Property ITERATION; - static Property REALTIMEMSGID; - static Property BATCHMSGID; - static Property REALTIMEINTERVAL; - static Property BATCHINTERVAL; - static Property BATCHMAXBUFFERSIZE; - //! Supported Relationships - static Relationship Success; - //! Connect to the socket - int connectServer(const char *host, uint16_t port); - int sendData(int socket, const char *buf, int buflen); - void onTriggerRealTime(ProcessContext *context, ProcessSession *session); - void onTriggerBatch(ProcessContext *context, ProcessSession *session); - -public: - //! OnTrigger method, implemented by NiFi RealTimeDataCollector - virtual void onTrigger(ProcessContext *context, ProcessSession *session); - //! Initialize, over write by NiFi RealTimeDataCollector - virtual void initialize(void); - -protected: - -private: - //! realtime server Name - std::string _realTimeServerName; - int64_t _realTimeServerPort; - std::string _batchServerName; - int64_t _batchServerPort; - int64_t _realTimeInterval; - int64_t _batchInterval; - int64_t _batchMaxBufferSize; - //! Match pattern for Real time Message ID - std::vector _realTimeMsgID; - //! Match pattern for Batch Message ID - std::vector _batchMsgID; - //! file for which the realTime collector will tail - std::string _fileName; - //! Whether we need to iterate from the beginning for demo - bool _iteration; - int _realTimeSocket; - int _batchSocket; - //! Logger - std::shared_ptr logger_; - //! Mutex for protection - std::mutex _mtx; - //! Queued data size - uint64_t _queuedDataSize; - //! Queue for the batch process - std::queue _queue; - std::thread::id _realTimeThreadId; - std::thread::id _batchThreadId; - std::atomic _firstInvoking; - int64_t _realTimeAccumulated; - int64_t _batchAcccumulated; - std::ifstream _fileStream; -}; - -#endif diff --git a/libminifi/include/Relationship.h b/libminifi/include/Relationship.h deleted file mode 100644 index 3454ee5c3c..0000000000 --- a/libminifi/include/Relationship.h +++ /dev/null @@ -1,87 +0,0 @@ -/** - * @file Relationship.h - * Relationship class declaration - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __RELATIONSHIP_H__ -#define __RELATIONSHIP_H__ - -#include -#include -#include -#include -#include -#include -#include - -//! undefined relationship for remote process group outgoing port and root process group incoming port -#define UNDEFINED_RELATIONSHIP "undefined" - -inline bool isRelationshipNameUndefined(std::string name) -{ - if (name == UNDEFINED_RELATIONSHIP) - return true; - else - return false; -} - -//! Relationship Class -class Relationship { - -public: - //! Constructor - /*! - * Create a new relationship - */ - Relationship(const std::string name, const std::string description) - : _name(name), _description(description) { - } - Relationship() - : _name(UNDEFINED_RELATIONSHIP) { - } - //! Destructor - virtual ~Relationship() { - } - //! Get Name for the relationship - std::string getName() { - return _name; - } - //! Get Description for the relationship - std::string getDescription() { - return _description; - } - //! Compare - bool operator < (const Relationship & right) const { - return _name < right._name; - } - //! Whether it is a undefined relationship - bool isRelationshipUndefined() - { - return isRelationshipNameUndefined(_name); - } - -protected: - - //! Name - std::string _name; - //! Description - std::string _description; - -private: -}; - -#endif diff --git a/libminifi/include/RemoteProcessorGroupPort.h b/libminifi/include/RemoteProcessorGroupPort.h index 05ecd17ca4..f28469bcc5 100644 --- a/libminifi/include/RemoteProcessorGroupPort.h +++ b/libminifi/include/RemoteProcessorGroupPort.h @@ -22,69 +22,83 @@ #include #include "FlowFileRecord.h" -#include "Processor.h" -#include "ProcessSession.h" +#include "core/Processor.h" +#include "core/ProcessSession.h" #include "Site2SiteClientProtocol.h" -//! RemoteProcessorGroupPort Class -class RemoteProcessorGroupPort: public Processor { -public: - //! Constructor - /*! - * Create a new processor - */ - RemoteProcessorGroupPort(std::string name, uuid_t uuid = NULL) : - Processor(name, uuid), direction_(SEND), transmitting_(false), peer_() { - logger_ = Logger::getLogger(); - protocol_ = std::unique_ptr( - new Site2SiteClientProtocol(0)); - protocol_->setPortId(uuid); - } - //! Destructor - virtual ~RemoteProcessorGroupPort() { +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +// RemoteProcessorGroupPort Class +class RemoteProcessorGroupPort : + public core::Processor { + public: + // Constructor + /*! + * Create a new processor + */ + RemoteProcessorGroupPort(std::string name, uuid_t uuid = NULL) + : core::Processor(name, uuid), + direction_(SEND), + transmitting_(false), + peer_() { + logger_ = logging::Logger::getLogger(); + protocol_ = std::unique_ptr( + new Site2SiteClientProtocol(0)); + protocol_->setPortId(uuid); + } + // Destructor + virtual ~RemoteProcessorGroupPort() { - } - //! Processor Name - static const std::string ProcessorName; - //! Supported Properties - static Property hostName; - static Property port; - //! Supported Relationships - static Relationship relation; -public: - //! OnTrigger method, implemented by NiFi RemoteProcessorGroupPort - virtual void onTrigger(ProcessContext *context, ProcessSession *session); - //! Initialize, over write by NiFi RemoteProcessorGroupPort - virtual void initialize(void); - //! Set Direction - void setDirection(TransferDirection direction) { - direction_ = direction; - if (direction_ == RECEIVE) - this->setTriggerWhenEmpty(true); - } - //! Set Timeout - void setTimeOut(uint64_t timeout) { - protocol_->setTimeOut(timeout); - } - //! SetTransmitting - void setTransmitting(bool val) { - transmitting_ = val; - } + } + // Processor Name + static const std::string ProcessorName; + // Supported Properties + static core::Property hostName; + static core::Property port; + // Supported Relationships + static core::Relationship relation; + public: + // OnTrigger method, implemented by NiFi RemoteProcessorGroupPort + virtual void onTrigger( + core::ProcessContext *context, + core::ProcessSession *session); + // Initialize, over write by NiFi RemoteProcessorGroupPort + virtual void initialize(void); + // Set Direction + void setDirection(TransferDirection direction) { + direction_ = direction; + if (direction_ == RECEIVE) + this->setTriggerWhenEmpty(true); + } + // Set Timeout + void setTimeOut(uint64_t timeout) { + protocol_->setTimeOut(timeout); + } + // SetTransmitting + void setTransmitting(bool val) { + transmitting_ = val; + } -protected: + protected: -private: - //! Logger - std::shared_ptr logger_; - //! Peer Connection - Site2SitePeer peer_; - //! Peer Protocol - std::unique_ptr protocol_; - //! Transaction Direction - TransferDirection direction_; - //! Transmitting - bool transmitting_; + private: + // Logger + std::shared_ptr logger_; + // Peer Connection + Site2SitePeer peer_; + // Peer Protocol + std::unique_ptr protocol_; + // Transaction Direction + TransferDirection direction_; + // Transmitting + bool transmitting_; }; +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ #endif diff --git a/libminifi/include/Repository.h b/libminifi/include/Repository.h deleted file mode 100644 index 55fb442741..0000000000 --- a/libminifi/include/Repository.h +++ /dev/null @@ -1,318 +0,0 @@ -/** - * @file Repository - * Repository class declaration - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __REPOSITORY_H__ -#define __REPOSITORY_H__ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef LEVELDB_SUPPORT -#include "leveldb/db.h" -#include "leveldb/options.h" -#include "leveldb/slice.h" -#include "leveldb/status.h" -#endif -#include "Configure.h" -#include "Connection.h" -#include "FlowFileRecord.h" -#include "Logger.h" -#include "Property.h" -#include "ResourceClaim.h" -#include "io/Serializable.h" -#include "utils/TimeUtil.h" -#include "utils/StringUtils.h" - -//! Repository -class Repository -{ -public: - enum RepositoryType { - //! Provenance Repo Type - PROVENANCE, - //! FlowFile Repo Type - FLOWFILE, - MAX_REPO_TYPE - }; - static const char *RepositoryTypeStr[MAX_REPO_TYPE]; - //! Constructor - /*! - * Create a new provenance repository - */ - Repository(RepositoryType type, std::string directory, - int64_t maxPartitionMillis, int64_t maxPartitionBytes, uint64_t purgePeriod) { - _type = type; - _directory = directory; - _maxPartitionMillis = maxPartitionMillis; - _maxPartitionBytes = maxPartitionBytes; - _purgePeriod = purgePeriod; - logger_ = Logger::getLogger(); - configure_ = Configure::getConfigure(); -#ifdef LEVELDB_SUPPORT - _db = NULL; -#endif - _thread = NULL; - _running = false; - _repoFull = false; - _enable = true; - } - - //! Destructor - virtual ~Repository() { - stop(); - if (this->_thread) - delete this->_thread; - destroy(); - } - - //! initialize - virtual bool initialize() - { - std::string value; - -#ifdef LEVELDB_SUPPORT - if (_type == PROVENANCE) - { - if (!(configure_->get(Configure::nifi_provenance_repository_enable, value) - && StringUtils::StringToBool(value, _enable))) { - _enable = true; - } - if (!_enable) - return false; - if (configure_->get(Configure::nifi_provenance_repository_directory_default, value)) - { - _directory = value; - } - logger_->log_info("NiFi Provenance Repository Directory %s", _directory.c_str()); - if (configure_->get(Configure::nifi_provenance_repository_max_storage_size, value)) - { - Property::StringToInt(value, _maxPartitionBytes); - } - logger_->log_info("NiFi Provenance Max Partition Bytes %d", _maxPartitionBytes); - if (configure_->get(Configure::nifi_provenance_repository_max_storage_time, value)) - { - TimeUnit unit; - if (Property::StringToTime(value, _maxPartitionMillis, unit) && - Property::ConvertTimeUnitToMS(_maxPartitionMillis, unit, _maxPartitionMillis)) - { - } - } - logger_->log_info("NiFi Provenance Max Storage Time: [%d] ms", _maxPartitionMillis); - leveldb::Options options; - options.create_if_missing = true; - leveldb::Status status = leveldb::DB::Open(options, _directory.c_str(), &_db); - if (status.ok()) - { - logger_->log_info("NiFi Provenance Repository database open %s success", _directory.c_str()); - } - else - { - logger_->log_error("NiFi Provenance Repository database open %s fail", _directory.c_str()); - return false; - } - } - - if (_type == FLOWFILE) - { - if (!(configure_->get(Configure::nifi_flowfile_repository_enable, value) - && StringUtils::StringToBool(value, _enable))) { - _enable = true; - } - if (!_enable) - return false; - if (configure_->get(Configure::nifi_flowfile_repository_directory_default, value)) - { - _directory = value; - } - logger_->log_info("NiFi FlowFile Repository Directory %s", _directory.c_str()); - if (configure_->get(Configure::nifi_flowfile_repository_max_storage_size, value)) - { - Property::StringToInt(value, _maxPartitionBytes); - } - logger_->log_info("NiFi FlowFile Max Partition Bytes %d", _maxPartitionBytes); - if (configure_->get(Configure::nifi_flowfile_repository_max_storage_time, value)) - { - TimeUnit unit; - if (Property::StringToTime(value, _maxPartitionMillis, unit) && - Property::ConvertTimeUnitToMS(_maxPartitionMillis, unit, _maxPartitionMillis)) - { - } - } - logger_->log_info("NiFi FlowFile Max Storage Time: [%d] ms", _maxPartitionMillis); - leveldb::Options options; - options.create_if_missing = true; - leveldb::Status status = leveldb::DB::Open(options, _directory.c_str(), &_db); - if (status.ok()) - { - logger_->log_info("NiFi FlowFile Repository database open %s success", _directory.c_str()); - } - else - { - logger_->log_error("NiFi FlowFile Repository database open %s fail", _directory.c_str()); - return false; - } - } - - return true; -#else - return false; -#endif - } - //! Put - virtual bool Put(std::string key, uint8_t *buf, int bufLen) - { -#ifdef LEVELDB_SUPPORT - if (!_enable) - return false; - - // persistent to the DB - leveldb::Slice value((const char *) buf, bufLen); - leveldb::Status status; - status = _db->Put(leveldb::WriteOptions(), key, value); - if (status.ok()) - return true; - else - return false; -#else - return false; -#endif - } - //! Delete - virtual bool Delete(std::string key) - { -#ifdef LEVELDB_SUPPORT - if (!_enable) - return false; - leveldb::Status status; - status = _db->Delete(leveldb::WriteOptions(), key); - if (status.ok()) - return true; - else - return false; -#else - return false; -#endif - } - //! Get - virtual bool Get(std::string key, std::string &value) - { -#ifdef LEVELDB_SUPPORT - if (!_enable) - return false; - leveldb::Status status; - status = _db->Get(leveldb::ReadOptions(), key, &value); - if (status.ok()) - return true; - else - return false; -#else - return false; -#endif - } - //! Run function for the thread - static void run(Repository *repo); - //! Start the repository monitor thread - virtual void start(); - //! Stop the repository monitor thread - virtual void stop(); - //! whether the repo is full - virtual bool isFull() - { - return _repoFull; - } - //! whether the repo is enable - virtual bool isEnable() - { - return _enable; - } - -protected: - //! Repo Type - RepositoryType _type; - //! Mutex for protection - std::mutex _mtx; - //! repository directory - std::string _directory; - //! Logger - std::shared_ptr logger_; - //! Configure - //! max db entry life time - Configure *configure_; - int64_t _maxPartitionMillis; - //! max db size - int64_t _maxPartitionBytes; - //! purge period - uint64_t _purgePeriod; -#ifdef LEVELDB_SUPPORT - //! level DB database - leveldb::DB* _db; -#endif - //! thread - std::thread *_thread; - //! whether the monitoring thread is running for the repo while it was enabled - bool _running; - //! whether it is enabled by minfi property for the repo - bool _enable; - //! whether stop accepting provenace event - std::atomic _repoFull; - //! repoSize - uint64_t repoSize(); - //! size of the directory - static uint64_t _repoSize[MAX_REPO_TYPE]; - //! call back for directory size - static int repoSumProvenance(const char *fpath, const struct stat *sb, int typeflag) - { - _repoSize[PROVENANCE] += sb->st_size; - return 0; - } - //! call back for directory size - static int repoSumFlowFile(const char *fpath, const struct stat *sb, int typeflag) - { - _repoSize[FLOWFILE] += sb->st_size; - return 0; - } - -private: - //! destroy - void destroy() - { -#ifdef LEVELDB_SUPPORT - if (_db) - { - delete _db; - _db = NULL; - } -#endif - } - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - Repository(const Repository &parent); - Repository &operator=(const Repository &parent); -}; - -#endif diff --git a/libminifi/include/ResourceClaim.h b/libminifi/include/ResourceClaim.h index 7ca79a3505..6178ee2a54 100644 --- a/libminifi/include/ResourceClaim.h +++ b/libminifi/include/ResourceClaim.h @@ -29,73 +29,83 @@ #include #include "Configure.h" -//! Default content directory +namespace org { +namespace apache { +namespace nifi { +namespace minifi { + + +// Default content directory #define DEFAULT_CONTENT_DIRECTORY "./content_repository" -//! ResourceClaim Class +// ResourceClaim Class class ResourceClaim { public: static std::string default_directory_path; - //! Constructor + // Constructor /*! * Create a new resource claim */ ResourceClaim(const std::string contentDirectory = default_directory_path); - //! Destructor + // Destructor virtual ~ResourceClaim() {} - //! increaseFlowFileRecordOwnedCount + // increaseFlowFileRecordOwnedCount void increaseFlowFileRecordOwnedCount() { ++_flowFileRecordOwnedCount; } - //! decreaseFlowFileRecordOwenedCount + // decreaseFlowFileRecordOwenedCount void decreaseFlowFileRecordOwnedCount() { --_flowFileRecordOwnedCount; } - //! getFlowFileRecordOwenedCount + // getFlowFileRecordOwenedCount uint64_t getFlowFileRecordOwnedCount() { return _flowFileRecordOwnedCount; } - //! Get the content full path + // Get the content full path std::string getContentFullPath() { return _contentFullPath; } - //! Set the content full path + // Set the content full path void setContentFullPath(std::string path) { _contentFullPath = path; } protected: - //! A global unique identifier + // A global unique identifier uuid_t _uuid; - //! A local unique identifier + // A local unique identifier uint64_t _id; - //! Full path to the content + // Full path to the content std::string _contentFullPath; - //! How many FlowFileRecord Own this cliam + // How many FlowFileRecord Own this cliam std::atomic _flowFileRecordOwnedCount; private: - //! Configure + // Configure Configure *configure_; - //! Logger - std::shared_ptr logger_; + // Logger + std::shared_ptr logger_; // Prevent default copy constructor and assignment operation // Only support pass by reference or pointer ResourceClaim(const ResourceClaim &parent); ResourceClaim &operator=(const ResourceClaim &parent); - //! Local resource claim number + // Local resource claim number static std::atomic _localResourceClaimNumber; }; +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ #endif diff --git a/libminifi/include/SchedulingAgent.h b/libminifi/include/SchedulingAgent.h index f6d5a1cdad..67cab47e1f 100644 --- a/libminifi/include/SchedulingAgent.h +++ b/libminifi/include/SchedulingAgent.h @@ -28,72 +28,90 @@ #include #include #include "utils/TimeUtil.h" -#include "Logger.h" +#include "core/logging/Logger.h" #include "Configure.h" #include "FlowFileRecord.h" -#include "Logger.h" -#include "Processor.h" -#include "ProcessContext.h" +#include "core/logging/Logger.h" +#include "core/Processor.h" +#include "core/ProcessContext.h" -//! SchedulingAgent Class -class SchedulingAgent -{ -public: - //! Constructor - /*! - * Create a new processor - */ - SchedulingAgent() { - configure_ = Configure::getConfigure(); - logger_ = Logger::getLogger(); - _running = false; - } - //! Destructor - virtual ~SchedulingAgent() - { - } - //! onTrigger, return whether the yield is need - bool onTrigger(Processor *processor, ProcessContext *processContext, ProcessSessionFactory *sessionFactory); - //! Whether agent has work to do - bool hasWorkToDo(Processor *processor); - //! Whether the outgoing need to be backpressure - bool hasTooMuchOutGoing(Processor *processor); - //! start - void start() { - _running = true; - } - //! stop - void stop() { - _running = false; - } +namespace org { +namespace apache { +namespace nifi { +namespace minifi { -public: - //! schedule, overwritten by different DrivenSchedulingAgent - virtual void schedule(Processor *processor) = 0; - //! unschedule, overwritten by different DrivenSchedulingAgent - virtual void unschedule(Processor *processor) = 0; -protected: - //! Logger - std::shared_ptr logger_; - //! Configure - Configure *configure_; - //! Mutex for protection - std::mutex _mtx; - //! Whether it is running - std::atomic _running; - //! AdministrativeYieldDuration - int64_t _administrativeYieldDuration; - //! BoredYieldDuration - int64_t _boredYieldDuration; +// SchedulingAgent Class +class SchedulingAgent { + public: + // Constructor + /*! + * Create a new processor + */ + SchedulingAgent(std::shared_ptr repo) { + configure_ = Configure::getConfigure(); + logger_ = logging::Logger::getLogger(); + running_ = false; + repo_ = repo; + } + // Destructor + virtual ~SchedulingAgent() { -private: - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - SchedulingAgent(const SchedulingAgent &parent); - SchedulingAgent &operator=(const SchedulingAgent &parent); + } + // onTrigger, return whether the yield is need + bool onTrigger( + std::shared_ptr processor, + core::ProcessContext *processContext, + core::ProcessSessionFactory *sessionFactory); + // Whether agent has work to do + bool hasWorkToDo(std::shared_ptr processor); + // Whether the outgoing need to be backpressure + bool hasTooMuchOutGoing( + std::shared_ptr processor); + // start + void start() { + running_ = true; + } + // stop + void stop() { + running_ = false; + } + + public: + // schedule, overwritten by different DrivenSchedulingAgent + virtual void schedule( + std::shared_ptr processor) = 0; + // unschedule, overwritten by different DrivenSchedulingAgent + virtual void unschedule( + std::shared_ptr processor) = 0; + + SchedulingAgent(const SchedulingAgent &parent) = delete; + SchedulingAgent &operator=(const SchedulingAgent &parent) = delete; + protected: + // Logger + std::shared_ptr logger_; + // Configure + Configure *configure_; + // Mutex for protection + std::mutex mutex_; + // Whether it is running + std::atomic running_; + // AdministrativeYieldDuration + int64_t _administrativeYieldDuration; + // BoredYieldDuration + int64_t _boredYieldDuration; + + std::shared_ptr repo_; + + private: + // Prevent default copy constructor and assignment operation + // Only support pass by reference or pointer }; +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ #endif diff --git a/libminifi/include/Site2SiteClientProtocol.h b/libminifi/include/Site2SiteClientProtocol.h index 444eed50d7..705909e5a4 100644 --- a/libminifi/include/Site2SiteClientProtocol.h +++ b/libminifi/include/Site2SiteClientProtocol.h @@ -34,16 +34,23 @@ #include #include #include + +#include "core/Property.h" #include "Configure.h" -#include "Property.h" #include "Site2SitePeer.h" #include "FlowFileRecord.h" -#include "Logger.h" -#include "ProcessContext.h" -#include "ProcessSession.h" +#include "core/logging/Logger.h" +#include "core/ProcessContext.h" +#include "core/ProcessSession.h" #include "io/CRCStream.h" -//! Resource Negotiated Status Code + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { + +// Resource Negotiated Status Code #define RESOURCE_OK 20 #define DIFFERENT_RESOURCE_VERSION 21 #define NEGOTIATED_ABORT 255 @@ -55,143 +62,144 @@ * transferred between a client and a remote NiFi instance. */ typedef enum { - /** - * * The client is to send data to the remote instance. - * */ - SEND, - /** - * * The client is to receive data from the remote instance. - * */ - RECEIVE + /** + * * The client is to send data to the remote instance. + * */ + SEND, + /** + * * The client is to receive data from the remote instance. + * */ + RECEIVE } TransferDirection; -//! Peer State +// Peer State typedef enum { - /** - * * IDLE - * */ - IDLE = 0, - /** - * * Socket Established - * */ - ESTABLISHED, - /** - * * HandShake Done - * */ - HANDSHAKED, - /** - * * After CodeDec Completion - * */ - READY + /** + * * IDLE + * */ + IDLE = 0, + /** + * * Socket Established + * */ + ESTABLISHED, + /** + * * HandShake Done + * */ + HANDSHAKED, + /** + * * After CodeDec Completion + * */ + READY } PeerState; -//! Transaction State +// Transaction State typedef enum { - /** - * * Transaction has been started but no data has been sent or received. - * */ - TRANSACTION_STARTED, - /** - * * Transaction has been started and data has been sent or received. - * */ - DATA_EXCHANGED, - /** - * * Data that has been transferred has been confirmed via its CRC. - * * Transaction is ready to be completed. - * */ - TRANSACTION_CONFIRMED, - /** - * * Transaction has been successfully completed. - * */ - TRANSACTION_COMPLETED, - /** - * * The Transaction has been canceled. - * */ - TRANSACTION_CANCELED, - /** - * * The Transaction ended in an error. - * */ - TRANSACTION_ERROR + /** + * * Transaction has been started but no data has been sent or received. + * */ + TRANSACTION_STARTED, + /** + * * Transaction has been started and data has been sent or received. + * */ + DATA_EXCHANGED, + /** + * * Data that has been transferred has been confirmed via its CRC. + * * Transaction is ready to be completed. + * */ + TRANSACTION_CONFIRMED, + /** + * * Transaction has been successfully completed. + * */ + TRANSACTION_COMPLETED, + /** + * * The Transaction has been canceled. + * */ + TRANSACTION_CANCELED, + /** + * * The Transaction ended in an error. + * */ + TRANSACTION_ERROR } TransactionState; -//! Request Type +// Request Type typedef enum { - NEGOTIATE_FLOWFILE_CODEC = 0, - REQUEST_PEER_LIST, - SEND_FLOWFILES, - RECEIVE_FLOWFILES, - SHUTDOWN, - MAX_REQUEST_TYPE + NEGOTIATE_FLOWFILE_CODEC = 0, + REQUEST_PEER_LIST, + SEND_FLOWFILES, + RECEIVE_FLOWFILES, + SHUTDOWN, + MAX_REQUEST_TYPE } RequestType; -//! Request Type Str +// Request Type Str static const char *RequestTypeStr[MAX_REQUEST_TYPE] = { - "NEGOTIATE_FLOWFILE_CODEC", "REQUEST_PEER_LIST", "SEND_FLOWFILES", - "RECEIVE_FLOWFILES", "SHUTDOWN" }; + "NEGOTIATE_FLOWFILE_CODEC", "REQUEST_PEER_LIST", "SEND_FLOWFILES", + "RECEIVE_FLOWFILES", "SHUTDOWN" }; -//! Respond Code +// Respond Code typedef enum { - RESERVED = 0, - // ResponseCode, so that we can indicate a 0 followed by some other bytes - - // handshaking properties - PROPERTIES_OK = 1, - UNKNOWN_PROPERTY_NAME = 230, - ILLEGAL_PROPERTY_VALUE = 231, - MISSING_PROPERTY = 232, - // transaction indicators - CONTINUE_TRANSACTION = 10, - FINISH_TRANSACTION = 11, - CONFIRM_TRANSACTION = 12, // "Explanation" of this code is the checksum - TRANSACTION_FINISHED = 13, - TRANSACTION_FINISHED_BUT_DESTINATION_FULL = 14, - CANCEL_TRANSACTION = 15, - BAD_CHECKSUM = 19, - // data availability indicators - MORE_DATA = 20, - NO_MORE_DATA = 21, - // port state indicators - UNKNOWN_PORT = 200, - PORT_NOT_IN_VALID_STATE = 201, - PORTS_DESTINATION_FULL = 202, - // authorization - UNAUTHORIZED = 240, - // error indicators - ABORT = 250, - UNRECOGNIZED_RESPONSE_CODE = 254, - END_OF_STREAM = 255 + RESERVED = 0, + // ResponseCode, so that we can indicate a 0 followed by some other bytes + + // handshaking properties + PROPERTIES_OK = 1, + UNKNOWN_PROPERTY_NAME = 230, + ILLEGAL_PROPERTY_VALUE = 231, + MISSING_PROPERTY = 232, + // transaction indicators + CONTINUE_TRANSACTION = 10, + FINISH_TRANSACTION = 11, + CONFIRM_TRANSACTION = 12, // "Explanation" of this code is the checksum + TRANSACTION_FINISHED = 13, + TRANSACTION_FINISHED_BUT_DESTINATION_FULL = 14, + CANCEL_TRANSACTION = 15, + BAD_CHECKSUM = 19, + // data availability indicators + MORE_DATA = 20, + NO_MORE_DATA = 21, + // port state indicators + UNKNOWN_PORT = 200, + PORT_NOT_IN_VALID_STATE = 201, + PORTS_DESTINATION_FULL = 202, + // authorization + UNAUTHORIZED = 240, + // error indicators + ABORT = 250, + UNRECOGNIZED_RESPONSE_CODE = 254, + END_OF_STREAM = 255 } RespondCode; -//! Respond Code Class +// Respond Code Class typedef struct { - RespondCode code; - const char *description; - bool hasDescription; + RespondCode code; + const char *description; + bool hasDescription; } RespondCodeContext; -//! Respond Code Context +// Respond Code Context static RespondCodeContext respondCodeContext[] = { { RESERVED, - "Reserved for Future Use", false }, { PROPERTIES_OK, "Properties OK", - false }, { UNKNOWN_PROPERTY_NAME, "Unknown Property Name", true }, { - ILLEGAL_PROPERTY_VALUE, "Illegal Property Value", true }, { - MISSING_PROPERTY, "Missing Property", true }, { CONTINUE_TRANSACTION, - "Continue Transaction", false }, { FINISH_TRANSACTION, - "Finish Transaction", false }, { CONFIRM_TRANSACTION, - "Confirm Transaction", true }, { TRANSACTION_FINISHED, - "Transaction Finished", false }, { - TRANSACTION_FINISHED_BUT_DESTINATION_FULL, - "Transaction Finished But Destination is Full", false }, { - CANCEL_TRANSACTION, "Cancel Transaction", true }, { BAD_CHECKSUM, - "Bad Checksum", false }, { MORE_DATA, "More Data Exists", false }, { - NO_MORE_DATA, "No More Data Exists", false }, { UNKNOWN_PORT, - "Unknown Port", false }, { PORT_NOT_IN_VALID_STATE, - "Port Not in a Valid State", true }, { PORTS_DESTINATION_FULL, - "Port's Destination is Full", false }, { UNAUTHORIZED, - "User Not Authorized", true }, { ABORT, "Abort", true }, { - UNRECOGNIZED_RESPONSE_CODE, "Unrecognized Response Code", false }, { - END_OF_STREAM, "End of Stream", false } }; - -//! Respond Code Sequence Pattern + "Reserved for Future Use", false }, + { PROPERTIES_OK, "Properties OK", false }, { UNKNOWN_PROPERTY_NAME, + "Unknown Property Name", true }, { ILLEGAL_PROPERTY_VALUE, + "Illegal Property Value", true }, { MISSING_PROPERTY, + "Missing Property", true }, { CONTINUE_TRANSACTION, + "Continue Transaction", false }, { FINISH_TRANSACTION, + "Finish Transaction", false }, { CONFIRM_TRANSACTION, + "Confirm Transaction", true }, { TRANSACTION_FINISHED, + "Transaction Finished", false }, { + TRANSACTION_FINISHED_BUT_DESTINATION_FULL, + "Transaction Finished But Destination is Full", false }, { + CANCEL_TRANSACTION, "Cancel Transaction", true }, { BAD_CHECKSUM, + "Bad Checksum", false }, { MORE_DATA, "More Data Exists", false }, { + NO_MORE_DATA, "No More Data Exists", false }, { UNKNOWN_PORT, + "Unknown Port", false }, { PORT_NOT_IN_VALID_STATE, + "Port Not in a Valid State", true }, { PORTS_DESTINATION_FULL, + "Port's Destination is Full", false }, { UNAUTHORIZED, + "User Not Authorized", true }, { ABORT, "Abort", true }, { + UNRECOGNIZED_RESPONSE_CODE, "Unrecognized Response Code", false }, { + END_OF_STREAM, "End of Stream", false } }; + +// Respond Code Sequence Pattern static const uint8_t CODE_SEQUENCE_VALUE_1 = (uint8_t) 'R'; static const uint8_t CODE_SEQUENCE_VALUE_2 = (uint8_t) 'C'; @@ -200,43 +208,44 @@ static const uint8_t CODE_SEQUENCE_VALUE_2 = (uint8_t) 'C'; * Protocol. */ typedef enum { - /** - * Boolean value indicating whether or not the contents of a FlowFile should - * be GZipped when transferred. - */ - GZIP, - /** - * The unique identifier of the port to communicate with - */ - PORT_IDENTIFIER, - /** - * Indicates the number of milliseconds after the request was made that the - * client will wait for a response. If no response has been received by the - * time this value expires, the server can move on without attempting to - * service the request because the client will have already disconnected. - */ - REQUEST_EXPIRATION_MILLIS, - /** - * The preferred number of FlowFiles that the server should send to the - * client when pulling data. This property was introduced in version 5 of - * the protocol. - */ - BATCH_COUNT, - /** - * The preferred number of bytes that the server should send to the client - * when pulling data. This property was introduced in version 5 of the - * protocol. - */ - BATCH_SIZE, - /** - * The preferred amount of time that the server should send data to the - * client when pulling data. This property was introduced in version 5 of - * the protocol. Value is in milliseconds. - */ - BATCH_DURATION, MAX_HANDSHAKE_PROPERTY + /** + * Boolean value indicating whether or not the contents of a FlowFile should + * be GZipped when transferred. + */ + GZIP, + /** + * The unique identifier of the port to communicate with + */ + PORT_IDENTIFIER, + /** + * Indicates the number of milliseconds after the request was made that the + * client will wait for a response. If no response has been received by the + * time this value expires, the server can move on without attempting to + * service the request because the client will have already disconnected. + */ + REQUEST_EXPIRATION_MILLIS, + /** + * The preferred number of FlowFiles that the server should send to the + * client when pulling data. This property was introduced in version 5 of + * the protocol. + */ + BATCH_COUNT, + /** + * The preferred number of bytes that the server should send to the client + * when pulling data. This property was introduced in version 5 of the + * protocol. + */ + BATCH_SIZE, + /** + * The preferred amount of time that the server should send data to the + * client when pulling data. This property was introduced in version 5 of + * the protocol. Value is in milliseconds. + */ + BATCH_DURATION, + MAX_HANDSHAKE_PROPERTY } HandshakeProperty; -//! HandShakeProperty Str +// HandShakeProperty Str static const char *HandShakePropertyStr[MAX_HANDSHAKE_PROPERTY] = { /** * Boolean value indicating whether or not the contents of a FlowFile should @@ -275,89 +284,89 @@ static const char *HandShakePropertyStr[MAX_HANDSHAKE_PROPERTY] = { class Site2SiteClientProtocol; -//! Transaction Class +// Transaction Class class Transaction { - friend class Site2SiteClientProtocol; -public: - //! Constructor - /*! - * Create a new transaction - */ - explicit Transaction(TransferDirection direction, - CRCStream &stream) : - crcStream(std::move(stream)) { - _state = TRANSACTION_STARTED; - _direction = direction; - _dataAvailable = false; - _transfers = 0; - _bytes = 0; - - char uuidStr[37]; - - // Generate the global UUID for the transaction - uuid_generate(_uuid); - uuid_unparse_lower(_uuid, uuidStr); - _uuidStr = uuidStr; - } - //! Destructor - virtual ~Transaction() { - } - //! getUUIDStr - std::string getUUIDStr() { - return _uuidStr; - } - //! getState - TransactionState getState() { - return _state; - } - //! isDataAvailable - bool isDataAvailable() { - return _dataAvailable; - } - //! setDataAvailable() - void setDataAvailable(bool value) { - _dataAvailable = value; - } - //! getDirection - TransferDirection getDirection() { - return _direction; - } - //! getCRC - long getCRC() { - return crcStream.getCRC(); - } - //! updateCRC - void updateCRC(uint8_t *buffer, uint32_t length) { - crcStream.updateCRC(buffer, length); - } - - CRCStream &getStream() { - return crcStream; - } - - - Transaction(const Transaction &parent) = delete; - Transaction &operator=(const Transaction &parent) = delete; - -protected: - -private: - - CRCStream crcStream; - //! Transaction State - TransactionState _state; - //! Transaction Direction - TransferDirection _direction; - //! Whether received data is available - bool _dataAvailable; - //! A global unique identifier - uuid_t _uuid; - //! UUID string - std::string _uuidStr; - //! Number of transfer - int _transfers; - //! Number of content bytes - uint64_t _bytes; + friend class Site2SiteClientProtocol; + public: + // Constructor + /*! + * Create a new transaction + */ + explicit Transaction( + TransferDirection direction, + org::apache::nifi::minifi::io::CRCStream &stream) + : crcStream(std::move(stream)) { + _state = TRANSACTION_STARTED; + _direction = direction; + _dataAvailable = false; + _transfers = 0; + _bytes = 0; + + char uuidStr[37]; + + // Generate the global UUID for the transaction + uuid_generate(_uuid); + uuid_unparse_lower(_uuid, uuidStr); + _uuidStr = uuidStr; + } + // Destructor + virtual ~Transaction() { + } + // getUUIDStr + std::string getUUIDStr() { + return _uuidStr; + } + // getState + TransactionState getState() { + return _state; + } + // isDataAvailable + bool isDataAvailable() { + return _dataAvailable; + } + // setDataAvailable() + void setDataAvailable(bool value) { + _dataAvailable = value; + } + // getDirection + TransferDirection getDirection() { + return _direction; + } + // getCRC + long getCRC() { + return crcStream.getCRC(); + } + // updateCRC + void updateCRC(uint8_t *buffer, uint32_t length) { + crcStream.updateCRC(buffer, length); + } + + org::apache::nifi::minifi::io::CRCStream &getStream() { + return crcStream; + } + + Transaction(const Transaction &parent) = delete; + Transaction &operator=(const Transaction &parent) = delete; + + protected: + + private: + + org::apache::nifi::minifi::io::CRCStream crcStream; + // Transaction State + TransactionState _state; + // Transaction Direction + TransferDirection _direction; + // Whether received data is available + bool _dataAvailable; + // A global unique identifier + uuid_t _uuid; + // UUID string + std::string _uuidStr; + // Number of transfer + int _transfers; + // Number of content bytes + uint64_t _bytes; }; @@ -366,264 +375,270 @@ class Transaction { * NiFi instance. */ class DataPacket { -public: - DataPacket(Site2SiteClientProtocol *protocol, Transaction *transaction, - std::map attributes) { - _protocol = protocol; - _size = 0; - _transaction = transaction; - _attributes = attributes; - } - std::map _attributes; - uint64_t _size; - Site2SiteClientProtocol *_protocol; - Transaction *_transaction; + public: + DataPacket(Site2SiteClientProtocol *protocol, Transaction *transaction, + std::map attributes) { + _protocol = protocol; + _size = 0; + _transaction = transaction; + _attributes = attributes; + } + std::map _attributes; + uint64_t _size; + Site2SiteClientProtocol *_protocol; + Transaction *_transaction; }; -//! Site2SiteClientProtocol Class +// Site2SiteClientProtocol Class class Site2SiteClientProtocol { -public: - //! Constructor - /*! - * Create a new control protocol - */ - Site2SiteClientProtocol(Site2SitePeer *peer) { - logger_ = Logger::getLogger(); - configure_ = Configure::getConfigure(); - peer_ = peer; - _batchSize = 0; - _batchCount = 0; - _batchDuration = 0; - _batchSendNanos = 5000000000; // 5 seconds - _timeOut = 30000; // 30 seconds - _peerState = IDLE; - _supportedVersion[0] = 5; - _supportedVersion[1] = 4; - _supportedVersion[2] = 3; - _supportedVersion[3] = 2; - _supportedVersion[4] = 1; - _currentVersion = _supportedVersion[0]; - _currentVersionIndex = 0; - _supportedCodecVersion[0] = 1; - _currentCodecVersion = _supportedCodecVersion[0]; - _currentCodecVersionIndex = 0; - } - //! Destructor - virtual ~Site2SiteClientProtocol() { - } - -public: - //! setBatchSize - void setBatchSize(uint64_t size) { - _batchSize = size; - } - //! setBatchCount - void setBatchCount(uint64_t count) { - _batchCount = count; - } - //! setBatchDuration - void setBatchDuration(uint64_t duration) { - _batchDuration = duration; - } - //! setTimeOut - void setTimeOut(uint64_t time) { - _timeOut = time; - if (peer_) - peer_->setTimeOut(time); - - } - - void setPeer(Site2SitePeer *peer) - { - peer_ = peer; - } - /** - * Provides a reference to the time out - * @returns timeout - */ - const uint64_t getTimeOut() const { - return _timeOut; - } - - /** - * Provides a reference to the port identifier - * @returns port identifier - */ - const std::string getPortId() const { - return _portIdStr; - } - //! setPortId - void setPortId(uuid_t id) { - uuid_copy(_portId, id); - char idStr[37]; - uuid_unparse_lower(id, idStr); - _portIdStr = idStr; - } - //! getResourceName - std::string getResourceName() { - return "SocketFlowFileProtocol"; - } - //! getCodecResourceName - std::string getCodecResourceName() { - return "StandardFlowFileCodec"; - } - //! bootstrap the protocol to the ready for transaction state by going through the state machine - bool bootstrap(); - //! establish - bool establish(); - //! handShake - bool handShake(); - //! negotiateCodec - bool negotiateCodec(); - //! initiateResourceNegotiation - bool initiateResourceNegotiation(); - //! initiateCodecResourceNegotiation - bool initiateCodecResourceNegotiation(); - //! tearDown - void tearDown(); - //! write Request Type - int writeRequestType(RequestType type); - //! read Request Type - int readRequestType(RequestType &type); - //! read Respond - int readRespond(RespondCode &code, std::string &message); - //! write respond - int writeRespond(RespondCode code, std::string message); - //! getRespondCodeContext - RespondCodeContext *getRespondCodeContext(RespondCode code) { - for (unsigned int i = 0; - i < sizeof(respondCodeContext) / sizeof(RespondCodeContext); - i++) { - if (respondCodeContext[i].code == code) { - return &respondCodeContext[i]; - } - } - return NULL; - } - //! getPeer - Site2SitePeer *getPeer() { - return peer_; - } - //! Creation of a new transaction, return the transaction ID if success, - //! Return NULL when any error occurs - Transaction *createTransaction(std::string &transactionID, - TransferDirection direction); - //! Receive the data packet from the transaction - //! Return false when any error occurs - bool receive(std::string transactionID, DataPacket *packet, bool &eof); - //! Send the data packet from the transaction - //! Return false when any error occurs - bool send(std::string transactionID, DataPacket *packet, - FlowFileRecord *flowFile, ProcessSession *session); - //! Confirm the data that was sent or received by comparing CRC32's of the data sent and the data received. - bool confirm(std::string transactionID); - //! Cancel the transaction - void cancel(std::string transactionID); - //! Complete the transaction - bool complete(std::string transactionID); - //! Error the transaction - void error(std::string transactionID); - //! Receive flow files for the process session - void receiveFlowFiles(ProcessContext *context, ProcessSession *session); - //! Transfer flow files for the process session - void transferFlowFiles(ProcessContext *context, ProcessSession *session); - //! deleteTransaction - void deleteTransaction(std::string transactionID); - //! Nest Callback Class for write stream - class WriteCallback: public OutputStreamCallback { - public: - WriteCallback(DataPacket *packet) : - _packet(packet) { - } - DataPacket *_packet; - void process(std::ofstream *stream) { - uint8_t buffer[8192]; - int len = _packet->_size; - while (len > 0) { - int size = std::min(len, (int) sizeof(buffer)); - int ret = _packet->_transaction->getStream().readData(buffer,size); - if (ret != size) { - _packet->_protocol->logger_->log_error( - "Site2Site Receive Flow Size %d Failed %d", size, - ret); - break; - } - stream->write((const char *) buffer, size); - len -= size; - } - } - }; - //! Nest Callback Class for read stream - class ReadCallback: public InputStreamCallback { - public: - ReadCallback(DataPacket *packet) : - _packet(packet) { - } - DataPacket *_packet; - void process(std::ifstream *stream) { - _packet->_size = 0; - uint8_t buffer[8192]; - int readSize; - while (stream->good()) { - if (!stream->read((char *) buffer, 8192)) - readSize = stream->gcount(); - else - readSize = 8192; - int ret = _packet->_transaction->getStream().writeData(buffer,readSize); - if (ret != readSize) { - _packet->_protocol->logger_->log_error( - "Site2Site Send Flow Size %d Failed %d", readSize, - ret); - break; - } - _packet->_size += readSize; - } - } - }; - -protected: - -private: - - //! Mutex for protection - std::mutex _mtx; - //! Logger - std::shared_ptr logger_; - //! Configure - Configure *configure_; - //! Batch Count - std::atomic _batchCount; - //! Batch Size - std::atomic _batchSize; - //! Batch Duration in msec - std::atomic _batchDuration; - //! Timeout in msec - std::atomic _timeOut; - //! Peer Connection - Site2SitePeer *peer_; - //! portId - uuid_t _portId; - //! portIDStr - std::string _portIdStr; - //! BATCH_SEND_NANOS - uint64_t _batchSendNanos; - //! Peer State - PeerState _peerState; - uint32_t _supportedVersion[5]; - uint32_t _currentVersion; - int _currentVersionIndex; - uint32_t _supportedCodecVersion[1]; - uint32_t _currentCodecVersion; - int _currentCodecVersionIndex; - //! commsIdentifier - std::string _commsIdentifier; - //! transaction map - std::map _transactionMap; - - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - Site2SiteClientProtocol(const Site2SiteClientProtocol &parent); - Site2SiteClientProtocol &operator=(const Site2SiteClientProtocol &parent); + public: + // Constructor + /*! + * Create a new control protocol + */ + Site2SiteClientProtocol(Site2SitePeer *peer) { + logger_ = logging::Logger::getLogger(); + configure_ = Configure::getConfigure(); + peer_ = peer; + _batchSize = 0; + _batchCount = 0; + _batchDuration = 0; + _batchSendNanos = 5000000000; // 5 seconds + _timeOut = 30000; // 30 seconds + _peerState = IDLE; + _supportedVersion[0] = 5; + _supportedVersion[1] = 4; + _supportedVersion[2] = 3; + _supportedVersion[3] = 2; + _supportedVersion[4] = 1; + _currentVersion = _supportedVersion[0]; + _currentVersionIndex = 0; + _supportedCodecVersion[0] = 1; + _currentCodecVersion = _supportedCodecVersion[0]; + _currentCodecVersionIndex = 0; + } + // Destructor + virtual ~Site2SiteClientProtocol() { + } + + public: + // setBatchSize + void setBatchSize(uint64_t size) { + _batchSize = size; + } + // setBatchCount + void setBatchCount(uint64_t count) { + _batchCount = count; + } + // setBatchDuration + void setBatchDuration(uint64_t duration) { + _batchDuration = duration; + } + // setTimeOut + void setTimeOut(uint64_t time) { + _timeOut = time; + if (peer_) + peer_->setTimeOut(time); + + } + + void setPeer(Site2SitePeer *peer) { + peer_ = peer; + } + /** + * Provides a reference to the time out + * @returns timeout + */ + const uint64_t getTimeOut() const { + return _timeOut; + } + + /** + * Provides a reference to the port identifier + * @returns port identifier + */ + const std::string getPortId() const { + return _portIdStr; + } + // setPortId + void setPortId(uuid_t id) { + uuid_copy(_portId, id); + char idStr[37]; + uuid_unparse_lower(id, idStr); + _portIdStr = idStr; + } + // getResourceName + std::string getResourceName() { + return "SocketFlowFileProtocol"; + } + // getCodecResourceName + std::string getCodecResourceName() { + return "StandardFlowFileCodec"; + } + // bootstrap the protocol to the ready for transaction state by going through the state machine + bool bootstrap(); + // establish + bool establish(); + // handShake + bool handShake(); + // negotiateCodec + bool negotiateCodec(); + // initiateResourceNegotiation + bool initiateResourceNegotiation(); + // initiateCodecResourceNegotiation + bool initiateCodecResourceNegotiation(); + // tearDown + void tearDown(); + // write Request Type + int writeRequestType(RequestType type); + // read Request Type + int readRequestType(RequestType &type); + // read Respond + int readRespond(RespondCode &code, std::string &message); + // write respond + int writeRespond(RespondCode code, std::string message); + // getRespondCodeContext + RespondCodeContext *getRespondCodeContext(RespondCode code) { + for (unsigned int i = 0; + i < sizeof(respondCodeContext) / sizeof(RespondCodeContext); i++) { + if (respondCodeContext[i].code == code) { + return &respondCodeContext[i]; + } + } + return NULL; + } + // getPeer + Site2SitePeer *getPeer() { + return peer_; + } + // Creation of a new transaction, return the transaction ID if success, + // Return NULL when any error occurs + Transaction *createTransaction(std::string &transactionID, + TransferDirection direction); + // Receive the data packet from the transaction + // Return false when any error occurs + bool receive(std::string transactionID, DataPacket *packet, bool &eof); + // Send the data packet from the transaction + // Return false when any error occurs + bool send(std::string transactionID, DataPacket *packet, + std::shared_ptr flowFile, + core::ProcessSession *session); + // Confirm the data that was sent or received by comparing CRC32's of the data sent and the data received. + bool confirm(std::string transactionID); + // Cancel the transaction + void cancel(std::string transactionID); + // Complete the transaction + bool complete(std::string transactionID); + // Error the transaction + void error(std::string transactionID); + // Receive flow files for the process session + void receiveFlowFiles( + core::ProcessContext *context, + core::ProcessSession *session); + // Transfer flow files for the process session + void transferFlowFiles( + core::ProcessContext *context, + core::ProcessSession *session); + // deleteTransaction + void deleteTransaction(std::string transactionID); + // Nest Callback Class for write stream + class WriteCallback : public OutputStreamCallback { + public: + WriteCallback(DataPacket *packet) + : _packet(packet) { + } + DataPacket *_packet; + void process(std::ofstream *stream) { + uint8_t buffer[8192]; + int len = _packet->_size; + while (len > 0) { + int size = std::min(len, (int) sizeof(buffer)); + int ret = _packet->_transaction->getStream().readData(buffer, size); + if (ret != size) { + _packet->_protocol->logger_->log_error( + "Site2Site Receive Flow Size %d Failed %d", size, ret); + break; + } + stream->write((const char *) buffer, size); + len -= size; + } + } + }; + // Nest Callback Class for read stream + class ReadCallback : public InputStreamCallback { + public: + ReadCallback(DataPacket *packet) + : _packet(packet) { + } + DataPacket *_packet; + void process(std::ifstream *stream) { + _packet->_size = 0; + uint8_t buffer[8192]; + int readSize; + while (stream->good()) { + if (!stream->read((char *) buffer, 8192)) + readSize = stream->gcount(); + else + readSize = 8192; + int ret = _packet->_transaction->getStream().writeData(buffer, + readSize); + if (ret != readSize) { + _packet->_protocol->logger_->log_error( + "Site2Site Send Flow Size %d Failed %d", readSize, ret); + break; + } + _packet->_size += readSize; + } + } + }; + + protected: + + private: + + // Mutex for protection + std::mutex mutex_; + // Logger + std::shared_ptr logger_; + // Configure + Configure *configure_; + // Batch Count + std::atomic _batchCount; + // Batch Size + std::atomic _batchSize; + // Batch Duration in msec + std::atomic _batchDuration; + // Timeout in msec + std::atomic _timeOut; + // Peer Connection + Site2SitePeer *peer_; + // portId + uuid_t _portId; + // portIDStr + std::string _portIdStr; + // BATCH_SEND_NANOS + uint64_t _batchSendNanos; + // Peer State + PeerState _peerState; + uint32_t _supportedVersion[5]; + uint32_t _currentVersion; + int _currentVersionIndex; + uint32_t _supportedCodecVersion[1]; + uint32_t _currentCodecVersion; + int _currentCodecVersionIndex; + // commsIdentifier + std::string _commsIdentifier; + // transaction map + std::map _transactionMap; + + // Prevent default copy constructor and assignment operation + // Only support pass by reference or pointer + Site2SiteClientProtocol(const Site2SiteClientProtocol &parent); + Site2SiteClientProtocol &operator=(const Site2SiteClientProtocol &parent); }; +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ #endif diff --git a/libminifi/include/Site2SitePeer.h b/libminifi/include/Site2SitePeer.h index e89bb74271..f14993bbfc 100644 --- a/libminifi/include/Site2SitePeer.h +++ b/libminifi/include/Site2SitePeer.h @@ -29,242 +29,261 @@ #include #include #include -#include "Logger.h" + +#include "core/Property.h" +#include "core/logging/Logger.h" #include "Configure.h" -#include "Property.h" #include "io/ClientSocket.h" #include "io/BaseStream.h" #include "utils/TimeUtil.h" +namespace org { +namespace apache { +namespace nifi { +namespace minifi { static const char MAGIC_BYTES[] = { 'N', 'i', 'F', 'i' }; -//! Site2SitePeer Class -class Site2SitePeer : public BaseStream{ -public: - +// Site2SitePeer Class +class Site2SitePeer : public org::apache::nifi::minifi::io::BaseStream { + public: + + Site2SitePeer() + : stream_(nullptr), + host_(""), + port_(-1) { + + } + /* + * Create a new site2site peer + */ + explicit Site2SitePeer( + std::unique_ptr injected_socket, + const std::string host_, uint16_t port_) + : host_(host_), + port_(port_), + stream_(injected_socket.release()) { + logger_ = logging::Logger::getLogger(); + configure_ = Configure::getConfigure(); + _yieldExpiration = 0; + _timeOut = 30000; // 30 seconds + _url = "nifi://" + host_ + ":" + std::to_string(port_); + } - Site2SitePeer() : stream_(nullptr),host_(""),port_(-1){ - - } - /* - * Create a new site2site peer - */ - explicit Site2SitePeer(std::unique_ptr injected_socket, const std::string host_, uint16_t port_ ) : - host_(host_), port_(port_), stream_(injected_socket.release()){ - logger_ = Logger::getLogger(); - configure_ = Configure::getConfigure(); - _yieldExpiration = 0; - _timeOut = 30000; // 30 seconds - _url = "nifi://" + host_ + ":" + std::to_string(port_); - } - - explicit Site2SitePeer(Site2SitePeer &&ss) : stream_( ss.stream_.release()), host_( std::move(ss.host_) ), port_ (std::move(ss.port_) ) - { - logger_ = Logger::getLogger(); - configure_ = Configure::getConfigure(); - _yieldExpiration.store(ss._yieldExpiration); - _timeOut.store(ss._timeOut); - _url = std::move(ss._url); - } - //! Destructor - virtual ~Site2SitePeer() { - Close(); - } - //! Set Processor yield period in MilliSecond - void setYieldPeriodMsec(uint64_t period) { - _yieldPeriodMsec = period; - } - //! get URL - std::string getURL() { - return _url; - } - //! Get Processor yield period in MilliSecond - uint64_t getYieldPeriodMsec(void) { - return (_yieldPeriodMsec); - } - //! Yield based on the yield period - void yield() { - _yieldExpiration = (getTimeMillis() + _yieldPeriodMsec); - } - //! setHostName - void setHostName(std::string host_) { - this->host_ = host_; - _url = "nifi://" + host_ + ":" + std::to_string(port_); - } - //! setPort - void setPort(uint16_t port_) { - this->port_ = port_; - _url = "nifi://" + host_ + ":" + std::to_string(port_); - } - //! getHostName - std::string getHostName() { - return host_; - } - //! getPort - uint16_t getPort() { - return port_; - } - //! Yield based on the input time - void yield(uint64_t time) { - _yieldExpiration = (getTimeMillis() + time); - } - //! whether need be to yield - bool isYield() { - if (_yieldExpiration > 0) - return (_yieldExpiration >= getTimeMillis()); - else - return false; - } - // clear yield expiration - void clearYield() { - _yieldExpiration = 0; - } - //! Yield based on the yield period - void yield(std::string portId) { - std::lock_guard lock(_mtx); - uint64_t yieldExpiration = (getTimeMillis() + _yieldPeriodMsec); - _yieldExpirationPortIdMap[portId] = yieldExpiration; - } - //! Yield based on the input time - void yield(std::string portId, uint64_t time) { - std::lock_guard lock(_mtx); - uint64_t yieldExpiration = (getTimeMillis() + time); - _yieldExpirationPortIdMap[portId] = yieldExpiration; - } - //! whether need be to yield - bool isYield(std::string portId) { - std::lock_guard lock(_mtx); - std::map::iterator it = - this->_yieldExpirationPortIdMap.find(portId); - if (it != _yieldExpirationPortIdMap.end()) { - uint64_t yieldExpiration = it->second; - return (yieldExpiration >= getTimeMillis()); - } else { - return false; - } - } - //! clear yield expiration - void clearYield(std::string portId) { - std::lock_guard lock(_mtx); - std::map::iterator it = - this->_yieldExpirationPortIdMap.find(portId); - if (it != _yieldExpirationPortIdMap.end()) { - _yieldExpirationPortIdMap.erase(portId); - } - } - //! setTimeOut - void setTimeOut(uint64_t time) { - _timeOut = time; - } - //! getTimeOut - uint64_t getTimeOut() { - return _timeOut; - } - int write(uint8_t value) { - return Serializable::write(value,stream_.get()); - } - int write(char value) { - return Serializable::write(value,stream_.get()); - } - int write(uint32_t value) { + explicit Site2SitePeer(Site2SitePeer &&ss) + : stream_(ss.stream_.release()), + host_(std::move(ss.host_)), + port_(std::move(ss.port_)) { + logger_ = logging::Logger::getLogger(); + configure_ = Configure::getConfigure(); + _yieldExpiration.store(ss._yieldExpiration); + _timeOut.store(ss._timeOut); + _url = std::move(ss._url); + } + // Destructor + virtual ~Site2SitePeer() { + Close(); + } + // Set Processor yield period in MilliSecond + void setYieldPeriodMsec(uint64_t period) { + _yieldPeriodMsec = period; + } + // get URL + std::string getURL() { + return _url; + } + // Get Processor yield period in MilliSecond + uint64_t getYieldPeriodMsec(void) { + return (_yieldPeriodMsec); + } + // Yield based on the yield period + void yield() { + _yieldExpiration = (getTimeMillis() + _yieldPeriodMsec); + } + // setHostName + void setHostName(std::string host_) { + this->host_ = host_; + _url = "nifi://" + host_ + ":" + std::to_string(port_); + } + // setPort + void setPort(uint16_t port_) { + this->port_ = port_; + _url = "nifi://" + host_ + ":" + std::to_string(port_); + } + // getHostName + std::string getHostName() { + return host_; + } + // getPort + uint16_t getPort() { + return port_; + } + // Yield based on the input time + void yield(uint64_t time) { + _yieldExpiration = (getTimeMillis() + time); + } + // whether need be to yield + bool isYield() { + if (_yieldExpiration > 0) + return (_yieldExpiration >= getTimeMillis()); + else + return false; + } + // clear yield expiration + void clearYield() { + _yieldExpiration = 0; + } + // Yield based on the yield period + void yield(std::string portId) { + std::lock_guard lock(mutex_); + uint64_t yieldExpiration = (getTimeMillis() + _yieldPeriodMsec); + _yieldExpirationPortIdMap[portId] = yieldExpiration; + } + // Yield based on the input time + void yield(std::string portId, uint64_t time) { + std::lock_guard lock(mutex_); + uint64_t yieldExpiration = (getTimeMillis() + time); + _yieldExpirationPortIdMap[portId] = yieldExpiration; + } + // whether need be to yield + bool isYield(std::string portId) { + std::lock_guard lock(mutex_); + std::map::iterator it = this + ->_yieldExpirationPortIdMap.find(portId); + if (it != _yieldExpirationPortIdMap.end()) { + uint64_t yieldExpiration = it->second; + return (yieldExpiration >= getTimeMillis()); + } else { + return false; + } + } + // clear yield expiration + void clearYield(std::string portId) { + std::lock_guard lock(mutex_); + std::map::iterator it = this + ->_yieldExpirationPortIdMap.find(portId); + if (it != _yieldExpirationPortIdMap.end()) { + _yieldExpirationPortIdMap.erase(portId); + } + } + // setTimeOut + void setTimeOut(uint64_t time) { + _timeOut = time; + } + // getTimeOut + uint64_t getTimeOut() { + return _timeOut; + } + int write(uint8_t value) { + return Serializable::write(value, stream_.get()); + } + int write(char value) { + return Serializable::write(value, stream_.get()); + } + int write(uint32_t value) { - return Serializable::write(value,stream_.get()); + return Serializable::write(value, stream_.get()); - } - int write(uint16_t value) { - return Serializable::write(value,stream_.get()); - } - int write(uint8_t *value, int len) { - return Serializable::write(value,len,stream_.get()); - } - int write(uint64_t value) { - return Serializable::write(value,stream_.get()); - } - int write(bool value) { - uint8_t temp = value; - return Serializable::write(temp,stream_.get()); - } - int writeUTF(std::string str, bool widen = false){ - return Serializable::writeUTF(str,stream_.get(),widen); - } - int read(uint8_t &value) { - return Serializable::read(value,stream_.get()); - } - int read(uint16_t &value) { - return Serializable::read(value,stream_.get()); - } - int read(char &value) { - return Serializable::read(value,stream_.get()); - } - int read(uint8_t *value, int len) { - return Serializable::read(value,len,stream_.get()); - } - int read(uint32_t &value) { - return Serializable::read(value,stream_.get()); - } - int read(uint64_t &value) { - return Serializable::read(value,stream_.get()); - } - int readUTF(std::string &str, bool widen = false) - { - return Serializable::readUTF(str,stream_.get(),widen); - } - //! open connection to the peer - bool Open(); - //! close connection to the peer - void Close(); - - /** - * Move assignment operator. - */ - Site2SitePeer& operator=(Site2SitePeer&& other) - { - stream_ = std::unique_ptr( other.stream_.release()); - host_ = std::move(other.host_); - port_ = std::move(other.port_); - logger_ = Logger::getLogger(); - configure_ = Configure::getConfigure(); - _yieldExpiration = 0; - _timeOut = 30000; // 30 seconds - _url = "nifi://" + host_ + ":" + std::to_string(port_); - - return *this; - } + } + int write(uint16_t value) { + return Serializable::write(value, stream_.get()); + } + int write(uint8_t *value, int len) { + return Serializable::write(value, len, stream_.get()); + } + int write(uint64_t value) { + return Serializable::write(value, stream_.get()); + } + int write(bool value) { + uint8_t temp = value; + return Serializable::write(temp, stream_.get()); + } + int writeUTF(std::string str, bool widen = false) { + return Serializable::writeUTF(str, stream_.get(), widen); + } + int read(uint8_t &value) { + return Serializable::read(value, stream_.get()); + } + int read(uint16_t &value) { + return Serializable::read(value, stream_.get()); + } + int read(char &value) { + return Serializable::read(value, stream_.get()); + } + int read(uint8_t *value, int len) { + return Serializable::read(value, len, stream_.get()); + } + int read(uint32_t &value) { + return Serializable::read(value, stream_.get()); + } + int read(uint64_t &value) { + return Serializable::read(value, stream_.get()); + } + int readUTF(std::string &str, bool widen = false) { + return org::apache::nifi::minifi::io::Serializable::readUTF(str, + stream_.get(), + widen); + } + // open connection to the peer + bool Open(); + // close connection to the peer + void Close(); - Site2SitePeer(const Site2SitePeer &parent) = delete; - Site2SitePeer &operator=(const Site2SitePeer &parent) = delete; + /** + * Move assignment operator. + */ + Site2SitePeer& operator=(Site2SitePeer&& other) { + stream_ = std::unique_ptr( + other.stream_.release()); + host_ = std::move(other.host_); + port_ = std::move(other.port_); + logger_ = logging::Logger::getLogger(); + configure_ = Configure::getConfigure(); + _yieldExpiration = 0; + _timeOut = 30000; // 30 seconds + _url = "nifi://" + host_ + ":" + std::to_string(port_); -protected: + return *this; + } -private: + Site2SitePeer(const Site2SitePeer &parent) = delete; + Site2SitePeer &operator=(const Site2SitePeer &parent) = delete; - std::unique_ptr stream_; + protected: - std::string host_; - uint16_t port_; + private: - //! Mutex for protection - std::mutex _mtx; - //! URL - std::string _url; - //! socket timeout; - std::atomic _timeOut; - //! Logger - std::shared_ptr logger_; - //! Configure - Configure *configure_; - //! Yield Period in Milliseconds - std::atomic _yieldPeriodMsec; - //! Yield Expiration - std::atomic _yieldExpiration; - //! Yield Expiration per destination PortID - std::map _yieldExpirationPortIdMap; - //! OpenSSL connection state - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer + std::unique_ptr stream_; + + std::string host_; + uint16_t port_; + + // Mutex for protection + std::mutex mutex_; + // URL + std::string _url; + // socket timeout; + std::atomic _timeOut; + // Logger + std::shared_ptr logger_; + // Configure + Configure *configure_; + // Yield Period in Milliseconds + std::atomic _yieldPeriodMsec; + // Yield Expiration + std::atomic _yieldExpiration; + // Yield Expiration per destination PortID + std::map _yieldExpirationPortIdMap; + // OpenSSL connection state + // Prevent default copy constructor and assignment operation + // Only support pass by reference or pointer }; + +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ #endif diff --git a/libminifi/include/TailFile.h b/libminifi/include/TailFile.h deleted file mode 100644 index d68748e37b..0000000000 --- a/libminifi/include/TailFile.h +++ /dev/null @@ -1,93 +0,0 @@ -/** - * @file TailFile.h - * TailFile class declaration - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef __TAIL_FILE_H__ -#define __TAIL_FILE_H__ - -#include "FlowFileRecord.h" -#include "Processor.h" -#include "ProcessSession.h" - -//! TailFile Class -class TailFile : public Processor -{ -public: - //! Constructor - /*! - * Create a new processor - */ - TailFile(std::string name, uuid_t uuid = NULL) - : Processor(name, uuid) - { - logger_ = Logger::getLogger(); - _stateRecovered = false; - } - //! Destructor - virtual ~TailFile() - { - storeState(); - } - //! Processor Name - static const std::string ProcessorName; - //! Supported Properties - static Property FileName; - static Property StateFile; - //! Supported Relationships - static Relationship Success; - -public: - //! OnTrigger method, implemented by NiFi TailFile - virtual void onTrigger(ProcessContext *context, ProcessSession *session); - //! Initialize, over write by NiFi TailFile - virtual void initialize(void); - //! recoverState - void recoverState(); - //! storeState - void storeState(); - -protected: - -private: - //! Logger - std::shared_ptr logger_; - std::string _fileLocation; - //! Property Specified Tailed File Name - std::string _fileName; - //! File to save state - std::string _stateFile; - //! State related to the tailed file - std::string _currentTailFileName; - uint64_t _currentTailFilePosition; - bool _stateRecovered; - uint64_t _currentTailFileCreatedTime; - //! Utils functions for parse state file - std::string trimLeft(const std::string& s); - std::string trimRight(const std::string& s); - void parseStateFileLine(char *buf); - void checkRollOver(); - -}; - -//! Matched File Item for Roll over check -typedef struct { - std::string fileName; - uint64_t modifiedTime; -} TailMatchedFileItem; - -#endif diff --git a/libminifi/include/ThreadedSchedulingAgent.h b/libminifi/include/ThreadedSchedulingAgent.h index 5eb5d8a902..8c1c3e8c18 100644 --- a/libminifi/include/ThreadedSchedulingAgent.h +++ b/libminifi/include/ThreadedSchedulingAgent.h @@ -21,50 +21,58 @@ #define __THREADED_SCHEDULING_AGENT_H__ #include "Configure.h" -#include "Logger.h" -#include "Processor.h" -#include "ProcessContext.h" +#include "core/logging/Logger.h" +#include "core/Processor.h" +#include "core/ProcessContext.h" #include "SchedulingAgent.h" +namespace org { +namespace apache { +namespace nifi { +namespace minifi { + /** * An abstract scheduling agent which creates and manages a pool of threads for * each processor scheduled. */ -class ThreadedSchedulingAgent : public SchedulingAgent -{ -public: - //! Constructor - /*! - * Create a new processor - */ - ThreadedSchedulingAgent() - : SchedulingAgent() - { - } - //! Destructor - virtual ~ThreadedSchedulingAgent() - { - } +class ThreadedSchedulingAgent : public SchedulingAgent { + public: + // Constructor + /*! + * Create a new processor + */ + ThreadedSchedulingAgent(std::shared_ptr repo) + : SchedulingAgent(repo) { + } + // Destructor + virtual ~ThreadedSchedulingAgent() { + } - //! Run function for the thread - virtual void run(Processor *processor, ProcessContext *processContext, ProcessSessionFactory *sessionFactory) = 0; + // Run function for the thread + virtual void run(std::shared_ptr processor, + core::ProcessContext *processContext, + core::ProcessSessionFactory *sessionFactory) = 0; -public: - //! schedule, overwritten by different DrivenTimerDrivenSchedulingAgent - virtual void schedule(Processor *processor); - //! unschedule, overwritten by different DrivenTimerDrivenSchedulingAgent - virtual void unschedule(Processor *processor); + public: + // schedule, overwritten by different DrivenTimerDrivenSchedulingAgent + virtual void schedule(std::shared_ptr processor); + // unschedule, overwritten by different DrivenTimerDrivenSchedulingAgent + virtual void unschedule(std::shared_ptr processor); -protected: - //! Threads - std::map> _threads; + protected: + // Threads + std::map> _threads; -private: - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - ThreadedSchedulingAgent(const ThreadedSchedulingAgent &parent); - ThreadedSchedulingAgent &operator=(const ThreadedSchedulingAgent &parent); + private: + // Prevent default copy constructor and assignment operation + // Only support pass by reference or pointer + ThreadedSchedulingAgent(const ThreadedSchedulingAgent &parent); + ThreadedSchedulingAgent &operator=(const ThreadedSchedulingAgent &parent); }; +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ #endif diff --git a/libminifi/include/TimerDrivenSchedulingAgent.h b/libminifi/include/TimerDrivenSchedulingAgent.h index 389ccf62d9..b34f9238b4 100644 --- a/libminifi/include/TimerDrivenSchedulingAgent.h +++ b/libminifi/include/TimerDrivenSchedulingAgent.h @@ -20,36 +20,46 @@ #ifndef __TIMER_DRIVEN_SCHEDULING_AGENT_H__ #define __TIMER_DRIVEN_SCHEDULING_AGENT_H__ -#include "Logger.h" -#include "Processor.h" -#include "ProcessContext.h" +#include "core/logging/Logger.h" +#include "core/Processor.h" +#include "core/ProcessContext.h" #include "ThreadedSchedulingAgent.h" -//! TimerDrivenSchedulingAgent Class -class TimerDrivenSchedulingAgent : public ThreadedSchedulingAgent -{ -public: - //! Constructor - /*! - * Create a new processor - */ - TimerDrivenSchedulingAgent() - : ThreadedSchedulingAgent() - { - } - //! Destructor - virtual ~TimerDrivenSchedulingAgent() - { - } - //! Run function for the thread - void run(Processor *processor, ProcessContext *processContext, ProcessSessionFactory *sessionFactory); +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +// TimerDrivenSchedulingAgent Class +class TimerDrivenSchedulingAgent : public ThreadedSchedulingAgent { + public: + // Constructor + /*! + * Create a new processor + */ + TimerDrivenSchedulingAgent(std::shared_ptr repo) + : ThreadedSchedulingAgent(repo) { + } + // Destructor + virtual ~TimerDrivenSchedulingAgent() { + } + /** + * Run function that accepts the processor, context and session factory. + */ + void run(std::shared_ptr processor, + core::ProcessContext *processContext, + core::ProcessSessionFactory *sessionFactory); -private: - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - TimerDrivenSchedulingAgent(const TimerDrivenSchedulingAgent &parent); - TimerDrivenSchedulingAgent &operator=(const TimerDrivenSchedulingAgent &parent); + private: + // Prevent default copy constructor and assignment operation + // Only support pass by reference or pointer + TimerDrivenSchedulingAgent(const TimerDrivenSchedulingAgent &parent); + TimerDrivenSchedulingAgent &operator=( + const TimerDrivenSchedulingAgent &parent); }; +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ #endif diff --git a/libminifi/include/core/ConfigurableComponent.h b/libminifi/include/core/ConfigurableComponent.h new file mode 100644 index 0000000000..c0cc623651 --- /dev/null +++ b/libminifi/include/core/ConfigurableComponent.h @@ -0,0 +1,104 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBMINIFI_INCLUDE_CORE_CONFIGURABLECOMPONENT_H_ +#define LIBMINIFI_INCLUDE_CORE_CONFIGURABLECOMPONENT_H_ + +#include "Property.h" +#include "core/logging/Logger.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +/** + * Represents a configurable component + * Purpose: Extracts configuration items for all components and localized them + */ +class ConfigurableComponent { + public: + + + ConfigurableComponent() = delete; + + + explicit ConfigurableComponent(std::shared_ptr logger); + + explicit ConfigurableComponent(const ConfigurableComponent &&other); + + /** + * Get property using the provided name. + * @param name property name. + * @param value value passed in by reference + * @return result of getting property. + */ + bool getProperty(const std::string name, std::string &value); + /** + * Sets the property using the provided name + * @param property name + * @param value property value. + * @return result of setting property. + */ + bool setProperty(const std::string name, std::string value); + /** + * Sets the property using the provided name + * @param property name + * @param value property value. + * @return whether property was set or not + */ + bool setProperty(Property &prop, std::string value); + + /** + * Sets supported properties for the ConfigurableComponent + * @param supported properties + * @return result of set operation. + */ + bool setSupportedProperties(std::set properties); + /** + * Sets supported properties for the ConfigurableComponent + * @param supported properties + * @return result of set operation. + */ + + virtual ~ConfigurableComponent(); + + protected: + + + /** + * Returns true if the instance can be edited. + * @return true/false + */ + virtual bool canEdit()= 0; + + std::mutex configuration_mutex_; + std::shared_ptr logger_; + // Supported properties + std::map properties_; + +}; + +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif /* LIBMINIFI_INCLUDE_CORE_CONFIGURABLECOMPONENT_H_ */ diff --git a/libminifi/include/core/Connectable.h b/libminifi/include/core/Connectable.h new file mode 100644 index 0000000000..15e618fbf5 --- /dev/null +++ b/libminifi/include/core/Connectable.h @@ -0,0 +1,165 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBMINIFI_INCLUDE_CORE_CONNECTABLE_H_ +#define LIBMINIFI_INCLUDE_CORE_CONNECTABLE_H_ + +#include +#include "core.h" +#include +#include "core/logging/Logger.h" +#include "Relationship.h" +#include "Scheduling.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +/** + * Represents the base connectable component + * Purpose: As in NiFi, this represents a connection point and allows the derived + * object to be connected to other connectables. + */ +class Connectable : public CoreComponent { + public: + + explicit Connectable(std::string name, uuid_t uuid); + + explicit Connectable(const Connectable &&other); + + bool setSupportedRelationships(std::set relationships); + + // Whether the relationship is supported + bool isSupportedRelationship(Relationship relationship); + + /** + * Sets auto terminated relationships + * @param relationships + * @return result of set operation. + */ + bool setAutoTerminatedRelationships(std::set relationships); + + // Check whether the relationship is auto terminated + bool isAutoTerminated(Relationship relationship); + + // Get Processor penalization period in MilliSecond + uint64_t getPenalizationPeriodMsec(void) { + return (_penalizationPeriodMsec); + } + + /** + * Get outgoing connection based on relationship + * @return set of outgoing connections. + */ + std::set> getOutGoingConnections( + std::string relationship); + + /** + * Get next incoming connection + * @return next incoming connection + */ + std::shared_ptr getNextIncomingConnection(); + + /** + * @return true if incoming connections > 0 + */ + bool hasIncomingConnections() { + return (_incomingConnections.size() > 0); + } + + uint8_t getMaxConcurrentTasks() { + return max_concurrent_tasks_; + } + + void setMaxConcurrentTasks(const uint8_t tasks) { + max_concurrent_tasks_ = tasks; + } + /** + * Yield + */ + virtual void yield() = 0; + + virtual ~Connectable(); + + /** + * Determines if we are connected and operating + */ + virtual bool isRunning() = 0; + + /** + * Block until work is available on any input connection, or the given duration elapses + * @param timeoutMs timeout in milliseconds + */ + void waitForWork(uint64_t timeoutMs); + /** + * Notify this processor that work may be available + */ + + void notifyWork(); + + /** + * Determines if work is available by this connectable + * @return boolean if work is available. + */ + virtual bool isWorkAvailable() = 0; + + protected: + + // Penalization Period in MilliSecond + std::atomic _penalizationPeriodMsec; + + uint8_t max_concurrent_tasks_; + + // Supported relationships + std::map relationships_; + // Autoterminated relationships + std::map auto_terminated_relationships_; + + // Incoming connection Iterator + std::set>::iterator incoming_connections_Iter; + // Incoming connections + std::set> _incomingConnections; + // Outgoing connections map based on Relationship name + std::map>>_outGoingConnections; + + // Mutex for protection + std::mutex relationship_mutex_; + + ///// work conditionals and locking mechanisms + + // Concurrent condition mutex for whether there is incoming work to do + std::mutex work_available_mutex_; + // Condition for whether there is incoming work to do + std::atomic has_work_; + // Scheduling Strategy + std::atomic strategy_; + // Concurrent condition variable for whether there is incoming work to do + std::condition_variable work_condition_; + +}; + +} +/* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif /* LIBMINIFI_INCLUDE_CORE_CONNECTABLE_H_ */ diff --git a/libminifi/include/core/FlowFile.h b/libminifi/include/core/FlowFile.h new file mode 100644 index 0000000000..229a4aa069 --- /dev/null +++ b/libminifi/include/core/FlowFile.h @@ -0,0 +1,266 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RECORD_H +#define RECORD_H + +#include "utils/TimeUtil.h" +#include "ResourceClaim.h" +#include "Connectable.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +class FlowFile { + public: + FlowFile(); + ~FlowFile(); + FlowFile& operator=(const FlowFile& other); + + /** + * Returns a pointer to this flow file record's + * claim + */ + std::shared_ptr getResourceClaim(); + /** + * Sets _claim to the inbound claim argument + */ + void setResourceClaim(std::shared_ptr &claim); + + /** + * clear the resource claim + */ + void clearResourceClaim(); + + /** + * Get lineage identifiers + */ + std::set &getlineageIdentifiers(); + + /** + * Returns whether or not this flow file record + * is marked as deleted. + * @return marked deleted + */ + bool isDeleted(); + + /** + * Sets whether to mark this flow file record + * as deleted + * @param deleted deleted flag + */ + void setDeleted(const bool deleted); + + /** + * Get entry date for this record + * @return entry date uint64_t + */ + uint64_t getEntryDate(); + /** + * Get lineage start date + * @return lineage start date uint64_t + */ + uint64_t getlineageStartDate(); + + /** + * Sets the lineage start date + * @param date new lineage start date + */ + void setLineageStartDate(const uint64_t date); + + void setLineageIdentifiers(std::set lineage_Identifiers) { + lineage_Identifiers_ = lineage_Identifiers; + } + /** + * Obtains an attribute if it exists. If it does the value is + * copied into value + * @param key key to look for + * @param value value to set + * @return result of finding key + */ + bool getAttribute(std::string key, std::string &value); + + /** + * Updates the value in the attribute map that corresponds + * to key + * @param key attribute name + * @param value value to set to attribute name + * @return result of finding key + */ + bool updateAttribute(const std::string key, const std::string value); + + /** + * Removes the attribute + * @param key attribute name to remove + * @return result of finding key + */ + bool removeAttribute(const std::string key); + + /** + * setAttribute, if attribute already there, update it, else, add it + */ + void setAttribute(const std::string &key, const std::string &value) { + attributes_[key] = value; + } + + /** + * Returns the map of attributes + * @return attributes. + */ + std::map getAttributes() { + return attributes_; + } + + /** + * adds an attribute if it does not exist + * + */ + bool addAttribute(const std::string &key, const std::string &value); + + /** + * Set the size of this record. + * @param size size of record to set.Ï + */ + void setSize(const uint64_t size) { + size_ = size; + } + /** + * Returns the size of corresponding flow file + * @return size as a uint64_t + */ + uint64_t getSize(); + + /** + * Sets the offset + * @param offset offset to apply to this record. + */ + void setOffset(const uint64_t offset) { + offset_ = offset; + } + + /** + * Sets the penalty expiration + * @param penaltyExp new penalty expiration + */ + void setPenaltyExpiration(const uint64_t penaltyExp) { + penaltyExpiration_ms_ = penaltyExp; + } + + uint64_t getPenaltyExpiration() { + return penaltyExpiration_ms_; + } + + /** + * Gets the offset within the flow file + * @return size as a uint64_t + */ + uint64_t getOffset(); + + // Get the UUID as string + std::string getUUIDStr() { + return uuid_str_; + } + + // Check whether it is still being penalized + bool isPenalized() { + return ( + penaltyExpiration_ms_ > 0 ? + penaltyExpiration_ms_ > getTimeMillis() : false); + } + + /** + * Sets the original connection with a shared pointer. + * @param connection shared connection. + */ + void setConnection(std::shared_ptr &connection); + + /** + * Sets the original connection with a shared pointer. + * @param connection shared connection. + */ + void setConnection(std::shared_ptr &&connection); + + /** + * Returns the connection referenced by this record. + * @return shared connection pointer. + */ + std::shared_ptr getConnection(); + /** + * Sets the original connection with a shared pointer. + * @param connection shared connection. + */ + void setOriginalConnection(std::shared_ptr &connection); + /** + * Returns the original connection referenced by this record. + * @return shared original connection pointer. + */ + std::shared_ptr getOriginalConnection(); + + void setStoredToRepository(bool storedInRepository) { + stored = storedInRepository; + } + + bool isStored() { + return stored; + } + + protected: + bool stored; + // Mark for deletion + bool marked_delete_; + // Date at which the flow file entered the flow + uint64_t entry_date_; + // Date at which the origin of this flow file entered the flow + uint64_t lineage_start_date_; + // Date at which the flow file was queued + uint64_t last_queue_date_; + // Size in bytes of the data corresponding to this flow file + uint64_t size_; + // A global unique identifier + uuid_t uuid_; + // A local unique identifier + uint64_t id_; + // Offset to the content + uint64_t offset_; + // Penalty expiration + uint64_t penaltyExpiration_ms_; + // Attributes key/values pairs for the flow record + std::map attributes_; + // Pointer to the associated content resource claim + std::shared_ptr claim_; + // UUID string + std::string uuid_str_; + // UUID string for all parents + std::set lineage_Identifiers_; + + // Connection queue that this flow file will be transfer or current in + std::shared_ptr connection_; + // Orginal connection queue that this flow file was dequeued from + std::shared_ptr original_connection_; + +}; + +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif // RECORD_H diff --git a/libminifi/include/core/ProcessContext.h b/libminifi/include/core/ProcessContext.h new file mode 100644 index 0000000000..1da85cd271 --- /dev/null +++ b/libminifi/include/core/ProcessContext.h @@ -0,0 +1,114 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __PROCESS_CONTEXT_H__ +#define __PROCESS_CONTEXT_H__ + +#include +#include +#include +#include +#include +#include +#include + +#include "Property.h" +#include "core/logging/Logger.h" +#include "ProcessorNode.h" +#include "core/Repository.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +// ProcessContext Class +class ProcessContext { + public: + // Constructor + /*! + * Create a new process context associated with the processor/controller service/state manager + */ + ProcessContext(ProcessorNode &processor, + std::shared_ptr repo) + : processor_node_(processor) { + logger_ = logging::Logger::getLogger(); + repo_ = repo; + } + // Destructor + virtual ~ProcessContext() { + } + // Get Processor associated with the Process Context + ProcessorNode &getProcessorNode() { + return processor_node_; + } + bool getProperty(std::string name, std::string &value) { + return processor_node_.getProperty(name, value); + } + // Sets the property value using the property's string name + bool setProperty(std::string name, std::string value) { + return processor_node_.setProperty(name, value); + } + // Sets the property value using the Property object + bool setProperty(Property prop, std::string value) { + return processor_node_.setProperty(prop, value); + } + // Whether the relationship is supported + bool isSupportedRelationship(Relationship relationship) { + return processor_node_.isSupportedRelationship(relationship); + } + + // Check whether the relationship is auto terminated + bool isAutoTerminated(Relationship relationship) { + return processor_node_.isAutoTerminated(relationship); + } + // Get ProcessContext Maximum Concurrent Tasks + uint8_t getMaxConcurrentTasks(void) { + return processor_node_.getMaxConcurrentTasks(); + } + // Yield based on the yield period + void yield() { + processor_node_.yield(); + } + + std::shared_ptr getProvenanceRepository() { + return repo_; + } + + // Prevent default copy constructor and assignment operation + // Only support pass by reference or pointer + ProcessContext(const ProcessContext &parent) = delete; + ProcessContext &operator=(const ProcessContext &parent) = delete; + + private: + + // repository shared pointer. + std::shared_ptr repo_; + // Processor + ProcessorNode processor_node_; + // Logger + std::shared_ptr logger_; + +}; + +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ +#endif diff --git a/libminifi/include/core/ProcessGroup.h b/libminifi/include/core/ProcessGroup.h new file mode 100644 index 0000000000..05a50a0f3c --- /dev/null +++ b/libminifi/include/core/ProcessGroup.h @@ -0,0 +1,189 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __PROCESS_GROUP_H__ +#define __PROCESS_GROUP_H__ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Processor.h" +#include "Exception.h" +#include "TimerDrivenSchedulingAgent.h" +#include "EventDrivenSchedulingAgent.h" +#include "core/logging/Logger.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +// Process Group Type +enum ProcessGroupType { + ROOT_PROCESS_GROUP = 0, + REMOTE_PROCESS_GROUP, + MAX_PROCESS_GROUP_TYPE +}; + +// ProcessGroup Class +class ProcessGroup { + public: + // Constructor + /*! + * Create a new process group + */ + ProcessGroup(ProcessGroupType type, std::string name, uuid_t uuid = NULL, + ProcessGroup *parent = NULL); + // Destructor + virtual ~ProcessGroup(); + // Set Processor Name + void setName(std::string name) { + name_ = name; + } + // Get Process Name + std::string getName(void) { + return (name_); + } + // Set URL + void setURL(std::string url) { + url_ = url; + } + // Get URL + std::string getURL(void) { + return (url_); + } + // SetTransmitting + void setTransmitting(bool val) { + transmitting_ = val; + } + // Get Transmitting + bool getTransmitting() { + return transmitting_; + } + // setTimeOut + void setTimeOut(uint64_t time) { + timeOut_ = time; + } + uint64_t getTimeOut() { + return timeOut_; + } + // Set Processor yield period in MilliSecond + void setYieldPeriodMsec(uint64_t period) { + yield_period_msec_ = period; + } + // Get Processor yield period in MilliSecond + uint64_t getYieldPeriodMsec(void) { + return (yield_period_msec_); + } + // Set UUID + void setUUID(uuid_t uuid) { + uuid_copy(uuid_, uuid); + } + // Get UUID + bool getUUID(uuid_t uuid) { + if (uuid) { + uuid_copy(uuid, uuid_); + return true; + } else + return false; + } + // Start Processing + void startProcessing(TimerDrivenSchedulingAgent *timeScheduler, + EventDrivenSchedulingAgent *eventScheduler); + // Stop Processing + void stopProcessing(TimerDrivenSchedulingAgent *timeScheduler, + EventDrivenSchedulingAgent *eventScheduler); + // Whether it is root process group + bool isRootProcessGroup(); + // set parent process group + void setParent(ProcessGroup *parent) { + std::lock_guard lock(mutex_); + parent_process_group_ = parent; + } + // get parent process group + ProcessGroup *getParent(void) { + std::lock_guard lock(mutex_); + return parent_process_group_; + } + // Add processor + void addProcessor(std::shared_ptr processor); + // Remove processor + void removeProcessor(std::shared_ptr processor); + // Add child processor group + void addProcessGroup(ProcessGroup *child); + // Remove child processor group + void removeProcessGroup(ProcessGroup *child); + // ! Add connections + void addConnection(std::shared_ptr connection); + // findProcessor based on UUID + std::shared_ptr findProcessor(uuid_t uuid); + // findProcessor based on name + std::shared_ptr findProcessor(const std::string &processorName); + // removeConnection + void removeConnection(std::shared_ptr connection); + // update property value + void updatePropertyValue(std::string processorName, std::string propertyName, + std::string propertyValue); + + void getConnections(std::map &connectionMap); + + protected: + // A global unique identifier + uuid_t uuid_; + // Processor Group Name + std::string name_; + // Process Group Type + ProcessGroupType type_; + // Processors (ProcessNode) inside this process group which include Input/Output Port, Remote Process Group input/Output port + std::set > processors_; + std::set child_process_groups_; + // Connections between the processor inside the group; + std::set > connections_; + // Parent Process Group + ProcessGroup* parent_process_group_; + // Yield Period in Milliseconds + std::atomic yield_period_msec_; + std::atomic timeOut_; + // URL + std::string url_; + // Transmitting + std::atomic transmitting_; + + private: + + // Mutex for protection + std::mutex mutex_; + // Logger + std::shared_ptr logger_; + // Prevent default copy constructor and assignment operation + // Only support pass by reference or pointer + ProcessGroup(const ProcessGroup &parent); + ProcessGroup &operator=(const ProcessGroup &parent); +}; +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ +#endif diff --git a/libminifi/include/core/ProcessSession.h b/libminifi/include/core/ProcessSession.h new file mode 100644 index 0000000000..d50c8ec4ec --- /dev/null +++ b/libminifi/include/core/ProcessSession.h @@ -0,0 +1,163 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __PROCESS_SESSION_H__ +#define __PROCESS_SESSION_H__ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ProcessContext.h" +#include "FlowFileRecord.h" +#include "Exception.h" +#include "core/logging/Logger.h" +#include "FlowFile.h" +#include "provenance/Provenance.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +// ProcessSession Class +class ProcessSession { + public: + // Constructor + /*! + * Create a new process session + */ + ProcessSession(ProcessContext *processContext = NULL) + : process_context_(processContext) { + logger_ = logging::Logger::getLogger(); + logger_->log_trace("ProcessSession created for %s", + process_context_->getProcessorNode().getName().c_str()); + auto repo = processContext->getProvenanceRepository(); + provenance_report_ = new provenance::ProvenanceReporter( + repo, process_context_->getProcessorNode().getUUIDStr(), + process_context_->getProcessorNode().getName()); + } + } + // Destructor + virtual ~ProcessSession() { + if (provenance_report_) + delete provenance_report_; + } + // Commit the session + void commit(); + // Roll Back the session + void rollback(); + // Get Provenance Report + provenance::ProvenanceReporter *getProvenanceReporter() { + return provenance_report_; + } + // + // Get the FlowFile from the highest priority queue + std::shared_ptr get(); + // Create a new UUID FlowFile with no content resource claim and without parent + std::shared_ptr create(); + // Create a new UUID FlowFile with no content resource claim and inherit all attributes from parent + std::shared_ptr create(std::shared_ptr &parent); + // Clone a new UUID FlowFile from parent both for content resource claim and attributes + std::shared_ptr clone(std::shared_ptr &parent); + // Clone a new UUID FlowFile from parent for attributes and sub set of parent content resource claim + std::shared_ptr clone(std::shared_ptr &parent, + long offset, long size); + // Duplicate a FlowFile with the same UUID and all attributes and content resource claim for the roll back of the session + std::shared_ptr duplicate( + std::shared_ptr &original); + // Transfer the FlowFile to the relationship + void transfer(std::shared_ptr &flow, Relationship relationship); + void transfer(std::shared_ptr &&flow, + Relationship relationship); + // Put Attribute + void putAttribute(std::shared_ptr &flow, std::string key, + std::string value); + void putAttribute(std::shared_ptr &&flow, std::string key, + std::string value); + // Remove Attribute + void removeAttribute(std::shared_ptr &flow, std::string key); + void removeAttribute(std::shared_ptr &&flow, std::string key); + // Remove Flow File + void remove(std::shared_ptr &flow); + void remove(std::shared_ptr &&flow); + // Execute the given read callback against the content + void read(std::shared_ptr &flow, InputStreamCallback *callback); + void read(std::shared_ptr &&flow, + InputStreamCallback *callback); + // Execute the given write callback against the content + void write(std::shared_ptr &flow, + OutputStreamCallback *callback); + void write(std::shared_ptr &&flow, + OutputStreamCallback *callback); + // Execute the given write/append callback against the content + void append(std::shared_ptr &flow, + OutputStreamCallback *callback); + void append(std::shared_ptr &&flow, + OutputStreamCallback *callback); + // Penalize the flow + void penalize(std::shared_ptr &flow); + void penalize(std::shared_ptr &&flow); + // Import the existed file into the flow + void import(std::string source, std::shared_ptr &flow, + bool keepSource = true, uint64_t offset = 0); + void import(std::string source, std::shared_ptr &&flow, + bool keepSource = true, uint64_t offset = 0); + + // Prevent default copy constructor and assignment operation + // Only support pass by reference or pointer + ProcessSession(const ProcessSession &parent) = delete; + ProcessSession &operator=(const ProcessSession &parent) = delete; + + protected: + // FlowFiles being modified by current process session + std::map > _updatedFlowFiles; + // Copy of the original FlowFiles being modified by current process session as above + std::map > _originalFlowFiles; + // FlowFiles being added by current process session + std::map > _addedFlowFiles; + // FlowFiles being deleted by current process session + std::map > _deletedFlowFiles; + // FlowFiles being transfered to the relationship + std::map _transferRelationship; + // FlowFiles being cloned for multiple connections per relationship + std::map > _clonedFlowFiles; + + private: + // Clone the flow file during transfer to multiple connections for a relationship + std::shared_ptr cloneDuringTransfer( + std::shared_ptr &parent); + // ProcessContext + ProcessContext *process_context_; + // Logger + std::shared_ptr logger_; + // Provenance Report + provenance::ProvenanceReporter *provenance_report_; + +}; +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ +#endif diff --git a/libminifi/include/ProcessSessionFactory.h b/libminifi/include/core/ProcessSessionFactory.h similarity index 54% rename from libminifi/include/ProcessSessionFactory.h rename to libminifi/include/core/ProcessSessionFactory.h index 7fc34106a0..e0ebe18058 100644 --- a/libminifi/include/ProcessSessionFactory.h +++ b/libminifi/include/core/ProcessSessionFactory.h @@ -25,28 +25,40 @@ #include "ProcessContext.h" #include "ProcessSession.h" -//! ProcessSessionFactory Class -class ProcessSessionFactory -{ -public: - //! Constructor - /*! - * Create a new process session factory - */ - explicit ProcessSessionFactory(ProcessContext *processContext) : _processContext(processContext) {} - - //! Create the session - std::unique_ptr createSession(); - - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - ProcessSessionFactory(const ProcessSessionFactory &parent) = delete; - ProcessSessionFactory &operator=(const ProcessSessionFactory &parent) = delete; - -private: - //! ProcessContext - ProcessContext *_processContext; +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +// ProcessSessionFactory Class +class ProcessSessionFactory { + public: + // Constructor + /*! + * Create a new process session factory + */ + explicit ProcessSessionFactory(ProcessContext *processContext) + : process_context_(processContext) { + } + + // Create the session + std::unique_ptr createSession(); + + // Prevent default copy constructor and assignment operation + // Only support pass by reference or pointer + ProcessSessionFactory(const ProcessSessionFactory &parent) = delete; + ProcessSessionFactory &operator=(const ProcessSessionFactory &parent) = delete; + + private: + // ProcessContext + ProcessContext *process_context_; }; +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ #endif diff --git a/libminifi/include/core/Processor.h b/libminifi/include/core/Processor.h new file mode 100644 index 0000000000..fd0411f73b --- /dev/null +++ b/libminifi/include/core/Processor.h @@ -0,0 +1,270 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __PROCESSOR_H__ +#define __PROCESSOR_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Connectable.h" +#include "ConfigurableComponent.h" +#include "Property.h" +#include "utils/TimeUtil.h" +#include "Relationship.h" +#include "Connection.h" +#include "ProcessContext.h" +#include "ProcessSession.h" +#include "ProcessSessionFactory.h" +#include "Scheduling.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +// Minimum scheduling period in Nano Second +#define MINIMUM_SCHEDULING_NANOS 30000 + +// Default yield period in second +#define DEFAULT_YIELD_PERIOD_SECONDS 1 + +// Default penalization period in second +#define DEFAULT_PENALIZATION_PERIOD_SECONDS 30 + +// Processor Class +class Processor : public Connectable, public ConfigurableComponent, + public std::enable_shared_from_this { + + public: + // Constructor + /*! + * Create a new processor + */ + Processor(std::string name, uuid_t uuid = NULL); + // Destructor + virtual ~Processor() { + } + + bool isRunning(); + // Set Processor Scheduled State + void setScheduledState(ScheduledState state); + // Get Processor Scheduled State + ScheduledState getScheduledState(void) { + return state_; + } + // Set Processor Scheduling Strategy + void setSchedulingStrategy(SchedulingStrategy strategy) { + strategy_ = strategy; + } + // Get Processor Scheduling Strategy + SchedulingStrategy getSchedulingStrategy(void) { + return strategy_; + } + // Set Processor Loss Tolerant + void setlossTolerant(bool lossTolerant) { + loss_tolerant_ = lossTolerant; + } + // Get Processor Loss Tolerant + bool getlossTolerant(void) { + return loss_tolerant_; + } + // Set Processor Scheduling Period in Nano Second + void setSchedulingPeriodNano(uint64_t period) { + uint64_t minPeriod = MINIMUM_SCHEDULING_NANOS; + scheduling_period_nano_ = std::max(period, minPeriod); + } + // Get Processor Scheduling Period in Nano Second + uint64_t getSchedulingPeriodNano(void) { + return scheduling_period_nano_; + } + // Set Processor Run Duration in Nano Second + void setRunDurationNano(uint64_t period) { + run_durantion_nano_ = period; + } + // Get Processor Run Duration in Nano Second + uint64_t getRunDurationNano(void) { + return (run_durantion_nano_); + } + // Set Processor yield period in MilliSecond + void setYieldPeriodMsec(uint64_t period) { + yield_period_msec_ = period; + } + // Get Processor yield period in MilliSecond + uint64_t getYieldPeriodMsec(void) { + return (yield_period_msec_); + } + // Set Processor penalization period in MilliSecond + void setPenalizationPeriodMsec(uint64_t period) { + _penalizationPeriodMsec = period; + } + + + // Set Processor Maximum Concurrent Tasks + void setMaxConcurrentTasks(uint8_t tasks) { + max_concurrent_tasks_ = tasks; + } + // Get Processor Maximum Concurrent Tasks + uint8_t getMaxConcurrentTasks(void) { + return (max_concurrent_tasks_); + } + // Set Trigger when empty + void setTriggerWhenEmpty(bool value) { + _triggerWhenEmpty = value; + } + // Get Trigger when empty + bool getTriggerWhenEmpty(void) { + return (_triggerWhenEmpty); + } + // Get Active Task Counts + uint8_t getActiveTasks(void) { + return (active_tasks_); + } + // Increment Active Task Counts + void incrementActiveTasks(void) { + active_tasks_++; + } + // decrement Active Task Counts + void decrementActiveTask(void) { + active_tasks_--; + } + void clearActiveTask(void) { + active_tasks_ = 0; + } + // Yield based on the yield period + void yield() { + yield_expiration_ = (getTimeMillis() + yield_period_msec_); + } + // Yield based on the input time + void yield(uint64_t time) { + yield_expiration_ = (getTimeMillis() + time); + } + // whether need be to yield + bool isYield() { + if (yield_expiration_ > 0) + return (yield_expiration_ >= getTimeMillis()); + else + return false; + } + // clear yield expiration + void clearYield() { + yield_expiration_ = 0; + } + // get yield time + uint64_t getYieldTime() { + uint64_t curTime = getTimeMillis(); + if (yield_expiration_ > curTime) + return (yield_expiration_ - curTime); + else + return 0;; + } + // Whether flow file queued in incoming connection + bool flowFilesQueued(); + // Whether flow file queue full in any of the outgoin connection + bool flowFilesOutGoingFull(); + + // Get outgoing connections based on relationship name + std::set > getOutGoingConnections( + std::string relationship); + // Add connection + bool addConnection(std::shared_ptr connection); + // Remove connection + void removeConnection(std::shared_ptr connection); + // Get the UUID as string + std::string getUUIDStr() { + return uuidStr_; + } + // Get the Next RoundRobin incoming connection + std::shared_ptr getNextIncomingConnection(); + // On Trigger + void onTrigger(ProcessContext *context, + ProcessSessionFactory *sessionFactory); + + virtual bool canEdit() { + return !isRunning(); + } + + public: + + + // OnTrigger method, implemented by NiFi Processor Designer + virtual void onTrigger(ProcessContext *context, ProcessSession *session) = 0; + // Initialize, overridden by NiFi Process Designer + virtual void initialize() { + } + // Scheduled event hook, overridden by NiFi Process Designer + virtual void onSchedule(ProcessContext *context, + ProcessSessionFactory *sessionFactory) { + } + + protected: + + // Processor state + std::atomic state_; + + // lossTolerant + std::atomic loss_tolerant_; + // SchedulePeriod in Nano Seconds + std::atomic scheduling_period_nano_; + // Run Duration in Nano Seconds + std::atomic run_durantion_nano_; + // Yield Period in Milliseconds + std::atomic yield_period_msec_; + + // Active Tasks + std::atomic active_tasks_; + // Trigger the Processor even if the incoming connection is empty + std::atomic _triggerWhenEmpty; + +private: + + // Mutex for protection + std::mutex mutex_; + // Yield Expiration + std::atomic yield_expiration_; + + + // Check all incoming connections for work + bool isWorkAvailable(); + // Logger + std::shared_ptr logger_; + // Prevent default copy constructor and assignment operation + // Only support pass by reference or pointer + Processor(const Processor &parent); + Processor &operator=(const Processor &parent); + +}; + +} +/* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif diff --git a/libminifi/include/core/ProcessorNode.h b/libminifi/include/core/ProcessorNode.h new file mode 100644 index 0000000000..8836f62b0d --- /dev/null +++ b/libminifi/include/core/ProcessorNode.h @@ -0,0 +1,246 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBMINIFI_INCLUDE_PROCESSOR_PROCESSORNODE_H_ +#define LIBMINIFI_INCLUDE_PROCESSOR_PROCESSORNODE_H_ + +#include "ConfigurableComponent.h" +#include "Connectable.h" +#include "Property.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +/** + * Processor node functions as a pass through to the implementing Connectables + * ProcessorNode can be used by itself or with a pass through object, in which case + * we need to function as a passthrough or not. + */ +class ProcessorNode : public ConfigurableComponent, public Connectable { + public: + explicit ProcessorNode(const std::shared_ptr processor); + + explicit ProcessorNode(const ProcessorNode &other); + + /** + * Get property using the provided name. + * @param name property name. + * @param value value passed in by reference + * @return result of getting property. + */ + std::shared_ptr getProcessor() const { + return processor_; + } + + void yield() { + processor_->yield(); + } + + /** + * Get property using the provided name. + * @param name property name. + * @param value value passed in by reference + * @return result of getting property. + */ + bool getProperty(const std::string name, std::string &value) { + const std::shared_ptr processor_cast = + std::dynamic_pointer_cast(processor_); + if (nullptr != processor_cast) + return processor_cast->getProperty(name, value); + else { + return ConfigurableComponent::getProperty(name, value); + } + } + /** + * Sets the property using the provided name + * @param property name + * @param value property value. + * @return result of setting property. + */ + bool setProperty(const std::string name, std::string value) { + const std::shared_ptr processor_cast = + std::dynamic_pointer_cast(processor_); + bool ret = ConfigurableComponent::setProperty(name, value); + if (nullptr != processor_cast) + ret = processor_cast->setProperty(name, value); + + return ret; + + } + + /** + * Sets the property using the provided name + * @param property name + * @param value property value. + * @return whether property was set or not + */ + bool setProperty(Property &prop, std::string value) { + const std::shared_ptr processor_cast = + std::dynamic_pointer_cast(processor_); + bool ret = ConfigurableComponent::setProperty(prop, value); + if (nullptr != processor_cast) + ret = processor_cast->setProperty(prop, value); + + return ret; + } + + /** + * Sets supported properties for the ConfigurableComponent + * @param supported properties + * @return result of set operation. + */ + bool setSupportedProperties(std::set properties) { + const std::shared_ptr processor_cast = + std::dynamic_pointer_cast(processor_); + bool ret = ConfigurableComponent::setSupportedProperties(properties); + if (nullptr != processor_cast) + ret = processor_cast->setSupportedProperties(properties); + + return ret; + } + /** + * Sets supported properties for the ConfigurableComponent + * @param supported properties + * @return result of set operation. + */ + + bool setAutoTerminatedRelationships(std::set relationships) { + return processor_->setAutoTerminatedRelationships(relationships); + } + + bool isAutoTerminated(Relationship relationship) { + return processor_->isAutoTerminated(relationship); + } + + bool setSupportedRelationships(std::set relationships) { + return processor_->setSupportedRelationships(relationships); + } + + bool isSupportedRelationship(Relationship relationship) { + return processor_->isSupportedRelationship(relationship); + } + + /** + * Set name. + * @param name + */ + void setName(const std::string name) { + Connectable::setName(name); + processor_->setName(name); + } + + /** + * Set UUID in this instance + * @param uuid uuid to apply to the internal representation. + */ + void setUUID(uuid_t uuid) { + Connectable::setUUID(uuid); + processor_->setUUID(uuid); + } + +// Get Processor penalization period in MilliSecond + uint64_t getPenalizationPeriodMsec(void) { + return processor_->getPenalizationPeriodMsec(); + } + + /** + * Get outgoing connection based on relationship + * @return set of outgoing connections. + */ + std::set> getOutGoingConnections( + std::string relationship) { + return processor_->getOutGoingConnections(relationship); + } + + /** + * Get next incoming connection + * @return next incoming connection + */ + std::shared_ptr getNextIncomingConnection() { + return processor_->getNextIncomingConnection(); + } + + /** + * @return true if incoming connections > 0 + */ + bool hasIncomingConnections() { + return processor_->hasIncomingConnections(); + } + + /** + * Returns the UUID through the provided object. + * @param uuid uuid struct to which we will copy the memory + * @return success of request + */ + bool getUUID(uuid_t uuid) { + return processor_->getUUID(uuid); + } + + unsigned const char *getUUID() { + return processor_->getUUID(); + } + /** + * Return the UUID string + * @param constant reference to the UUID str + */ + const std::string & getUUIDStr() const { + return processor_->getUUIDStr(); + } + +// Get Process Name + std::string getName() const { + return processor_->getName(); + } + + uint8_t getMaxConcurrentTasks() { + return processor_->getMaxConcurrentTasks(); + } + + void setMaxConcurrentTasks(const uint8_t tasks) { + processor_->setMaxConcurrentTasks(tasks); + } + + virtual bool isRunning(); + + virtual bool isWorkAvailable(); + + virtual ~ProcessorNode(); + + protected: + + virtual bool canEdit() { + return !processor_->isRunning(); + } + + /** + * internal connectable. + */ + std::shared_ptr processor_; + +} +; + +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif /* LIBMINIFI_INCLUDE_PROCESSOR_PROCESSORNODE_H_ */ diff --git a/libminifi/include/core/Property.h b/libminifi/include/core/Property.h new file mode 100644 index 0000000000..c681449b34 --- /dev/null +++ b/libminifi/include/core/Property.h @@ -0,0 +1,264 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __PROPERTY_H__ +#define __PROPERTY_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +// Time Unit +enum TimeUnit { + DAY, + HOUR, + MINUTE, + SECOND, + MILLISECOND, + NANOSECOND +}; + +// Property Class +class Property { + + public: + // Constructor + /*! + * Create a new property + */ + Property(const std::string name, const std::string description, + const std::string value) + : name_(name), + description_(description), + value_(value) { + } + Property() { + } + // Destructor + virtual ~Property() { + } + // Get Name for the property + std::string getName() const; + // Get Description for the property + std::string getDescription(); + // Get value for the property + std::string getValue() const; + // Set value for the property + void setValue(std::string value); + const Property &operator=(const Property &other); + // Compare + bool operator <(const Property & right) const; + + // Convert TimeUnit to MilliSecond + static bool ConvertTimeUnitToMS(int64_t input, TimeUnit unit, int64_t &out) { + if (unit == MILLISECOND) { + out = input; + return true; + } else if (unit == SECOND) { + out = input * 1000; + return true; + } else if (unit == MINUTE) { + out = input * 60 * 1000; + return true; + } else if (unit == HOUR) { + out = input * 60 * 60 * 1000; + return true; + } else if (unit == DAY) { + out = 24 * 60 * 60 * 1000; + return true; + } else if (unit == NANOSECOND) { + out = input / 1000 / 1000; + return true; + } else { + return false; + } + } + // Convert TimeUnit to NanoSecond + static bool ConvertTimeUnitToNS(int64_t input, TimeUnit unit, int64_t &out) { + if (unit == MILLISECOND) { + out = input * 1000 * 1000; + return true; + } else if (unit == SECOND) { + out = input * 1000 * 1000 * 1000; + return true; + } else if (unit == MINUTE) { + out = input * 60 * 1000 * 1000 * 1000; + return true; + } else if (unit == HOUR) { + out = input * 60 * 60 * 1000 * 1000 * 1000; + return true; + } else if (unit == NANOSECOND) { + out = input; + return true; + } else { + return false; + } + } + // Convert String + static bool StringToTime(std::string input, int64_t &output, + TimeUnit &timeunit) { + if (input.size() == 0) { + return false; + } + + const char *cvalue = input.c_str(); + char *pEnd; + long int ival = strtol(cvalue, &pEnd, 0); + + if (pEnd[0] == '\0') { + return false; + } + + while (*pEnd == ' ') { + // Skip the space + pEnd++; + } + + std::string unit(pEnd); + + if (unit == "sec" || unit == "s" || unit == "second" || unit == "seconds" + || unit == "secs") { + timeunit = SECOND; + output = ival; + return true; + } else if (unit == "min" || unit == "m" || unit == "mins" + || unit == "minute" || unit == "minutes") { + timeunit = MINUTE; + output = ival; + return true; + } else if (unit == "ns" || unit == "nano" || unit == "nanos" + || unit == "nanoseconds") { + timeunit = NANOSECOND; + output = ival; + return true; + } else if (unit == "ms" || unit == "milli" || unit == "millis" + || unit == "milliseconds") { + timeunit = MILLISECOND; + output = ival; + return true; + } else if (unit == "h" || unit == "hr" || unit == "hour" || unit == "hrs" + || unit == "hours") { + timeunit = HOUR; + output = ival; + return true; + } else if (unit == "d" || unit == "day" || unit == "days") { + timeunit = DAY; + output = ival; + return true; + } else + return false; + } + + // Convert String to Integer + static bool StringToInt(std::string input, int64_t &output) { + if (input.size() == 0) { + return false; + } + + const char *cvalue = input.c_str(); + char *pEnd; + long int ival = strtol(cvalue, &pEnd, 0); + + if (pEnd[0] == '\0') { + output = ival; + return true; + } + + while (*pEnd == ' ') { + // Skip the space + pEnd++; + } + + char end0 = toupper(pEnd[0]); + if ((end0 == 'K') || (end0 == 'M') || (end0 == 'G') || (end0 == 'T') + || (end0 == 'P')) { + if (pEnd[1] == '\0') { + unsigned long int multiplier = 1000; + + if ((end0 != 'K')) { + multiplier *= 1000; + if (end0 != 'M') { + multiplier *= 1000; + if (end0 != 'G') { + multiplier *= 1000; + if (end0 != 'T') { + multiplier *= 1000; + } + } + } + } + output = ival * multiplier; + return true; + + } else if ((pEnd[1] == 'b' || pEnd[1] == 'B') && (pEnd[2] == '\0')) { + + unsigned long int multiplier = 1024; + + if ((end0 != 'K')) { + multiplier *= 1024; + if (end0 != 'M') { + multiplier *= 1024; + if (end0 != 'G') { + multiplier *= 1024; + if (end0 != 'T') { + multiplier *= 1024; + } + } + } + } + output = ival * multiplier; + return true; + } + } + + return false; + } + + protected: + // Name + std::string name_; + // Description + std::string description_; + // Value + std::string value_; + + private: + +}; + +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif diff --git a/libminifi/include/core/Relationship.h b/libminifi/include/core/Relationship.h new file mode 100644 index 0000000000..416ede6568 --- /dev/null +++ b/libminifi/include/core/Relationship.h @@ -0,0 +1,96 @@ +/** + * @file Relationship.h + * Relationship class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __RELATIONSHIP_H__ +#define __RELATIONSHIP_H__ + +#include +#include +#include +#include +#include +#include +#include + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +// undefined relationship for remote process group outgoing port and root process group incoming port +#define UNDEFINED_RELATIONSHIP "undefined" + +inline bool isRelationshipNameUndefined(std::string name) { + if (name == UNDEFINED_RELATIONSHIP) + return true; + else + return false; +} + +// Relationship Class +class Relationship { + + public: + /* + * Create a new relationship + */ + Relationship(const std::string name, const std::string description) + : name_(name), + description_(description) { + } + Relationship() + : name_(UNDEFINED_RELATIONSHIP) { + } + // Destructor + virtual ~Relationship() { + } + // Get Name for the relationship + std::string getName() const { + return name_; + } + // Get Description for the relationship + std::string getDescription() const { + return description_; + } + // Compare + bool operator <(const Relationship & right) const { + return name_ < right.name_; + } + // Whether it is a undefined relationship + bool isRelationshipUndefined() { + return isRelationshipNameUndefined(name_); + } + + protected: + + // Name + std::string name_; + // Description + std::string description_; + + private: +}; + +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ +#endif diff --git a/libminifi/include/core/Repository.h b/libminifi/include/core/Repository.h new file mode 100644 index 0000000000..d0a1d56044 --- /dev/null +++ b/libminifi/include/core/Repository.h @@ -0,0 +1,149 @@ +/** + * @file Repository + * Repository class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __REPOSITORY_H__ +#define __REPOSITORY_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "leveldb/db.h" +#include "leveldb/options.h" +#include "leveldb/slice.h" +#include "leveldb/status.h" +#include "Configure.h" +#include "Connection.h" +#include "FlowFileRecord.h" +#include "core/logging/Logger.h" +#include "core/Property.h" +#include "ResourceClaim.h" +#include "io/Serializable.h" +#include "utils/TimeUtil.h" +#include "utils/StringUtils.h" +#include "core.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +class Repository : public CoreComponent { + public: + /* + * Constructor for the repository + */ + Repository(std::string repo_name, std::string directory, + int64_t maxPartitionMillis, int64_t maxPartitionBytes, + uint64_t purgePeriod) + : CoreComponent(repo_name), + thread_() { + directory_ = directory; + max_partition_millis_ = maxPartitionMillis; + max_partition_bytes_ = maxPartitionBytes; + purge_period_ = purgePeriod; + configure_ = Configure::getConfigure(); + running_ = false; + repo_full_ = false; + } + + // Destructor + virtual ~Repository() { + stop(); + } + + // initialize + virtual bool initialize() = 0; + // Put + virtual bool Put(std::string key, uint8_t *buf, int bufLen) = 0; + // Delete + virtual bool Delete(std::string key) = 0; + + virtual bool Get(std::string key, std::string &value) = 0; + + // Run function for the thread + void run() = 0; + // Start the repository monitor thread + virtual void start(); + // Stop the repository monitor thread + virtual void stop(); + // whether the repo is full + virtual bool isFull() { + return repo_full_; + } + // whether the repo is enable + virtual bool isRunning() { + return running_; + } + uint64_t incrementSize(const char *fpath, const struct stat *sb, + int typeflag) { + return (repo_full_ += sb->st_size); + } + + // Prevent default copy constructor and assignment operation + // Only support pass by reference or pointer + Repository(const Repository &parent) = delete; + Repository &operator=(const Repository &parent) = delete; + + protected: + // Mutex for protection + std::mutex mutex_; + // repository directory + std::string directory_; + // Configure + Configure *configure_; + // max db entry life time + int64_t max_partition_millis_; + // max db size + int64_t max_partition_bytes_; + // purge period + uint64_t purge_period_; + // thread + std::thread thread_; + // whether the monitoring thread is running for the repo while it was enabled + bool running_; + // whether stop accepting provenace event + std::atomic repo_full_; + // repoSize + uint64_t repoSize(); + // size of the directory + std::atomic repo_size_; + + private: + // Run function for the thread + void threadExecutor(){ + run(); + } +}; + +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ +#endif diff --git a/libminifi/include/core/Scheduling.h b/libminifi/include/core/Scheduling.h new file mode 100644 index 0000000000..0c983df999 --- /dev/null +++ b/libminifi/include/core/Scheduling.h @@ -0,0 +1,64 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBMINIFI_INCLUDE_CORE_SCHEDULING_H_ +#define LIBMINIFI_INCLUDE_CORE_SCHEDULING_H_ + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +/* + * Indicates the valid values for the state of a entity + * with respect to scheduling the entity to run. + */ +enum ScheduledState { + + /** + * Entity cannot be scheduled to run + */ + DISABLED, + /** + * Entity can be scheduled to run but currently is not + */ + STOPPED, + /** + * Entity is currently scheduled to run + */ + RUNNING +}; + +/* + * Scheduling Strategy + */ +enum SchedulingStrategy { + // Event driven + EVENT_DRIVEN, + // Timer driven + TIMER_DRIVEN, + // Cron Driven + CRON_DRIVEN +}; + +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ +#endif /* LIBMINIFI_INCLUDE_CORE_SCHEDULING_H_ */ diff --git a/libminifi/include/core/core.h b/libminifi/include/core/core.h new file mode 100644 index 0000000000..b639b2d6ae --- /dev/null +++ b/libminifi/include/core/core.h @@ -0,0 +1,141 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBMINIFI_INCLUDE_CORE_CORE_H_ +#define LIBMINIFI_INCLUDE_CORE_CORE_H_ +/** + * namespace aliasing + */ +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace utils { +} +namespace processors { +} +namespace provenance { + +} +namespace core { + +template +static inline std::string getClassName() { + std::string name = abi::__cxa_demangle(typeid(T).name(), 0, 0, 0); + return name; +} + +/** + * Base component within MiNiFi + * Purpose: Many objects store a name and UUID, therefore + * the functionality is localized here to avoid duplication + */ +class CoreComponent { + + public: + + /** + * Constructor that sets the name and uuid. + */ + explicit CoreComponent(const std::string name, uuid_t uuid = 0) + : logger_(logging::Logger::getLogger()), + name_(name) { + if (!uuid) + // Generate the global UUID for the flow record + uuid_generate(uuid_); + else + uuid_copy(uuid_, uuid); + + char uuidStr[37]; + uuid_unparse_lower(uuid_, uuidStr); + uuidStr_ = uuidStr; + } + + /** + * Move Constructor. + */ + explicit CoreComponent(const CoreComponent &&other) + : name_(std::move(other.name_)), + logger_(logging::Logger::getLogger()) { + uuid_copy(uuid_, other.uuid_); + } + + // Get component name Name + std::string getName() const; + + /** + * Set name. + * @param name + */ + void setName(const std::string name); + + /** + * Set UUID in this instance + * @param uuid uuid to apply to the internal representation. + */ + void setUUID(uuid_t uuid); + + /** + * Returns the UUID through the provided object. + * @param uuid uuid struct to which we will copy the memory + * @return success of request + */ + bool getUUID(uuid_t uuid) const; + + unsigned const char *getUUID() const; + /** + * Return the UUID string + * @param constant reference to the UUID str + */ + const std::string & getUUIDStr() const { + return uuidStr_; + } + + protected: + // A global unique identifier + uuid_t uuid_; + // UUID string + std::string uuidStr_; + + // logger shared ptr + std::shared_ptr logger_; + + // Connectable's name + std::string name_; +}; + +namespace logging { +} +} +} +} +} +} + +namespace minifi = org::apache::nifi::minifi; + +namespace core = org::apache::nifi::minifi::core; + +namespace processors = org::apache::nifi::minifi::processors; + +namespace logging = org::apache::nifi::minifi::core::logging; + +namespace utils = org::apache::nifi::minifi::utils; + +namespace provenance = org::apache::nifi::minifi::provenance; + +#endif /* LIBMINIFI_INCLUDE_CORE_CORE_H_ */ diff --git a/libminifi/include/core/logging/BaseLogger.h b/libminifi/include/core/logging/BaseLogger.h new file mode 100644 index 0000000000..bfdf26fd56 --- /dev/null +++ b/libminifi/include/core/logging/BaseLogger.h @@ -0,0 +1,224 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBMINIFI_INCLUDE_BASELOGGER_H_ +#define LIBMINIFI_INCLUDE_BASELOGGER_H_ + +#include +#include +#include "spdlog/spdlog.h" +#include + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { +namespace logging { + +// 5M default log file size +#define DEFAULT_LOG_FILE_SIZE (5*1024*1024) +// 3 log files rotation +#define DEFAULT_LOG_FILE_NUMBER 3 +#define LOG_NAME "minifi log" +#define LOG_FILE_NAME "minifi-app.log" + +/** + * Log level enumeration. + */ +typedef enum { + trace = 0, + debug = 1, + info = 2, + warn = 3, + err = 4, + critical = 5, + off = 6 +} LOG_LEVEL_E; + +#define LOG_BUFFER_SIZE 1024 +#define FILL_BUFFER char buffer[LOG_BUFFER_SIZE]; \ + va_list args; \ + va_start(args, format); \ + std::vsnprintf(buffer, LOG_BUFFER_SIZE,format, args); \ + va_end(args); + +/** + * Base class that represents a logger configuration. + */ +class BaseLogger { + + public: + static const char *nifi_log_level; + static const char *nifi_log_appender; + + /** + * Base Constructor + */ + BaseLogger() { + setLogLevel("info"); + logger_ = nullptr; + stderr_ = nullptr; + } + + /** + * Logger configuration constructorthat will set the base log level. + * @param config incoming configuration. + */ + BaseLogger(std::string log_level, std::shared_ptr logger) + : logger_(logger) { + setLogLevel(log_level); + + } + + virtual ~BaseLogger() { + + } + + /** + * Move constructor that will atomically swap configuration + * shared pointers. + */ + BaseLogger(const BaseLogger &&other) + : configured_level_(other.configured_level_.load()) { + // must atomically exchange the pointers + logger_ = std::move(other.logger_); + set_error_logger(other.stderr_); + + } + + /** + * Returns the log level for this instance. + */ + virtual LOG_LEVEL_E getLogLevel() const { + return configured_level_; + } + + /** + * @brief Log error message + * @param format format string ('man printf' for syntax) + * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match + */ + virtual void log_error(const char * const format, ...); + /** + * @brief Log warn message + * @param format format string ('man printf' for syntax) + * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match + */ + virtual void log_warn(const char * const format, ...); + /** + * @brief Log info message + * @param format format string ('man printf' for syntax) + * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match + */ + virtual void log_info(const char * const format, ...); + /** + * @brief Log debug message + * @param format format string ('man printf' for syntax) + * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match + */ + virtual void log_debug(const char * const format, ...); + /** + * @brief Log trace message + * @param format format string ('man printf' for syntax) + * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match + */ + virtual void log_trace(const char * const format, ...); + + /** + * @brief Log error message + * @param format format string ('man printf' for syntax) + * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match + */ + virtual void log_str(LOG_LEVEL_E level, const std::string &buffer); + + /** + * Sets the log level for this instance based on the string + * @param level desired log leve. + * @param defaultLevel default level if we cannot match level. + */ + virtual void setLogLevel(const std::string &level, LOG_LEVEL_E defaultLevel = + info); + + /** + * Sets the log level atomic and sets it + * within logger if it can + * @param level desired log level. + */ + virtual void setLogLevel(LOG_LEVEL_E level) { + configured_level_ = level; + setLogLevel(); + } + + bool shouldLog(LOG_LEVEL_E level) { + return level >= configured_level_.load(std::memory_order_relaxed); + } + + /** + * Move operator overload + */ + BaseLogger &operator=(const BaseLogger &&other) { + configured_level_ = (other.configured_level_.load()); + // must atomically exchange the pointers + logger_ = std::move(other.logger_); + set_error_logger(other.stderr_); + return *this; + } + + protected: + + /** + * Logger configuration constructorthat will set the base log level. + * @param config incoming configuration. + */ + BaseLogger(std::string log_level) + : logger_(nullptr) { + setLogLevel(log_level); + } + + void setLogger(std::shared_ptr logger) { + logger_ = logger; + } + + /** + * Since a thread may be using stderr and it can be null, + * we must atomically exchange the shared pointers. + * @param other other shared pointer. can be null ptr + */ + void set_error_logger(std::shared_ptr other); + + /** + * Sets the log level on the spdlogger if it is not null. + */ + void setLogLevel() { + if (logger_ != nullptr) + logger_->set_level((spdlog::level::level_enum) configured_level_.load()); + + } + + std::atomic configured_level_; + std::shared_ptr logger_; + std::shared_ptr stderr_; +}; + +} /* namespace logging */ +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ +#endif /* LIBMINIFI_INCLUDE_BASELOGGER_H_ */ diff --git a/libminifi/include/core/logging/LogAppenders.h b/libminifi/include/core/logging/LogAppenders.h new file mode 100644 index 0000000000..36892e9880 --- /dev/null +++ b/libminifi/include/core/logging/LogAppenders.h @@ -0,0 +1,301 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBMINIFI_INCLUDE_LOGAPPENDERS_H_ +#define LIBMINIFI_INCLUDE_LOGAPPENDERS_H_ + +#include "BaseLogger.h" +#include "spdlog/sinks/null_sink.h" +#include "spdlog/sinks/ostream_sink.h" +#include +#include "Configure.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { +namespace logging { + +template +static std::string getUniqueName() { + std::string name = LOG_NAME; + name += " -- "; + name += abi::__cxa_demangle(typeid(T).name(), 0, 0, 0); + spdlog::drop(name); + return name; +} + +/** + * Null appender sets a null sink, thereby performing no logging. + */ +class NullAppender : public BaseLogger { + public: + /** + * Base constructor that creates the null sink. + */ + explicit NullAppender() + : BaseLogger("off") { + auto null_sink = std::make_shared(); + std::string unique_name = getUniqueName(); + logger_ = std::make_shared(unique_name, null_sink); + configured_level_ = off; + setLogLevel(); + } + + /** + * Move constructor for the null appender. + */ + explicit NullAppender(const NullAppender &&other) + : BaseLogger(std::move(other)) { + + } + +}; + +/** + * Basic output stream configuration that uses a supplied ostream + * + * Design : extends LoggerConfiguration using the logger and log level + * encapsulated within the base configuration class. + */ +class OutputStreamAppender : public BaseLogger { + + public: + + static const char *nifi_log_output_stream_error_stderr; + + /** + * Output stream move constructor. + */ + explicit OutputStreamAppender(const OutputStreamAppender &&other) + : BaseLogger(std::move(other)) { + + } + + /** + * Base constructor. Creates a ostream sink. + * @param stream incoming stream reference. + * @param config configuration. + */ + explicit OutputStreamAppender(Configure *config) + : BaseLogger("info") { + auto ostream_sink = std::make_shared( + std::cout); + + std::string unique_name = getUniqueName(); + logger_ = std::make_shared(unique_name, ostream_sink); + + std::string use_std_err; + + if (NULL != config + && config->get(nifi_log_output_stream_error_stderr, use_std_err)) { + + std::transform(use_std_err.begin(), use_std_err.end(), + use_std_err.begin(), ::tolower); + + if (use_std_err == "true") { + std::string err_unique_name = getUniqueName(); + auto error_ostream_sink = std::make_shared< + spdlog::sinks::ostream_sink_mt>(std::cerr); + stderr_ = std::make_shared(err_unique_name, + error_ostream_sink); + } + } else { + stderr_ = nullptr; + } + + std::string log_level; + if (NULL != config && config->get(BaseLogger::nifi_log_level, log_level)) { + setLogLevel(log_level); + } else { + setLogLevel("info"); + } + + } + + /** + * Base constructor. Creates a ostream sink. + * @param stream incoming stream reference. + * @param config configuration. + */ + OutputStreamAppender(std::ostream &stream, Configure *config) + : BaseLogger("info") { + auto ostream_sink = std::make_shared( + stream); + std::string unique_name = getUniqueName(); + logger_ = std::make_shared(unique_name, ostream_sink); + + stderr_ = nullptr; + + std::string log_level; + if (NULL != config && config->get(BaseLogger::nifi_log_level, log_level)) { + setLogLevel(log_level); + } else { + setLogLevel("info"); + } + + } + + protected: + +}; + +/** + * Rolling configuration + * Design : extends LoggerConfiguration using the logger and log level + * encapsulated within the base configuration class. + */ +class RollingAppender : public BaseLogger { + public: + static const char *nifi_log_rolling_apender_file; + static const char *nifi_log_rolling_appender_max_files; + static const char *nifi_log_rolling_appender_max_file_size; + + /** + * RollingAppenderConfiguration move constructor. + */ + explicit RollingAppender(const RollingAppender&& other) + : BaseLogger(std::move(other)), + max_files_(std::move(other.max_files_)), + file_name_(std::move(other.file_name_)), + max_file_size_(std::move(other.max_file_size_)) { + } + /** + * Base Constructor. + * @param config pointer to the configuration for this instance. + */ + explicit RollingAppender(Configure * config = 0) + : BaseLogger("info") { + std::string file_name = ""; + if (NULL != config + && config->get(nifi_log_rolling_apender_file, file_name)) { + file_name_ = file_name; + } else { + file_name_ = LOG_FILE_NAME; + } + + std::string max_files = ""; + if (NULL != config + && config->get(nifi_log_rolling_appender_max_files, max_files)) { + try { + max_files_ = std::stoi(max_files); + } catch (const std::invalid_argument &ia) { + max_files_ = DEFAULT_LOG_FILE_NUMBER; + } catch (const std::out_of_range &oor) { + max_files_ = DEFAULT_LOG_FILE_NUMBER; + } + } else { + max_files_ = DEFAULT_LOG_FILE_NUMBER; + } + + std::string max_file_size = ""; + if (NULL != config + && config->get(nifi_log_rolling_appender_max_file_size, + max_file_size)) { + try { + max_file_size_ = std::stoi(max_file_size); + } catch (const std::invalid_argument &ia) { + max_file_size_ = DEFAULT_LOG_FILE_SIZE; + } catch (const std::out_of_range &oor) { + max_file_size_ = DEFAULT_LOG_FILE_SIZE; + } + } else { + max_file_size_ = DEFAULT_LOG_FILE_SIZE; + } + + std::string unique_name = getUniqueName(); + logger_ = spdlog::rotating_logger_mt(unique_name, file_name_, + max_file_size_, max_files_); + + std::string log_level; + if (NULL != config && config->get(BaseLogger::nifi_log_level, log_level)) { + setLogLevel(log_level); + } + } + + /** + * To maintain current functionality we will flush on write. + */ + void log_str(LOG_LEVEL_E level, const std::string &buffer) { + BaseLogger::log_str(level, buffer); + logger_->flush(); + } + + protected: + + /** + * file name. + */ + std::string file_name_; + /** + * maximum number of files to keep in the rotation. + */ + size_t max_files_; + /** + * Maximum file size per rotated file. + */ + size_t max_file_size_; + +}; + +class LogInstance { + public: + /** + * Returns a logger configuration based on + * the configuration within this instance. + * @param config configuration for this instance. + */ + static std::unique_ptr getConfiguredLogger(Configure *config) { + std::string appender = ""; + + if (config->get(BaseLogger::nifi_log_appender, appender)) { + std::transform(appender.begin(), appender.end(), appender.begin(), + ::tolower); + + if ("nullappender" == appender || "null appender" == appender + || "null" == appender) { + + return std::move(std::unique_ptr(new NullAppender())); + + } else if ("rollingappender" == appender || "rolling appender" == appender + || "rolling" == appender) { + + return std::move( + std::unique_ptr(new RollingAppender(config))); + + } else if ("outputstream" == appender + || "outputstreamappender" == appender + || "outputstream appender" == appender) { + + return std::move( + std::unique_ptr(new OutputStreamAppender(config))); + + } + } + return nullptr; + + } +}; + +} /* namespace logging */ +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ +#endif /* LIBMINIFI_INCLUDE_LOGAPPENDERS_H_ */ diff --git a/libminifi/include/core/logging/Logger.h b/libminifi/include/core/logging/Logger.h new file mode 100644 index 0000000000..08ef70239f --- /dev/null +++ b/libminifi/include/core/logging/Logger.h @@ -0,0 +1,214 @@ +/** + * @file Logger.h + * Logger class declaration + * This is a C++ wrapper for spdlog, a lightweight C++ logging library + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __LOGGER_H__ +#define __LOGGER_H__ + +#include +#include +#include +#include +#include +#include +#include + +#include "BaseLogger.h" +#include "spdlog/spdlog.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { +namespace logging { + +/** + * Logger class + * Design: Extends BaseLogger, leaving this class to be the facade to the underlying + * logging mechanism. Is a facade to BaseLogger's underlying log stream. This allows + * the underlying implementation to be replaced real time. + */ +class Logger : public BaseLogger { + protected: + struct singleton; + public: + + /** + * Returns a shared pointer to the logger instance. + * Note that while there is no synchronization this is expected + * to be called and initialized first + * @returns shared pointer to the base logger. + */ + static std::shared_ptr getLogger() { + + if (singleton_logger_ == nullptr) + singleton_logger_ = std::make_shared(singleton { 0 }); + return singleton_logger_; + } + + /** + * Returns the log level for this instance. + */ + LOG_LEVEL_E getLogLevel() const { + return current_logger_.load()->getLogLevel(); + } + + /** + * Sets the log level atomic and sets it + * within logger if it can + * @param level desired log level. + */ + void setLogLevel(LOG_LEVEL_E level) { + current_logger_.load()->setLogLevel(level); + } + + /** + * Sets the log level for this instance based on the string + * @param level desired log leve. + * @param defaultLevel default level if we cannot match level. + */ + void setLogLevel(const std::string &level, LOG_LEVEL_E defaultLevel = info) { + current_logger_.load()->setLogLevel(level, info); + } + + void updateLogger(std::unique_ptr logger) { + + if (logger == nullptr) + return; + current_logger_.store(logger.release()); + } + + /** + * @brief Log error message + * @param format format string ('man printf' for syntax) + * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match + */ + void log_error(const char * const format, ...) { + if (!current_logger_.load()->shouldLog(err)) + return; + FILL_BUFFER + current_logger_.load()->log_str(err, buffer); + } + /** + * @brief Log warn message + * @param format format string ('man printf' for syntax) + * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match + */ + void log_warn(const char * const format, ...) { + if (!current_logger_.load()->shouldLog(warn)) + return; + FILL_BUFFER + current_logger_.load()->log_str(warn, buffer); + } + /** + * @brief Log info message + * @param format format string ('man printf' for syntax) + * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match + */ + void log_info(const char * const format, ...) { + if (!current_logger_.load()->shouldLog(info)) + return; + FILL_BUFFER + current_logger_.load()->log_str(info, buffer); + } + /** + * @brief Log debug message + * @param format format string ('man printf' for syntax) + * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match + */ + void log_debug(const char * const format, ...) { + + if (!current_logger_.load()->shouldLog(debug)) + return; + FILL_BUFFER + current_logger_.load()->log_str(debug, buffer); + } + /** + * @brief Log trace message + * @param format format string ('man printf' for syntax) + * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match + */ + void log_trace(const char * const format, ...) { + + if (!current_logger_.load()->shouldLog(trace)) + return; + FILL_BUFFER + current_logger_.load()->log_str(trace, buffer); + } + + /** + * @brief Log message + * @param format format string ('man printf' for syntax) + * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match + */ + virtual void log_str(LOG_LEVEL_E level, const std::string &buffer) { + current_logger_.load()->log_str(level, buffer); + } + + // Destructor + ~Logger() { + } + + explicit Logger(const singleton &a) { + + /** + * flush on info to maintain current functionality + */ + std::shared_ptr defaultsink = spdlog::rotating_logger_mt( + LOG_NAME, + LOG_FILE_NAME, + DEFAULT_LOG_FILE_SIZE, DEFAULT_LOG_FILE_NUMBER); + defaultsink->flush_on(spdlog::level::level_enum::info); + + std::unique_ptr new_logger_ = std::unique_ptr( + new BaseLogger("info", defaultsink)); + + new_logger_->setLogLevel(info); + current_logger_.store(new_logger_.release()); + } + + Logger(const Logger &parent) = delete; + Logger &operator=(const Logger &parent) = delete; + + protected: + + /** + * Allows for a null constructor above so that we can have a public constructor that + * effectively limits us to being a singleton by having a protected argument in the constructor + */ + struct singleton { + explicit singleton(int) { + } + }; + + std::atomic current_logger_; + +// Singleton logger instance + static std::shared_ptr singleton_logger_; +}; + +} /* namespace logging */ +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif diff --git a/libminifi/include/core/repository/FlowFIleRepository.h b/libminifi/include/core/repository/FlowFIleRepository.h new file mode 100644 index 0000000000..231148a51b --- /dev/null +++ b/libminifi/include/core/repository/FlowFIleRepository.h @@ -0,0 +1,116 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBMINIFI_INCLUDE_CORE_REPOSITORY_FLOWFILEREPOSITORY_H_ +#define LIBMINIFI_INCLUDE_CORE_REPOSITORY_FLOWFILEREPOSITORY_H_ + +#include "core/Repository.h" +#include "core/core.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { +namespace repository { + + + +#define FLOWFILE_REPOSITORY_DIRECTORY "./flowfile_repository" +#define MAX_FLOWFILE_REPOSITORY_STORAGE_SIZE (10*1024*1024) // 10M +#define MAX_FLOWFILE_REPOSITORY_ENTRY_LIFE_TIME (600000) // 10 minute +#define FLOWFILE_REPOSITORY_PURGE_PERIOD (2500) // 2500 msec + +/** + * Flow File repository + * Design: Extends Repository and implements the run function, using LevelDB as the primary substrate. + */ +class FlowFileRepository : public Repository { + public: + // Constructor + + FlowFileRepository(std::string directory, int64_t maxPartitionMillis, + int64_t maxPartitionBytes, uint64_t purgePeriod) + : Repository(getClassName(), directory, + maxPartitionMillis, maxPartitionBytes, purgePeriod) + + { + db_ = NULL; + } + + // Destructor + virtual ~FlowFileRepository() { + if (db_) + delete db_; + } + + // initialize + virtual bool initialize() { + + std::string value; + + if (configure_->get(Configure::nifi_flowfile_repository_directory_default, + value)) { + directory_ = value; + } + logger_->log_info("NiFi FlowFile Repository Directory %s", + directory_.c_str()); + if (configure_->get(Configure::nifi_flowfile_repository_max_storage_size, + value)) { + Property::StringToInt(value, max_partition_bytes_); + } + logger_->log_info("NiFi FlowFile Max Partition Bytes %d", + max_partition_bytes_); + if (configure_->get(Configure::nifi_flowfile_repository_max_storage_time, + value)) { + TimeUnit unit; + if (Property::StringToTime(value, max_partition_millis_, unit) + && Property::ConvertTimeUnitToMS(max_partition_millis_, unit, + max_partition_millis_)) { + } + } + logger_->log_info("NiFi FlowFile Max Storage Time: [%d] ms", + max_partition_millis_); + leveldb::Options options; + options.create_if_missing = true; + leveldb::Status status = leveldb::DB::Open(options, directory_.c_str(), + &db_); + if (status.ok()) { + logger_->log_info("NiFi FlowFile Repository database open %s success", + directory_.c_str()); + } else { + logger_->log_error("NiFi FlowFile Repository database open %s fail", + directory_.c_str()); + return false; + } + return true; + } + + virtual void run(); + + private: + leveldb::DB* db_; +}; + +} /* namespace repository */ +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif /* LIBMINIFI_INCLUDE_CORE_REPOSITORY_FLOWFILEREPOSITORY_H_ */ diff --git a/libminifi/include/io/BaseStream.h b/libminifi/include/io/BaseStream.h index c3ebe4277b..b0b358990d 100644 --- a/libminifi/include/io/BaseStream.h +++ b/libminifi/include/io/BaseStream.h @@ -19,132 +19,144 @@ #ifndef LIBMINIFI_INCLUDE_IO_BASESTREAM_H_ #define LIBMINIFI_INCLUDE_IO_BASESTREAM_H_ - #include #include "EndianCheck.h" #include "DataStream.h" #include "Serializable.h" -class BaseStream: public DataStream, public Serializable { - -public: - BaseStream() { - - } - virtual ~BaseStream() { - - } - /** - * write 4 bytes to stream - * @param base_value non encoded value - * @param stream output stream - * @param is_little_endian endianness determination - * @return resulting write size - **/ - virtual int write(uint32_t base_value, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - /** - * write 2 bytes to stream - * @param base_value non encoded value - * @param stream output stream - * @param is_little_endian endianness determination - * @return resulting write size - **/ - virtual int write(uint16_t base_value, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - /** - * write valueto stream - * @param value non encoded value - * @param len length of value - * @param strema output stream - * @return resulting write size - **/ - virtual int write(uint8_t *value, int len); - - /** - * write 8 bytes to stream - * @param base_value non encoded value - * @param stream output stream - * @param is_little_endian endianness determination - * @return resulting write size - **/ - virtual int write(uint64_t base_value, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - /** - * write bool to stream - * @param value non encoded value - * @return resulting write size - **/ - virtual int write(bool value); - - /** - * write UTF string to stream - * @param str string to write - * @return resulting write size - **/ - virtual int writeUTF(std::string str, bool widen = false); - - /** - * reads a byte from the stream - * @param value reference in which will set the result - * @param stream stream from which we will read - * @return resulting read size - **/ - virtual int read(uint8_t &value); - - /** - * reads two bytes from the stream - * @param value reference in which will set the result - * @param stream stream from which we will read - * @return resulting read size - **/ - virtual int read(uint16_t &base_value, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - /** - * reads a byte from the stream - * @param value reference in which will set the result - * @param stream stream from which we will read - * @return resulting read size - **/ - virtual int read(char &value); - - /** - * reads a byte array from the stream - * @param value reference in which will set the result - * @param len length to read - * @param stream stream from which we will read - * @return resulting read size - **/ - virtual int read(uint8_t *value, int len); - - /** - * reads four bytes from the stream - * @param value reference in which will set the result - * @param stream stream from which we will read - * @return resulting read size - **/ - virtual int read(uint32_t &value, - bool is_little_endian = EndiannessCheck::IS_LITTLE); - - /** - * reads eight byte from the stream - * @param value reference in which will set the result - * @param stream stream from which we will read - * @return resulting read size - **/ - virtual int read(uint64_t &value, - bool is_little_endian = EndiannessCheck::IS_LITTLE); - - /** - * read UTF from stream - * @param str reference string - * @param stream stream from which we will read - * @return resulting read size - **/ - virtual int readUTF(std::string &str, bool widen = false); + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace io { + + +class BaseStream : public DataStream, public Serializable { + + public: + BaseStream() { + + } + virtual ~BaseStream() { + + } + /** + * write 4 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ + virtual int write(uint32_t base_value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * write 2 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ + virtual int write(uint16_t base_value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * write valueto stream + * @param value non encoded value + * @param len length of value + * @param strema output stream + * @return resulting write size + **/ + virtual int write(uint8_t *value, int len); + + /** + * write 8 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ + virtual int write(uint64_t base_value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * write bool to stream + * @param value non encoded value + * @return resulting write size + **/ + virtual int write(bool value); + + /** + * write UTF string to stream + * @param str string to write + * @return resulting write size + **/ + virtual int writeUTF(std::string str, bool widen = false); + + /** + * reads a byte from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ + virtual int read(uint8_t &value); + + /** + * reads two bytes from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ + virtual int read(uint16_t &base_value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * reads a byte from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ + virtual int read(char &value); + + /** + * reads a byte array from the stream + * @param value reference in which will set the result + * @param len length to read + * @param stream stream from which we will read + * @return resulting read size + **/ + virtual int read(uint8_t *value, int len); + + /** + * reads four bytes from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ + virtual int read(uint32_t &value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * reads eight byte from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ + virtual int read(uint64_t &value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * read UTF from stream + * @param str reference string + * @param stream stream from which we will read + * @return resulting read size + **/ + virtual int readUTF(std::string &str, bool widen = false); }; +} /* namespace io */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ #endif /* LIBMINIFI_INCLUDE_IO_BASESTREAM_H_ */ diff --git a/libminifi/include/io/CRCStream.h b/libminifi/include/io/CRCStream.h index 01b61995ae..99fdfc33ab 100644 --- a/libminifi/include/io/CRCStream.h +++ b/libminifi/include/io/CRCStream.h @@ -25,282 +25,279 @@ #include "BaseStream.h" #include "Serializable.h" -#define htonll_r(x) ((((uint64_t)htonl(x)) << 32) + htonl((x) >> 32)) +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace io { +#define htonll_r(x) ((((uint64_t)htonl(x)) << 32) + htonl((x) >> 32)) template -class CRCStream: public BaseStream { -public: - /** - * Raw pointer because the caller guarantees that - * it will exceed our lifetime. - */ - explicit CRCStream(T *stream); - - explicit CRCStream( CRCStream &&move ); - - virtual ~CRCStream() { - - } - - /** - * Reads data and places it into buf - * @param buf buffer in which we extract data - * @param buflen - */ - virtual int readData(std::vector &buf, int buflen); - /** - * Reads data and places it into buf - * @param buf buffer in which we extract data - * @param buflen - */ - virtual int readData(uint8_t *buf, int buflen); - - /** - * Write value to the stream using std::vector - * @param buf incoming buffer - * @param buflen buffer to write - * - */ - virtual int writeData(std::vector &buf, int buflen); - - /** - * writes value to stream - * @param value value to write - * @param size size of value - */ - virtual int writeData(uint8_t *value, int size); - - /** - * write 4 bytes to stream - * @param base_value non encoded value - * @param stream output stream - * @param is_little_endian endianness determination - * @return resulting write size - **/ - virtual int write(uint32_t base_value, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - /** - * write 2 bytes to stream - * @param base_value non encoded value - * @param stream output stream - * @param is_little_endian endianness determination - * @return resulting write size - **/ - virtual int write(uint16_t base_value, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - - /** - * write 8 bytes to stream - * @param base_value non encoded value - * @param stream output stream - * @param is_little_endian endianness determination - * @return resulting write size - **/ - virtual int write(uint64_t base_value, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - - - /** - * Reads a system word - * @param value value to write - */ - virtual int read(uint64_t &value, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - /** - * Reads a uint32_t - * @param value value to write - */ - virtual int read(uint32_t &value, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - /** - * Reads a system short - * @param value value to write - */ - virtual int read(uint16_t &value, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - - virtual short initialize() { - child_stream_->initialize(); - reset(); - return 0; - } - - - void updateCRC(uint8_t *buffer, uint32_t length); - - uint64_t getCRC() { - return crc_; - } - - void reset(); -protected: - +class CRCStream : public BaseStream { + public: /** - * Creates a vector and returns the vector using the provided - * type name. - * @param t incoming object - * @returns vector. - */ - template - std::vector readBuffer(const K& t){ - std::vector buf; - buf.resize(sizeof t); - readData((uint8_t*) &buf[0], sizeof(t)); - return buf; - } - - - uint64_t crc_; - T *child_stream_; -}; + * Raw pointer because the caller guarantees that + * it will exceed our lifetime. + */ + explicit CRCStream(T *stream); + + explicit CRCStream(CRCStream &&move); + + virtual ~CRCStream() { + + } + + /** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ + virtual int readData(std::vector &buf, int buflen); + /** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ + virtual int readData(uint8_t *buf, int buflen); + + /** + * Write value to the stream using std::vector + * @param buf incoming buffer + * @param buflen buffer to write + * + */ + virtual int writeData(std::vector &buf, int buflen); + + /** + * writes value to stream + * @param value value to write + * @param size size of value + */ + virtual int writeData(uint8_t *value, int size); + + /** + * write 4 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ + virtual int write(uint32_t base_value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + /** + * write 2 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ + virtual int write(uint16_t base_value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * write 8 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ + virtual int write(uint64_t base_value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + /** + * Reads a system word + * @param value value to write + */ + virtual int read(uint64_t &value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * Reads a uint32_t + * @param value value to write + */ + virtual int read(uint32_t &value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * Reads a system short + * @param value value to write + */ + virtual int read(uint16_t &value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + virtual short initialize() { + child_stream_->initialize(); + reset(); + return 0; + } + + void updateCRC(uint8_t *buffer, uint32_t length); + + uint64_t getCRC() { + return crc_; + } + + void reset(); + protected: + + /** + * Creates a vector and returns the vector using the provided + * type name. + * @param t incoming object + * @returns vector. + */ + template + std::vector readBuffer(const K& t) { + std::vector buf; + buf.resize(sizeof t); + readData((uint8_t*) &buf[0], sizeof(t)); + return buf; + } + + uint64_t crc_; + T *child_stream_; +}; template -CRCStream::CRCStream(T *other) : - child_stream_(other) { - crc_ = crc32(0L, Z_NULL, 0); +CRCStream::CRCStream(T *other) + : child_stream_(other) { + crc_ = crc32(0L, Z_NULL, 0); } template -CRCStream::CRCStream(CRCStream &&move) : - crc_(std::move(move.crc_)), child_stream_(std::move(move.child_stream_)) { +CRCStream::CRCStream(CRCStream &&move) + : crc_(std::move(move.crc_)), + child_stream_(std::move(move.child_stream_)) { } template int CRCStream::readData(std::vector &buf, int buflen) { - if (buf.capacity() < buflen) - buf.resize(buflen); - return readData((uint8_t*) &buf[0], buflen); + if (buf.capacity() < buflen) + buf.resize(buflen); + return readData((uint8_t*) &buf[0], buflen); } template int CRCStream::readData(uint8_t *buf, int buflen) { - int ret = child_stream_->read(buf, buflen); - crc_ = crc32(crc_, buf, buflen); - return ret; + int ret = child_stream_->read(buf, buflen); + crc_ = crc32(crc_, buf, buflen); + return ret; } template int CRCStream::writeData(std::vector &buf, int buflen) { - if (buf.capacity() < buflen) - buf.resize(buflen); - return writeData((uint8_t*) &buf[0], buflen); + if (buf.capacity() < buflen) + buf.resize(buflen); + return writeData((uint8_t*) &buf[0], buflen); } template int CRCStream::writeData(uint8_t *value, int size) { - int ret = child_stream_->write(value, size); - crc_ = crc32(crc_, value, size); - return ret; + int ret = child_stream_->write(value, size); + crc_ = crc32(crc_, value, size); + return ret; } template void CRCStream::reset() { - crc_ = crc32(0L, Z_NULL, 0); + crc_ = crc32(0L, Z_NULL, 0); } template void CRCStream::updateCRC(uint8_t *buffer, uint32_t length) { - crc_ = crc32(crc_, buffer, length); + crc_ = crc32(crc_, buffer, length); } template -int CRCStream::write(uint64_t base_value, bool is_little_endian){ - - const uint64_t value = - is_little_endian == 1 ? htonll_r(base_value) : base_value; - uint8_t bytes[sizeof value]; - std::copy(static_cast(static_cast(&value)), - static_cast(static_cast(&value)) + sizeof value, - bytes); - return writeData(bytes,sizeof value); -} +int CRCStream::write(uint64_t base_value, bool is_little_endian) { + const uint64_t value = + is_little_endian == 1 ? htonll_r(base_value) : base_value; + uint8_t bytes[sizeof value]; + std::copy( + static_cast(static_cast(&value)), + static_cast(static_cast(&value)) + sizeof value, + bytes); + return writeData(bytes, sizeof value); +} template -int CRCStream::write(uint32_t base_value, bool is_little_endian){ - const uint32_t value = is_little_endian ? htonl(base_value) : base_value; - uint8_t bytes[sizeof value]; - std::copy(static_cast(static_cast(&value)), - static_cast(static_cast(&value)) + sizeof value, - bytes); - return writeData(bytes,sizeof value); +int CRCStream::write(uint32_t base_value, bool is_little_endian) { + const uint32_t value = is_little_endian ? htonl(base_value) : base_value; + uint8_t bytes[sizeof value]; + std::copy( + static_cast(static_cast(&value)), + static_cast(static_cast(&value)) + sizeof value, + bytes); + return writeData(bytes, sizeof value); } template -int CRCStream::write(uint16_t base_value, bool is_little_endian){ - const uint16_t value = - is_little_endian == 1 ? htons(base_value) : base_value; +int CRCStream::write(uint16_t base_value, bool is_little_endian) { + const uint16_t value = is_little_endian == 1 ? htons(base_value) : base_value; uint8_t bytes[sizeof value]; - std::copy(static_cast(static_cast(&value)), - static_cast(static_cast(&value)) + sizeof value, - bytes); - return writeData(bytes,sizeof value); + std::copy( + static_cast(static_cast(&value)), + static_cast(static_cast(&value)) + sizeof value, + bytes); + return writeData(bytes, sizeof value); } - template int CRCStream::read(uint64_t &value, bool is_little_endian) { - auto buf = readBuffer(value); - - if (is_little_endian) { - value = ((uint64_t) buf[0] << 56) | ((uint64_t) (buf[1] & 255) << 48) - | ((uint64_t) (buf[2] & 255) << 40) - | ((uint64_t) (buf[3] & 255) << 32) - | ((uint64_t) (buf[4] & 255) << 24) - | ((uint64_t) (buf[5] & 255) << 16) - | ((uint64_t) (buf[6] & 255) << 8) - | ((uint64_t) (buf[7] & 255) << 0); - } else { - value = ((uint64_t) buf[0] << 0) | ((uint64_t) (buf[1] & 255) << 8) - | ((uint64_t) (buf[2] & 255) << 16) - | ((uint64_t) (buf[3] & 255) << 24) - | ((uint64_t) (buf[4] & 255) << 32) - | ((uint64_t) (buf[5] & 255) << 40) - | ((uint64_t) (buf[6] & 255) << 48) - | ((uint64_t) (buf[7] & 255) << 56); - } - return sizeof(value); + auto buf = readBuffer(value); + + if (is_little_endian) { + value = ((uint64_t) buf[0] << 56) | ((uint64_t) (buf[1] & 255) << 48) + | ((uint64_t) (buf[2] & 255) << 40) | ((uint64_t) (buf[3] & 255) << 32) + | ((uint64_t) (buf[4] & 255) << 24) | ((uint64_t) (buf[5] & 255) << 16) + | ((uint64_t) (buf[6] & 255) << 8) | ((uint64_t) (buf[7] & 255) << 0); + } else { + value = ((uint64_t) buf[0] << 0) | ((uint64_t) (buf[1] & 255) << 8) + | ((uint64_t) (buf[2] & 255) << 16) | ((uint64_t) (buf[3] & 255) << 24) + | ((uint64_t) (buf[4] & 255) << 32) | ((uint64_t) (buf[5] & 255) << 40) + | ((uint64_t) (buf[6] & 255) << 48) | ((uint64_t) (buf[7] & 255) << 56); + } + return sizeof(value); } template int CRCStream::read(uint32_t &value, bool is_little_endian) { - auto buf = readBuffer(value); + auto buf = readBuffer(value); - if (is_little_endian) { - value = (buf[0] << 24) | (buf[1] << 16) | (buf[2] << 8) | buf[3]; - } else { - value = buf[0] | buf[1] << 8 | buf[2] << 16 | buf[3] << 24; + if (is_little_endian) { + value = (buf[0] << 24) | (buf[1] << 16) | (buf[2] << 8) | buf[3]; + } else { + value = buf[0] | buf[1] << 8 | buf[2] << 16 | buf[3] << 24; - } + } - return sizeof(value); + return sizeof(value); } template int CRCStream::read(uint16_t &value, bool is_little_endian) { - auto buf = readBuffer(value); + auto buf = readBuffer(value); - if (is_little_endian) { - value = (buf[0] << 8) | buf[1]; - } else { - value = buf[0] | buf[1] << 8; + if (is_little_endian) { + value = (buf[0] << 8) | buf[1]; + } else { + value = buf[0] | buf[1] << 8; - } - return sizeof(value); + } + return sizeof(value); } - +} /* namespace io */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ #endif /* LIBMINIFI_INCLUDE_IO_CRCSTREAM_H_ */ diff --git a/libminifi/include/io/ClientSocket.h b/libminifi/include/io/ClientSocket.h index 3f8aae198e..97cace2256 100644 --- a/libminifi/include/io/ClientSocket.h +++ b/libminifi/include/io/ClientSocket.h @@ -26,10 +26,17 @@ #include #include #include "io/BaseStream.h" -#include "Logger.h" +#include "core/core.h" +#include "core/logging/Logger.h" #include "io/validation.h" +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace io { + /** * Socket class. * Purpose: Provides a general purpose socket interface that abstracts @@ -39,207 +46,209 @@ * * */ -class Socket: public BaseStream { -public: - /** - * Constructor that accepts host name, port and listeners. With this - * contructor we will be creating a server socket - * @param hostname our host name - * @param port connecting port - * @param listeners number of listeners in the queue - */ - explicit Socket(const std::string &hostname, const uint16_t port, - const uint16_t listeners); - - /** - * Constructor that creates a client socket. - * @param hostname hostname we are connecting to. - * @param port port we are connecting to. - */ - explicit Socket(const std::string &hostname, const uint16_t port); - - /** - * Move constructor. - */ - explicit Socket(const Socket &&); - - static std::string HOSTNAME; - - /** - * Static function to return the current machine's host name - */ - static std::string getMyHostName(std::string *str = &HOSTNAME) { - if (__builtin_expect(!IsNullOrEmpty(str), 0)) - return *str; - else { - char hostname[1024]; - gethostname(hostname, 1024); - Socket mySock(hostname, 0); - mySock.initialize(); - return mySock.getHostname(); - } - } - - /** - * Destructor - */ - - virtual ~Socket(); - - virtual void closeStream(); - /** - * Initializes the socket - * @return result of the creation operation. - */ - virtual short initialize(); - - std::string getHostname() const; - - /** - * Return the port for this socket - * @returns port - */ - uint16_t getPort(); - - // data stream extensions - /** - * Reads data and places it into buf - * @param buf buffer in which we extract data - * @param buflen - */ - virtual int readData(std::vector &buf, int buflen); - /** - * Reads data and places it into buf - * @param buf buffer in which we extract data - * @param buflen - */ - virtual int readData(uint8_t *buf, int buflen); - - /** - * Write value to the stream using std::vector - * @param buf incoming buffer - * @param buflen buffer to write - * - */ - virtual int writeData(std::vector &buf, int buflen); - - /** - * writes value to stream - * @param value value to write - * @param size size of value - */ - virtual int writeData(uint8_t *value, int size); - - - - /** - * Writes a system word - * @param value value to write - */ - virtual int write(uint64_t value, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - /** - * Writes a uint32_t - * @param value value to write - */ - virtual int write(uint32_t value, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - /** - * Writes a system short - * @param value value to write - */ - virtual int write(uint16_t value, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - - /** - * Reads a system word - * @param value value to write - */ - virtual int read(uint64_t &value, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - /** - * Reads a uint32_t - * @param value value to write - */ - virtual int read(uint32_t &value, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - /** - * Reads a system short - * @param value value to write - */ - virtual int read(uint16_t &value, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - /** - * Returns the underlying buffer - * @return vector's array - **/ - const uint8_t *getBuffer() const { - return ::DataStream::getBuffer(); - } - - /** - * Retrieve size of data stream - * @return size of data stream - **/ - const uint32_t getSize() const { - return ::DataStream::getSize(); - } - -protected: - - /** - * Creates a vector and returns the vector using the provided - * type name. - * @param t incoming object - * @returns vector. - */ - template - std::vector readBuffer(const T&); - - /** - * Creates a connection using the address info object. - * @param p addrinfo structure. - * @returns fd. - */ - virtual int8_t createConnection(const addrinfo *p,in_addr_t &addr); - - /** - * Sets socket options depending on the instance. - * @param sock socket file descriptor. - */ - virtual short setSocketOptions(const int sock); - - /** - * Attempt to select the socket file descriptor - * @param msec timeout interval to wait - * @returns file descriptor - */ - virtual short select_descriptor(const uint16_t msec); - - std::shared_ptr logger_; - - addrinfo *addr_info_; - - std::recursive_mutex selection_mutex_; - - std::string requested_hostname_; - std::string canonical_hostname_; - uint16_t port_; - - // connection information - int32_t socket_file_descriptor_; - - fd_set total_list_; - fd_set read_fds_; - std::atomic socket_max_; - uint16_t listeners_; +class Socket : public BaseStream { + public: + /** + * Constructor that accepts host name, port and listeners. With this + * contructor we will be creating a server socket + * @param hostname our host name + * @param port connecting port + * @param listeners number of listeners in the queue + */ + explicit Socket(const std::string &hostname, const uint16_t port, + const uint16_t listeners); + + /** + * Constructor that creates a client socket. + * @param hostname hostname we are connecting to. + * @param port port we are connecting to. + */ + explicit Socket(const std::string &hostname, const uint16_t port); + + /** + * Move constructor. + */ + explicit Socket(const Socket &&); + + static std::string HOSTNAME; + + /** + * Static function to return the current machine's host name + */ + static std::string getMyHostName(std::string *str = &HOSTNAME) { + if (__builtin_expect(!IsNullOrEmpty(str), 0)) + return *str; + else { + char hostname[1024]; + gethostname(hostname, 1024); + Socket mySock(hostname, 0); + mySock.initialize(); + return mySock.getHostname(); + } + } + + /** + * Destructor + */ + + virtual ~Socket(); + + virtual void closeStream(); + /** + * Initializes the socket + * @return result of the creation operation. + */ + virtual short initialize(); + + std::string getHostname() const; + + /** + * Return the port for this socket + * @returns port + */ + uint16_t getPort(); + + // data stream extensions + /** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ + virtual int readData(std::vector &buf, int buflen); + /** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ + virtual int readData(uint8_t *buf, int buflen); + + /** + * Write value to the stream using std::vector + * @param buf incoming buffer + * @param buflen buffer to write + * + */ + virtual int writeData(std::vector &buf, int buflen); + + /** + * writes value to stream + * @param value value to write + * @param size size of value + */ + virtual int writeData(uint8_t *value, int size); + + /** + * Writes a system word + * @param value value to write + */ + virtual int write(uint64_t value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * Writes a uint32_t + * @param value value to write + */ + virtual int write(uint32_t value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * Writes a system short + * @param value value to write + */ + virtual int write(uint16_t value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * Reads a system word + * @param value value to write + */ + virtual int read(uint64_t &value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * Reads a uint32_t + * @param value value to write + */ + virtual int read(uint32_t &value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * Reads a system short + * @param value value to write + */ + virtual int read(uint16_t &value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * Returns the underlying buffer + * @return vector's array + **/ + const uint8_t *getBuffer() const { + return DataStream::getBuffer(); + } + + /** + * Retrieve size of data stream + * @return size of data stream + **/ + const uint32_t getSize() const { + return DataStream::getSize(); + } + + protected: + + /** + * Creates a vector and returns the vector using the provided + * type name. + * @param t incoming object + * @returns vector. + */ + template + std::vector readBuffer(const T&); + + /** + * Creates a connection using the address info object. + * @param p addrinfo structure. + * @returns fd. + */ + virtual int8_t createConnection(const addrinfo *p, in_addr_t &addr); + + /** + * Sets socket options depending on the instance. + * @param sock socket file descriptor. + */ + virtual short setSocketOptions(const int sock); + + /** + * Attempt to select the socket file descriptor + * @param msec timeout interval to wait + * @returns file descriptor + */ + virtual short select_descriptor(const uint16_t msec); + + std::shared_ptr logger_; + + addrinfo *addr_info_; + + std::recursive_mutex selection_mutex_; + + std::string requested_hostname_; + std::string canonical_hostname_; + uint16_t port_; + + // connection information + int32_t socket_file_descriptor_; + + fd_set total_list_; + fd_set read_fds_; + std::atomic socket_max_; + uint16_t listeners_; }; +} /* namespace io */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ #endif /* LIBMINIFI_INCLUDE_IO_CLIENTSOCKET_H_ */ diff --git a/libminifi/include/io/DataStream.h b/libminifi/include/io/DataStream.h index 2f793ff410..e37dbf789a 100644 --- a/libminifi/include/io/DataStream.h +++ b/libminifi/include/io/DataStream.h @@ -22,99 +22,111 @@ #include #include #include "EndianCheck.h" - +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace io { /** * DataStream defines the mechanism through which * binary data will be written to a sink */ class DataStream { -public: - - DataStream() : - readBuffer(0) { - - } - - /** - * Constructor - **/ - explicit DataStream(const uint8_t *buf, const uint32_t buflen) : - DataStream() { - writeData((uint8_t*) buf, buflen); - - } - - virtual short initialize() { - buffer.clear(); - readBuffer = 0; - return 0; - } - - virtual void closeStream() - { - - } - /** - * Reads data and places it into buf - * @param buf buffer in which we extract data - * @param buflen - */ - virtual int readData(std::vector &buf, int buflen); - /** - * Reads data and places it into buf - * @param buf buffer in which we extract data - * @param buflen - */ - virtual int readData(uint8_t *buf, int buflen); - - /** - * writes valiue to buffer - * @param value value to write - * @param size size of value - */ - virtual int writeData(uint8_t *value, int size); - - /** - * Reads a system word - * @param value value to write - */ - virtual int read(uint64_t &value, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - /** - * Reads a uint32_t - * @param value value to write - */ - virtual int read(uint32_t &value, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - /** - * Reads a system short - * @param value value to write - */ - virtual int read(uint16_t &value, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - /** - * Returns the underlying buffer - * @return vector's array - **/ - const uint8_t *getBuffer() const { - return &buffer[0]; - } - - /** - * Retrieve size of data stream - * @return size of data stream - **/ - const uint32_t getSize() const { - return buffer.size(); - } - -protected: - // All serialization related method and internal buf - std::vector buffer; - uint32_t readBuffer; + public: + + DataStream() + : readBuffer(0) { + + } + + ~DataStream() { + + } + + /** + * Constructor + **/ + explicit DataStream(const uint8_t *buf, const uint32_t buflen) + : DataStream() { + writeData((uint8_t*) buf, buflen); + + } + + virtual short initialize() { + buffer.clear(); + readBuffer = 0; + return 0; + } + + virtual void closeStream() { + + } + /** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ + virtual int readData(std::vector &buf, int buflen); + /** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ + virtual int readData(uint8_t *buf, int buflen); + + /** + * writes valiue to buffer + * @param value value to write + * @param size size of value + */ + virtual int writeData(uint8_t *value, int size); + + /** + * Reads a system word + * @param value value to write + */ + virtual int read(uint64_t &value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * Reads a uint32_t + * @param value value to write + */ + virtual int read(uint32_t &value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * Reads a system short + * @param value value to write + */ + virtual int read(uint16_t &value, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * Returns the underlying buffer + * @return vector's array + **/ + const uint8_t *getBuffer() const { + return &buffer[0]; + } + + /** + * Retrieve size of data stream + * @return size of data stream + **/ + const uint32_t getSize() const { + return buffer.size(); + } + + protected: + // All serialization related method and internal buf + std::vector buffer; + uint32_t readBuffer; }; +} /* namespace io */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ #endif /* LIBMINIFI_INCLUDE_IO_DATASTREAM_H_ */ diff --git a/libminifi/include/io/EndianCheck.h b/libminifi/include/io/EndianCheck.h index ef900e0eed..3ceb19ca03 100644 --- a/libminifi/include/io/EndianCheck.h +++ b/libminifi/include/io/EndianCheck.h @@ -1,5 +1,5 @@ /** - * + * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -16,32 +16,34 @@ * limitations under the License. */ - #ifndef LIBMINIFI_INCLUDE_IO_ENDIANCHECK_H_ #define LIBMINIFI_INCLUDE_IO_ENDIANCHECK_H_ - - - +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace io { /** * Mechanism to determine endianness of host. * Accounts for only BIG/LITTLE/BIENDIAN **/ -class EndiannessCheck -{ -public: - static bool IS_LITTLE; -private: - - static bool is_little_endian() { - /* do whatever is needed at static init time */ - unsigned int x = 1; - char *c = (char*) &x; - IS_LITTLE=*c==1; - return IS_LITTLE; - } +class EndiannessCheck { + public: + static bool IS_LITTLE; + private: + + static bool is_little_endian() { + /* do whatever is needed at static init time */ + unsigned int x = 1; + char *c = (char*) &x; + IS_LITTLE = *c == 1; + return IS_LITTLE; + } }; - - - +} /* namespace io */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ #endif /* LIBMINIFI_INCLUDE_IO_ENDIANCHECK_H_ */ diff --git a/libminifi/include/io/Serializable.h b/libminifi/include/io/Serializable.h index 59d3a7338e..5ee886b71f 100644 --- a/libminifi/include/io/Serializable.h +++ b/libminifi/include/io/Serializable.h @@ -1,5 +1,5 @@ /** - * + * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -22,9 +22,11 @@ #include #include "EndianCheck.h" #include "DataStream.h" - - - +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace io { /** * Serializable instances provide base functionality to * write certain objects/primitives to a data stream. @@ -32,155 +34,158 @@ */ class Serializable { -public: - - /** - * Inline function to write T to stream - **/ - template - inline int writeData(const T &t,DataStream *stream); - - /** - * Inline function to write T to to_vec - **/ - template - inline int writeData(const T &t, uint8_t *to_vec); - - /** - * Inline function to write T to to_vec - **/ - template - inline int writeData(const T &t, std::vector &to_vec); - - - /** - * write byte to stream - * @return resulting write size - **/ - int write(uint8_t value,DataStream *stream); - - /** - * write byte to stream - * @return resulting write size - **/ - int write(char value,DataStream *stream); - - /** - * write 4 bytes to stream - * @param base_value non encoded value - * @param stream output stream - * @param is_little_endian endianness determination - * @return resulting write size - **/ - int write(uint32_t base_value,DataStream *stream, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - /** - * write 2 bytes to stream - * @param base_value non encoded value - * @param stream output stream - * @param is_little_endian endianness determination - * @return resulting write size - **/ - int write(uint16_t base_value,DataStream *stream, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - /** - * write valueto stream - * @param value non encoded value - * @param len length of value - * @param strema output stream - * @return resulting write size - **/ - int write(uint8_t *value, int len,DataStream *stream); - - /** - * write 8 bytes to stream - * @param base_value non encoded value - * @param stream output stream - * @param is_little_endian endianness determination - * @return resulting write size - **/ - int write(uint64_t base_value,DataStream *stream, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - /** - * write bool to stream - * @param value non encoded value - * @return resulting write size - **/ - int write(bool value); - - /** - * write UTF string to stream - * @param str string to write - * @return resulting write size - **/ - int writeUTF(std::string str,DataStream *stream, bool widen = false); - - /** - * reads a byte from the stream - * @param value reference in which will set the result - * @param stream stream from which we will read - * @return resulting read size - **/ - int read(uint8_t &value,DataStream *stream); - - /** - * reads two bytes from the stream - * @param value reference in which will set the result - * @param stream stream from which we will read - * @return resulting read size - **/ - int read(uint16_t &base_value,DataStream *stream, bool is_little_endian = - EndiannessCheck::IS_LITTLE); - - /** - * reads a byte from the stream - * @param value reference in which will set the result - * @param stream stream from which we will read - * @return resulting read size - **/ - int read(char &value,DataStream *stream); - - /** - * reads a byte array from the stream - * @param value reference in which will set the result - * @param len length to read - * @param stream stream from which we will read - * @return resulting read size - **/ - int read(uint8_t *value, int len,DataStream *stream); - - /** - * reads four bytes from the stream - * @param value reference in which will set the result - * @param stream stream from which we will read - * @return resulting read size - **/ - int read(uint32_t &value,DataStream *stream, - bool is_little_endian = EndiannessCheck::IS_LITTLE); - - /** - * reads eight byte from the stream - * @param value reference in which will set the result - * @param stream stream from which we will read - * @return resulting read size - **/ - int read(uint64_t &value,DataStream *stream, - bool is_little_endian = EndiannessCheck::IS_LITTLE); - - /** - * read UTF from stream - * @param str reference string - * @param stream stream from which we will read - * @return resulting read size - **/ - int readUTF(std::string &str,DataStream *stream, bool widen = false); - -protected: + public: + + /** + * Inline function to write T to stream + **/ + template + inline int writeData(const T &t, DataStream *stream); + + /** + * Inline function to write T to to_vec + **/ + template + inline int writeData(const T &t, uint8_t *to_vec); + + /** + * Inline function to write T to to_vec + **/ + template + inline int writeData(const T &t, std::vector &to_vec); + + /** + * write byte to stream + * @return resulting write size + **/ + int write(uint8_t value, DataStream *stream); + + /** + * write byte to stream + * @return resulting write size + **/ + int write(char value, DataStream *stream); + + /** + * write 4 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ + int write(uint32_t base_value, DataStream *stream, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * write 2 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ + int write(uint16_t base_value, DataStream *stream, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * write valueto stream + * @param value non encoded value + * @param len length of value + * @param strema output stream + * @return resulting write size + **/ + int write(uint8_t *value, int len, DataStream *stream); + + /** + * write 8 bytes to stream + * @param base_value non encoded value + * @param stream output stream + * @param is_little_endian endianness determination + * @return resulting write size + **/ + int write(uint64_t base_value, DataStream *stream, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * write bool to stream + * @param value non encoded value + * @return resulting write size + **/ + int write(bool value); + + /** + * write UTF string to stream + * @param str string to write + * @return resulting write size + **/ + int writeUTF(std::string str, DataStream *stream, bool widen = false); + + /** + * reads a byte from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ + int read(uint8_t &value, DataStream *stream); + + /** + * reads two bytes from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ + int read(uint16_t &base_value, DataStream *stream, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * reads a byte from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ + int read(char &value, DataStream *stream); + + /** + * reads a byte array from the stream + * @param value reference in which will set the result + * @param len length to read + * @param stream stream from which we will read + * @return resulting read size + **/ + int read(uint8_t *value, int len, DataStream *stream); + + /** + * reads four bytes from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ + int read(uint32_t &value, DataStream *stream, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * reads eight byte from the stream + * @param value reference in which will set the result + * @param stream stream from which we will read + * @return resulting read size + **/ + int read(uint64_t &value, DataStream *stream, bool is_little_endian = + EndiannessCheck::IS_LITTLE); + + /** + * read UTF from stream + * @param str reference string + * @param stream stream from which we will read + * @return resulting read size + **/ + int readUTF(std::string &str, DataStream *stream, bool widen = false); + + protected: }; - +} /* namespace io */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ #endif /* LIBMINIFI_INCLUDE_IO_SERIALIZABLE_H_ */ diff --git a/libminifi/include/io/SocketFactory.h b/libminifi/include/io/SocketFactory.h index c8cbcb19c2..d2349a5607 100644 --- a/libminifi/include/io/SocketFactory.h +++ b/libminifi/include/io/SocketFactory.h @@ -24,36 +24,42 @@ #include "Configure.h" #include "utils/StringUtils.h" +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace io { + /** - Purpose: Due to the current design this is the only mechanism by which we can - inject different socket types - -**/ -class SocketFactory{ -public: + Purpose: Due to the current design this is the only mechanism by which we can + inject different socket types + + **/ +class SocketFactory { + public: - /** - * Build an instance, creating a memory fence, which - * allows us to avoid locking. This is tantamount to double checked locking. - * @returns new SocketFactory; - */ - static SocketFactory *getInstance() { - SocketFactory* atomic_context = context_instance_.load( - std::memory_order_relaxed); - std::atomic_thread_fence(std::memory_order_acquire); - if (atomic_context == nullptr) { - std::lock_guard lock(context_mutex_); - atomic_context = context_instance_.load(std::memory_order_relaxed); - if (atomic_context == nullptr) { - atomic_context = new SocketFactory(); - std::atomic_thread_fence(std::memory_order_release); - context_instance_.store(atomic_context, - std::memory_order_relaxed); - } - } - return atomic_context; - } + /** + * Build an instance, creating a memory fence, which + * allows us to avoid locking. This is tantamount to double checked locking. + * @returns new SocketFactory; + */ + static SocketFactory *getInstance() { + SocketFactory* atomic_context = context_instance_.load( + std::memory_order_relaxed); + std::atomic_thread_fence(std::memory_order_acquire); + if (atomic_context == nullptr) { + std::lock_guard lock(context_mutex_); + atomic_context = context_instance_.load(std::memory_order_relaxed); + if (atomic_context == nullptr) { + atomic_context = new SocketFactory(); + std::atomic_thread_fence(std::memory_order_release); + context_instance_.store(atomic_context, std::memory_order_relaxed); + } + } + return atomic_context; + } +<<<<<<< HEAD /** * Creates a socket and returns a unique ptr * @@ -80,12 +86,43 @@ class SocketFactory{ StringUtils::StringToBool(secureStr, is_secure_); } } +======= + /** + * Creates a socket and returns a unique ptr + * + */ + std::unique_ptr createSocket(const std::string &host, + const uint16_t port) { + Socket *socket = 0; + if (is_secure_) { + socket = new TLSSocket(host, port); + } else { + socket = new Socket(host, port); + } + return std::unique_ptr(socket); + } + protected: + SocketFactory() + : configure_(Configure::getConfigure()) { + std::string secureStr; + is_secure_ = false; + if (configure_->get(Configure::nifi_remote_input_secure, secureStr)) { + org::apache::nifi::minifi::utils::StringUtils::StringToBool(secureStr, is_secure_); + } + } +>>>>>>> MINIFI-217: First commit. Updates namespaces and removes - bool is_secure_; - static std::atomic context_instance_; - static std::mutex context_mutex_; + bool is_secure_; + static std::atomic context_instance_; + static std::mutex context_mutex_; - Configure *configure_; + Configure *configure_; }; +} /* namespace io */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + #endif diff --git a/libminifi/include/io/TLSSocket.h b/libminifi/include/io/TLSSocket.h index 32645ca6e5..469fcf7477 100644 --- a/libminifi/include/io/TLSSocket.h +++ b/libminifi/include/io/TLSSocket.h @@ -25,6 +25,13 @@ #include #include "Configure.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace io { + #include #include @@ -36,6 +43,7 @@ class TLSContext { +<<<<<<< HEAD public: /** @@ -112,76 +120,159 @@ class TLSContext { static std::atomic context_instance; static std::mutex context_mutex; +======= + public: + + /** + * Build an instance, creating a memory fence, which + * allows us to avoid locking. This is tantamount to double checked locking. + * @returns new TLSContext; + */ + static TLSContext *getInstance() { + TLSContext* atomic_context = context_instance.load( + std::memory_order_relaxed); + std::atomic_thread_fence(std::memory_order_acquire); + if (atomic_context == nullptr) { + std::lock_guard lock(context_mutex); + atomic_context = context_instance.load(std::memory_order_relaxed); + if (atomic_context == nullptr) { + atomic_context = new TLSContext(); + atomic_context->initialize(); + std::atomic_thread_fence(std::memory_order_release); + context_instance.store(atomic_context, std::memory_order_relaxed); + } + } + return atomic_context; + } + + virtual ~TLSContext() { + if (0 != ctx) + SSL_CTX_free(ctx); + } + + SSL_CTX *getContext() { + return ctx; + } + + short getError() { + return error_value; + } + + short initialize(); + + private: + + static int pemPassWordCb(char *buf, int size, int rwflag, void *userdata) { + std::string passphrase; + + if (Configure::getConfigure()->get( + Configure::nifi_security_client_pass_phrase, passphrase)) { + + std::ifstream file(passphrase.c_str(), std::ifstream::in); + if (!file.good()) { + memset(buf, 0x00, size); + return 0; + } + + std::string password; + password.assign((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + file.close(); + memset(buf, 0x00, size); + memcpy(buf, password.c_str(), password.length() - 1); + + return password.length() - 1; + } + return 0; + } + + TLSContext(); + + std::shared_ptr logger_; + Configure *configuration; + SSL_CTX *ctx; + + short error_value; + + static std::atomic context_instance; + static std::mutex context_mutex; +>>>>>>> MINIFI-217: First commit. Updates namespaces and removes }; -class TLSSocket: public Socket { -public: - - /** - * Constructor that accepts host name, port and listeners. With this - * contructor we will be creating a server socket - * @param hostname our host name - * @param port connecting port - * @param listeners number of listeners in the queue - */ - explicit TLSSocket(const std::string &hostname, const uint16_t port, - const uint16_t listeners); - - /** - * Constructor that creates a client socket. - * @param hostname hostname we are connecting to. - * @param port port we are connecting to. - */ - explicit TLSSocket(const std::string &hostname, const uint16_t port); - - /** - * Move constructor. - */ - explicit TLSSocket(const TLSSocket &&); - - virtual ~TLSSocket(); - - /** - * Initializes the socket - * @return result of the creation operation. - */ - short initialize(); - - /** - * Attempt to select the socket file descriptor - * @param msec timeout interval to wait - * @returns file descriptor - */ - virtual short select_descriptor(const uint16_t msec); - - /** - * Reads data and places it into buf - * @param buf buffer in which we extract data - * @param buflen - */ - virtual int readData(uint8_t *buf, int buflen); - - /** - * Write value to the stream using std::vector - * @param buf incoming buffer - * @param buflen buffer to write - * - */ - int writeData(std::vector &buf, int buflen); - - /** - * Write value to the stream using uint8_t ptr - * @param buf incoming buffer - * @param buflen buffer to write - * - */ - int writeData(uint8_t *value, int size); - -protected: - - SSL* ssl; +class TLSSocket : public Socket { + public: + + /** + * Constructor that accepts host name, port and listeners. With this + * contructor we will be creating a server socket + * @param hostname our host name + * @param port connecting port + * @param listeners number of listeners in the queue + */ + explicit TLSSocket(const std::string &hostname, const uint16_t port, + const uint16_t listeners); + + /** + * Constructor that creates a client socket. + * @param hostname hostname we are connecting to. + * @param port port we are connecting to. + */ + explicit TLSSocket(const std::string &hostname, const uint16_t port); + + /** + * Move constructor. + */ + explicit TLSSocket(const TLSSocket &&); + + virtual ~TLSSocket(); + + /** + * Initializes the socket + * @return result of the creation operation. + */ + short initialize(); + + /** + * Attempt to select the socket file descriptor + * @param msec timeout interval to wait + * @returns file descriptor + */ + virtual short select_descriptor(const uint16_t msec); + + /** + * Reads data and places it into buf + * @param buf buffer in which we extract data + * @param buflen + */ + virtual int readData(uint8_t *buf, int buflen); + + /** + * Write value to the stream using std::vector + * @param buf incoming buffer + * @param buflen buffer to write + * + */ + int writeData(std::vector &buf, int buflen); + + /** + * Write value to the stream using uint8_t ptr + * @param buf incoming buffer + * @param buflen buffer to write + * + */ + int writeData(uint8_t *value, int size); + + protected: + + SSL* ssl; }; #endif +} /* namespace io */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + #endif /* LIBMINIFI_INCLUDE_IO_TLSSOCKET_H_ */ diff --git a/libminifi/include/io/validation.h b/libminifi/include/io/validation.h index e1b4bb6f29..9db554472c 100644 --- a/libminifi/include/io/validation.h +++ b/libminifi/include/io/validation.h @@ -21,23 +21,24 @@ #include #include + /** * A checker that will, at compile time, tell us * if the declared type has a size method. */ template class size_function_functor_checker { - typedef char hasit; - typedef long doesnothaveit; + typedef char hasit; + typedef long doesnothaveit; - // look for the declared type - template static hasit test(decltype(&O::size)); - template static doesnothaveit test(...); + // look for the declared type + template static hasit test(decltype(&O::size)); + template static doesnothaveit test(...); -public: - enum { - has_size_function = sizeof(test(0)) == sizeof(char) - }; + public: + enum { + has_size_function = sizeof(test(0)) == sizeof(char) + }; }; /** @@ -46,7 +47,7 @@ class size_function_functor_checker { template static auto IsNullOrEmpty( T &object) -> typename std::enable_if::has_size_function==1, bool>::type { - return object.size() == 0; + return object.size() == 0; } /** @@ -55,7 +56,7 @@ static auto IsNullOrEmpty( template static auto IsNullOrEmpty( T *object) -> typename std::enable_if::has_size_function==1, bool>::type { - return (0 == object || object->size() == 0); + return (0 == object || object->size() == 0); } /** @@ -64,13 +65,13 @@ static auto IsNullOrEmpty( template static auto IsNullOrEmpty( T *object) -> typename std::enable_if::has_size_function , bool>::type { - return (0 == object); + return (0 == object); } /** * Determines if the variable is null or strlen(str) == 0 */ static auto IsNullOrEmpty(char *str)-> decltype(NULL !=str, bool()) { - return (NULL == str || strlen(str) == 0); + return (NULL == str || strlen(str) == 0); } diff --git a/libminifi/include/processors/AppendHostInfo.h b/libminifi/include/processors/AppendHostInfo.h new file mode 100644 index 0000000000..6515918a89 --- /dev/null +++ b/libminifi/include/processors/AppendHostInfo.h @@ -0,0 +1,80 @@ +/** + * @file AppendHostInfo.h + * AppendHostInfo class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __APPEND_HOSTINFO_H__ +#define __APPEND_HOSTINFO_H__ + +#include "core/Property.h" +#include "FlowFileRecord.h" +#include "core/Processor.h" +#include "core/ProcessSession.h" +#include "core/core.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +// AppendHostInfo Class +class AppendHostInfo : public core::Processor { + public: + // Constructor + /*! + * Create a new processor + */ + AppendHostInfo(std::string name, uuid_t uuid = NULL) + : core::Processor(name, uuid) { + logger_ = logging::Logger::getLogger(); + } + // Destructor + virtual ~AppendHostInfo() { + } + // Processor Name + static const std::string ProcessorName; + // Supported Properties + static core::Property InterfaceName; + static core::Property HostAttribute; + static core::Property IPAttribute; + + // Supported Relationships + static core::Relationship Success; + + public: + // OnTrigger method, implemented by NiFi AppendHostInfo + virtual void onTrigger( + core::ProcessContext *context, + core::ProcessSession *session); + // Initialize, over write by NiFi AppendHostInfo + virtual void initialize(void); + + protected: + + private: + // Logger + std::shared_ptr logger_; +}; + +} /* namespace processors */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif diff --git a/libminifi/include/processors/ExecuteProcess.h b/libminifi/include/processors/ExecuteProcess.h new file mode 100644 index 0000000000..123eed37f6 --- /dev/null +++ b/libminifi/include/processors/ExecuteProcess.h @@ -0,0 +1,125 @@ +/** + * @file ExecuteProcess.h + * ExecuteProcess class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __EXECUTE_PROCESS_H__ +#define __EXECUTE_PROCESS_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "FlowFileRecord.h" +#include "core/Processor.h" +#include "core/ProcessSession.h" +#include "core/core.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +// ExecuteProcess Class +class ExecuteProcess : public core::Processor { + public: + // Constructor + /*! + * Create a new processor + */ + ExecuteProcess(std::string name, uuid_t uuid = NULL) + : Processor(name, uuid) { + logger_ = logging::Logger::getLogger(); + _redirectErrorStream = false; + _batchDuration = 0; + _workingDir = "."; + _processRunning = false; + _pid = 0; + } + // Destructor + virtual ~ExecuteProcess() { + if (_processRunning && _pid > 0) + kill(_pid, SIGTERM); + } + // Processor Name + static const std::string ProcessorName; + // Supported Properties + static core::Property Command; + static core::Property CommandArguments; + static core::Property WorkingDir; + static core::Property BatchDuration; + static core::Property RedirectErrorStream; + // Supported Relationships + static core::Relationship Success; + + // Nest Callback Class for write stream + class WriteCallback : public OutputStreamCallback { + public: + WriteCallback(char *data, uint64_t size) + : _data(data), + _dataSize(size) { + } + char *_data; + uint64_t _dataSize; + void process(std::ofstream *stream) { + if (_data && _dataSize > 0) + stream->write(_data, _dataSize); + } + }; + + public: + // OnTrigger method, implemented by NiFi ExecuteProcess + virtual void onTrigger( + core::ProcessContext *context, + core::ProcessSession *session); + // Initialize, over write by NiFi ExecuteProcess + virtual void initialize(void); + + protected: + + private: + // Logger + std::shared_ptr logger_; + // Property + std::string _command; + std::string _commandArgument; + std::string _workingDir; + int64_t _batchDuration; + bool _redirectErrorStream; + // Full command + std::string _fullCommand; + // whether the process is running + bool _processRunning; + int _pipefd[2]; + pid_t _pid; +}; + +} /* namespace processors */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif diff --git a/libminifi/include/processors/GenerateFlowFile.h b/libminifi/include/processors/GenerateFlowFile.h new file mode 100644 index 0000000000..c4ab6fefac --- /dev/null +++ b/libminifi/include/processors/GenerateFlowFile.h @@ -0,0 +1,98 @@ +/** + * @file GenerateFlowFile.h + * GenerateFlowFile class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __GENERATE_FLOW_FILE_H__ +#define __GENERATE_FLOW_FILE_H__ + +#include "FlowFileRecord.h" +#include "core/Processor.h" +#include "core/ProcessSession.h" +#include "core/core.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { +// GenerateFlowFile Class +class GenerateFlowFile : public core::Processor { + public: + // Constructor + /*! + * Create a new processor + */ + GenerateFlowFile(std::string name, uuid_t uuid = NULL) + : Processor(name, uuid) { + _data = NULL; + _dataSize = 0; + } + // Destructor + virtual ~GenerateFlowFile() { + if (_data) + delete[] _data; + } + // Processor Name + static const std::string ProcessorName; + // Supported Properties + static core::Property FileSize; + static core::Property BatchSize; + static core::Property DataFormat; + static core::Property UniqueFlowFiles; + static const char *DATA_FORMAT_BINARY; + static const char *DATA_FORMAT_TEXT; + // Supported Relationships + static core::Relationship Success; + // Nest Callback Class for write stream + class WriteCallback : public OutputStreamCallback { + public: + WriteCallback(char *data, uint64_t size) + : _data(data), + _dataSize(size) { + } + char *_data; + uint64_t _dataSize; + void process(std::ofstream *stream) { + if (_data && _dataSize > 0) + stream->write(_data, _dataSize); + } + }; + + public: + // OnTrigger method, implemented by NiFi GenerateFlowFile + virtual void onTrigger( + core::ProcessContext *context, + core::ProcessSession *session); + // Initialize, over write by NiFi GenerateFlowFile + virtual void initialize(void); + + protected: + + private: + // Generated data + char * _data; + // Size of the generate data + uint64_t _dataSize; +}; + +} /* namespace processors */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ +#endif diff --git a/libminifi/include/processors/GetFile.h b/libminifi/include/processors/GetFile.h new file mode 100644 index 0000000000..f1f069491c --- /dev/null +++ b/libminifi/include/processors/GetFile.h @@ -0,0 +1,129 @@ +/** + * @file GetFile.h + * GetFile class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __GET_FILE_H__ +#define __GET_FILE_H__ + +#include "FlowFileRecord.h" +#include "core/Processor.h" +#include "core/ProcessSession.h" +#include "core/core.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +// GetFile Class +class GetFile : public core::Processor { + public: + // Constructor + /*! + * Create a new processor + */ + GetFile(std::string name, uuid_t uuid = NULL) + : Processor(name, uuid) { + logger_ = logging::Logger::getLogger(); + _directory = "."; + _recursive = true; + _keepSourceFile = false; + _minAge = 0; + _maxAge = 0; + _minSize = 0; + _maxSize = 0; + _ignoreHiddenFile = true; + _pollInterval = 0; + _batchSize = 10; + _lastDirectoryListingTime = getTimeMillis(); + _fileFilter = "[^\\.].*"; + } + // Destructor + virtual ~GetFile() { + } + // Processor Name + static const std::string ProcessorName; + // Supported Properties + static core::Property Directory; + static core::Property Recurse; + static core::Property KeepSourceFile; + static core::Property MinAge; + static core::Property MaxAge; + static core::Property MinSize; + static core::Property MaxSize; + static core::Property IgnoreHiddenFile; + static core::Property PollInterval; + static core::Property BatchSize; + static core::Property FileFilter; + // Supported Relationships + static core::Relationship Success; + + public: + // OnTrigger method, implemented by NiFi GetFile + virtual void onTrigger( + core::ProcessContext *context, + core::ProcessSession *session); + // Initialize, over write by NiFi GetFile + virtual void initialize(void); + // perform directory listing + void performListing(std::string dir); + + protected: + + private: + // Logger + std::shared_ptr logger_; + // Queue for store directory list + std::queue _dirList; + // Get Listing size + uint64_t getListingSize() { + std::lock_guard lock(mutex_); + return _dirList.size(); + } + // Whether the directory listing is empty + bool isListingEmpty(); + // Put full path file name into directory listing + void putListing(std::string fileName); + // Poll directory listing for files + void pollListing(std::queue &list, int maxSize); + // Check whether file can be added to the directory listing + bool acceptFile(std::string fullName, std::string name); + // Mutex for protection of the directory listing + std::mutex mutex_; + std::string _directory; + bool _recursive; + bool _keepSourceFile; + int64_t _minAge; + int64_t _maxAge; + int64_t _minSize; + int64_t _maxSize; + bool _ignoreHiddenFile; + int64_t _pollInterval; + int64_t _batchSize; + uint64_t _lastDirectoryListingTime; + std::string _fileFilter; +}; + +} /* namespace processors */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif diff --git a/libminifi/include/processors/ListenHTTP.h b/libminifi/include/processors/ListenHTTP.h new file mode 100644 index 0000000000..adaefb13a0 --- /dev/null +++ b/libminifi/include/processors/ListenHTTP.h @@ -0,0 +1,126 @@ +/** + * @file ListenHTTP.h + * ListenHTTP class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __LISTEN_HTTP_H__ +#define __LISTEN_HTTP_H__ + +#include +#include + +#include + +#include "FlowFileRecord.h" +#include "core/Processor.h" +#include "core/ProcessSession.h" +#include "core/core.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +// ListenHTTP Class +class ListenHTTP : public core::Processor { + public: + + // Constructor + /*! + * Create a new processor + */ + ListenHTTP(std::string name, uuid_t uuid = NULL) + : Processor(name, uuid) { + _logger = logging::Logger::getLogger(); + } + // Destructor + virtual ~ListenHTTP(); + // Processor Name + static const std::string ProcessorName; + // Supported Properties + static core::Property BasePath; + static core::Property Port; + static core::Property AuthorizedDNPattern; + static core::Property SSLCertificate; + static core::Property SSLCertificateAuthority; + static core::Property SSLVerifyPeer; + static core::Property SSLMinimumVersion; + static core::Property HeadersAsAttributesRegex; + // Supported Relationships + static core::Relationship Success; + + void onTrigger(core::ProcessContext *context, + core::ProcessSession *session); + void initialize(); + void onSchedule( + core::ProcessContext *context, + core::ProcessSessionFactory *sessionFactory); + + // HTTP request handler + class Handler : public CivetHandler { + public: + Handler( + core::ProcessContext *context, + core::ProcessSessionFactory *sessionFactory, + std::string &&authDNPattern, std::string &&headersAsAttributesPattern); + bool handlePost(CivetServer *server, struct mg_connection *conn); + + private: + // Send HTTP 500 error response to client + void sendErrorResponse(struct mg_connection *conn); + // Logger + std::shared_ptr _logger; + + std::regex _authDNRegex; + std::regex _headersAsAttributesRegex; + core::ProcessContext *_processContext; + core::ProcessSessionFactory *_processSessionFactory; + }; + + // Write callback for transferring data from HTTP request to content repo + class WriteCallback : public OutputStreamCallback { + public: + WriteCallback(struct mg_connection *conn, + const struct mg_request_info *reqInfo); + void process(std::ofstream *stream); + + private: + // Logger + std::shared_ptr _logger; + + struct mg_connection *_conn; + const struct mg_request_info *_reqInfo; + }; + + protected: + + private: + // Logger + std::shared_ptr _logger; + + std::unique_ptr _server; + std::unique_ptr _handler; +}; + +} /* namespace processors */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif diff --git a/libminifi/include/processors/ListenSyslog.h b/libminifi/include/processors/ListenSyslog.h new file mode 100644 index 0000000000..1e1e11f628 --- /dev/null +++ b/libminifi/include/processors/ListenSyslog.h @@ -0,0 +1,216 @@ +/** + * @file ListenSyslog.h + * ListenSyslog class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __LISTEN_SYSLOG_H__ +#define __LISTEN_SYSLOG_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "FlowFileRecord.h" +#include "core/Processor.h" +#include "core/ProcessSession.h" +#include "core/core.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +// SyslogEvent +typedef struct { + uint8_t *payload; + uint64_t len; +} SysLogEvent; + +// ListenSyslog Class +class ListenSyslog : public core::Processor { + public: + // Constructor + /*! + * Create a new processor + */ + ListenSyslog(std::string name, uuid_t uuid = NULL) + : Processor(name, uuid) { + logger_ = logging::Logger::getLogger(); + _eventQueueByteSize = 0; + _serverSocket = 0; + _recvBufSize = 65507; + _maxSocketBufSize = 1024 * 1024; + _maxConnections = 2; + _maxBatchSize = 1; + _messageDelimiter = "\n"; + _protocol = "UDP"; + _port = 514; + _parseMessages = false; + _serverSocket = 0; + _maxFds = 0; + FD_ZERO(&_readfds); + _thread = NULL; + _resetServerSocket = false; + _serverTheadRunning = false; + } + // Destructor + virtual ~ListenSyslog() { + _serverTheadRunning = false; + if (this->_thread) + delete this->_thread; + // need to reset the socket + std::vector::iterator it; + for (it = _clientSockets.begin(); it != _clientSockets.end(); ++it) { + int clientSocket = *it; + close(clientSocket); + } + _clientSockets.clear(); + if (_serverSocket > 0) { + logger_->log_info("ListenSysLog Server socket %d close", _serverSocket); + close(_serverSocket); + _serverSocket = 0; + } + } + // Processor Name + static const std::string ProcessorName; + // Supported Properties + static core::Property RecvBufSize; + static core::Property MaxSocketBufSize; + static core::Property MaxConnections; + static core::Property MaxBatchSize; + static core::Property MessageDelimiter; + static core::Property ParseMessages; + static core::Property Protocol; + static core::Property Port; + // Supported Relationships + static core::Relationship Success; + static core::Relationship Invalid; + // Nest Callback Class for write stream + class WriteCallback : public OutputStreamCallback { + public: + WriteCallback(char *data, uint64_t size) + : _data(data), + _dataSize(size) { + } + char *_data; + uint64_t _dataSize; + void process(std::ofstream *stream) { + if (_data && _dataSize > 0) + stream->write(_data, _dataSize); + } + }; + + public: + // OnTrigger method, implemented by NiFi ListenSyslog + virtual void onTrigger( + core::ProcessContext *context, + core::ProcessSession *session); + // Initialize, over write by NiFi ListenSyslog + virtual void initialize(void); + + protected: + + private: + // Logger + std::shared_ptr logger_; + // Run function for the thread + static void run(ListenSyslog *process); + // Run Thread + void runThread(); + // Queue for store syslog event + std::queue _eventQueue; + // Size of Event queue in bytes + uint64_t _eventQueueByteSize; + // Get event queue size + uint64_t getEventQueueSize() { + std::lock_guard lock(mutex_); + return _eventQueue.size(); + } + // Get event queue byte size + uint64_t getEventQueueByteSize() { + std::lock_guard lock(mutex_); + return _eventQueueByteSize; + } + // Whether the event queue is empty + bool isEventQueueEmpty() { + std::lock_guard lock(mutex_); + return _eventQueue.empty(); + } + // Put event into directory listing + void putEvent(uint8_t *payload, uint64_t len) { + std::lock_guard lock(mutex_); + SysLogEvent event; + event.payload = payload; + event.len = len; + _eventQueue.push(event); + _eventQueueByteSize += len; + } + // Read \n terminated line from TCP socket + int readline(int fd, char *bufptr, size_t len); + // start server socket and handling client socket + void startSocketThread(); + // Poll event + void pollEvent(std::queue &list, int maxSize) { + std::lock_guard lock(mutex_); + + while (!_eventQueue.empty() && (maxSize == 0 || list.size() < maxSize)) { + SysLogEvent event = _eventQueue.front(); + _eventQueue.pop(); + _eventQueueByteSize -= event.len; + list.push(event); + } + return; + } + // Mutex for protection of the directory listing + std::mutex mutex_; + int64_t _recvBufSize; + int64_t _maxSocketBufSize; + int64_t _maxConnections; + int64_t _maxBatchSize; + std::string _messageDelimiter; + std::string _protocol; + int64_t _port; + bool _parseMessages; + int _serverSocket; + std::vector _clientSockets; + int _maxFds; + fd_set _readfds; + // thread + std::thread *_thread; + // whether to reset the server socket + bool _resetServerSocket; + bool _serverTheadRunning; + // buffer for read socket + uint8_t _buffer[2048]; +}; + +} /* namespace processors */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif diff --git a/libminifi/include/processors/LogAttribute.h b/libminifi/include/processors/LogAttribute.h new file mode 100644 index 0000000000..37c0ec3874 --- /dev/null +++ b/libminifi/include/processors/LogAttribute.h @@ -0,0 +1,130 @@ +/** + * @file LogAttribute.h + * LogAttribute class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __LOG_ATTRIBUTE_H__ +#define __LOG_ATTRIBUTE_H__ + +#include "FlowFileRecord.h" +#include "core/Processor.h" +#include "core/ProcessSession.h" +#include "core/core.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +// LogAttribute Class +class LogAttribute : public core::Processor { + public: + // Constructor + /*! + * Create a new processor + */ + LogAttribute(std::string name, uuid_t uuid = NULL) + : Processor(name, uuid) { + logger_ = logging::Logger::getLogger(); + } + // Destructor + virtual ~LogAttribute() { + } + // Processor Name + static const std::string ProcessorName; + // Supported Properties + static core::Property LogLevel; + static core::Property AttributesToLog; + static core::Property AttributesToIgnore; + static core::Property LogPayload; + static core::Property LogPrefix; + // Supported Relationships + static core::Relationship Success; + enum LogAttrLevel { + LogAttrLevelTrace, + LogAttrLevelDebug, + LogAttrLevelInfo, + LogAttrLevelWarn, + LogAttrLevelError + }; + // Convert log level from string to enum + bool logLevelStringToEnum(std::string logStr, LogAttrLevel &level) { + if (logStr == "trace") { + level = LogAttrLevelTrace; + return true; + } else if (logStr == "debug") { + level = LogAttrLevelDebug; + return true; + } else if (logStr == "info") { + level = LogAttrLevelInfo; + return true; + } else if (logStr == "warn") { + level = LogAttrLevelWarn; + return true; + } else if (logStr == "error") { + level = LogAttrLevelError; + return true; + } else + return false; + } + // Nest Callback Class for read stream + class ReadCallback : public InputStreamCallback { + public: + ReadCallback(uint64_t size) { + _bufferSize = size; + _buffer = new char[_bufferSize]; + } + ~ReadCallback() { + if (_buffer) + delete[] _buffer; + } + void process(std::ifstream *stream) { + + stream->read(_buffer, _bufferSize); + if (!stream) + _readSize = stream->gcount(); + else + _readSize = _bufferSize; + } + char *_buffer; + uint64_t _bufferSize; + uint64_t _readSize; + }; + + public: + // OnTrigger method, implemented by NiFi LogAttribute + virtual void onTrigger( + core::ProcessContext *context, + core::ProcessSession *session); + // Initialize, over write by NiFi LogAttribute + virtual void initialize(void); + + protected: + + private: + // Logger + std::shared_ptr logger_; +}; + +} /* namespace processors */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif diff --git a/libminifi/include/processors/PutFile.h b/libminifi/include/processors/PutFile.h new file mode 100644 index 0000000000..7653faca9c --- /dev/null +++ b/libminifi/include/processors/PutFile.h @@ -0,0 +1,101 @@ +/** + * @file PutFile.h + * PutFile class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __PUT_FILE_H__ +#define __PUT_FILE_H__ + +#include "FlowFileRecord.h" +#include "core/Processor.h" +#include "core/ProcessSession.h" +#include "core/core.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +// PutFile Class +class PutFile : public core::Processor { + public: + + static const std::string CONFLICT_RESOLUTION_STRATEGY_REPLACE; + static const std::string CONFLICT_RESOLUTION_STRATEGY_IGNORE; + static const std::string CONFLICT_RESOLUTION_STRATEGY_FAIL; + + // Constructor + /*! + * Create a new processor + */ + PutFile(std::string name, uuid_t uuid = NULL) + : core::Processor(name, uuid) { + logger_ = logging::Logger::getLogger(); + } + // Destructor + virtual ~PutFile() { + } + // Processor Name + static const std::string ProcessorName; + // Supported Properties + static core::Property Directory; + static core::Property ConflictResolution; + // Supported Relationships + static core::Relationship Success; + static core::Relationship Failure; + + // OnTrigger method, implemented by NiFi PutFile + virtual void onTrigger( + core::ProcessContext *context, + core::ProcessSession *session); + // Initialize, over write by NiFi PutFile + virtual void initialize(void); + + class ReadCallback : public InputStreamCallback { + public: + ReadCallback(const std::string &tmpFile, const std::string &destFile); + ~ReadCallback(); + virtual void process(std::ifstream *stream); + bool commit(); + + private: + std::shared_ptr logger_; + std::ofstream _tmpFileOs; + bool _writeSucceeded = false; + std::string _tmpFile; + std::string _destFile; + }; + + protected: + + private: + // Logger + std::shared_ptr logger_; + + bool putFile(core::ProcessSession *session, + std::shared_ptr flowFile, + const std::string &tmpFile, const std::string &destFile); +}; + +} /* namespace processors */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif diff --git a/libminifi/include/processors/RealTimeDataCollector.h b/libminifi/include/processors/RealTimeDataCollector.h new file mode 100644 index 0000000000..41bd814bb0 --- /dev/null +++ b/libminifi/include/processors/RealTimeDataCollector.h @@ -0,0 +1,145 @@ +/** + * @file RealTimeDataCollector.h + * RealTimeDataCollector class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __REAL_TIME_DATA_COLLECTOR_H__ +#define __REAL_TIME_DATA_COLLECTOR_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "FlowFileRecord.h" +#include "core/Processor.h" +#include "core/ProcessSession.h" +#include "core/core.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +// RealTimeDataCollector Class +class RealTimeDataCollector : public core::Processor { + public: + // Constructor + /*! + * Create a new processor + */ + explicit RealTimeDataCollector(std::string name, uuid_t uuid = NULL) + : core::Processor(name, uuid) { + _realTimeSocket = 0; + _batchSocket = 0; + logger_ = logging::Logger::getLogger(); + _firstInvoking = false; + _realTimeAccumulated = 0; + _batchAcccumulated = 0; + _queuedDataSize = 0; + } + // Destructor + virtual ~RealTimeDataCollector() { + if (_realTimeSocket) + close(_realTimeSocket); + if (_batchSocket) + close(_batchSocket); + if (_fileStream.is_open()) + _fileStream.close(); + } + // Processor Name + static const std::string ProcessorName; + // Supported Properties + static core::Property REALTIMESERVERNAME; + static core::Property REALTIMESERVERPORT; + static core::Property BATCHSERVERNAME; + static core::Property BATCHSERVERPORT; + static core::Property FILENAME; + static core::Property ITERATION; + static core::Property REALTIMEMSGID; + static core::Property BATCHMSGID; + static core::Property REALTIMEINTERVAL; + static core::Property BATCHINTERVAL; + static core::Property BATCHMAXBUFFERSIZE; + // Supported Relationships + static core::Relationship Success; + // Connect to the socket + int connectServer(const char *host, uint16_t port); + int sendData(int socket, const char *buf, int buflen); + void onTriggerRealTime( + core::ProcessContext *context, + core::ProcessSession *session); + void onTriggerBatch(core::ProcessContext *context, + core::ProcessSession *session); + + public: + // OnTrigger method, implemented by NiFi RealTimeDataCollector + virtual void onTrigger( + core::ProcessContext *context, + core::ProcessSession *session); + // Initialize, over write by NiFi RealTimeDataCollector + virtual void initialize(void); + + protected: + + private: + // realtime server Name + std::string _realTimeServerName; + int64_t _realTimeServerPort; + std::string _batchServerName; + int64_t _batchServerPort; + int64_t _realTimeInterval; + int64_t _batchInterval; + int64_t _batchMaxBufferSize; + // Match pattern for Real time Message ID + std::vector _realTimeMsgID; + // Match pattern for Batch Message ID + std::vector _batchMsgID; + // file for which the realTime collector will tail + std::string _fileName; + // Whether we need to iterate from the beginning for demo + bool _iteration; + int _realTimeSocket; + int _batchSocket; + // Logger + std::shared_ptr logger_; + // Mutex for protection + std::mutex mutex_; + // Queued data size + uint64_t _queuedDataSize; + // Queue for the batch process + std::queue _queue; + std::thread::id _realTimeThreadId; + std::thread::id _batchThreadId; + std::atomic _firstInvoking; + int64_t _realTimeAccumulated; + int64_t _batchAcccumulated; + std::ifstream _fileStream; +}; + +} /* namespace processors */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ +#endif diff --git a/libminifi/include/processors/TailFile.h b/libminifi/include/processors/TailFile.h new file mode 100644 index 0000000000..5be76e4b72 --- /dev/null +++ b/libminifi/include/processors/TailFile.h @@ -0,0 +1,105 @@ +/** + * @file TailFile.h + * TailFile class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __TAIL_FILE_H__ +#define __TAIL_FILE_H__ + +#include "FlowFileRecord.h" +#include "core/Processor.h" +#include "core/ProcessSession.h" +#include "core/core.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +// TailFile Class +class TailFile : public core::Processor { + public: + // Constructor + /*! + * Create a new processor + */ + TailFile(std::string name, uuid_t uuid = NULL) + : core::Processor(name, uuid) { + logger_ = logging::Logger::getLogger(); + _stateRecovered = false; + } + // Destructor + virtual ~TailFile() { + storeState(); + } + // Processor Name + static const std::string ProcessorName; + // Supported Properties + static core::Property FileName; + static core::Property StateFile; + // Supported Relationships + static core::Relationship Success; + + public: + // OnTrigger method, implemented by NiFi TailFile + virtual void onTrigger( + core::ProcessContext *context, + core::ProcessSession *session); + // Initialize, over write by NiFi TailFile + virtual void initialize(void); + // recoverState + void recoverState(); + // storeState + void storeState(); + + protected: + + private: + // Logger + std::shared_ptr logger_; + std::string _fileLocation; + // Property Specified Tailed File Name + std::string _fileName; + // File to save state + std::string _stateFile; + // State related to the tailed file + std::string _currentTailFileName; + uint64_t _currentTailFilePosition; + bool _stateRecovered; + uint64_t _currentTailFileCreatedTime; + // Utils functions for parse state file + std::string trimLeft(const std::string& s); + std::string trimRight(const std::string& s); + void parseStateFileLine(char *buf); + void checkRollOver(); + +}; + +// Matched File Item for Roll over check +typedef struct { + std::string fileName; + uint64_t modifiedTime; +} TailMatchedFileItem; + +} /* namespace processors */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif diff --git a/libminifi/include/Configure.h b/libminifi/include/properties/Configure.h similarity index 71% rename from libminifi/include/Configure.h rename to libminifi/include/properties/Configure.h index 6f0d1986b7..8f598539f8 100644 --- a/libminifi/include/Configure.h +++ b/libminifi/include/properties/Configure.h @@ -27,11 +27,18 @@ #include #include #include -#include "Logger.h" +#include "core/core.h" +#include "core/logging/Logger.h" + + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { class Configure { public: - //! Get the singleton logger instance + // Get the singleton logger instance static Configure * getConfigure() { if (!configure_) @@ -40,7 +47,7 @@ class Configure { } return configure_; } - //! nifi.flow.configuration.file + // nifi.flow.configuration.file static const char *nifi_flow_configuration_file; static const char *nifi_administrative_yield_duration; static const char *nifi_bored_yield_duration; @@ -64,55 +71,55 @@ class Configure { static const char *nifi_security_client_pass_phrase; static const char *nifi_security_client_ca_certificate; - //! Clear the load config + // Clear the load config void clear() { - std::lock_guard lock(_mtx); - _properties.clear(); + std::lock_guard lock(mutex_); + properties_.clear(); } - //! Set the config value + // Set the config value void set(std::string key, std::string value) { - std::lock_guard lock(_mtx); - _properties[key] = value; + std::lock_guard lock(mutex_); + properties_[key] = value; } - //! Check whether the config value existed + // Check whether the config value existed bool has(std::string key) { - std::lock_guard lock(_mtx); - return (_properties.find(key) != _properties.end()); + std::lock_guard lock(mutex_); + return (properties_.find(key) != properties_.end()); } - //! Get the config value + // Get the config value bool get(std::string key, std::string &value); - //! Parse one line in configure file like key=value + // Parse one line in configure file like key=value void parseConfigureFileLine(char *buf); - //! Load Configure File + // Load Configure File void loadConfigureFile(const char *fileName); - //! Set the determined MINIFI_HOME + // Set the determined MINIFI_HOME void setHome(std::string minifiHome) { - _minifiHome = minifiHome; + minifi_home_ = minifiHome; } - //! Get the determined MINIFI_HOME + // Get the determined MINIFI_HOME std::string getHome() { - return _minifiHome; + return minifi_home_; } - //! Parse Command Line + // Parse Command Line void parseCommandLine(int argc, char **argv); private: - //! Mutex for protection - std::mutex _mtx; - //! Logger - std::shared_ptr logger_; - //! Home location for this executable - std::string _minifiHome; + // Mutex for protection + std::mutex mutex_; + // Logger + std::shared_ptr logger_; + // Home location for this executable + std::string minifi_home_; Configure() { - logger_ = Logger::getLogger(); + logger_ = logging::Logger::getLogger(); } virtual ~Configure() { @@ -121,7 +128,11 @@ class Configure { static Configure *configure_; protected: - std::map _properties; + std::map properties_; }; +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ #endif diff --git a/libminifi/include/provenance/Provenance.h b/libminifi/include/provenance/Provenance.h new file mode 100644 index 0000000000..bdbffac894 --- /dev/null +++ b/libminifi/include/provenance/Provenance.h @@ -0,0 +1,564 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __PROVENANCE_H__ +#define __PROVENANCE_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "leveldb/db.h" +#include "leveldb/options.h" +#include "leveldb/slice.h" +#include "leveldb/status.h" + +#include "core/Repository.h" +#include "core/Property.h" +#include "Configure.h" +#include "Connection.h" +#include "FlowFileRecord.h" +#include "core/logging/Logger.h" +#include "ResourceClaim.h" +#include "io/Serializable.h" +#include "utils/TimeUtil.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace provenance { +// Provenance Event Record Serialization Seg Size +#define PROVENANCE_EVENT_RECORD_SEG_SIZE 2048 + +// Provenance Event Record +class ProvenanceEventRecord : + protected org::apache::nifi::minifi::io::Serializable { + public: + enum ProvenanceEventType { + + /** + * A CREATE event is used when a FlowFile is generated from data that was + * not received from a remote system or external process + */ + CREATE, + + /** + * Indicates a provenance event for receiving data from an external process. This Event Type + * is expected to be the first event for a FlowFile. As such, a Processor that receives data + * from an external source and uses that data to replace the content of an existing FlowFile + * should use the {@link #FETCH} event type, rather than the RECEIVE event type. + */ + RECEIVE, + + /** + * Indicates that the contents of a FlowFile were overwritten using the contents of some + * external resource. This is similar to the {@link #RECEIVE} event but varies in that + * RECEIVE events are intended to be used as the event that introduces the FlowFile into + * the system, whereas FETCH is used to indicate that the contents of an existing FlowFile + * were overwritten. + */ + FETCH, + + /** + * Indicates a provenance event for sending data to an external process + */ + SEND, + + /** + * Indicates that the contents of a FlowFile were downloaded by a user or external entity. + */ + DOWNLOAD, + + /** + * Indicates a provenance event for the conclusion of an object's life for + * some reason other than object expiration + */ + DROP, + + /** + * Indicates a provenance event for the conclusion of an object's life due + * to the fact that the object could not be processed in a timely manner + */ + EXPIRE, + + /** + * FORK is used to indicate that one or more FlowFile was derived from a + * parent FlowFile. + */ + FORK, + + /** + * JOIN is used to indicate that a single FlowFile is derived from joining + * together multiple parent FlowFiles. + */ + JOIN, + + /** + * CLONE is used to indicate that a FlowFile is an exact duplicate of its + * parent FlowFile. + */ + CLONE, + + /** + * CONTENT_MODIFIED is used to indicate that a FlowFile's content was + * modified in some way. When using this Event Type, it is advisable to + * provide details about how the content is modified. + */ + CONTENT_MODIFIED, + + /** + * ATTRIBUTES_MODIFIED is used to indicate that a FlowFile's attributes were + * modified in some way. This event is not needed when another event is + * reported at the same time, as the other event will already contain all + * FlowFile attributes. + */ + ATTRIBUTES_MODIFIED, + + /** + * ROUTE is used to show that a FlowFile was routed to a specified + * {@link org.apache.nifi.processor.Relationship Relationship} and should provide + * information about why the FlowFile was routed to this relationship. + */ + ROUTE, + + /** + * Indicates a provenance event for adding additional information such as a + * new linkage to a new URI or UUID + */ + ADDINFO, + + /** + * Indicates a provenance event for replaying a FlowFile. The UUID of the + * event will indicate the UUID of the original FlowFile that is being + * replayed. The event will contain exactly one Parent UUID that is also the + * UUID of the FlowFile that is being replayed and exactly one Child UUID + * that is the UUID of the a newly created FlowFile that will be re-queued + * for processing. + */ + REPLAY + }; + public: + // Constructor + /*! + * Create a new provenance event record + */ + ProvenanceEventRecord(ProvenanceEventType event, std::string componentId, + std::string componentType) { + _eventType = event; + _componentId = componentId; + _componentType = componentType; + _eventTime = getTimeMillis(); + char eventIdStr[37]; + // Generate the global UUID for th event + uuid_generate(_eventId); + uuid_unparse_lower(_eventId, eventIdStr); + _eventIdStr = eventIdStr; + logger_ = logging::Logger::getLogger(); + } + + ProvenanceEventRecord() { + _eventTime = getTimeMillis(); + logger_ = logging::Logger::getLogger(); + } + + // Destructor + virtual ~ProvenanceEventRecord() { + } + // Get the Event ID + std::string getEventId() { + return _eventIdStr; + } + // Get Attributes + std::map getAttributes() { + return _attributes; + } + // Get Size + uint64_t getFileSize() { + return _size; + } + // ! Get Offset + uint64_t getFileOffset() { + return _offset; + } + // ! Get Entry Date + uint64_t getFlowFileEntryDate() { + return _entryDate; + } + // ! Get Lineage Start Date + uint64_t getlineageStartDate() { + return _lineageStartDate; + } + // ! Get Event Time + uint64_t getEventTime() { + return _eventTime; + } + // ! Get Event Duration + uint64_t getEventDuration() { + return _eventDuration; + } + // Set Event Duration + void setEventDuration(uint64_t duration) { + _eventDuration = duration; + } + // ! Get Event Type + ProvenanceEventType getEventType() { + return _eventType; + } + // Get Component ID + std::string getComponentId() { + return _componentId; + } + // Get Component Type + std::string getComponentType() { + return _componentType; + } + // Get FlowFileUuid + std::string getFlowFileUuid() { + return uuid_; + } + // Get content full path + std::string getContentFullPath() { + return _contentFullPath; + } + // Get LineageIdentifiers + std::set getLineageIdentifiers() { + return _lineageIdentifiers; + } + // Get Details + std::string getDetails() { + return _details; + } + // Set Details + void setDetails(std::string details) { + _details = details; + } + // Get TransitUri + std::string getTransitUri() { + return _transitUri; + } + // Set TransitUri + void setTransitUri(std::string uri) { + _transitUri = uri; + } + // Get SourceSystemFlowFileIdentifier + std::string getSourceSystemFlowFileIdentifier() { + return _sourceSystemFlowFileIdentifier; + } + // Set SourceSystemFlowFileIdentifier + void setSourceSystemFlowFileIdentifier(std::string identifier) { + _sourceSystemFlowFileIdentifier = identifier; + } + // Get Parent UUIDs + std::vector getParentUuids() { + return _parentUuids; + } + // Add Parent UUID + void addParentUuid(std::string uuid) { + if (std::find(_parentUuids.begin(), _parentUuids.end(), uuid) + != _parentUuids.end()) + return; + else + _parentUuids.push_back(uuid); + } + // Add Parent Flow File + void addParentFlowFile(std::shared_ptr flow) { + addParentUuid(flow->getUUIDStr()); + return; + } + // Remove Parent UUID + void removeParentUuid(std::string uuid) { + _parentUuids.erase( + std::remove(_parentUuids.begin(), _parentUuids.end(), uuid), + _parentUuids.end()); + } + // Remove Parent Flow File + void removeParentFlowFile(std::shared_ptr flow) { + removeParentUuid(flow->getUUIDStr()); + return; + } + // Get Children UUIDs + std::vector getChildrenUuids() { + return _childrenUuids; + } + // Add Child UUID + void addChildUuid(std::string uuid) { + if (std::find(_childrenUuids.begin(), _childrenUuids.end(), uuid) + != _childrenUuids.end()) + return; + else + _childrenUuids.push_back(uuid); + } + // Add Child Flow File + void addChildFlowFile(std::shared_ptr flow) { + addChildUuid(flow->getUUIDStr()); + return; + } + // Remove Child UUID + void removeChildUuid(std::string uuid) { + _childrenUuids.erase( + std::remove(_childrenUuids.begin(), _childrenUuids.end(), uuid), + _childrenUuids.end()); + } + // Remove Child Flow File + void removeChildFlowFile(std::shared_ptr flow) { + removeChildUuid(flow->getUUIDStr()); + return; + } + // Get AlternateIdentifierUri + std::string getAlternateIdentifierUri() { + return _alternateIdentifierUri; + } + // Set AlternateIdentifierUri + void setAlternateIdentifierUri(std::string uri) { + _alternateIdentifierUri = uri; + } + // Get Relationship + std::string getRelationship() { + return _relationship; + } + // Set Relationship + void setRelationship(std::string relation) { + _relationship = relation; + } + // Get sourceQueueIdentifier + std::string getSourceQueueIdentifier() { + return _sourceQueueIdentifier; + } + // Set sourceQueueIdentifier + void setSourceQueueIdentifier(std::string identifier) { + _sourceQueueIdentifier = identifier; + } + // fromFlowFile + void fromFlowFile(std::shared_ptr &flow) { + _entryDate = flow->getEntryDate(); + _lineageStartDate = flow->getlineageStartDate(); + _lineageIdentifiers = flow->getlineageIdentifiers(); + uuid_ = flow->getUUIDStr(); + _attributes = flow->getAttributes(); + _size = flow->getSize(); + _offset = flow->getOffset(); + if (flow->getOriginalConnection()) + _sourceQueueIdentifier = flow->getOriginalConnection()->getName(); + if (flow->getResourceClaim()) { + _contentFullPath = flow->getResourceClaim()->getContentFullPath(); + } + } + // Serialize and Persistent to the repository + bool Serialize(const std::shared_ptr &repo); + // DeSerialize + bool DeSerialize(const uint8_t *buffer, const int bufferSize); + // DeSerialize + bool DeSerialize(org::apache::nifi::minifi::io::DataStream &stream) { + return DeSerialize(stream.getBuffer(), stream.getSize()); + } + // DeSerialize + bool DeSerialize(const std::shared_ptr &repo, + std::string key); + + protected: + + // Event type + ProvenanceEventType _eventType; + // Date at which the event was created + uint64_t _eventTime; + // Date at which the flow file entered the flow + uint64_t _entryDate; + // Date at which the origin of this flow file entered the flow + uint64_t _lineageStartDate; + // Event Duration + uint64_t _eventDuration; + // Component ID + std::string _componentId; + // Component Type + std::string _componentType; + // Size in bytes of the data corresponding to this flow file + uint64_t _size; + // flow uuid + std::string uuid_; + // Offset to the content + uint64_t _offset; + // Full path to the content + std::string _contentFullPath; + // Attributes key/values pairs for the flow record + std::map _attributes; + // provenance ID + uuid_t _eventId; + // UUID string for all parents + std::set _lineageIdentifiers; + // transitUri + std::string _transitUri; + // sourceSystemFlowFileIdentifier + std::string _sourceSystemFlowFileIdentifier; + // parent UUID + std::vector _parentUuids; + // child UUID + std::vector _childrenUuids; + // detail + std::string _details; + // sourceQueueIdentifier + std::string _sourceQueueIdentifier; + // event ID Str + std::string _eventIdStr; + // relationship + std::string _relationship; + // alternateIdentifierUri; + std::string _alternateIdentifierUri; + + private: + + // Logger + std::shared_ptr logger_; + + // Prevent default copy constructor and assignment operation + // Only support pass by reference or pointer + ProvenanceEventRecord(const ProvenanceEventRecord &parent); + ProvenanceEventRecord &operator=(const ProvenanceEventRecord &parent); + +}; + +// Provenance Reporter +class ProvenanceReporter { + public: + // Constructor + /*! + * Create a new provenance reporter associated with the process session + */ + ProvenanceReporter(std::shared_ptr repo, + std::string componentId, std::string componentType) { + logger_ = logging::Logger::getLogger(); + _componentId = componentId; + _componentType = componentType; + repo_ = repo; + } + + // Destructor + virtual ~ProvenanceReporter() { + clear(); + } + // Get events + std::set getEvents() { + return _events; + } + // Add event + void add(ProvenanceEventRecord *event) { + _events.insert(event); + } + // Remove event + void remove(ProvenanceEventRecord *event) { + if (_events.find(event) != _events.end()) { + _events.erase(event); + } + } + // + // clear + void clear() { + for (auto it : _events) { + delete it; + } + _events.clear(); + } + // allocate + ProvenanceEventRecord *allocate( + ProvenanceEventRecord::ProvenanceEventType eventType, + std::shared_ptr flow) { + ProvenanceEventRecord *event = new ProvenanceEventRecord(eventType, + _componentId, + _componentType); + if (event) + event->fromFlowFile(flow); + + return event; + } + // commit + void commit(); + // create + void create(std::shared_ptr flow, std::string detail); + // route + void route(std::shared_ptr flow, core::Relationship relation, + std::string detail, uint64_t processingDuration); + // modifyAttributes + void modifyAttributes(std::shared_ptr flow, + std::string detail); + // modifyContent + void modifyContent(std::shared_ptr flow, std::string detail, + uint64_t processingDuration); + // clone + void clone(std::shared_ptr parent, + std::shared_ptr child); + // join + void join(std::vector > parents, + std::shared_ptr child, std::string detail, + uint64_t processingDuration); + // fork + void fork(std::vector > child, + std::shared_ptr parent, std::string detail, + uint64_t processingDuration); + // expire + void expire(std::shared_ptr flow, std::string detail); + // drop + void drop(std::shared_ptr flow, std::string reason); + // send + void send(std::shared_ptr flow, std::string transitUri, + std::string detail, uint64_t processingDuration, bool force); + // fetch + void fetch(std::shared_ptr flow, std::string transitUri, + std::string detail, uint64_t processingDuration); + // receive + void receive(std::shared_ptr flow, std::string transitUri, + std::string sourceSystemFlowFileIdentifier, std::string detail, + uint64_t processingDuration); + + protected: + + // Component ID + std::string _componentId; + // Component Type + std::string _componentType; + + private: + + // Incoming connection Iterator + std::set _events; + // Logger + std::shared_ptr logger_; + // provenance repository. + std::shared_ptr repo_; + + // Prevent default copy constructor and assignment operation + // Only support pass by reference or pointer + ProvenanceReporter(const ProvenanceReporter &parent); + ProvenanceReporter &operator=(const ProvenanceReporter &parent); +}; + +// Provenance Repository + +} /* namespace provenance */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif diff --git a/libminifi/include/provenance/ProvenanceRepository.h b/libminifi/include/provenance/ProvenanceRepository.h new file mode 100644 index 0000000000..de5ce2680c --- /dev/null +++ b/libminifi/include/provenance/ProvenanceRepository.h @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBMINIFI_INCLUDE_PROVENANCE_PROVENANCEREPOSITORY_H_ +#define LIBMINIFI_INCLUDE_PROVENANCE_PROVENANCEREPOSITORY_H_ + +#include "core/Repository.h" +#include "core/core.h" +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace provenance { + +#define PROVENANCE_DIRECTORY "./provenance_repository" +#define MAX_PROVENANCE_STORAGE_SIZE (10*1024*1024) // 10M +#define MAX_PROVENANCE_ENTRY_LIFE_TIME (60000) // 1 minute +#define PROVENANCE_PURGE_PERIOD (2500) // 2500 msec + +class ProvenanceRepository : public core::Repository, + public std::enable_shared_from_this { + public: + // Constructor + /*! + * Create a new provenance repository + */ + ProvenanceRepository(std::string directory = PROVENANCE_DIRECTORY, + int64_t maxPartitionMillis = + MAX_PROVENANCE_ENTRY_LIFE_TIME, + int64_t maxPartitionBytes = MAX_PROVENANCE_STORAGE_SIZE, + uint64_t purgePeriod = PROVENANCE_PURGE_PERIOD) + : Repository(core::getClassName(), directory, + maxPartitionMillis, maxPartitionBytes, purgePeriod) { + + db_ = NULL; + } + + // Destructor + virtual ~ProvenanceRepository() { + if (db_) + delete db_; + } + + // initialize + virtual bool initialize() { + std::string value; + if (configure_->get(Configure::nifi_provenance_repository_directory_default, + value)) { + directory_ = value; + } + logger_->log_info("NiFi Provenance Repository Directory %s", + directory_.c_str()); + if (configure_->get(Configure::nifi_provenance_repository_max_storage_size, + value)) { + core::Property::StringToInt(value, max_partition_bytes_); + } + logger_->log_info("NiFi Provenance Max Partition Bytes %d", + max_partition_bytes_); + if (configure_->get(Configure::nifi_provenance_repository_max_storage_time, + value)) { + core::TimeUnit unit; + if (core::Property::StringToTime(value, max_partition_millis_, unit) + && core::Property::ConvertTimeUnitToMS(max_partition_millis_, unit, + max_partition_millis_)) { + } + } + logger_->log_info("NiFi Provenance Max Storage Time: [%d] ms", + max_partition_millis_); + leveldb::Options options; + options.create_if_missing = true; + leveldb::Status status = leveldb::DB::Open(options, directory_.c_str(), + &db_); + if (status.ok()) { + logger_->log_info("NiFi Provenance Repository database open %s success", + directory_.c_str()); + } else { + logger_->log_error("NiFi Provenance Repository database open %s fail", + directory_.c_str()); + return false; + } + + return true; + } + // Put + virtual bool Put(std::string key, uint8_t *buf, int bufLen) { + + // persistent to the DB + leveldb::Slice value((const char *) buf, bufLen); + leveldb::Status status; + status = db_->Put(leveldb::WriteOptions(), key, value); + if (status.ok()) + return true; + else + return false; + } + // Delete + virtual bool Delete(std::string key) { + leveldb::Status status; + status = db_->Delete(leveldb::WriteOptions(), key); + if (status.ok()) + return true; + else + return false; + } + // Get + virtual bool Get(std::string key, std::string &value) { + leveldb::Status status; + status = db_->Get(leveldb::ReadOptions(), key, &value); + if (status.ok()) + return true; + else + return false; + } + // Persistent event + void registerEvent(std::shared_ptr &event) { + event->Serialize( + std::static_pointer_cast(shared_from_this())); + } + // Remove event + void removeEvent(ProvenanceEventRecord *event) { + Delete(event->getEventId()); + } + // destroy + void destroy() { + if (db_) { + delete db_; + db_ = NULL; + } + } + // Run function for the thread + static void run(); + + // Prevent default copy constructor and assignment operation + // Only support pass by reference or pointer + ProvenanceRepository(const ProvenanceRepository &parent) = delete; + ProvenanceRepository &operator=(const ProvenanceRepository &parent) = delete; + + private: + leveldb::DB* db_; + +}; + +} /* namespace provenance */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ +#endif /* LIBMINIFI_INCLUDE_PROVENANCE_PROVENANCEREPOSITORY_H_ */ diff --git a/libminifi/include/utils/FailurePolicy.h b/libminifi/include/utils/FailurePolicy.h index a4a7f9e225..98ec18a61c 100644 --- a/libminifi/include/utils/FailurePolicy.h +++ b/libminifi/include/utils/FailurePolicy.h @@ -17,6 +17,12 @@ #ifndef LIBMINIFI_INCLUDE_UTILS_FAILUREPOLICY_H_ #define LIBMINIFI_INCLUDE_UTILS_FAILUREPOLICY_H_ +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace utils { + /** * Basic failure policy enumeration * @@ -42,4 +48,11 @@ enum FailurePolicy { EXIT }; +} /* namespace utils */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + + #endif /* LIBMINIFI_INCLUDE_UTILS_FAILUREPOLICY_H_ */ diff --git a/libminifi/include/utils/StringUtils.h b/libminifi/include/utils/StringUtils.h index 30858c8ea3..82459dbe35 100644 --- a/libminifi/include/utils/StringUtils.h +++ b/libminifi/include/utils/StringUtils.h @@ -19,7 +19,13 @@ #include #include -#include "../utils/FailurePolicy.h" +#include "utils/FailurePolicy.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace utils { /** * Stateless String utility class. @@ -122,4 +128,11 @@ class StringUtils { }; +} /* namespace utils */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + + #endif /* LIBMINIFI_INCLUDE_IO_STRINGUTILS_H_ */ diff --git a/libminifi/src/AppendHostInfo.cpp b/libminifi/src/AppendHostInfo.cpp deleted file mode 100644 index d0769c17a1..0000000000 --- a/libminifi/src/AppendHostInfo.cpp +++ /dev/null @@ -1,97 +0,0 @@ -/** - * @file AppendHostInfo.cpp - * AppendHostInfo class implementation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include "AppendHostInfo.h" -#include "ProcessContext.h" -#include "ProcessSession.h" - -#include -#include -#include -#include -#include -#include - -#include "io/ClientSocket.h" - -#define __USE_POSIX -#include - -#ifndef HOST_NAME_MAX -#define HOST_NAME_MAX 255 -#endif - -const std::string AppendHostInfo::ProcessorName("AppendHostInfo"); -Property AppendHostInfo::InterfaceName("Network Interface Name", "Network interface from which to read an IP v4 address", "eth0"); -Property AppendHostInfo::HostAttribute("Hostname Attribute", "Flowfile attribute to used to record the agent's hostname", "source.hostname"); -Property AppendHostInfo::IPAttribute("IP Attribute", "Flowfile attribute to used to record the agent's IP address", "source.ipv4"); -Relationship AppendHostInfo::Success("success", "success operational on the flow record"); - -void AppendHostInfo::initialize() -{ - //! Set the supported properties - std::set properties; - properties.insert(InterfaceName); - properties.insert(HostAttribute); - properties.insert(IPAttribute); - setSupportedProperties(properties); - - //! Set the supported relationships - std::set relationships; - relationships.insert(Success); - setSupportedRelationships(relationships); -} - -void AppendHostInfo::onTrigger(ProcessContext *context, ProcessSession *session) -{ - FlowFileRecord *flow = session->get(); - if (!flow) - return; - - //Get Hostname - - std::string hostAttribute = ""; - context->getProperty(HostAttribute.getName(), hostAttribute); - flow->addAttribute(hostAttribute.c_str(), Socket::getMyHostName()); - - //Get IP address for the specified interface - std::string iface; - context->getProperty(InterfaceName.getName(), iface); - //Confirm the specified interface name exists on this device - if (if_nametoindex(iface.c_str()) != 0){ - struct ifreq ifr; - int fd = socket(AF_INET, SOCK_DGRAM, 0); - //Type of address to retrieve - IPv4 IP address - ifr.ifr_addr.sa_family = AF_INET; - //Copy the interface name in the ifreq structure - strncpy(ifr.ifr_name , iface.c_str(), IFNAMSIZ-1); - ioctl(fd, SIOCGIFADDR, &ifr); - close(fd); - - std::string ipAttribute; - context->getProperty(IPAttribute.getName(), ipAttribute); - flow->addAttribute(ipAttribute.c_str(), inet_ntoa(((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr)); - } - - // Transfer to the relationship - session->transfer(flow, Success); -} diff --git a/libminifi/src/Configure.cpp b/libminifi/src/Configure.cpp index 6f5c08d706..76bd03ba03 100644 --- a/libminifi/src/Configure.cpp +++ b/libminifi/src/Configure.cpp @@ -19,12 +19,18 @@ */ #include "Configure.h" #include "utils/StringUtils.h" +#include "core/core.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { Configure *Configure::configure_(NULL); const char *Configure::nifi_flow_configuration_file = "nifi.flow.configuration.file"; const char *Configure::nifi_administrative_yield_duration = "nifi.administrative.yield.duration"; const char *Configure::nifi_bored_yield_duration = "nifi.bored.yield.duration"; -const char *Configure::nifi_graceful_shutdown_seconds = "nifi.graceful.shutdown.seconds"; +const char *Configure::nifi_graceful_shutdown_seconds = "nifi.flowcontroller.graceful.shutdown.period"; const char *Configure::nifi_log_level = "nifi.log.level"; const char *Configure::nifi_server_name = "nifi.server.name"; const char *Configure::nifi_server_port = "nifi.server.port"; @@ -32,11 +38,9 @@ const char *Configure::nifi_server_report_interval= "nifi.server.report.interval const char *Configure::nifi_provenance_repository_max_storage_size = "nifi.provenance.repository.max.storage.size"; const char *Configure::nifi_provenance_repository_max_storage_time = "nifi.provenance.repository.max.storage.time"; const char *Configure::nifi_provenance_repository_directory_default = "nifi.provenance.repository.directory.default"; -const char *Configure::nifi_provenance_repository_enable = "nifi.provenance.repository.enable"; const char *Configure::nifi_flowfile_repository_max_storage_size = "nifi.flowfile.repository.max.storage.size"; const char *Configure::nifi_flowfile_repository_max_storage_time = "nifi.flowfile.repository.max.storage.time"; const char *Configure::nifi_flowfile_repository_directory_default = "nifi.flowfile.repository.directory.default"; -const char *Configure::nifi_flowfile_repository_enable = "nifi.flowfile.repository.enable"; const char *Configure::nifi_remote_input_secure = "nifi.remote.input.secure"; const char *Configure::nifi_security_need_ClientAuth = "nifi.security.need.ClientAuth"; const char *Configure::nifi_security_client_certificate = "nifi.security.client.certificate"; @@ -44,13 +48,13 @@ const char *Configure::nifi_security_client_private_key = "nifi.security.client. const char *Configure::nifi_security_client_pass_phrase = "nifi.security.client.pass.phrase"; const char *Configure::nifi_security_client_ca_certificate = "nifi.security.client.ca.certificate"; -//! Get the config value +// Get the config value bool Configure::get(std::string key, std::string &value) { - std::lock_guard lock(_mtx); - auto it = _properties.find(key); + std::lock_guard lock(mutex_); + auto it = properties_.find(key); - if (it != _properties.end()) + if (it != properties_.end()) { value = it->second; return true; @@ -62,7 +66,7 @@ bool Configure::get(std::string key, std::string &value) } -//! Parse one line in configure file like key=value +// Parse one line in configure file like key=value void Configure::parseConfigureFileLine(char *buf) { char *line = buf; @@ -96,12 +100,12 @@ void Configure::parseConfigureFileLine(char *buf) } std::string value = equal; - key = StringUtils::trimRight(key); - value = StringUtils::trimRight(value); + key = org::apache::nifi::minifi::utils::StringUtils::trimRight(key); + value = org::apache::nifi::minifi::utils::StringUtils::trimRight(value); set(key, value); } -//! Load Configure File +// Load Configure File void Configure::loadConfigureFile(const char *fileName) { @@ -138,7 +142,7 @@ void Configure::loadConfigureFile(const char *fileName) } } -//! Parse Command Line +// Parse Command Line void Configure::parseCommandLine(int argc, char **argv) { int i; @@ -162,3 +166,8 @@ void Configure::parseCommandLine(int argc, char **argv) } return; } + +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/Connection.cpp b/libminifi/src/Connection.cpp index 42dbfe4880..bf216ace9c 100644 --- a/libminifi/src/Connection.cpp +++ b/libminifi/src/Connection.cpp @@ -27,185 +27,162 @@ #include #include +#include "core/FlowFile.h" #include "Connection.h" -#include "Processor.h" -#include "FlowFileRepository.h" -#include "FlowController.h" - -Connection::Connection(std::string name, uuid_t uuid, uuid_t srcUUID, uuid_t destUUID) -: _name(name) -{ - if (!uuid) - // Generate the global UUID for the flow record - uuid_generate(_uuid); - else - uuid_copy(_uuid, uuid); - - if (srcUUID) - uuid_copy(_srcUUID, srcUUID); - if (destUUID) - uuid_copy(_destUUID, destUUID); - - _srcProcessor = NULL; - _destProcessor = NULL; - _maxQueueSize = 0; - _maxQueueDataSize = 0; - _expiredDuration = 0; - _queuedDataSize = 0; - - logger_ = Logger::getLogger(); - - char uuidStr[37]; - uuid_unparse_lower(_uuid, uuidStr); - _uuidStr = uuidStr; - - logger_->log_info("Connection %s created", _name.c_str()); +#include "core/Processor.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { + +Connection::Connection(std::shared_ptr flow_repository, + std::string name, uuid_t uuid, uuid_t srcUUID, + uuid_t destUUID) + : core::Connectable(name, uuid), + flow_repository_(flow_repository) { + + if (srcUUID) + uuid_copy(src_uuid_, srcUUID); + if (destUUID) + uuid_copy(dest_uuid_, destUUID); + + source_connectable_ = nullptr; + dest_connectable_ = nullptr; + max_queue_size_ = 0; + max_data_queue_size_ = 0; + expired_duration_ = 0; + queued_data_size_ = 0; + + logger_ = logging::Logger::getLogger(); + + logger_->log_info("Connection %s created", name_.c_str()); } -bool Connection::isEmpty() -{ - std::lock_guard lock(_mtx); +bool Connection::isEmpty() { + std::lock_guard lock(mutex_); - return _queue.empty(); + return queue_.empty(); } -bool Connection::isFull() -{ - std::lock_guard lock(_mtx); +bool Connection::isFull() { + std::lock_guard lock(mutex_); - if (_maxQueueSize <= 0 && _maxQueueDataSize <= 0) - // No back pressure setting - return false; + if (max_queue_size_ <= 0 && max_data_queue_size_ <= 0) + // No back pressure setting + return false; - if (_maxQueueSize > 0 && _queue.size() >= _maxQueueSize) - return true; + if (max_queue_size_ > 0 && queue_.size() >= max_queue_size_) + return true; - if (_maxQueueDataSize > 0 && _queuedDataSize >= _maxQueueDataSize) - return true; + if (max_data_queue_size_ > 0 && queued_data_size_ >= max_data_queue_size_) + return true; - return false; + return false; } -void Connection::put(FlowFileRecord *flow) -{ - { - std::lock_guard lock(_mtx); - - _queue.push(flow); - - _queuedDataSize += flow->getSize(); - - logger_->log_debug("Enqueue flow file UUID %s to connection %s", - flow->getUUIDStr().c_str(), _name.c_str()); - } - - - if (FlowControllerFactory::getFlowController()->getFlowFileRepository() && - FlowControllerFactory::getFlowController()->getFlowFileRepository()->isEnable() && - !flow->isStoredToRepository()) - { - // Save to the flowfile repo - FlowFileEventRecord event; - event.fromFlowFile(flow, this->_uuidStr); - if (event.Serialize( - FlowControllerFactory::getFlowController()->getFlowFileRepository())) - { - flow->setStoredToRepository(true); - } - } - - // Notify receiving processor that work may be available - if(_destProcessor) - { - _destProcessor->notifyWork(); - } +void Connection::put(std::shared_ptr flow) { + { + std::lock_guard lock(mutex_); + + queue_.push(flow); + + queued_data_size_ += flow->getSize(); + + logger_->log_debug("Enqueue flow file UUID %s to connection %s", + flow->getUUIDStr().c_str(), name_.c_str()); + } + + if (!flow->isStored()) { + // Save to the flowfile repo + FlowFileEventRecord event; + event.fromFlowFile(flow, this->uuidStr_); + if (event.Serialize(flow_repository_)) { + flow->setStoredToRepository(true); + } + } + + // Notify receiving processor that work may be available + if (dest_connectable_) { + dest_connectable_->notifyWork(); + } } -FlowFileRecord* Connection::poll(std::set &expiredFlowRecords) -{ - std::lock_guard lock(_mtx); - - while (!_queue.empty()) - { - FlowFileRecord *item = _queue.front(); - _queue.pop(); - _queuedDataSize -= item->getSize(); - - if (_expiredDuration > 0) - { - // We need to check for flow expiration - if (getTimeMillis() > (item->getEntryDate() + _expiredDuration)) - { - // Flow record expired - expiredFlowRecords.insert(item); - if (FlowControllerFactory::getFlowController()->getFlowFileRepository() && - FlowControllerFactory::getFlowController()->getFlowFileRepository()->isEnable()) - { - // delete from the flowfile repo - FlowControllerFactory::getFlowController()->getFlowFileRepository()->Delete(item->getUUIDStr()); - item->setStoredToRepository(false); - } - } - else - { - // Flow record not expired - if (item->isPenalized()) - { - // Flow record was penalized - _queue.push(item); - _queuedDataSize += item->getSize(); - break; - } - item->setOriginalConnection(this); - logger_->log_debug("Dequeue flow file UUID %s from connection %s", - item->getUUIDStr().c_str(), _name.c_str()); - if (FlowControllerFactory::getFlowController()->getFlowFileRepository() && - FlowControllerFactory::getFlowController()->getFlowFileRepository()->isEnable()) - { - // delete from the flowfile repo - FlowControllerFactory::getFlowController()->getFlowFileRepository()->Delete(item->getUUIDStr()); - item->setStoredToRepository(false); - } - return item; - } - } - else - { - // Flow record not expired - if (item->isPenalized()) - { - // Flow record was penalized - _queue.push(item); - _queuedDataSize += item->getSize(); - break; - } - item->setOriginalConnection(this); - logger_->log_debug("Dequeue flow file UUID %s from connection %s", - item->getUUIDStr().c_str(), _name.c_str()); - if (FlowControllerFactory::getFlowController()->getFlowFileRepository() && - FlowControllerFactory::getFlowController()->getFlowFileRepository()->isEnable()) - { - // delete from the flowfile repo - FlowControllerFactory::getFlowController()->getFlowFileRepository()->Delete(item->getUUIDStr()); - item->setStoredToRepository(false); - } - return item; - } - } - - return NULL; +std::shared_ptr Connection::poll( + std::set> &expiredFlowRecords) { + std::lock_guard lock(mutex_); + + while (!queue_.empty()) { + std::shared_ptr item = queue_.front(); + queue_.pop(); + queued_data_size_ -= item->getSize(); + + if (expired_duration_ > 0) { + // We need to check for flow expiration + if (getTimeMillis() > (item->getEntryDate() + expired_duration_)) { + // Flow record expired + expiredFlowRecords.insert(item); + if (flow_repository_->Delete(item->getUUIDStr())) { + item->setStoredToRepository(false); + } + } else { + // Flow record not expired + if (item->isPenalized()) { + // Flow record was penalized + queue_.push(item); + queued_data_size_ += item->getSize(); + break; + } + std::shared_ptr connectable = std::static_pointer_cast< + Connectable>(shared_from_this()); + item->setOriginalConnection(connectable); + logger_->log_debug("Dequeue flow file UUID %s from connection %s", + item->getUUIDStr().c_str(), name_.c_str()); + + // delete from the flowfile repo + if (flow_repository_->Delete(item->getUUIDStr())) { + item->setStoredToRepository(false); + } + + return item; + } + } else { + // Flow record not expired + if (item->isPenalized()) { + // Flow record was penalized + queue_.push(item); + queued_data_size_ += item->getSize(); + break; + } + std::shared_ptr connectable = std::static_pointer_cast< + Connectable>(shared_from_this()); + item->setOriginalConnection(connectable); + logger_->log_debug("Dequeue flow file UUID %s from connection %s", + item->getUUIDStr().c_str(), name_.c_str()); + // delete from the flowfile repo + if (flow_repository_->Delete(item->getUUIDStr())) { + item->setStoredToRepository(false); + } + + return item; + } + } + + return NULL; } -void Connection::drain() -{ - std::lock_guard lock(_mtx); +void Connection::drain() { + std::lock_guard lock(mutex_); - while (!_queue.empty()) - { - FlowFileRecord *item = _queue.front(); - _queue.pop(); - delete item; - } + while (!queue_.empty()) { + auto &&item = queue_.front(); + queue_.pop(); + } - logger_->log_debug("Drain connection %s", _name.c_str()); + logger_->log_debug("Drain connection %s", name_.c_str()); } + +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/EventDrivenSchedulingAgent.cpp b/libminifi/src/EventDrivenSchedulingAgent.cpp index 53bde4e6e2..0484139fa5 100644 --- a/libminifi/src/EventDrivenSchedulingAgent.cpp +++ b/libminifi/src/EventDrivenSchedulingAgent.cpp @@ -20,28 +20,43 @@ #include #include #include -#include "Property.h" #include "EventDrivenSchedulingAgent.h" +#include "core/Processor.h" +#include "core/ProcessContext.h" +#include "core/ProcessSessionFactory.h" +#include "core/Property.h" -void EventDrivenSchedulingAgent::run(Processor *processor, ProcessContext *processContext, ProcessSessionFactory *sessionFactory) -{ - while (this->_running) - { - bool shouldYield = this->onTrigger(processor, processContext, sessionFactory); +namespace org { +namespace apache { +namespace nifi { +namespace minifi { - if (processor->isYield()) - { - // Honor the yield - std::this_thread::sleep_for(std::chrono::milliseconds(processor->getYieldTime())); - } - else if (shouldYield && this->_boredYieldDuration > 0) - { - // No work to do or need to apply back pressure - std::this_thread::sleep_for(std::chrono::milliseconds(this->_boredYieldDuration)); - } - // Block until work is available - processor->waitForWork(1000); - } - return; +void EventDrivenSchedulingAgent::run( + std::shared_ptr processor, + core::ProcessContext *processContext, + core::ProcessSessionFactory *sessionFactory) { + while (this->running_) { + bool shouldYield = this->onTrigger(processor, processContext, + sessionFactory); + + if (processor->isYield()) { + // Honor the yield + std::this_thread::sleep_for( + std::chrono::milliseconds(processor->getYieldTime())); + } else if (shouldYield && this->_boredYieldDuration > 0) { + // No work to do or need to apply back pressure + std::this_thread::sleep_for( + std::chrono::milliseconds(this->_boredYieldDuration)); + } + + // Block until work is available + processor->waitForWork(1000); + } + return; } + +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/ExecuteProcess.cpp b/libminifi/src/ExecuteProcess.cpp deleted file mode 100644 index 61f96d52a3..0000000000 --- a/libminifi/src/ExecuteProcess.cpp +++ /dev/null @@ -1,251 +0,0 @@ -/** - * @file ExecuteProcess.cpp - * ExecuteProcess class implementation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "ExecuteProcess.h" -#include "ProcessContext.h" -#include "ProcessSession.h" -#include -#include "utils/StringUtils.h" -#include "utils/TimeUtil.h" - -const std::string ExecuteProcess::ProcessorName("ExecuteProcess"); -Property ExecuteProcess::Command("Command", "Specifies the command to be executed; if just the name of an executable is provided, it must be in the user's environment PATH.", ""); -Property ExecuteProcess::CommandArguments("Command Arguments", - "The arguments to supply to the executable delimited by white space. White space can be escaped by enclosing it in double-quotes.", ""); -Property ExecuteProcess::WorkingDir("Working Directory", - "The directory to use as the current working directory when executing the command", ""); -Property ExecuteProcess::BatchDuration("Batch Duration", - "If the process is expected to be long-running and produce textual output, a batch duration can be specified.", "0"); -Property ExecuteProcess::RedirectErrorStream("Redirect Error Stream", - "If true will redirect any error stream output of the process to the output stream.", "false"); -Relationship ExecuteProcess::Success("success", "All created FlowFiles are routed to this relationship."); - -void ExecuteProcess::initialize() -{ - //! Set the supported properties - std::set properties; - properties.insert(Command); - properties.insert(CommandArguments); - properties.insert(WorkingDir); - properties.insert(BatchDuration); - properties.insert(RedirectErrorStream); - setSupportedProperties(properties); - //! Set the supported relationships - std::set relationships; - relationships.insert(Success); - setSupportedRelationships(relationships); -} - - -void ExecuteProcess::onTrigger(ProcessContext *context, ProcessSession *session) -{ - std::string value; - if (context->getProperty(Command.getName(), value)) - { - this->_command = value; - } - if (context->getProperty(CommandArguments.getName(), value)) - { - this->_commandArgument = value; - } - if (context->getProperty(WorkingDir.getName(), value)) - { - this->_workingDir = value; - } - if (context->getProperty(BatchDuration.getName(), value)) - { - TimeUnit unit; - if (Property::StringToTime(value, _batchDuration, unit) && - Property::ConvertTimeUnitToMS(_batchDuration, unit, _batchDuration)) - { - - } - } - if (context->getProperty(RedirectErrorStream.getName(), value)) - { - StringUtils::StringToBool(value, _redirectErrorStream); - } - this->_fullCommand = _command + " " + _commandArgument; - if (_fullCommand.length() == 0) - { - yield(); - return; - } - if (_workingDir.length() > 0 && _workingDir != ".") - { - // change to working directory - if (chdir(_workingDir.c_str()) != 0) - { - logger_->log_error("Execute Command can not chdir %s", _workingDir.c_str()); - yield(); - return; - } - } - logger_->log_info("Execute Command %s", _fullCommand.c_str()); - // split the command into array - char cstr[_fullCommand.length()+1]; - std::strcpy(cstr, _fullCommand.c_str()); - char *p = std::strtok (cstr, " "); - int argc = 0; - char *argv[64]; - while (p != 0 && argc < 64) - { - argv[argc] = p; - p = std::strtok(NULL, " "); - argc++; - } - argv[argc] = NULL; - int status, died; - if (!_processRunning) - { - _processRunning = true; - // if the process has not launched yet - // create the pipe - if (pipe(_pipefd) == -1) - { - _processRunning = false; - yield(); - return; - } - switch (_pid = fork()) - { - case -1: - logger_->log_error("Execute Process fork failed"); - _processRunning = false; - close(_pipefd[0]); - close(_pipefd[1]); - yield(); - break; - case 0 : // this is the code the child runs - close(1); // close stdout - dup(_pipefd[1]); // points pipefd at file descriptor - if (_redirectErrorStream) - // redirect stderr - dup2(_pipefd[1], 2); - close(_pipefd[0]); - execvp(argv[0], argv); - exit(1); - break; - default: // this is the code the parent runs - // the parent isn't going to write to the pipe - close(_pipefd[1]); - if (_batchDuration > 0) - { - while (1) - { - std::this_thread::sleep_for(std::chrono::milliseconds(_batchDuration)); - char buffer[4096]; - int numRead = read(_pipefd[0], buffer, sizeof(buffer)); - if (numRead <= 0) - break; - logger_->log_info("Execute Command Respond %d", numRead); - ExecuteProcess::WriteCallback callback(buffer, numRead); - FlowFileRecord *flowFile = session->create(); - if (!flowFile) - continue; - flowFile->addAttribute("command", _command.c_str()); - flowFile->addAttribute("command.arguments", _commandArgument.c_str()); - session->write(flowFile, &callback); - session->transfer(flowFile, Success); - session->commit(); - } - } - else - { - char buffer[4096]; - char *bufPtr = buffer; - int totalRead = 0; - FlowFileRecord *flowFile = NULL; - while (1) - { - int numRead = read(_pipefd[0], bufPtr, (sizeof(buffer) - totalRead)); - if (numRead <= 0) - { - if (totalRead > 0) - { - logger_->log_info("Execute Command Respond %d", totalRead); - // child exits and close the pipe - ExecuteProcess::WriteCallback callback(buffer, totalRead); - if (!flowFile) - { - flowFile = session->create(); - if (!flowFile) - break; - flowFile->addAttribute("command", _command.c_str()); - flowFile->addAttribute("command.arguments", _commandArgument.c_str()); - session->write(flowFile, &callback); - } - else - { - session->append(flowFile, &callback); - } - session->transfer(flowFile, Success); - } - break; - } - else - { - if (numRead == (sizeof(buffer) - totalRead)) - { - // we reach the max buffer size - logger_->log_info("Execute Command Max Respond %d", sizeof(buffer)); - ExecuteProcess::WriteCallback callback(buffer, sizeof(buffer)); - if (!flowFile) - { - flowFile = session->create(); - if (!flowFile) - continue; - flowFile->addAttribute("command", _command.c_str()); - flowFile->addAttribute("command.arguments", _commandArgument.c_str()); - session->write(flowFile, &callback); - } - else - { - session->append(flowFile, &callback); - } - // Rewind - totalRead = 0; - bufPtr = buffer; - } - else - { - totalRead += numRead; - bufPtr += numRead; - } - } - } - } - - died= wait(&status); - if (WIFEXITED(status)) - { - logger_->log_info("Execute Command Complete %s status %d pid %d", _fullCommand.c_str(), WEXITSTATUS(status), _pid); - } - else - { - logger_->log_info("Execute Command Complete %s status %d pid %d", _fullCommand.c_str(), WTERMSIG(status), _pid); - } - - close(_pipefd[0]); - _processRunning = false; - break; - } - } -} - diff --git a/libminifi/src/FlowControlProtocol.cpp b/libminifi/src/FlowControlProtocol.cpp index 22ef1f90c2..50fc0e21a3 100644 --- a/libminifi/src/FlowControlProtocol.cpp +++ b/libminifi/src/FlowControlProtocol.cpp @@ -27,493 +27,471 @@ #include #include "FlowController.h" #include "FlowControlProtocol.h" - -int FlowControlProtocol::connectServer(const char *host, uint16_t port) -{ - in_addr_t addr; - int sock = 0; - struct hostent *h; +#include "core/core.h" +namespace org { +namespace apache { +namespace nifi { +namespace minifi { + +int FlowControlProtocol::connectServer(const char *host, uint16_t port) { + in_addr_t addr; + int sock = 0; + struct hostent *h; #ifdef __MACH__ - h = gethostbyname(host); + h = gethostbyname(host); #else - char buf[1024]; - struct hostent he; - int hh_errno; - gethostbyname_r(host, &he, buf, sizeof(buf), &h, &hh_errno); + char buf[1024]; + struct hostent he; + int hh_errno; + gethostbyname_r(host, &he, buf, sizeof(buf), &h, &hh_errno); #endif - memcpy((char *) &addr, h->h_addr_list[0], h->h_length); - sock = socket(AF_INET, SOCK_STREAM, 0); - if (sock < 0) - { - logger_->log_error("Could not create socket to hostName %s", host); - return 0; - } + memcpy((char *) &addr, h->h_addr_list[0], h->h_length); + sock = socket(AF_INET, SOCK_STREAM, 0); + if (sock < 0) { + logger_->log_error("Could not create socket to hostName %s", host); + return 0; + } #ifndef __MACH__ - int opt = 1; - bool nagle_off = true; - - if (nagle_off) - { - if (setsockopt(sock, SOL_TCP, TCP_NODELAY, (void *)&opt, sizeof(opt)) < 0) - { - logger_->log_error("setsockopt() TCP_NODELAY failed"); - close(sock); - return 0; - } - if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, - (char *)&opt, sizeof(opt)) < 0) - { - logger_->log_error("setsockopt() SO_REUSEADDR failed"); - close(sock); - return 0; - } - } - - int sndsize = 256*1024; - if (setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (char *)&sndsize, (int)sizeof(sndsize)) < 0) - { - logger_->log_error("setsockopt() SO_SNDBUF failed"); - close(sock); - return 0; - } + int opt = 1; + bool nagle_off = true; + + if (nagle_off) + { + if (setsockopt(sock, SOL_TCP, TCP_NODELAY, (void *)&opt, sizeof(opt)) < 0) + { + logger_->log_error("setsockopt() TCP_NODELAY failed"); + close(sock); + return 0; + } + if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, + (char *)&opt, sizeof(opt)) < 0) + { + logger_->log_error("setsockopt() SO_REUSEADDR failed"); + close(sock); + return 0; + } + } + + int sndsize = 256*1024; + if (setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (char *)&sndsize, (int)sizeof(sndsize)) < 0) + { + logger_->log_error("setsockopt() SO_SNDBUF failed"); + close(sock); + return 0; + } #endif - struct sockaddr_in sa; - socklen_t socklen; - int status; - - memset(&sa, 0, sizeof(sa)); - sa.sin_family = AF_INET; - sa.sin_addr.s_addr = htonl(INADDR_ANY); - sa.sin_port = htons(0); - socklen = sizeof(sa); - if (bind(sock, (struct sockaddr *)&sa, socklen) < 0) - { - logger_->log_error("socket bind failed"); - close(sock); - return 0; - } - - memset(&sa, 0, sizeof(sa)); - sa.sin_family = AF_INET; - sa.sin_addr.s_addr = addr; - sa.sin_port = htons(port); - socklen = sizeof(sa); - - status = connect(sock, (struct sockaddr *)&sa, socklen); - - if (status < 0) - { - logger_->log_error("socket connect failed to %s %d", host, port); - close(sock); - return 0; - } - - logger_->log_info("Flow Control Protocol socket %d connect to server %s port %d success", sock, host, port); - - return sock; + struct sockaddr_in sa; + socklen_t socklen; + int status; + + memset(&sa, 0, sizeof(sa)); + sa.sin_family = AF_INET; + sa.sin_addr.s_addr = htonl(INADDR_ANY); + sa.sin_port = htons(0); + socklen = sizeof(sa); + if (bind(sock, (struct sockaddr *) &sa, socklen) < 0) { + logger_->log_error("socket bind failed"); + close(sock); + return 0; + } + + memset(&sa, 0, sizeof(sa)); + sa.sin_family = AF_INET; + sa.sin_addr.s_addr = addr; + sa.sin_port = htons(port); + socklen = sizeof(sa); + + status = connect(sock, (struct sockaddr *) &sa, socklen); + + if (status < 0) { + logger_->log_error("socket connect failed to %s %d", host, port); + close(sock); + return 0; + } + + logger_->log_info( + "Flow Control Protocol socket %d connect to server %s port %d success", + sock, host, port); + + return sock; } -int FlowControlProtocol::sendData(uint8_t *buf, int buflen) -{ - int ret = 0, bytes = 0; - - while (bytes < buflen) - { - ret = send(_socket, buf+bytes, buflen-bytes, 0); - //check for errors - if (ret == -1) - { - return ret; - } - bytes+=ret; - } - - return bytes; +int FlowControlProtocol::sendData(uint8_t *buf, int buflen) { + int ret = 0, bytes = 0; + + while (bytes < buflen) { + ret = send(_socket, buf + bytes, buflen - bytes, 0); + //check for errors + if (ret == -1) { + return ret; + } + bytes += ret; + } + + return bytes; } -int FlowControlProtocol::selectClient(int msec) -{ - fd_set fds; - struct timeval tv; - int retval; - int fd = _socket; - - FD_ZERO(&fds); - FD_SET(fd, &fds); - - tv.tv_sec = msec/1000; - tv.tv_usec = (msec % 1000) * 1000; - - if (msec > 0) - retval = select(fd+1, &fds, NULL, NULL, &tv); - else - retval = select(fd+1, &fds, NULL, NULL, NULL); - - if (retval <= 0) - return retval; - if (FD_ISSET(fd, &fds)) - return retval; - else - return 0; +int FlowControlProtocol::selectClient(int msec) { + fd_set fds; + struct timeval tv; + int retval; + int fd = _socket; + + FD_ZERO(&fds); + FD_SET(fd, &fds); + + tv.tv_sec = msec / 1000; + tv.tv_usec = (msec % 1000) * 1000; + + if (msec > 0) + retval = select(fd + 1, &fds, NULL, NULL, &tv); + else + retval = select(fd + 1, &fds, NULL, NULL, NULL); + + if (retval <= 0) + return retval; + if (FD_ISSET(fd, &fds)) + return retval; + else + return 0; } -int FlowControlProtocol::readData(uint8_t *buf, int buflen) -{ - int sendSize = buflen; - - while (buflen) - { - int status; - status = selectClient(MAX_READ_TIMEOUT); - if (status <= 0) - { - return status; - } +int FlowControlProtocol::readData(uint8_t *buf, int buflen) { + int sendSize = buflen; + + while (buflen) { + int status; + status = selectClient(MAX_READ_TIMEOUT); + if (status <= 0) { + return status; + } #ifndef __MACH__ - status = read(_socket, buf, buflen); + status = read(_socket, buf, buflen); #else - status = recv(_socket, buf, buflen, 0); + status = recv(_socket, buf, buflen, 0); #endif - if (status <= 0) - { - return status; - } - buflen -= status; - buf += status; - } - - return sendSize; + if (status <= 0) { + return status; + } + buflen -= status; + buf += status; + } + + return sendSize; } -int FlowControlProtocol::readHdr(FlowControlProtocolHeader *hdr) -{ - uint8_t buffer[sizeof(FlowControlProtocolHeader)]; +int FlowControlProtocol::readHdr(FlowControlProtocolHeader *hdr) { + uint8_t buffer[sizeof(FlowControlProtocolHeader)]; - uint8_t *data = buffer; + uint8_t *data = buffer; - int status = readData(buffer, sizeof(FlowControlProtocolHeader)); - if (status <= 0) - return status; + int status = readData(buffer, sizeof(FlowControlProtocolHeader)); + if (status <= 0) + return status; - uint32_t value; - data = this->decode(data, value); - hdr->msgType = value; + uint32_t value; + data = this->decode(data, value); + hdr->msgType = value; - data = this->decode(data, value); - hdr->seqNumber = value; + data = this->decode(data, value); + hdr->seqNumber = value; - data = this->decode(data, value); - hdr->status = value; + data = this->decode(data, value); + hdr->status = value; - data = this->decode(data, value); - hdr->payloadLen = value; + data = this->decode(data, value); + hdr->payloadLen = value; - return sizeof(FlowControlProtocolHeader); + return sizeof(FlowControlProtocolHeader); } -void FlowControlProtocol::start() -{ - if (_reportInterval <= 0) - return; - if (_running) - return; - _running = true; - logger_->log_info("FlowControl Protocol Start"); - _thread = new std::thread(run, this); - _thread->detach(); +void FlowControlProtocol::start() { + if (_reportInterval <= 0) + return; + if (running_) + return; + running_ = true; + logger_->log_info("FlowControl Protocol Start"); + _thread = new std::thread(run, this); + _thread->detach(); } -void FlowControlProtocol::stop() -{ - if (!_running) - return; - _running = false; - logger_->log_info("FlowControl Protocol Stop"); +void FlowControlProtocol::stop() { + if (!running_) + return; + running_ = false; + logger_->log_info("FlowControl Protocol Stop"); } -void FlowControlProtocol::run(FlowControlProtocol *protocol) -{ - while (protocol->_running) - { - std::this_thread::sleep_for(std::chrono::milliseconds(protocol->_reportInterval)); - if (!protocol->_registered) - { - // if it is not register yet - protocol->sendRegisterReq(); - } - else - protocol->sendReportReq(); - } - return; +void FlowControlProtocol::run(FlowControlProtocol *protocol) { + while (protocol->running_) { + std::this_thread::sleep_for( + std::chrono::milliseconds(protocol->_reportInterval)); + if (!protocol->_registered) { + // if it is not register yet + protocol->sendRegisterReq(); + } else + protocol->sendReportReq(); + } + return; } -int FlowControlProtocol::sendRegisterReq() -{ - if (_registered) - { - logger_->log_info("Already registered"); - return -1; - } - - uint16_t port = this->_serverPort; - - if (this->_socket <= 0) - this->_socket = connectServer(_serverName.c_str(), port); - - if (this->_socket <= 0) - return -1; - - // Calculate the total payload msg size - uint32_t payloadSize = FlowControlMsgIDEncodingLen(FLOW_SERIAL_NUMBER, 0) + - FlowControlMsgIDEncodingLen(FLOW_YML_NAME, this->_controller->getName().size()+1); - uint32_t size = sizeof(FlowControlProtocolHeader) + payloadSize; - - uint8_t *data = new uint8_t[size]; - uint8_t *start = data; - - // encode the HDR - FlowControlProtocolHeader hdr; - hdr.msgType = REGISTER_REQ; - hdr.payloadLen = payloadSize; - hdr.seqNumber = this->_seqNumber; - hdr.status = RESP_SUCCESS; - data = this->encode(data, hdr.msgType); - data = this->encode(data, hdr.seqNumber); - data = this->encode(data, hdr.status); - data = this->encode(data, hdr.payloadLen); - - // encode the serial number - data = this->encode(data, FLOW_SERIAL_NUMBER); - data = this->encode(data, this->_serialNumber, 8); - - // encode the YAML name - data = this->encode(data, FLOW_YML_NAME); - data = this->encode(data, this->_controller->getName()); - - // send it - int status = sendData(start, size); - delete[] start; - if (status <= 0) - { - close(_socket); - _socket = 0; - logger_->log_error("Flow Control Protocol Send Register Req failed"); - return -1; - } - - // Looking for register respond - status = readHdr(&hdr); - - if (status <= 0) - { - close(_socket); - _socket = 0; - logger_->log_error("Flow Control Protocol Read Register Resp header failed"); - return -1; - } - logger_->log_info("Flow Control Protocol receive MsgType %s", FlowControlMsgTypeToStr((FlowControlMsgType) hdr.msgType)); - logger_->log_info("Flow Control Protocol receive Seq Num %d", hdr.seqNumber); - logger_->log_info("Flow Control Protocol receive Resp Code %s", FlowControlRespCodeToStr((FlowControlRespCode) hdr.status)); - logger_->log_info("Flow Control Protocol receive Payload len %d", hdr.payloadLen); - - if (hdr.status == RESP_SUCCESS && hdr.seqNumber == this->_seqNumber) - { - this->_registered = true; - this->_seqNumber++; - logger_->log_info("Flow Control Protocol Register success"); - uint8_t *payload = new uint8_t[hdr.payloadLen]; - uint8_t *payloadPtr = payload; - status = readData(payload, hdr.payloadLen); - if (status <= 0) - { - delete[] payload; - logger_->log_info("Flow Control Protocol Register Read Payload fail"); - close(_socket); - _socket = 0; - return -1; - } - while (payloadPtr < (payload + hdr.payloadLen)) - { - uint32_t msgID; - payloadPtr = this->decode(payloadPtr, msgID); - if (((FlowControlMsgID) msgID) == REPORT_INTERVAL) - { - // Fixed 4 bytes - uint32_t reportInterval; - payloadPtr = this->decode(payloadPtr, reportInterval); - logger_->log_info("Flow Control Protocol receive report interval %d ms", reportInterval); - this->_reportInterval = reportInterval; - } - else - { - break; - } - } - delete[] payload; - close(_socket); - _socket = 0; - return 0; - } - else - { - logger_->log_info("Flow Control Protocol Register fail"); - close(_socket); - _socket = 0; - return -1; - } +int FlowControlProtocol::sendRegisterReq() { + if (_registered) { + logger_->log_info("Already registered"); + return -1; + } + + uint16_t port = this->_serverPort; + + if (this->_socket <= 0) + this->_socket = connectServer(_serverName.c_str(), port); + + if (this->_socket <= 0) + return -1; + + // Calculate the total payload msg size + uint32_t payloadSize = FlowControlMsgIDEncodingLen(FLOW_SERIAL_NUMBER, 0) + + FlowControlMsgIDEncodingLen(FLOW_YML_NAME, + this->_controller->getName().size() + 1); + uint32_t size = sizeof(FlowControlProtocolHeader) + payloadSize; + + uint8_t *data = new uint8_t[size]; + uint8_t *start = data; + + // encode the HDR + FlowControlProtocolHeader hdr; + hdr.msgType = REGISTER_REQ; + hdr.payloadLen = payloadSize; + hdr.seqNumber = this->_seqNumber; + hdr.status = RESP_SUCCESS; + data = this->encode(data, hdr.msgType); + data = this->encode(data, hdr.seqNumber); + data = this->encode(data, hdr.status); + data = this->encode(data, hdr.payloadLen); + + // encode the serial number + data = this->encode(data, FLOW_SERIAL_NUMBER); + data = this->encode(data, this->_serialNumber, 8); + + // encode the YAML name + data = this->encode(data, FLOW_YML_NAME); + data = this->encode(data, this->_controller->getName()); + + // send it + int status = sendData(start, size); + delete[] start; + if (status <= 0) { + close(_socket); + _socket = 0; + logger_->log_error("Flow Control Protocol Send Register Req failed"); + return -1; + } + + // Looking for register respond + status = readHdr(&hdr); + + if (status <= 0) { + close(_socket); + _socket = 0; + logger_->log_error( + "Flow Control Protocol Read Register Resp header failed"); + return -1; + } + logger_->log_info("Flow Control Protocol receive MsgType %s", + FlowControlMsgTypeToStr((FlowControlMsgType) hdr.msgType)); + logger_->log_info("Flow Control Protocol receive Seq Num %d", hdr.seqNumber); + logger_->log_info("Flow Control Protocol receive Resp Code %s", + FlowControlRespCodeToStr((FlowControlRespCode) hdr.status)); + logger_->log_info("Flow Control Protocol receive Payload len %d", + hdr.payloadLen); + + if (hdr.status == RESP_SUCCESS && hdr.seqNumber == this->_seqNumber) { + this->_registered = true; + this->_seqNumber++; + logger_->log_info("Flow Control Protocol Register success"); + uint8_t *payload = new uint8_t[hdr.payloadLen]; + uint8_t *payloadPtr = payload; + status = readData(payload, hdr.payloadLen); + if (status <= 0) { + delete[] payload; + logger_->log_info("Flow Control Protocol Register Read Payload fail"); + close(_socket); + _socket = 0; + return -1; + } + while (payloadPtr < (payload + hdr.payloadLen)) { + uint32_t msgID; + payloadPtr = this->decode(payloadPtr, msgID); + if (((FlowControlMsgID) msgID) == REPORT_INTERVAL) { + // Fixed 4 bytes + uint32_t reportInterval; + payloadPtr = this->decode(payloadPtr, reportInterval); + logger_->log_info("Flow Control Protocol receive report interval %d ms", + reportInterval); + this->_reportInterval = reportInterval; + } else { + break; + } + } + delete[] payload; + close(_socket); + _socket = 0; + return 0; + } else { + logger_->log_info("Flow Control Protocol Register fail"); + close(_socket); + _socket = 0; + return -1; + } } - -int FlowControlProtocol::sendReportReq() -{ - uint16_t port = this->_serverPort; - - if (this->_socket <= 0) - this->_socket = connectServer(_serverName.c_str(), port); - - if (this->_socket <= 0) - return -1; - - // Calculate the total payload msg size - uint32_t payloadSize = - FlowControlMsgIDEncodingLen(FLOW_YML_NAME, this->_controller->getName().size()+1); - uint32_t size = sizeof(FlowControlProtocolHeader) + payloadSize; - - uint8_t *data = new uint8_t[size]; - uint8_t *start = data; - - // encode the HDR - FlowControlProtocolHeader hdr; - hdr.msgType = REPORT_REQ; - hdr.payloadLen = payloadSize; - hdr.seqNumber = this->_seqNumber; - hdr.status = RESP_SUCCESS; - data = this->encode(data, hdr.msgType); - data = this->encode(data, hdr.seqNumber); - data = this->encode(data, hdr.status); - data = this->encode(data, hdr.payloadLen); - - // encode the YAML name - data = this->encode(data, FLOW_YML_NAME); - data = this->encode(data, this->_controller->getName()); - - // send it - int status = sendData(start, size); - delete[] start; - if (status <= 0) - { - close(_socket); - _socket = 0; - logger_->log_error("Flow Control Protocol Send Report Req failed"); - return -1; - } - - // Looking for report respond - status = readHdr(&hdr); - - if (status <= 0) - { - close(_socket); - _socket = 0; - logger_->log_error("Flow Control Protocol Read Report Resp header failed"); - return -1; - } - logger_->log_info("Flow Control Protocol receive MsgType %s", FlowControlMsgTypeToStr((FlowControlMsgType) hdr.msgType)); - logger_->log_info("Flow Control Protocol receive Seq Num %d", hdr.seqNumber); - logger_->log_info("Flow Control Protocol receive Resp Code %s", FlowControlRespCodeToStr((FlowControlRespCode) hdr.status)); - logger_->log_info("Flow Control Protocol receive Payload len %d", hdr.payloadLen); - - if (hdr.status == RESP_SUCCESS && hdr.seqNumber == this->_seqNumber) - { - this->_seqNumber++; - uint8_t *payload = new uint8_t[hdr.payloadLen]; - uint8_t *payloadPtr = payload; - status = readData(payload, hdr.payloadLen); - if (status <= 0) - { - delete[] payload; - logger_->log_info("Flow Control Protocol Report Resp Read Payload fail"); - close(_socket); - _socket = 0; - return -1; - } - std::string processor; - std::string propertyName; - std::string propertyValue; - while (payloadPtr < (payload + hdr.payloadLen)) - { - uint32_t msgID; - payloadPtr = this->decode(payloadPtr, msgID); - if (((FlowControlMsgID) msgID) == PROCESSOR_NAME) - { - uint32_t len; - payloadPtr = this->decode(payloadPtr, len); - processor = (const char *) payloadPtr; - payloadPtr += len; - logger_->log_info("Flow Control Protocol receive report resp processor %s", processor.c_str()); - } - else if (((FlowControlMsgID) msgID) == PROPERTY_NAME) - { - uint32_t len; - payloadPtr = this->decode(payloadPtr, len); - propertyName = (const char *) payloadPtr; - payloadPtr += len; - logger_->log_info("Flow Control Protocol receive report resp property name %s", propertyName.c_str()); - } - else if (((FlowControlMsgID) msgID) == PROPERTY_VALUE) - { - uint32_t len; - payloadPtr = this->decode(payloadPtr, len); - propertyValue = (const char *) payloadPtr; - payloadPtr += len; - logger_->log_info("Flow Control Protocol receive report resp property value %s", propertyValue.c_str()); - this->_controller->updatePropertyValue(processor, propertyName, propertyValue); - } - else - { - break; - } - } - delete[] payload; - close(_socket); - _socket = 0; - return 0; - } - else if (hdr.status == RESP_TRIGGER_REGISTER && hdr.seqNumber == this->_seqNumber) - { - logger_->log_info("Flow Control Protocol trigger reregister"); - this->_registered = false; - this->_seqNumber++; - close(_socket); - _socket = 0; - return 0; - } - else if (hdr.status == RESP_STOP_FLOW_CONTROLLER && hdr.seqNumber == this->_seqNumber) - { - logger_->log_info("Flow Control Protocol stop flow controller"); - this->_controller->stop(true); - this->_seqNumber++; - close(_socket); - _socket = 0; - return 0; - } - else if (hdr.status == RESP_START_FLOW_CONTROLLER && hdr.seqNumber == this->_seqNumber) - { - logger_->log_info("Flow Control Protocol start flow controller"); - this->_controller->start(); - this->_seqNumber++; - close(_socket); - _socket = 0; - return 0; - } - else - { - logger_->log_info("Flow Control Protocol Report fail"); - close(_socket); - _socket = 0; - return -1; - } +int FlowControlProtocol::sendReportReq() { + uint16_t port = this->_serverPort; + + if (this->_socket <= 0) + this->_socket = connectServer(_serverName.c_str(), port); + + if (this->_socket <= 0) + return -1; + + // Calculate the total payload msg size + uint32_t payloadSize = FlowControlMsgIDEncodingLen( + FLOW_YML_NAME, this->_controller->getName().size() + 1); + uint32_t size = sizeof(FlowControlProtocolHeader) + payloadSize; + + uint8_t *data = new uint8_t[size]; + uint8_t *start = data; + + // encode the HDR + FlowControlProtocolHeader hdr; + hdr.msgType = REPORT_REQ; + hdr.payloadLen = payloadSize; + hdr.seqNumber = this->_seqNumber; + hdr.status = RESP_SUCCESS; + data = this->encode(data, hdr.msgType); + data = this->encode(data, hdr.seqNumber); + data = this->encode(data, hdr.status); + data = this->encode(data, hdr.payloadLen); + + // encode the YAML name + data = this->encode(data, FLOW_YML_NAME); + data = this->encode(data, this->_controller->getName()); + + // send it + int status = sendData(start, size); + delete[] start; + if (status <= 0) { + close(_socket); + _socket = 0; + logger_->log_error("Flow Control Protocol Send Report Req failed"); + return -1; + } + + // Looking for report respond + status = readHdr(&hdr); + + if (status <= 0) { + close(_socket); + _socket = 0; + logger_->log_error("Flow Control Protocol Read Report Resp header failed"); + return -1; + } + logger_->log_info("Flow Control Protocol receive MsgType %s", + FlowControlMsgTypeToStr((FlowControlMsgType) hdr.msgType)); + logger_->log_info("Flow Control Protocol receive Seq Num %d", hdr.seqNumber); + logger_->log_info("Flow Control Protocol receive Resp Code %s", + FlowControlRespCodeToStr((FlowControlRespCode) hdr.status)); + logger_->log_info("Flow Control Protocol receive Payload len %d", + hdr.payloadLen); + + if (hdr.status == RESP_SUCCESS && hdr.seqNumber == this->_seqNumber) { + this->_seqNumber++; + uint8_t *payload = new uint8_t[hdr.payloadLen]; + uint8_t *payloadPtr = payload; + status = readData(payload, hdr.payloadLen); + if (status <= 0) { + delete[] payload; + logger_->log_info("Flow Control Protocol Report Resp Read Payload fail"); + close(_socket); + _socket = 0; + return -1; + } + std::string processor; + std::string propertyName; + std::string propertyValue; + while (payloadPtr < (payload + hdr.payloadLen)) { + uint32_t msgID; + payloadPtr = this->decode(payloadPtr, msgID); + if (((FlowControlMsgID) msgID) == PROCESSOR_NAME) { + uint32_t len; + payloadPtr = this->decode(payloadPtr, len); + processor = (const char *) payloadPtr; + payloadPtr += len; + logger_->log_info( + "Flow Control Protocol receive report resp processor %s", + processor.c_str()); + } else if (((FlowControlMsgID) msgID) == PROPERTY_NAME) { + uint32_t len; + payloadPtr = this->decode(payloadPtr, len); + propertyName = (const char *) payloadPtr; + payloadPtr += len; + logger_->log_info( + "Flow Control Protocol receive report resp property name %s", + propertyName.c_str()); + } else if (((FlowControlMsgID) msgID) == PROPERTY_VALUE) { + uint32_t len; + payloadPtr = this->decode(payloadPtr, len); + propertyValue = (const char *) payloadPtr; + payloadPtr += len; + logger_->log_info( + "Flow Control Protocol receive report resp property value %s", + propertyValue.c_str()); + this->_controller->updatePropertyValue(processor, propertyName, + propertyValue); + } else { + break; + } + } + delete[] payload; + close(_socket); + _socket = 0; + return 0; + } else if (hdr.status == RESP_TRIGGER_REGISTER + && hdr.seqNumber == this->_seqNumber) { + logger_->log_info("Flow Control Protocol trigger reregister"); + this->_registered = false; + this->_seqNumber++; + close(_socket); + _socket = 0; + return 0; + } else if (hdr.status == RESP_STOP_FLOW_CONTROLLER + && hdr.seqNumber == this->_seqNumber) { + logger_->log_info("Flow Control Protocol stop flow controller"); + this->_controller->stop(true); + this->_seqNumber++; + close(_socket); + _socket = 0; + return 0; + } else if (hdr.status == RESP_START_FLOW_CONTROLLER + && hdr.seqNumber == this->_seqNumber) { + logger_->log_info("Flow Control Protocol start flow controller"); + this->_controller->start(); + this->_seqNumber++; + close(_socket); + _socket = 0; + return 0; + } else { + logger_->log_info("Flow Control Protocol Report fail"); + close(_socket); + _socket = 0; + return -1; + } } +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/FlowController.cpp b/libminifi/src/FlowController.cpp index eca84be2af..74e0b2bb5f 100644 --- a/libminifi/src/FlowController.cpp +++ b/libminifi/src/FlowController.cpp @@ -30,221 +30,241 @@ #include #include #include "FlowController.h" -#include "ProcessContext.h" +#include "core/ProcessContext.h" +#include "core/ProcessGroup.h" #include "utils/StringUtils.h" +#include "core/core.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { + +FlowControllerImpl::FlowControllerImpl( + std::shared_ptr repo , std::shared_ptr flow_file_repo, std::string name) + : FlowController(repo,flow_file_repo) { + uuid_generate(uuid_); + setUUID(uuid_); + + // Setup the default values + configuration_filename_ = DEFAULT_FLOW_YAML_FILENAME; + max_event_driven_threads_ = DEFAULT_MAX_EVENT_DRIVEN_THREAD; + max_timer_driven_threads_ = DEFAULT_MAX_TIMER_DRIVEN_THREAD; + running_ = false; + initialized_ = false; + root_ = NULL; + logger_ = logging::Logger::getLogger(); + protocol_ = new FlowControlProtocol(this); + + // NiFi config properties + configure_ = Configure::getConfigure(); + + std::string rawConfigFileString; + configure_->get(Configure::nifi_flow_configuration_file, rawConfigFileString); + + if (!rawConfigFileString.empty()) { + configuration_filename_ = rawConfigFileString; + } + + char *path = NULL; + char full_path[PATH_MAX]; + + std::string adjustedFilename; + if (!configuration_filename_.empty()) { + // perform a naive determination if this is a relative path + if (configuration_filename_.c_str()[0] != '/') { + adjustedFilename = adjustedFilename + configure_->getHome() + "/" + + configuration_filename_; + } else { + adjustedFilename = configuration_filename_; + } + } + + path = realpath(adjustedFilename.c_str(), full_path); + + if (path == NULL) { + throw std::runtime_error( + "Path is not specified. Either manually set MINIFI_HOME or ensure ../conf exists"); + } + std::string pathString(path); + configuration_filename_ = pathString; + logger_->log_info("FlowController NiFi Configuration file %s", + pathString.c_str()); + + // Create the content repo directory if needed + struct stat contentDirStat; + + if (stat(ResourceClaim::default_directory_path.c_str(), &contentDirStat) + != -1&& S_ISDIR(contentDirStat.st_mode)) { + path = realpath(ResourceClaim::default_directory_path.c_str(), full_path); + logger_->log_info("FlowController content directory %s", full_path); + } else { + if (mkdir(ResourceClaim::default_directory_path.c_str(), 0777) == -1) { + logger_->log_error("FlowController content directory creation failed"); + exit(1); + } + } -FlowController *FlowControllerFactory::_flowController(NULL); - -FlowControllerImpl::FlowControllerImpl(std::string name) { - uuid_generate(_uuid); - - _name = name; - // Setup the default values - _configurationFileName = DEFAULT_FLOW_YAML_FILE_NAME; - _maxEventDrivenThreads = DEFAULT_MAX_EVENT_DRIVEN_THREAD; - _maxTimerDrivenThreads = DEFAULT_MAX_TIMER_DRIVEN_THREAD; - _running = false; - _initialized = false; - _root = NULL; - logger_ = Logger::getLogger(); - _protocol = new FlowControlProtocol(this); - - // NiFi config properties - configure_ = Configure::getConfigure(); - - std::string rawConfigFileString; - configure_->get(Configure::nifi_flow_configuration_file, - rawConfigFileString); - - if (!rawConfigFileString.empty()) { - _configurationFileName = rawConfigFileString; - } - - char *path = NULL; - char full_path[PATH_MAX]; - - std::string adjustedFilename; - if (!_configurationFileName.empty()) { - // perform a naive determination if this is a relative path - if (_configurationFileName.c_str()[0] != '/') { - adjustedFilename = adjustedFilename + configure_->getHome() + "/" - + _configurationFileName; - } else { - adjustedFilename = _configurationFileName; - } - } - - path = realpath(adjustedFilename.c_str(), full_path); - - std::string pathString(path); - _configurationFileName = pathString; - logger_->log_info("FlowController NiFi Configuration file %s", pathString.c_str()); - - // Create the content repo directory if needed - struct stat contentDirStat; - - if (stat(ResourceClaim::default_directory_path.c_str(), &contentDirStat) != -1 && S_ISDIR(contentDirStat.st_mode)) - { - path = realpath(ResourceClaim::default_directory_path.c_str(), full_path); - logger_->log_info("FlowController content directory %s", full_path); - } - else - { - if (mkdir(ResourceClaim::default_directory_path.c_str(), 0777) == -1) - { - logger_->log_error("FlowController content directory creation failed"); - exit(1); - } - } - - - std::string clientAuthStr; + std::string clientAuthStr; - if (!path) { - logger_->log_error( - "Could not locate path from provided configuration file name (%s). Exiting.", - full_path); - exit(1); - } + if (!path) { + logger_->log_error( + "Could not locate path from provided configuration file name (%s). Exiting.", + full_path); + exit(1); + } - // Create repos for flow record and provenance - _flowfileRepo = new FlowFileRepository(); - _flowfileRepo->initialize(); - _provenanceRepo = new ProvenanceRepository(); - _provenanceRepo->initialize(); } FlowControllerImpl::~FlowControllerImpl() { - - stop(true); - unload(); - if (NULL != _protocol) - delete _protocol; - if (NULL != _provenanceRepo) - delete _provenanceRepo; - if (NULL != _flowfileRepo) - delete _flowfileRepo; + stop(true); + unload(); + if (NULL != protocol_) + delete protocol_; } void FlowControllerImpl::stop(bool force) { - if (_running) { - // immediately indicate that we are not running - _running = false; - - logger_->log_info("Stop Flow Controller"); - this->_timerScheduler.stop(); - this->_eventScheduler.stop(); - this->_flowfileRepo->stop(); - this->_provenanceRepo->stop(); - // Wait for sometime for thread stop - std::this_thread::sleep_for(std::chrono::milliseconds(1000)); - if (this->_root) - this->_root->stopProcessing(&this->_timerScheduler, - &this->_eventScheduler); - - } + if (running_) { + // immediately indicate that we are not running + running_ = false; + + logger_->log_info("Stop Flow Controller"); + this->_timerScheduler.stop(); + this->_eventScheduler.stop(); + this->flow_file_repo_->stop(); + this->provenance_repo_->stop(); + // Wait for sometime for thread stop + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + if (this->root_) + this->root_->stopProcessing(&this->_timerScheduler, + &this->_eventScheduler); + + } } /** * This function will attempt to unload yaml and stop running Processors. * * If the latter attempt fails or does not complete within the prescribed - * period, _running will be set to false and we will return. + * period, running_ will be set to false and we will return. * * @param timeToWaitMs Maximum time to wait before manually * marking running as false. */ void FlowControllerImpl::waitUnload(const uint64_t timeToWaitMs) { - if (_running) { - // use the current time and increment with the provided argument. - std::chrono::system_clock::time_point wait_time = - std::chrono::system_clock::now() - + std::chrono::milliseconds(timeToWaitMs); - - // create an asynchronous future. - std::future unload_task = std::async(std::launch::async, - [this]() {unload();}); - - if (std::future_status::ready == unload_task.wait_until(wait_time)) { - _running = false; - } + if (running_) { + // use the current time and increment with the provided argument. + std::chrono::system_clock::time_point wait_time = + std::chrono::system_clock::now() + + std::chrono::milliseconds(timeToWaitMs); + + // create an asynchronous future. + std::future unload_task = std::async(std::launch::async, + [this]() {unload();}); + + if (std::future_status::ready == unload_task.wait_until(wait_time)) { + running_ = false; + } - } + } } - void FlowControllerImpl::unload() { - if (_running) { - stop(true); - } - if (_initialized) { - logger_->log_info("Unload Flow Controller"); - if (_root) - delete _root; - _root = NULL; - _initialized = false; - _name = ""; - } - - return; + if (running_) { + stop(true); + } + if (initialized_) { + logger_->log_info("Unload Flow Controller"); + if (root_) + delete root_; + root_ = NULL; + initialized_ = false; + name_ = ""; + } + + return; } -Processor *FlowControllerImpl::createProcessor(std::string name, uuid_t uuid) { - Processor *processor = NULL; - if (name == GenerateFlowFile::ProcessorName) { - processor = new GenerateFlowFile(name, uuid); - } else if (name == LogAttribute::ProcessorName) { - processor = new LogAttribute(name, uuid); - } else if (name == RealTimeDataCollector::ProcessorName) { - processor = new RealTimeDataCollector(name, uuid); - } else if (name == GetFile::ProcessorName) { - processor = new GetFile(name, uuid); - } else if (name == PutFile::ProcessorName) { - processor = new PutFile(name, uuid); - } else if (name == TailFile::ProcessorName) { - processor = new TailFile(name, uuid); - } else if (name == ListenSyslog::ProcessorName) { - processor = new ListenSyslog(name, uuid); - } else if (name == ListenHTTP::ProcessorName) { - processor = new ListenHTTP(name, uuid); - } else if (name == ExecuteProcess::ProcessorName) { - processor = new ExecuteProcess(name, uuid); - } else if (name == AppendHostInfo::ProcessorName) { - processor = new AppendHostInfo(name, uuid); - } else { - logger_->log_error("No Processor defined for %s", name.c_str()); - return NULL; - } - - //! initialize the processor - processor->initialize(); - - return processor; +std::shared_ptr FlowControllerImpl::createProcessor( + std::string name, uuid_t uuid) { + std::shared_ptr processor = nullptr; + if (name + == org::apache::nifi::minifi::processors::GenerateFlowFile::ProcessorName) { + processor = std::make_shared< + org::apache::nifi::minifi::processors::GenerateFlowFile>(name, uuid); + } else if (name + == org::apache::nifi::minifi::processors::LogAttribute::ProcessorName) { + processor = std::make_shared< + org::apache::nifi::minifi::processors::LogAttribute>(name, uuid); + } else if (name + == org::apache::nifi::minifi::processors::RealTimeDataCollector::ProcessorName) { + processor = std::make_shared< + org::apache::nifi::minifi::processors::RealTimeDataCollector>(name, + uuid); + } else if (name + == org::apache::nifi::minifi::processors::GetFile::ProcessorName) { + processor = + std::make_shared(name, + uuid); + } else if (name + == org::apache::nifi::minifi::processors::PutFile::ProcessorName) { + processor = + std::make_shared(name, + uuid); + } else if (name + == org::apache::nifi::minifi::processors::TailFile::ProcessorName) { + processor = + std::make_shared(name, + uuid); + } else if (name + == org::apache::nifi::minifi::processors::ListenSyslog::ProcessorName) { + processor = std::make_shared< + org::apache::nifi::minifi::processors::ListenSyslog>(name, uuid); + } else if (name + == org::apache::nifi::minifi::processors::ListenHTTP::ProcessorName) { + processor = std::make_shared< + org::apache::nifi::minifi::processors::ListenHTTP>(name, uuid); + } else if (name + == org::apache::nifi::minifi::processors::ExecuteProcess::ProcessorName) { + processor = std::make_shared< + org::apache::nifi::minifi::processors::ExecuteProcess>(name, uuid); + } else if (name + == org::apache::nifi::minifi::processors::AppendHostInfo::ProcessorName) { + processor = std::make_shared< + org::apache::nifi::minifi::processors::AppendHostInfo>(name, uuid); + } else { + logger_->log_error("No Processor defined for %s", name.c_str()); + return nullptr; + } + + // initialize the processor + processor->initialize(); + + return processor; } -ProcessGroup *FlowControllerImpl::createRootProcessGroup(std::string name, - uuid_t uuid) { - return new ProcessGroup(ROOT_PROCESS_GROUP, name, uuid); +core::ProcessGroup *FlowControllerImpl::createRootProcessGroup(std::string name, + uuid_t uuid) { + return new core::ProcessGroup(core::ROOT_PROCESS_GROUP, name, uuid); } -ProcessGroup *FlowControllerImpl::createRemoteProcessGroup(std::string name, - uuid_t uuid) { - return new ProcessGroup(REMOTE_PROCESS_GROUP, name, uuid); +core::ProcessGroup *FlowControllerImpl::createRemoteProcessGroup( + std::string name, uuid_t uuid) { + return new core::ProcessGroup(core::REMOTE_PROCESS_GROUP, name, uuid); } -Connection *FlowControllerImpl::createConnection(std::string name, - uuid_t uuid) { - return new Connection(name, uuid); +std::shared_ptr FlowControllerImpl::createConnection( + std::string name, uuid_t uuid) { + return std::make_shared(name, uuid); } #ifdef YAML_SUPPORT void FlowControllerImpl::parseRootProcessGroupYaml(YAML::Node rootFlowNode) { - uuid_t uuid; - ProcessGroup *group = NULL; - + uuid_t uuid; + core::ProcessGroup *group = NULL; std::string flowName = rootFlowNode["name"].as(); std::string id = rootFlowNode["id"].as(); @@ -253,18 +273,18 @@ void FlowControllerImpl::parseRootProcessGroupYaml(YAML::Node rootFlowNode) { logger_->log_debug("parseRootProcessGroup: id => [%s]", id.c_str()); logger_->log_debug("parseRootProcessGroup: name => [%s]", flowName.c_str()); group = this->createRootProcessGroup(flowName, uuid); - this->_root = group; - this->_name = flowName; + this->root_ = group; + this->name_ = flowName; } void FlowControllerImpl::parseProcessorNodeYaml(YAML::Node processorsNode, - ProcessGroup *parentGroup) { + core::ProcessGroup *parentGroup) { int64_t schedulingPeriod = -1; int64_t penalizationPeriod = -1; int64_t yieldPeriod = -1; int64_t runDurationNanos = -1; uuid_t uuid; - Processor *processor = NULL; + std::shared_ptr processor = nullptr; if (!parentGroup) { logger_->log_error("parseProcessNodeYaml: no parent group exists"); @@ -398,18 +418,18 @@ void FlowControllerImpl::parseProcessorNodeYaml(YAML::Node processorsNode, } // Default to running - processor->setScheduledState(RUNNING); + processor->setScheduledState(core::RUNNING); if (procCfg.schedulingStrategy == "TIMER_DRIVEN") { - processor->setSchedulingStrategy(TIMER_DRIVEN); + processor->setSchedulingStrategy(core::TIMER_DRIVEN); logger_->log_debug("setting scheduling strategy as %s", procCfg.schedulingStrategy.c_str()); } else if (procCfg.schedulingStrategy == "EVENT_DRIVEN") { - processor->setSchedulingStrategy(EVENT_DRIVEN); + processor->setSchedulingStrategy(core::EVENT_DRIVEN); logger_->log_debug("setting scheduling strategy as %s", procCfg.schedulingStrategy.c_str()); } else { - processor->setSchedulingStrategy(CRON_DRIVEN); + processor->setSchedulingStrategy(core::CRON_DRIVEN); logger_->log_debug("setting scheduling strategy as %s", procCfg.schedulingStrategy.c_str()); @@ -432,9 +452,9 @@ void FlowControllerImpl::parseProcessorNodeYaml(YAML::Node processorsNode, processor->setRunDurationNano(runDurationNanos); } - std::set autoTerminatedRelationships; + std::set autoTerminatedRelationships; for (auto &&relString : procCfg.autoTerminatedRelationships) { - Relationship relationship(relString, ""); + core::Relationship relationship(relString, ""); logger_->log_debug( "parseProcessorNode: autoTerminatedRelationship => [%s]", relString.c_str()); @@ -454,7 +474,7 @@ void FlowControllerImpl::parseProcessorNodeYaml(YAML::Node processorsNode, } void FlowControllerImpl::parseRemoteProcessGroupYaml(YAML::Node *rpgNode, - ProcessGroup *parentGroup) { + core::ProcessGroup *parentGroup) { uuid_t uuid; if (!parentGroup) { @@ -493,7 +513,7 @@ void FlowControllerImpl::parseRemoteProcessGroupYaml(YAML::Node *rpgNode, YAML::Node inputPorts = rpgNode["Input Ports"].as(); YAML::Node outputPorts = rpgNode["Output Ports"].as(); - ProcessGroup *group = NULL; + core::ProcessGroup *group = NULL; uuid_parse(id.c_str(), uuid); @@ -554,9 +574,9 @@ void FlowControllerImpl::parseRemoteProcessGroupYaml(YAML::Node *rpgNode, } void FlowControllerImpl::parseConnectionYaml(YAML::Node *connectionsNode, - ProcessGroup *parent) { + core::ProcessGroup *parent) { uuid_t uuid; - Connection *connection = NULL; + std::shared_ptr connection = nullptr; if (!parent) { logger_->log_error("parseProcessNode: no parent group was provided"); @@ -585,7 +605,7 @@ void FlowControllerImpl::parseConnectionYaml(YAML::Node *connectionsNode, auto rawRelationship = connectionNode["source relationship name"].as< std::string>(); - Relationship relationship(rawRelationship, ""); + core::Relationship relationship(rawRelationship, ""); logger_->log_debug("parseConnection: relationship => [%s]", rawRelationship.c_str()); if (connection) @@ -595,7 +615,7 @@ void FlowControllerImpl::parseConnectionYaml(YAML::Node *connectionsNode, uuid_t srcUUID; uuid_parse(connectionSrcProcId.c_str(), srcUUID); - Processor *srcProcessor = this->_root->findProcessor( + Processor *srcProcessor = this->root_->findProcessor( srcUUID); if (!srcProcessor) { @@ -609,12 +629,12 @@ void FlowControllerImpl::parseConnectionYaml(YAML::Node *connectionsNode, uuid_t destUUID; uuid_parse(destId.c_str(), destUUID); - Processor *destProcessor = this->_root->findProcessor(destUUID); + Processor *destProcessor = this->root_->findProcessor(destUUID); // If we could not find name, try by UUID if (!destProcessor) { uuid_t destUuid; uuid_parse(destId.c_str(), destUuid); - destProcessor = this->_root->findProcessor(destUuid); + destProcessor = this->root_->findProcessor(destUuid); } if (destProcessor) { std::string destUuid = destProcessor->getUUIDStr(); @@ -623,9 +643,9 @@ void FlowControllerImpl::parseConnectionYaml(YAML::Node *connectionsNode, uuid_t srcUuid; uuid_t destUuid; srcProcessor->getUUID(srcUuid); - connection->setSourceProcessorUUID(srcUuid); + connection->setSourceUUID(srcUuid); destProcessor->getUUID(destUuid); - connection->setDestinationProcessorUUID(destUuid); + connection->setDestinationUUID(destUuid); if (connection) { parent->addConnection(connection); @@ -641,76 +661,78 @@ void FlowControllerImpl::parseConnectionYaml(YAML::Node *connectionsNode, } void FlowControllerImpl::parsePortYaml(YAML::Node *portNode, - ProcessGroup *parent, TransferDirection direction) { - uuid_t uuid; - Processor *processor = NULL; - RemoteProcessorGroupPort *port = NULL; - - if (!parent) { - logger_->log_error("parseProcessNode: no parent group existed"); - return; - } - - YAML::Node inputPortsObj = portNode->as(); - - // generate the random UIID - uuid_generate(uuid); - - auto portId = inputPortsObj["id"].as(); - auto nameStr = inputPortsObj["name"].as(); - uuid_parse(portId.c_str(), uuid); - - port = new RemoteProcessorGroupPort(nameStr.c_str(), uuid); - - processor = (Processor *) port; - port->setDirection(direction); - port->setTimeOut(parent->getTimeOut()); - port->setTransmitting(true); - processor->setYieldPeriodMsec(parent->getYieldPeriodMsec()); - processor->initialize(); - - // handle port properties - YAML::Node nodeVal = portNode->as(); - YAML::Node propertiesNode = nodeVal["Properties"]; - - parsePropertiesNodeYaml(&propertiesNode, processor); - - // add processor to parent - parent->addProcessor(processor); - processor->setScheduledState(RUNNING); - auto rawMaxConcurrentTasks = inputPortsObj["max concurrent tasks"].as< - std::string>(); - int64_t maxConcurrentTasks; - if (Property::StringToInt(rawMaxConcurrentTasks, maxConcurrentTasks)) { - processor->setMaxConcurrentTasks(maxConcurrentTasks); - } - logger_->log_debug("parseProcessorNode: maxConcurrentTasks => [%d]", - maxConcurrentTasks); - processor->setMaxConcurrentTasks(maxConcurrentTasks); + core::ProcessGroup *parent, + TransferDirection direction) { + uuid_t uuid; + std::shared_ptr processor = NULL; + RemoteProcessorGroupPort *port = NULL; + + if (!parent) { + logger_->log_error("parseProcessNode: no parent group existed"); + return; + } + + YAML::Node inputPortsObj = portNode->as(); + + // generate the random UIID + uuid_generate(uuid); + + auto portId = inputPortsObj["id"].as(); + auto nameStr = inputPortsObj["name"].as(); + uuid_parse(portId.c_str(), uuid); + + port = new RemoteProcessorGroupPort(nameStr.c_str(), uuid); + + processor = (std::shared_ptr) port; + port->setDirection(direction); + port->setTimeOut(parent->getTimeOut()); + port->setTransmitting(true); + processor->setYieldPeriodMsec(parent->getYieldPeriodMsec()); + processor->initialize(); + + // handle port properties + YAML::Node nodeVal = portNode->as(); + YAML::Node propertiesNode = nodeVal["Properties"]; + + parsePropertiesNodeYaml(&propertiesNode, processor); + + // add processor to parent + parent->addProcessor(processor); + processor->setScheduledState(core::RUNNING); + auto rawMaxConcurrentTasks = inputPortsObj["max concurrent tasks"] + .as(); + int64_t maxConcurrentTasks; + if (core::Property::StringToInt(rawMaxConcurrentTasks, maxConcurrentTasks)) { + processor->setMaxConcurrentTasks(maxConcurrentTasks); + } + logger_->log_debug("parseProcessorNode: maxConcurrentTasks => [%d]", + maxConcurrentTasks); + processor->setMaxConcurrentTasks(maxConcurrentTasks); } -void FlowControllerImpl::parsePropertiesNodeYaml(YAML::Node *propertiesNode, - Processor *processor) { - // Treat generically as a YAML node so we can perform inspection on entries to ensure they are populated - for (YAML::const_iterator propsIter = propertiesNode->begin(); - propsIter != propertiesNode->end(); ++propsIter) { - std::string propertyName = propsIter->first.as(); - YAML::Node propertyValueNode = propsIter->second; - if (!propertyValueNode.IsNull() && propertyValueNode.IsDefined()) { - std::string rawValueString = propertyValueNode.as(); - if (!processor->setProperty(propertyName, rawValueString)) { - logger_->log_warn( - "Received property %s with value %s but is not one of the properties for %s", - propertyName.c_str(), rawValueString.c_str(), - processor->getName().c_str()); - } - } - } +void FlowControllerImpl::parsePropertiesNodeYaml( + YAML::Node *propertiesNode, std::shared_ptr processor) { + // Treat generically as a YAML node so we can perform inspection on entries to ensure they are populated + for (YAML::const_iterator propsIter = propertiesNode->begin(); + propsIter != propertiesNode->end(); ++propsIter) { + std::string propertyName = propsIter->first.as(); + YAML::Node propertyValueNode = propsIter->second; + if (!propertyValueNode.IsNull() && propertyValueNode.IsDefined()) { + std::string rawValueString = propertyValueNode.as(); + if (!processor->setProperty(propertyName, rawValueString)) { + logger_->log_warn( + "Received property %s with value %s but is not one of the properties for %s", + propertyName.c_str(), rawValueString.c_str(), + processor->getName().c_str()); + } + } + } } #endif /* ifdef YAML_SUPPORT */ void FlowControllerImpl::load() { +<<<<<<< HEAD if (_running) { stop(true); } @@ -719,76 +741,92 @@ void FlowControllerImpl::load() { #ifdef YAML_SUPPORT YAML::Node flow = YAML::LoadFile(_configurationFileName); +======= + if (running_) { + stop(true); + } + if (!initialized_) { + logger_->log_info("Load Flow Controller from file %s", + configuration_filename_.c_str()); - YAML::Node flowControllerNode = flow["Flow Controller"]; - YAML::Node processorsNode = flow[CONFIG_YAML_PROCESSORS_KEY]; - YAML::Node connectionsNode = flow["Connections"]; - YAML::Node remoteProcessingGroupNode = flow["Remote Processing Groups"]; + YAML::Node flow = YAML::LoadFile(configuration_filename_); +>>>>>>> MINIFI-217: First commit. Updates namespaces and removes - // Create the root process group - parseRootProcessGroupYaml(flowControllerNode); - parseProcessorNodeYaml(processorsNode, this->_root); - parseRemoteProcessGroupYaml(&remoteProcessingGroupNode, this->_root); - parseConnectionYaml(&connectionsNode, this->_root); + YAML::Node flowControllerNode = flow["Flow Controller"]; + YAML::Node processorsNode = flow[CONFIG_YAML_PROCESSORS_KEY]; + YAML::Node connectionsNode = flow["Connections"]; + YAML::Node remoteProcessingGroupNode = flow["Remote Processing Groups"]; + + // Create the root process group + parseRootProcessGroupYaml(flowControllerNode); + parseProcessorNodeYaml(processorsNode, this->root_); + parseRemoteProcessGroupYaml(&remoteProcessingGroupNode, this->root_); + parseConnectionYaml(&connectionsNode, this->root_); // Load Flow File from Repo loadFlowRepo(); #endif - _initialized = true; - } -} -void FlowControllerImpl::loadFlowRepo() -{ - if (this->_flowfileRepo && this->_flowfileRepo->isEnable()) - { - std::map connectionMap; - this->_root->getConnections(&connectionMap); - this->_flowfileRepo->loadFlowFileToConnections(&connectionMap); - } + } } -void FlowControllerImpl::reload(std::string yamlFile) -{ - logger_->log_info("Starting to reload Flow Controller with yaml %s", yamlFile.c_str()); +void FlowControllerImpl::reload(std::string yamlFile) { + logger_->log_info("Starting to reload Flow Controller with yaml %s", + yamlFile.c_str()); + stop(true); + unload(); + std::string oldYamlFile = this->configuration_filename_; + this->configuration_filename_ = yamlFile; + load(); + start(); + if (!this->root_) { + this->configuration_filename_ = oldYamlFile; + logger_->log_info("Rollback Flow Controller to YAML %s", + oldYamlFile.c_str()); stop(true); unload(); - std::string oldYamlFile = this->_configurationFileName; - this->_configurationFileName = yamlFile; load(); start(); - if (!this->_root) - { - this->_configurationFileName = oldYamlFile; - logger_->log_info("Rollback Flow Controller to YAML %s", oldYamlFile.c_str()); - stop(true); - unload(); - load(); - start(); - } + } } -bool FlowControllerImpl::start() { - if (!_initialized) { - logger_->log_error( - "Can not start Flow Controller because it has not been initialized"); - return false; - } else { +void FlowControllerImpl::loadFlowRepo() +{ + if (this->flow_file_repo_) + { + std::map connectionMap; + this->root_->getConnections(connectionMap); + this->flow_file_repo_->loadFlowFileToConnections(&connectionMap); + } +} - if (!_running) { - logger_->log_info("Starting Flow Controller"); - this->_timerScheduler.start(); - this->_eventScheduler.start(); - if (this->_root) - this->_root->startProcessing(&this->_timerScheduler, - &this->_eventScheduler); - _running = true; - this->_protocol->start(); - this->_provenanceRepo->start(); - this->_flowfileRepo->start(); - logger_->log_info("Started Flow Controller"); - } - return true; - } + +bool FlowControllerImpl::start() { + if (!initialized_) { + logger_->log_error( + "Can not start Flow Controller because it has not been initialized"); + return false; + } else { + + if (!running_) { + logger_->log_info("Starting Flow Controller"); + this->_timerScheduler.start(); + this->_eventScheduler.start(); + if (this->root_) + this->root_->startProcessing(&this->_timerScheduler, + &this->_eventScheduler); + running_ = true; + this->protocol_->start(); + this->provenance_repo_->start(); + this->flow_file_repo_->start(); + logger_->log_info("Started Flow Controller"); + } + return true; + } } + +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/FlowFileRecord.cpp b/libminifi/src/FlowFileRecord.cpp index a2f232394c..8bc132f2e0 100644 --- a/libminifi/src/FlowFileRecord.cpp +++ b/libminifi/src/FlowFileRecord.cpp @@ -27,253 +27,119 @@ #include #include "FlowFileRecord.h" -#include "Relationship.h" -#include "Logger.h" -#include "FlowController.h" -#include "FlowFileRepository.h" - -std::atomic FlowFileRecord::_localFlowSeqNumber(0); - -FlowFileRecord::FlowFileRecord(std::map attributes, ResourceClaim *claim) -: _size(0), - _id(_localFlowSeqNumber.load()), - _offset(0), - _penaltyExpirationMs(0), - _claim(claim), - _isStoredToRepo(false), - _markedDelete(false), - _connection(NULL), - _orginalConnection(NULL) -{ - _entryDate = getTimeMillis(); - _lineageStartDate = _entryDate; - - char uuidStr[37]; - - // Generate the global UUID for the flow record - uuid_generate(_uuid); - // Increase the local ID for the flow record - ++_localFlowSeqNumber; - uuid_unparse_lower(_uuid, uuidStr); - _uuidStr = uuidStr; - - // Populate the default attributes - addAttribute(FILENAME, std::to_string(getTimeNano())); - addAttribute(PATH, DEFAULT_FLOWFILE_PATH); - addAttribute(UUID, uuidStr); - // Populate the attributes from the input - std::map::iterator it; - for (it = attributes.begin(); it!= attributes.end(); it++) - { - addAttribute(it->first, it->second); - } - - _snapshot = false; - - if (_claim) - // Increase the flow file record owned count for the resource claim - _claim->increaseFlowFileRecordOwnedCount(); - logger_ = Logger::getLogger(); -} - -FlowFileRecord::FlowFileRecord(FlowFileEventRecord *event) -: _size(0), - _id(_localFlowSeqNumber.load()), - _offset(0), - _penaltyExpirationMs(0), - _claim(NULL), - _isStoredToRepo(false), - _markedDelete(false), - _connection(NULL), - _orginalConnection(NULL) -{ - _entryDate = event->getFlowFileEntryDate(); - _lineageStartDate = event->getlineageStartDate(); - _size = event->getFileSize(); - _offset = event->getFileOffset(); - _lineageIdentifiers = event->getLineageIdentifiers(); - _attributes = event->getAttributes(); - _snapshot = false; - _uuidStr = event->getFlowFileUuid(); - uuid_parse(_uuidStr.c_str(), _uuid); - - if (_size > 0) - { - _claim = new ResourceClaim(); - } - - if (_claim) - { - _claim->setContentFullPath(event->getContentFullPath()); - // Increase the flow file record owned count for the resource claim - _claim->increaseFlowFileRecordOwnedCount(); - } - logger_ = Logger::getLogger(); - ++_localFlowSeqNumber; -} - - -FlowFileRecord::~FlowFileRecord() -{ - if (!_snapshot) - logger_->log_debug("Delete FlowFile UUID %s", _uuidStr.c_str()); - else - logger_->log_debug("Delete SnapShot FlowFile UUID %s", _uuidStr.c_str()); - if (_claim) - { - // Decrease the flow file record owned count for the resource claim - _claim->decreaseFlowFileRecordOwnedCount(); - if (_claim->getFlowFileRecordOwnedCount() <= 0) - { - logger_->log_debug("Delete Resource Claim %s", _claim->getContentFullPath().c_str()); - std::string value; - if (!FlowControllerFactory::getFlowController()->getFlowFileRepository() || - !FlowControllerFactory::getFlowController()->getFlowFileRepository()->isEnable() || - !this->_isStoredToRepo || - !FlowControllerFactory::getFlowController()->getFlowFileRepository()->Get(_uuidStr, value)) - { - // if it is persistent to DB already while it is in the queue, we keep the content - std::remove(_claim->getContentFullPath().c_str()); - } - delete _claim; - } - } +#include "core/logging/Logger.h" +#include "core/Relationship.h" +#include "core/Repository.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { + +std::atomic FlowFileRecord::local_flow_seq_number_(0); + +FlowFileRecord::FlowFileRecord( + std::shared_ptr flow_repository, + std::map attributes, + std::shared_ptr claim) + : FlowFile(), + flow_repository_(flow_repository) { + + id_ = local_flow_seq_number_.load(); + claim_ = claim; + // Increase the local ID for the flow record + ++local_flow_seq_number_; + // Populate the default attributes + addKeyedAttribute(FILENAME, std::to_string(getTimeNano())); + addKeyedAttribute(PATH, DEFAULT_FLOWFILE_PATH); + addKeyedAttribute(UUID, getUUIDStr()); + // Populate the attributes from the input + std::map::iterator it; + for (it = attributes.begin(); it != attributes.end(); it++) { + FlowFile::addAttribute(it->first, it->second); + } + + snapshot_ = false; + + if (claim_ != nullptr) + // Increase the flow file record owned count for the resource claim + claim_->increaseFlowFileRecordOwnedCount(); + logger_ = logging::Logger::getLogger(); } -bool FlowFileRecord::addAttribute(FlowAttribute key, std::string value) -{ - const char *keyStr = FlowAttributeKey(key); - if (keyStr) - { - std::string keyString = keyStr; - return addAttribute(keyString, value); - } - else - { - return false; - } -} +FlowFileRecord::FlowFileRecord( + std::shared_ptr flow_repository, + std::shared_ptr event) + : FlowFile(), + flow_repository_(flow_repository) { -bool FlowFileRecord::addAttribute(std::string key, std::string value) -{ - std::map::iterator it = _attributes.find(key); - if (it != _attributes.end()) - { - // attribute already there in the map - return false; - } - else - { - _attributes[key] = value; - return true; - } } -bool FlowFileRecord::removeAttribute(FlowAttribute key) -{ - const char *keyStr = FlowAttributeKey(key); - if (keyStr) - { - std::string keyString = keyStr; - return removeAttribute(keyString); - } - else - { - return false; - } -} - -bool FlowFileRecord::removeAttribute(std::string key) -{ - std::map::iterator it = _attributes.find(key); - if (it != _attributes.end()) - { - _attributes.erase(key); - return true; - } - else - { - return false; - } +FlowFileRecord::~FlowFileRecord() { + if (!snapshot_) + logger_->log_debug("Delete FlowFile UUID %s", uuid_str_.c_str()); + else + logger_->log_debug("Delete SnapShot FlowFile UUID %s", uuid_str_.c_str()); + if (claim_) { + // Decrease the flow file record owned count for the resource claim + claim_->decreaseFlowFileRecordOwnedCount(); + if (claim_->getFlowFileRecordOwnedCount() <= 0) { + logger_->log_debug("Delete Resource Claim %s", + claim_->getContentFullPath().c_str()); + if (!this->stored || !flow_repository_->Get(_uuidStr, value)) { + std::remove(claim_->getContentFullPath().c_str()); + } + } + } } -bool FlowFileRecord::updateAttribute(FlowAttribute key, std::string value) -{ - const char *keyStr = FlowAttributeKey(key); - if (keyStr) - { - std::string keyString = keyStr; - return updateAttribute(keyString, value); - } - else - { - return false; - } +bool FlowFileRecord::addKeyedAttribute(FlowAttribute key, std::string value) { + const char *keyStr = FlowAttributeKey(key); + if (keyStr) { + const std::string keyString = keyStr; + return FlowFile::addAttribute(keyString, value); + } else { + return false; + } } -bool FlowFileRecord::updateAttribute(std::string key, std::string value) -{ - std::map::iterator it = _attributes.find(key); - if (it != _attributes.end()) - { - _attributes[key] = value; - return true; - } - else - { - return false; - } +bool FlowFileRecord::removeKeyedAttribute(FlowAttribute key) { + const char *keyStr = FlowAttributeKey(key); + if (keyStr) { + std::string keyString = keyStr; + return FlowFile::removeAttribute(keyString); + } else { + return false; + } } -bool FlowFileRecord::getAttribute(FlowAttribute key, std::string &value) -{ - const char *keyStr = FlowAttributeKey(key); - if (keyStr) - { - std::string keyString = keyStr; - return getAttribute(keyString, value); - } - else - { - return false; - } +bool FlowFileRecord::updateKeyedAttribute(FlowAttribute key, + std::string value) { + const char *keyStr = FlowAttributeKey(key); + if (keyStr) { + std::string keyString = keyStr; + return FlowFile::updateAttribute(keyString, value); + } else { + return false; + } } -bool FlowFileRecord::getAttribute(std::string key, std::string &value) -{ - std::map::iterator it = _attributes.find(key); - if (it != _attributes.end()) - { - value = it->second; - return true; - } - else - { - return false; - } +bool FlowFileRecord::getKeyedAttribute(FlowAttribute key, std::string &value) { + const char *keyStr = FlowAttributeKey(key); + if (keyStr) { + std::string keyString = keyStr; + return FlowFile::getAttribute(keyString, value); + } else { + return false; + } } -void FlowFileRecord::duplicate(FlowFileRecord *original) -{ - uuid_copy(this->_uuid, original->_uuid); - this->_attributes = original->_attributes; - this->_entryDate = original->_entryDate; - this->_id = original->_id; - this->_lastQueueDate = original->_lastQueueDate; - this->_lineageStartDate = original->_lineageStartDate; - this->_offset = original->_offset; - this->_penaltyExpirationMs = original->_penaltyExpirationMs; - this->_size = original->_size; - this->_lineageIdentifiers = original->_lineageIdentifiers; - this->_orginalConnection = original->_orginalConnection; - this->_uuidStr = original->_uuidStr; - this->_connection = original->_connection; - this->_markedDelete = original->_markedDelete; - - this->_claim = original->_claim; - if (this->_claim) - this->_claim->increaseFlowFileRecordOwnedCount(); - - this->_snapshot = true; +FlowFileRecord &FlowFileRecord::operator=(const FlowFileRecord &other) { + core::FlowFile::operator=(other); + return *this; } +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/FlowFileRepository.cpp b/libminifi/src/FlowFileRepository.cpp deleted file mode 100644 index 3388738273..0000000000 --- a/libminifi/src/FlowFileRepository.cpp +++ /dev/null @@ -1,282 +0,0 @@ -/** - * @file FlowFileRepository.cpp - * FlowFile implemenatation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include "io/DataStream.h" -#include "io/Serializable.h" -#include "FlowFileRecord.h" -#include "Relationship.h" -#include "Logger.h" -#include "FlowController.h" -#include "FlowFileRepository.h" - -//! DeSerialize -bool FlowFileEventRecord::DeSerialize(FlowFileRepository *repo, - std::string key) { - std::string value; - bool ret; - - ret = repo->Get(key, value); - - if (!ret) { - logger_->log_error("NiFi FlowFile Store event %s can not found", - key.c_str()); - return false; - } else - logger_->log_debug("NiFi FlowFile Read event %s length %d", - key.c_str(), value.length()); - - - DataStream stream((const uint8_t*)value.data(),value.length()); - - ret = DeSerialize(stream); - - if (ret) { - logger_->log_debug( - "NiFi FlowFile retrieve uuid %s size %d connection %s success", - _uuid.c_str(), stream.getSize(), _uuidConnection.c_str()); - } else { - logger_->log_debug( - "NiFi FlowFile retrieve uuid %s size %d connection %d fail", - _uuid.c_str(), stream.getSize(), _uuidConnection.c_str()); - } - - return ret; -} - -bool FlowFileEventRecord::Serialize(FlowFileRepository *repo) { - - DataStream outStream; - - int ret; - - ret = write(this->_eventTime,&outStream); - if (ret != 8) { - - return false; - } - - ret = write(this->_entryDate,&outStream); - if (ret != 8) { - return false; - } - - ret = write(this->_lineageStartDate,&outStream); - if (ret != 8) { - - return false; - } - - ret = writeUTF(this->_uuid,&outStream); - if (ret <= 0) { - - return false; - } - - ret = writeUTF(this->_uuidConnection,&outStream); - if (ret <= 0) { - - return false; - } - - // write flow attributes - uint32_t numAttributes = this->_attributes.size(); - ret = write(numAttributes,&outStream); - if (ret != 4) { - - return false; - } - - for (auto itAttribute : _attributes) { - ret = writeUTF(itAttribute.first,&outStream, true); - if (ret <= 0) { - - return false; - } - ret = writeUTF(itAttribute.second,&outStream, true); - if (ret <= 0) { - - return false; - } - } - - ret = writeUTF(this->_contentFullPath,&outStream); - if (ret <= 0) { - - return false; - } - - ret = write(this->_size,&outStream); - if (ret != 8) { - - return false; - } - - ret = write(this->_offset,&outStream); - if (ret != 8) { - - return false; - } - - // Persistent to the DB - - if (repo->Put(_uuid, const_cast(outStream.getBuffer()), outStream.getSize())) { - logger_->log_debug("NiFi FlowFile Store event %s size %d success", - _uuid.c_str(), outStream.getSize()); - return true; - } else { - logger_->log_error("NiFi FlowFile Store event %s size %d fail", - _uuid.c_str(), outStream.getSize()); - return false; - } - - // cleanup - - return true; -} - -bool FlowFileEventRecord::DeSerialize(const uint8_t *buffer, const int bufferSize) { - - int ret; - - DataStream outStream(buffer,bufferSize); - - ret = read(this->_eventTime,&outStream); - if (ret != 8) { - return false; - } - - ret = read(this->_entryDate,&outStream); - if (ret != 8) { - return false; - } - - ret = read(this->_lineageStartDate,&outStream); - if (ret != 8) { - return false; - } - - ret = readUTF(this->_uuid,&outStream); - if (ret <= 0) { - return false; - } - - ret = readUTF(this->_uuidConnection,&outStream); - if (ret <= 0) { - return false; - } - - // read flow attributes - uint32_t numAttributes = 0; - ret = read(numAttributes,&outStream); - if (ret != 4) { - return false; - } - - for (uint32_t i = 0; i < numAttributes; i++) { - std::string key; - ret = readUTF(key,&outStream, true); - if (ret <= 0) { - return false; - } - std::string value; - ret = readUTF(value,&outStream, true); - if (ret <= 0) { - return false; - } - this->_attributes[key] = value; - } - - ret = readUTF(this->_contentFullPath,&outStream); - if (ret <= 0) { - return false; - } - - ret = read(this->_size,&outStream); - if (ret != 8) { - return false; - } - - ret = read(this->_offset,&outStream); - if (ret != 8) { - return false; - } - - return true; -} - -void FlowFileRepository::loadFlowFileToConnections(std::map *connectionMap) -{ -#ifdef LEVELDB_SUPPORT - if (!_enable) - return; - - std::vector purgeList; - leveldb::Iterator* it = _db->NewIterator( - leveldb::ReadOptions()); - - for (it->SeekToFirst(); it->Valid(); it->Next()) - { - FlowFileEventRecord eventRead; - std::string key = it->key().ToString(); - if (eventRead.DeSerialize((uint8_t *) it->value().data(), - (int) it->value().size())) - { - auto search = connectionMap->find(eventRead.getConnectionUuid()); - if (search != connectionMap->end()) - { - // we find the connection for the persistent flowfile, create the flowfile and enqueue that - FlowFileRecord *record = new FlowFileRecord(&eventRead); - // set store to repo to true so that we do need to persistent again in enqueue - record->setStoredToRepository(true); - search->second->put(record); - } - else - { - if (eventRead.getContentFullPath().length() > 0) - { - std::remove(eventRead.getContentFullPath().c_str()); - } - purgeList.push_back(key); - } - } - else - { - purgeList.push_back(key); - } - } - - delete it; - std::vector::iterator itPurge; - for (itPurge = purgeList.begin(); itPurge != purgeList.end(); - itPurge++) - { - std::string eventId = *itPurge; - logger_->log_info("Repository Repo %s Purge %s", - RepositoryTypeStr[_type], - eventId.c_str()); - Delete(eventId); - } -#endif - - return; -} - diff --git a/libminifi/src/GetFile.cpp b/libminifi/src/GetFile.cpp deleted file mode 100644 index 40dd387a51..0000000000 --- a/libminifi/src/GetFile.cpp +++ /dev/null @@ -1,329 +0,0 @@ -/** - * @file GetFile.cpp - * GetFile class implementation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if (__GNUC__ >= 4) - #if (__GNUC_MINOR__ < 9) - #include - #endif -#endif -#include "utils/StringUtils.h" -#include -#include "utils/TimeUtil.h" -#include "GetFile.h" -#include "ProcessContext.h" -#include "ProcessSession.h" - -const std::string GetFile::ProcessorName("GetFile"); -Property GetFile::BatchSize("Batch Size", "The maximum number of files to pull in each iteration", "10"); -Property GetFile::Directory("Input Directory", "The input directory from which to pull files", "."); -Property GetFile::IgnoreHiddenFile("Ignore Hidden Files", "Indicates whether or not hidden files should be ignored", "true"); -Property GetFile::KeepSourceFile("Keep Source File", - "If true, the file is not deleted after it has been copied to the Content Repository", "false"); -Property GetFile::MaxAge("Maximum File Age", - "The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored", "0 sec"); -Property GetFile::MinAge("Minimum File Age", - "The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored", "0 sec"); -Property GetFile::MaxSize("Maximum File Size", "The maximum size that a file can be in order to be pulled", "0 B"); -Property GetFile::MinSize("Minimum File Size", "The minimum size that a file must be in order to be pulled", "0 B"); -Property GetFile::PollInterval("Polling Interval", "Indicates how long to wait before performing a directory listing", "0 sec"); -Property GetFile::Recurse("Recurse Subdirectories", "Indicates whether or not to pull files from subdirectories", "true"); -Property GetFile::FileFilter("File Filter", "Only files whose names match the given regular expression will be picked up", "[^\\.].*"); -Relationship GetFile::Success("success", "All files are routed to success"); - -void GetFile::initialize() -{ - //! Set the supported properties - std::set properties; - properties.insert(BatchSize); - properties.insert(Directory); - properties.insert(IgnoreHiddenFile); - properties.insert(KeepSourceFile); - properties.insert(MaxAge); - properties.insert(MinAge); - properties.insert(MaxSize); - properties.insert(MinSize); - properties.insert(PollInterval); - properties.insert(Recurse); - properties.insert(FileFilter); - setSupportedProperties(properties); - //! Set the supported relationships - std::set relationships; - relationships.insert(Success); - setSupportedRelationships(relationships); -} - -void GetFile::onTrigger(ProcessContext *context, ProcessSession *session) -{ - std::string value; - - logger_->log_info("onTrigger GetFile"); - if (context->getProperty(Directory.getName(), value)) - { - _directory = value; - } - if (context->getProperty(BatchSize.getName(), value)) - { - Property::StringToInt(value, _batchSize); - } - if (context->getProperty(IgnoreHiddenFile.getName(), value)) - { - StringUtils::StringToBool(value, _ignoreHiddenFile); - } - if (context->getProperty(KeepSourceFile.getName(), value)) - { - StringUtils::StringToBool(value, _keepSourceFile); - } - - logger_->log_info("onTrigger GetFile"); - if (context->getProperty(MaxAge.getName(), value)) - { - TimeUnit unit; - if (Property::StringToTime(value, _maxAge, unit) && - Property::ConvertTimeUnitToMS(_maxAge, unit, _maxAge)) - { - - } - } - if (context->getProperty(MinAge.getName(), value)) - { - TimeUnit unit; - if (Property::StringToTime(value, _minAge, unit) && - Property::ConvertTimeUnitToMS(_minAge, unit, _minAge)) - { - - } - } - if (context->getProperty(MaxSize.getName(), value)) - { - Property::StringToInt(value, _maxSize); - } - if (context->getProperty(MinSize.getName(), value)) - { - Property::StringToInt(value, _minSize); - } - if (context->getProperty(PollInterval.getName(), value)) - { - TimeUnit unit; - if (Property::StringToTime(value, _pollInterval, unit) && - Property::ConvertTimeUnitToMS(_pollInterval, unit, _pollInterval)) - { - - } - } - if (context->getProperty(Recurse.getName(), value)) - { - StringUtils::StringToBool(value, _recursive); - } - - if (context->getProperty(FileFilter.getName(), value)) - { - _fileFilter = value; - } - - // Perform directory list - logger_->log_info("Is listing empty %i",isListingEmpty()); - if (isListingEmpty()) - { - if (_pollInterval == 0 || (getTimeMillis() - _lastDirectoryListingTime) > _pollInterval) - { - performListing(_directory); - } - } - logger_->log_info("Is listing empty %i",isListingEmpty()); - - if (!isListingEmpty()) - { - try - { - std::queue list; - pollListing(list, _batchSize); - while (!list.empty()) - { - - std::string fileName = list.front(); - list.pop(); - logger_->log_info("GetFile process %s", fileName.c_str()); - FlowFileRecord *flowFile = session->create(); - if (!flowFile) - return; - std::size_t found = fileName.find_last_of("/\\"); - std::string path = fileName.substr(0,found); - std::string name = fileName.substr(found+1); - flowFile->updateAttribute(FILENAME, name); - flowFile->updateAttribute(PATH, path); - flowFile->addAttribute(ABSOLUTE_PATH, fileName); - session->import(fileName, flowFile, _keepSourceFile); - session->transfer(flowFile, Success); - } - } - catch (std::exception &exception) - { - logger_->log_debug("GetFile Caught Exception %s", exception.what()); - throw; - } - catch (...) - { - throw; - } - } - -} - -bool GetFile::isListingEmpty() -{ - std::lock_guard lock(_mtx); - - return _dirList.empty(); -} - -void GetFile::putListing(std::string fileName) -{ - std::lock_guard lock(_mtx); - - _dirList.push(fileName); -} - -void GetFile::pollListing(std::queue &list, int maxSize) -{ - std::lock_guard lock(_mtx); - - while (!_dirList.empty() && (maxSize == 0 || list.size() < maxSize)) - { - std::string fileName = _dirList.front(); - _dirList.pop(); - list.push(fileName); - } - - return; -} - -bool GetFile::acceptFile(std::string fullName, std::string name) -{ - struct stat statbuf; - - if (stat(fullName.c_str(), &statbuf) == 0) - { - if (_minSize > 0 && statbuf.st_size <_minSize) - return false; - - if (_maxSize > 0 && statbuf.st_size > _maxSize) - return false; - - uint64_t modifiedTime = ((uint64_t) (statbuf.st_mtime) * 1000); - uint64_t fileAge = getTimeMillis() - modifiedTime; - if (_minAge > 0 && fileAge < _minAge) - return false; - if (_maxAge > 0 && fileAge > _maxAge) - return false; - - if (_ignoreHiddenFile && fullName.c_str()[0] == '.') - return false; - - if (access(fullName.c_str(), R_OK) != 0) - return false; - - if (_keepSourceFile == false && access(fullName.c_str(), W_OK) != 0) - return false; - - - #ifdef __GNUC__ - #if (__GNUC__ >= 4) - #if (__GNUC_MINOR__ < 9) - regex_t regex; - int ret = regcomp(®ex, _fileFilter.c_str(),0); - if (ret) - return false; - ret = regexec(®ex,name.c_str(),(size_t)0,NULL,0); - regfree(®ex); - if (ret) - return false; - #else - try{ - std::regex re(_fileFilter); - - if (!std::regex_match(name, re)) { - return false; - } - } catch (std::regex_error e) { - logger_->log_error("Invalid File Filter regex: %s.", e.what()); - return false; - } - #endif - #endif - #else - logger_->log_info("Cannot support regex filtering"); - #endif - return true; - } - - return false; -} - -void GetFile::performListing(std::string dir) -{ - logger_->log_info("Performing file listing against %s",dir.c_str()); - DIR *d; - d = opendir(dir.c_str()); - if (!d) - return; - // only perform a listing while we are not empty - logger_->log_info("Performing file listing against %s",dir.c_str()); - while (isRunning()) - { - struct dirent *entry; - entry = readdir(d); - if (!entry) - break; - std::string d_name = entry->d_name; - if ((entry->d_type & DT_DIR)) - { - // if this is a directory - if (_recursive && strcmp(d_name.c_str(), "..") != 0 && strcmp(d_name.c_str(), ".") != 0) - { - std::string path = dir + "/" + d_name; - performListing(path); - } - } - else - { - std::string fileName = dir + "/" + d_name; - if (acceptFile(fileName, d_name)) - { - // check whether we can take this file - putListing(fileName); - } - } - } - closedir(d); -} diff --git a/libminifi/src/ListenHTTP.cpp b/libminifi/src/ListenHTTP.cpp deleted file mode 100644 index 89ce1d233f..0000000000 --- a/libminifi/src/ListenHTTP.cpp +++ /dev/null @@ -1,395 +0,0 @@ -/** - * @file ListenHTTP.cpp - * ListenHTTP class implementation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include - -#include - -#include "ListenHTTP.h" - -#include "utils/TimeUtil.h" -#include "ProcessContext.h" -#include "ProcessSession.h" -#include "ProcessSessionFactory.h" - -const std::string ListenHTTP::ProcessorName("ListenHTTP"); - -Property ListenHTTP::BasePath("Base Path", "Base path for incoming connections", "contentListener"); -Property ListenHTTP::Port("Listening Port", "The Port to listen on for incoming connections", ""); -Property ListenHTTP::AuthorizedDNPattern("Authorized DN Pattern", "A Regular Expression to apply against the Distinguished Name of incoming connections. If the Pattern does not match the DN, the connection will be refused.", ".*"); -Property ListenHTTP::SSLCertificate("SSL Certificate", "File containing PEM-formatted file including TLS/SSL certificate and key", ""); -Property ListenHTTP::SSLCertificateAuthority("SSL Certificate Authority", "File containing trusted PEM-formatted certificates", ""); -Property ListenHTTP::SSLVerifyPeer("SSL Verify Peer", "Whether or not to verify the client's certificate (yes/no)", "no"); -Property ListenHTTP::SSLMinimumVersion("SSL Minimum Version", "Minimum TLS/SSL version allowed (SSL2, SSL3, TLS1.0, TLS1.1, TLS1.2)", "SSL2"); -Property ListenHTTP::HeadersAsAttributesRegex("HTTP Headers to receive as Attributes (Regex)", "Specifies the Regular Expression that determines the names of HTTP Headers that should be passed along as FlowFile attributes", ""); - -Relationship ListenHTTP::Success("success", "All files are routed to success"); - -void ListenHTTP::initialize() -{ - _logger->log_info("Initializing ListenHTTP"); - - //! Set the supported properties - std::set properties; - properties.insert(BasePath); - properties.insert(Port); - properties.insert(AuthorizedDNPattern); - properties.insert(SSLCertificate); - properties.insert(SSLCertificateAuthority); - properties.insert(SSLVerifyPeer); - properties.insert(SSLMinimumVersion); - properties.insert(HeadersAsAttributesRegex); - setSupportedProperties(properties); - //! Set the supported relationships - std::set relationships; - relationships.insert(Success); - setSupportedRelationships(relationships); -} - -void ListenHTTP::onSchedule(ProcessContext *context, ProcessSessionFactory *sessionFactory) -{ - - std::string basePath; - - if (!context->getProperty(BasePath.getName(), basePath)) - { - _logger->log_info("%s attribute is missing, so default value of %s will be used", - BasePath.getName().c_str(), - BasePath.getValue().c_str()); - basePath = BasePath.getValue(); - } - - basePath.insert(0, "/"); - - std::string listeningPort; - - if (!context->getProperty(Port.getName(), listeningPort)) - { - _logger->log_error("%s attribute is missing or invalid", - Port.getName().c_str()); - return; - } - - std::string authDNPattern; - - if (context->getProperty(AuthorizedDNPattern.getName(), authDNPattern) && !authDNPattern.empty()) - { - _logger->log_info("ListenHTTP using %s: %s", - AuthorizedDNPattern.getName().c_str(), - authDNPattern.c_str()); - } - - std::string sslCertFile; - - if (context->getProperty(SSLCertificate.getName(), sslCertFile) && !sslCertFile.empty()) - { - _logger->log_info("ListenHTTP using %s: %s", - SSLCertificate.getName().c_str(), - sslCertFile.c_str()); - } - - // Read further TLS/SSL options only if TLS/SSL usage is implied by virtue of certificate value being set - std::string sslCertAuthorityFile; - std::string sslVerifyPeer; - std::string sslMinVer; - - if (!sslCertFile.empty()) - { - if (context->getProperty(SSLCertificateAuthority.getName(), sslCertAuthorityFile) - && !sslCertAuthorityFile.empty()) - { - _logger->log_info("ListenHTTP using %s: %s", - SSLCertificateAuthority.getName().c_str(), - sslCertAuthorityFile.c_str()); - } - - if (context->getProperty(SSLVerifyPeer.getName(), sslVerifyPeer)) - { - if (sslVerifyPeer.empty() || sslVerifyPeer.compare("no") == 0) - { - _logger->log_info("ListenHTTP will not verify peers"); - } - else - { - _logger->log_info("ListenHTTP will verify peers"); - } - } - else - { - _logger->log_info("ListenHTTP will not verify peers"); - } - - if (context->getProperty(SSLMinimumVersion.getName(), sslMinVer)) - { - _logger->log_info("ListenHTTP using %s: %s", - SSLMinimumVersion.getName().c_str(), - sslMinVer.c_str()); - } - } - - std::string headersAsAttributesPattern; - - if (context->getProperty(HeadersAsAttributesRegex.getName(),headersAsAttributesPattern) - && !headersAsAttributesPattern.empty()) - { - _logger->log_info("ListenHTTP using %s: %s", - HeadersAsAttributesRegex.getName().c_str(), - headersAsAttributesPattern.c_str()); - } - - auto numThreads = getMaxConcurrentTasks(); - - _logger->log_info("ListenHTTP starting HTTP server on port %s and path %s with %d threads", - listeningPort.c_str(), - basePath.c_str(), - numThreads); - - // Initialize web server - std::vector options; - options.push_back("enable_keep_alive"); - options.push_back("yes"); - options.push_back("keep_alive_timeout_ms"); - options.push_back("15000"); - options.push_back("num_threads"); - options.push_back(std::to_string(numThreads)); - - if (sslCertFile.empty()) - { - options.push_back("listening_ports"); - options.push_back(listeningPort); - } - else - { - listeningPort += "s"; - options.push_back("listening_ports"); - options.push_back(listeningPort); - - options.push_back("ssl_certificate"); - options.push_back(sslCertFile); - - if (!sslCertAuthorityFile.empty()) - { - options.push_back("ssl_ca_file"); - options.push_back(sslCertAuthorityFile); - } - - if (sslVerifyPeer.empty() || sslVerifyPeer.compare("no") == 0) - { - options.push_back("ssl_verify_peer"); - options.push_back("no"); - } - else - { - options.push_back("ssl_verify_peer"); - options.push_back("yes"); - } - - if (sslMinVer.compare("SSL2") == 0) - { - options.push_back("ssl_protocol_version"); - options.push_back(std::to_string(0)); - } - else if (sslMinVer.compare("SSL3") == 0) - { - options.push_back("ssl_protocol_version"); - options.push_back(std::to_string(1)); - } - else if (sslMinVer.compare("TLS1.0") == 0) - { - options.push_back("ssl_protocol_version"); - options.push_back(std::to_string(2)); - } - else if (sslMinVer.compare("TLS1.1") == 0) - { - options.push_back("ssl_protocol_version"); - options.push_back(std::to_string(3)); - } - else - { - options.push_back("ssl_protocol_version"); - options.push_back(std::to_string(4)); - } - } - - _server.reset(new CivetServer(options)); - _handler.reset(new Handler(context, - sessionFactory, - std::move(authDNPattern), - std::move(headersAsAttributesPattern))); - _server->addHandler(basePath, _handler.get()); -} - -void ListenHTTP::onTrigger(ProcessContext *context, ProcessSession *session) -{ - - FlowFileRecord *flowFile = session->get(); - - // Do nothing if there are no incoming files - if (!flowFile) - { - return; - } -} - -ListenHTTP::Handler::Handler(ProcessContext *context, - ProcessSessionFactory *sessionFactory, - std::string &&authDNPattern, - std::string &&headersAsAttributesPattern) -: _authDNRegex(std::move(authDNPattern)) -, _headersAsAttributesRegex(std::move(headersAsAttributesPattern)) -{ - _processContext = context; - _processSessionFactory = sessionFactory; -} - -void ListenHTTP::Handler::sendErrorResponse(struct mg_connection *conn) -{ - mg_printf(conn, - "HTTP/1.1 500 Internal Server Error\r\n" - "Content-Type: text/html\r\n" - "Content-Length: 0\r\n\r\n"); -} - -bool ListenHTTP::Handler::handlePost(CivetServer *server, struct mg_connection *conn) -{ - _logger = Logger::getLogger(); - - auto req_info = mg_get_request_info(conn); - _logger->log_info("ListenHTTP handling POST request of length %d", req_info->content_length); - - // If this is a two-way TLS connection, authorize the peer against the configured pattern - if (req_info->is_ssl && req_info->client_cert != nullptr) - { - if (!std::regex_match(req_info->client_cert->subject, _authDNRegex)) - { - mg_printf(conn, - "HTTP/1.1 403 Forbidden\r\n" - "Content-Type: text/html\r\n" - "Content-Length: 0\r\n\r\n"); - _logger->log_warn("ListenHTTP client DN not authorized: %s", req_info->client_cert->subject); - return true; - } - } - - // Always send 100 Continue, as allowed per standard to minimize client delay (https://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html) - mg_printf(conn, "HTTP/1.1 100 Continue\r\n\r\n"); - - auto session = _processSessionFactory->createSession(); - ListenHTTP::WriteCallback callback(conn, req_info); - auto flowFile = session->create(); - - if (!flowFile) - { - sendErrorResponse(conn); - return true; - } - - try - { - session->write(flowFile, &callback); - - // Add filename from "filename" header value (and pattern headers) - for (int i = 0; i < req_info->num_headers; i++) - { - auto header = &req_info->http_headers[i]; - - if (strcmp("filename", header->name) == 0) - { - if (!flowFile->updateAttribute("filename", header->value)) - { - flowFile->addAttribute("filename", header->value); - } - } - else if (std::regex_match(header->name, _headersAsAttributesRegex)) - { - if (!flowFile->updateAttribute(header->name, header->value)) - { - flowFile->addAttribute(header->name, header->value); - } - } - } - - session->transfer(flowFile, Success); - session->commit(); - } - catch (std::exception &exception) - { - _logger->log_debug("ListenHTTP Caught Exception %s", exception.what()); - sendErrorResponse(conn); - session->rollback(); - throw; - } - catch (...) - { - _logger->log_debug("ListenHTTP Caught Exception Processor::onTrigger"); - sendErrorResponse(conn); - session->rollback(); - throw; - } - - mg_printf(conn, - "HTTP/1.1 200 OK\r\n" - "Content-Type: text/html\r\n" - "Content-Length: 0\r\n\r\n"); - - return true; -} - -ListenHTTP::WriteCallback::WriteCallback(struct mg_connection *conn, const struct mg_request_info *reqInfo) -{ - _logger = Logger::getLogger(); - _conn = conn; - _reqInfo = reqInfo; -} - -void ListenHTTP::WriteCallback::process(std::ofstream *stream) -{ - long long rlen; - long long nlen = 0; - long long tlen = _reqInfo->content_length; - char buf[16384]; - - while (nlen < tlen) - { - rlen = tlen - nlen; - - if (rlen > sizeof(buf)) - { - rlen = sizeof(buf); - } - - // Read a buffer of data from client - rlen = mg_read(_conn, &buf[0], (size_t)rlen); - - if (rlen <= 0) - { - break; - } - - // Transfer buffer data to the output stream - stream->write(&buf[0], rlen); - - nlen += rlen; - } -} diff --git a/libminifi/src/ListenSyslog.cpp b/libminifi/src/ListenSyslog.cpp deleted file mode 100644 index f2901e049e..0000000000 --- a/libminifi/src/ListenSyslog.cpp +++ /dev/null @@ -1,343 +0,0 @@ -/** - * @file ListenSyslog.cpp - * ListenSyslog class implementation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include "utils/TimeUtil.h" -#include "utils/StringUtils.h" -#include "ListenSyslog.h" -#include "ProcessContext.h" -#include "ProcessSession.h" - -const std::string ListenSyslog::ProcessorName("ListenSyslog"); -Property ListenSyslog::RecvBufSize("Receive Buffer Size", "The size of each buffer used to receive Syslog messages.", "65507 B"); -Property ListenSyslog::MaxSocketBufSize("Max Size of Socket Buffer", "The maximum size of the socket buffer that should be used.", "1 MB"); -Property ListenSyslog::MaxConnections("Max Number of TCP Connections", "The maximum number of concurrent connections to accept Syslog messages in TCP mode.", "2"); -Property ListenSyslog::MaxBatchSize("Max Batch Size", - "The maximum number of Syslog events to add to a single FlowFile.", "1"); -Property ListenSyslog::MessageDelimiter("Message Delimiter", - "Specifies the delimiter to place between Syslog messages when multiple messages are bundled together (see property).", "\n"); -Property ListenSyslog::ParseMessages("Parse Messages", - "Indicates if the processor should parse the Syslog messages. If set to false, each outgoing FlowFile will only.", "false"); -Property ListenSyslog::Protocol("Protocol", "The protocol for Syslog communication.", "UDP"); -Property ListenSyslog::Port("Port", "The port for Syslog communication.", "514"); -Relationship ListenSyslog::Success("success", "All files are routed to success"); -Relationship ListenSyslog::Invalid("invalid", "SysLog message format invalid"); - -void ListenSyslog::initialize() -{ - //! Set the supported properties - std::set properties; - properties.insert(RecvBufSize); - properties.insert(MaxSocketBufSize); - properties.insert(MaxConnections); - properties.insert(MaxBatchSize); - properties.insert(MessageDelimiter); - properties.insert(ParseMessages); - properties.insert(Protocol); - properties.insert(Port); - setSupportedProperties(properties); - //! Set the supported relationships - std::set relationships; - relationships.insert(Success); - relationships.insert(Invalid); - setSupportedRelationships(relationships); -} - -void ListenSyslog::startSocketThread() -{ - if (_thread != NULL) - return; - - logger_->log_info("ListenSysLog Socket Thread Start"); - _serverTheadRunning = true; - _thread = new std::thread(run, this); - _thread->detach(); -} - -void ListenSyslog::run(ListenSyslog *process) -{ - process->runThread(); -} - -void ListenSyslog::runThread() -{ - while (_serverTheadRunning) - { - if (_resetServerSocket) - { - _resetServerSocket = false; - // need to reset the socket - std::vector::iterator it; - for (it = _clientSockets.begin(); it != _clientSockets.end(); ++it) - { - int clientSocket = *it; - close(clientSocket); - } - _clientSockets.clear(); - if (_serverSocket > 0) - { - close(_serverSocket); - _serverSocket = 0; - } - } - - if (_serverSocket <= 0) - { - uint16_t portno = _port; - struct sockaddr_in serv_addr; - int sockfd; - if (_protocol == "TCP") - sockfd = socket(AF_INET, SOCK_STREAM, 0); - else - sockfd = socket(AF_INET, SOCK_DGRAM, 0); - if (sockfd < 0) - { - logger_->log_info("ListenSysLog Server socket creation failed"); - break; - } - bzero((char *) &serv_addr, sizeof(serv_addr)); - serv_addr.sin_family = AF_INET; - serv_addr.sin_addr.s_addr = INADDR_ANY; - serv_addr.sin_port = htons(portno); - if (bind(sockfd, (struct sockaddr *) &serv_addr, - sizeof(serv_addr)) < 0) - { - logger_->log_error("ListenSysLog Server socket bind failed"); - break; - } - if (_protocol == "TCP") - listen(sockfd,5); - _serverSocket = sockfd; - logger_->log_error("ListenSysLog Server socket %d bind OK to port %d", _serverSocket, portno); - } - FD_ZERO(&_readfds); - FD_SET(_serverSocket, &_readfds); - _maxFds = _serverSocket; - std::vector::iterator it; - for (it = _clientSockets.begin(); it != _clientSockets.end(); ++it) - { - int clientSocket = *it; - if (clientSocket >= _maxFds) - _maxFds = clientSocket; - FD_SET(clientSocket, &_readfds); - } - fd_set fds; - struct timeval tv; - int retval; - fds = _readfds; - tv.tv_sec = 0; - // 100 msec - tv.tv_usec = 100000; - retval = select(_maxFds+1, &fds, NULL, NULL, &tv); - if (retval < 0) - break; - if (retval == 0) - continue; - if (FD_ISSET(_serverSocket, &fds)) - { - // server socket, either we have UDP datagram or TCP connection request - if (_protocol == "TCP") - { - socklen_t clilen; - struct sockaddr_in cli_addr; - clilen = sizeof(cli_addr); - int newsockfd = accept(_serverSocket, - (struct sockaddr *) &cli_addr, - &clilen); - if (newsockfd > 0) - { - if (_clientSockets.size() < _maxConnections) - { - _clientSockets.push_back(newsockfd); - logger_->log_info("ListenSysLog new client socket %d connection", newsockfd); - continue; - } - else - { - close(newsockfd); - } - } - } - else - { - socklen_t clilen; - struct sockaddr_in cli_addr; - clilen = sizeof(cli_addr); - int recvlen = recvfrom(_serverSocket, _buffer, sizeof(_buffer), 0, - (struct sockaddr *)&cli_addr, &clilen); - if (recvlen > 0 && (recvlen + getEventQueueByteSize()) <= _recvBufSize) - { - uint8_t *payload = new uint8_t[recvlen]; - memcpy(payload, _buffer, recvlen); - putEvent(payload, recvlen); - } - } - } - it = _clientSockets.begin(); - while (it != _clientSockets.end()) - { - int clientSocket = *it; - if (FD_ISSET(clientSocket, &fds)) - { - int recvlen = readline(clientSocket, (char *)_buffer, sizeof(_buffer)); - if (recvlen <= 0) - { - close(clientSocket); - logger_->log_info("ListenSysLog client socket %d close", clientSocket); - it = _clientSockets.erase(it); - } - else - { - if ((recvlen + getEventQueueByteSize()) <= _recvBufSize) - { - uint8_t *payload = new uint8_t[recvlen]; - memcpy(payload, _buffer, recvlen); - putEvent(payload, recvlen); - } - ++it; - } - } - } - } - return; -} - - -int ListenSyslog::readline( int fd, char *bufptr, size_t len ) -{ - char *bufx = bufptr; - static char *bp; - static int cnt = 0; - static char b[ 2048 ]; - char c; - - while ( --len > 0 ) - { - if ( --cnt <= 0 ) - { - cnt = recv( fd, b, sizeof( b ), 0 ); - if ( cnt < 0 ) - { - if ( errno == EINTR ) - { - len++; /* the while will decrement */ - continue; - } - return -1; - } - if ( cnt == 0 ) - return 0; - bp = b; - } - c = *bp++; - *bufptr++ = c; - if ( c == '\n' ) - { - *bufptr = '\n'; - return bufptr - bufx + 1; - } - } - return -1; -} - -void ListenSyslog::onTrigger(ProcessContext *context, ProcessSession *session) -{ - std::string value; - bool needResetServerSocket = false; - if (context->getProperty(Protocol.getName(), value)) - { - if (_protocol != value) - needResetServerSocket = true; - _protocol = value; - } - if (context->getProperty(RecvBufSize.getName(), value)) - { - Property::StringToInt(value, _recvBufSize); - } - if (context->getProperty(MaxSocketBufSize.getName(), value)) - { - Property::StringToInt(value, _maxSocketBufSize); - } - if (context->getProperty(MaxConnections.getName(), value)) - { - Property::StringToInt(value, _maxConnections); - } - if (context->getProperty(MessageDelimiter.getName(), value)) - { - _messageDelimiter = value; - } - if (context->getProperty(ParseMessages.getName(), value)) - { - StringUtils::StringToBool(value, _parseMessages); - } - if (context->getProperty(Port.getName(), value)) - { - int64_t oldPort = _port; - Property::StringToInt(value, _port); - if (_port != oldPort) - needResetServerSocket = true; - } - if (context->getProperty(MaxBatchSize.getName(), value)) - { - Property::StringToInt(value, _maxBatchSize); - } - - if (needResetServerSocket) - _resetServerSocket = true; - - startSocketThread(); - - // read from the event queue - if (isEventQueueEmpty()) - { - context->yield(); - return; - } - - std::queue eventQueue; - pollEvent(eventQueue, _maxBatchSize); - bool firstEvent = true; - FlowFileRecord *flowFile = NULL; - while(!eventQueue.empty()) - { - SysLogEvent event = eventQueue.front(); - eventQueue.pop(); - if (firstEvent) - { - flowFile = session->create(); - if (!flowFile) - return; - ListenSyslog::WriteCallback callback((char *)event.payload, event.len); - session->write(flowFile, &callback); - delete[] event.payload; - firstEvent = false; - } - else - { - ListenSyslog::WriteCallback callback((char *)event.payload, event.len); - session->append(flowFile, &callback); - delete[] event.payload; - } - } - flowFile->addAttribute("syslog.protocol", _protocol); - flowFile->addAttribute("syslog.port", std::to_string(_port)); - session->transfer(flowFile, Success); -} diff --git a/libminifi/src/LogAttribute.cpp b/libminifi/src/LogAttribute.cpp deleted file mode 100644 index 345eb699b1..0000000000 --- a/libminifi/src/LogAttribute.cpp +++ /dev/null @@ -1,159 +0,0 @@ -/** - * @file LogAttribute.cpp - * LogAttribute class implementation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "utils/TimeUtil.h" -#include "utils/StringUtils.h" -#include "LogAttribute.h" -#include "ProcessContext.h" -#include "ProcessSession.h" - -const std::string LogAttribute::ProcessorName("LogAttribute"); -Property LogAttribute::LogLevel("Log Level", "The Log Level to use when logging the Attributes", "info"); -Property LogAttribute::AttributesToLog("Attributes to Log", "A comma-separated list of Attributes to Log. If not specified, all attributes will be logged.", ""); -Property LogAttribute::AttributesToIgnore("Attributes to Ignore", "A comma-separated list of Attributes to ignore. If not specified, no attributes will be ignored.", ""); -Property LogAttribute::LogPayload("Log Payload", - "If true, the FlowFile's payload will be logged, in addition to its attributes; otherwise, just the Attributes will be logged.", "false"); -Property LogAttribute::LogPrefix("Log prefix", - "Log prefix appended to the log lines. It helps to distinguish the output of multiple LogAttribute processors.", ""); -Relationship LogAttribute::Success("success", "success operational on the flow record"); - -void LogAttribute::initialize() -{ - //! Set the supported properties - std::set properties; - properties.insert(LogLevel); - properties.insert(AttributesToLog); - properties.insert(AttributesToIgnore); - properties.insert(LogPayload); - properties.insert(LogPrefix); - setSupportedProperties(properties); - //! Set the supported relationships - std::set relationships; - relationships.insert(Success); - setSupportedRelationships(relationships); -} - -void LogAttribute::onTrigger(ProcessContext *context, ProcessSession *session) -{ - std::string dashLine = "--------------------------------------------------"; - LogAttrLevel level = LogAttrLevelInfo; - bool logPayload = false; - std::ostringstream message; - - FlowFileRecord *flow = session->get(); - - if (!flow) - return; - - std::string value; - if (context->getProperty(LogLevel.getName(), value)) - { - logLevelStringToEnum(value, level); - } - if (context->getProperty(LogPrefix.getName(), value)) - { - dashLine = "-----" + value + "-----"; - } - if (context->getProperty(LogPayload.getName(), value)) - { - StringUtils::StringToBool(value, logPayload); - } - - message << "Logging for flow file " << "\n"; - message << dashLine; - message << "\nStandard FlowFile Attributes"; - message << "\n" << "UUID:" << flow->getUUIDStr(); - message << "\n" << "EntryDate:" << getTimeStr(flow->getEntryDate()); - message << "\n" << "lineageStartDate:" << getTimeStr(flow->getlineageStartDate()); - message << "\n" << "Size:" << flow->getSize() << " Offset:" << flow->getOffset(); - message << "\nFlowFile Attributes Map Content"; - std::map attrs = flow->getAttributes(); - std::map::iterator it; - for (it = attrs.begin(); it!= attrs.end(); it++) - { - message << "\n" << "key:" << it->first << " value:" << it->second; - } - message << "\nFlowFile Resource Claim Content"; - ResourceClaim *claim = flow->getResourceClaim(); - if (claim) - { - message << "\n" << "Content Claim:" << claim->getContentFullPath(); - } - if (logPayload && flow->getSize() <= 1024*1024) - { - message << "\n" << "Payload:" << "\n"; - ReadCallback callback(flow->getSize()); - session->read(flow, &callback); - for (unsigned int i = 0, j = 0; i < callback._readSize; i++) - { - char temp[8]; - sprintf(temp, "%02x ", (unsigned char) (callback._buffer[i])); - message << temp; - j++; - if (j == 16) - { - message << '\n'; - j = 0; - } - } - } - message << "\n" << dashLine << std::ends; - std::string output = message.str(); - - switch (level) - { - case LogAttrLevelInfo: - logger_->log_info("%s", output.c_str()); - break; - case LogAttrLevelDebug: - logger_->log_debug("%s", output.c_str()); - break; - case LogAttrLevelError: - logger_->log_error("%s", output.c_str()); - break; - case LogAttrLevelTrace: - logger_->log_trace("%s", output.c_str()); - break; - case LogAttrLevelWarn: - logger_->log_warn("%s", output.c_str()); - break; - default: - break; - } - - // Test Import - /* - FlowFileRecord *importRecord = session->create(); - session->import(claim->getContentFullPath(), importRecord); - session->transfer(importRecord, Success); */ - - - // Transfer to the relationship - session->transfer(flow, Success); -} diff --git a/libminifi/src/ProcessGroup.cpp b/libminifi/src/ProcessGroup.cpp deleted file mode 100644 index 7e3527e298..0000000000 --- a/libminifi/src/ProcessGroup.cpp +++ /dev/null @@ -1,307 +0,0 @@ -/** - * @file ProcessGroup.cpp - * ProcessGroup class implementation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "ProcessGroup.h" -#include "Processor.h" - -ProcessGroup::ProcessGroup(ProcessGroupType type, std::string name, uuid_t uuid, - ProcessGroup *parent) : - name_(name), type_(type), parent_process_group_(parent) { - if (!uuid) - // Generate the global UUID for the flow record - uuid_generate(uuid_); - else - uuid_copy(uuid_, uuid); - - yield_period_msec_ = 0; - transmitting_ = false; - - logger_ = Logger::getLogger(); - logger_->log_info("ProcessGroup %s created", name_.c_str()); -} - -ProcessGroup::~ProcessGroup() { - for (std::set::iterator it = connections_.begin(); - it != connections_.end(); ++it) { - Connection *connection = *it; - connection->drain(); - delete connection; - } - - for (std::set::iterator it = child_process_groups_.begin(); - it != child_process_groups_.end(); ++it) { - ProcessGroup *processGroup(*it); - delete processGroup; - } - - for (std::set::iterator it = processors_.begin(); - it != processors_.end(); ++it) { - Processor *processor(*it); - delete processor; - } -} - -void ProcessGroup::getConnections(std::map *connectionMap) -{ - for (auto connection : connections_) - { - (*connectionMap)[connection->getUUIDStr()] = connection; - } - - for (auto processGroup: child_process_groups_) { - processGroup->getConnections(connectionMap); - } -} - -bool ProcessGroup::isRootProcessGroup() { - std::lock_guard lock(mtx_); - return (type_ == ROOT_PROCESS_GROUP); -} - -void ProcessGroup::addProcessor(Processor *processor) { - std::lock_guard lock(mtx_); - - if (processors_.find(processor) == processors_.end()) { - // We do not have the same processor in this process group yet - processors_.insert(processor); - logger_->log_info("Add processor %s into process group %s", - processor->getName().c_str(), name_.c_str()); - } -} - -void ProcessGroup::removeProcessor(Processor *processor) { - std::lock_guard lock(mtx_); - - if (processors_.find(processor) != processors_.end()) { - // We do have the same processor in this process group yet - processors_.erase(processor); - logger_->log_info("Remove processor %s from process group %s", - processor->getName().c_str(), name_.c_str()); - } -} - -void ProcessGroup::addProcessGroup(ProcessGroup *child) { - std::lock_guard lock(mtx_); - - if (child_process_groups_.find(child) == child_process_groups_.end()) { - // We do not have the same child process group in this process group yet - child_process_groups_.insert(child); - logger_->log_info("Add child process group %s into process group %s", - child->getName().c_str(), name_.c_str()); - } -} - -void ProcessGroup::removeProcessGroup(ProcessGroup *child) { - std::lock_guard lock(mtx_); - - if (child_process_groups_.find(child) != child_process_groups_.end()) { - // We do have the same child process group in this process group yet - child_process_groups_.erase(child); - logger_->log_info("Remove child process group %s from process group %s", - child->getName().c_str(), name_.c_str()); - } -} - -void ProcessGroup::startProcessing(TimerDrivenSchedulingAgent *timeScheduler, - EventDrivenSchedulingAgent *eventScheduler) { - std::lock_guard lock(mtx_); - - try { - // Start all the processor node, input and output ports - for (auto processor : processors_) { - logger_->log_debug("Starting %s", processor->getName().c_str()); - - if (!processor->isRunning() - && processor->getScheduledState() != DISABLED) { - if (processor->getSchedulingStrategy() == TIMER_DRIVEN) - timeScheduler->schedule(processor); - else if (processor->getSchedulingStrategy() == EVENT_DRIVEN) - eventScheduler->schedule(processor); - } - } - // Start processing the group - for (auto processGroup : child_process_groups_) { - processGroup->startProcessing(timeScheduler, eventScheduler); - } - } catch (std::exception &exception) { - logger_->log_debug("Caught Exception %s", exception.what()); - throw; - } catch (...) { - logger_->log_debug( - "Caught Exception during process group start processing"); - throw; - } -} - -void ProcessGroup::stopProcessing(TimerDrivenSchedulingAgent *timeScheduler, - EventDrivenSchedulingAgent *eventScheduler) { - std::lock_guard lock(mtx_); - - try { - // Stop all the processor node, input and output ports - for (std::set::iterator it = processors_.begin(); - it != processors_.end(); ++it) { - Processor *processor(*it); - if (processor->getSchedulingStrategy() == TIMER_DRIVEN) - timeScheduler->unschedule(processor); - else if (processor->getSchedulingStrategy() == EVENT_DRIVEN) - eventScheduler->unschedule(processor); - } - - for (std::set::iterator it = - child_process_groups_.begin(); it != child_process_groups_.end(); - ++it) { - ProcessGroup *processGroup(*it); - processGroup->stopProcessing(timeScheduler, eventScheduler); - } - } catch (std::exception &exception) { - logger_->log_debug("Caught Exception %s", exception.what()); - throw; - } catch (...) { - logger_->log_debug( - "Caught Exception during process group stop processing"); - throw; - } -} - -Processor *ProcessGroup::findProcessor(uuid_t uuid) { - - Processor *ret = NULL; - // std::lock_guard lock(_mtx); - - for(auto processor : processors_){ - logger_->log_info("find processor %s", processor->getName().c_str()); - uuid_t processorUUID; - - if (processor->getUUID(processorUUID)) { - - char uuid_str[37]; // ex. "1b4e28ba-2fa1-11d2-883f-0016d3cca427" + "\0" - uuid_unparse_lower(processorUUID, uuid_str); - std::string processorUUIDstr = uuid_str; - uuid_unparse_lower(uuid, uuid_str); - std::string uuidStr = uuid_str; - if (processorUUIDstr == uuidStr) { - return processor; - } - } - - } - for(auto processGroup : child_process_groups_){ - - logger_->log_info("find processor child %s", - processGroup->getName().c_str()); - Processor *processor = processGroup->findProcessor(uuid); - if (processor) - return processor; - } - - return ret; -} - -Processor *ProcessGroup::findProcessor(std::string processorName) { - Processor *ret = NULL; - - for(auto processor : processors_){ - logger_->log_debug("Current processor is %s", - processor->getName().c_str()); - if (processor->getName() == processorName) - return processor; - } - - for(auto processGroup : child_process_groups_){ - Processor *processor = processGroup->findProcessor(processorName); - if (processor) - return processor; - } - - return ret; -} - -void ProcessGroup::updatePropertyValue(std::string processorName, - std::string propertyName, std::string propertyValue) { - std::lock_guard lock(mtx_); - - for(auto processor : processors_){ - if (processor->getName() == processorName) { - processor->setProperty(propertyName, propertyValue); - } - } - - for(auto processGroup : child_process_groups_){ - processGroup->updatePropertyValue(processorName, propertyName, - propertyValue); - } - - return; -} - -void ProcessGroup::addConnection(Connection *connection) { - std::lock_guard lock(mtx_); - - if (connections_.find(connection) == connections_.end()) { - // We do not have the same connection in this process group yet - connections_.insert(connection); - logger_->log_info("Add connection %s into process group %s", - connection->getName().c_str(), name_.c_str()); - uuid_t sourceUUID; - Processor *source = NULL; - connection->getSourceProcessorUUID(sourceUUID); - source = this->findProcessor(sourceUUID); - if (source) - source->addConnection(connection); - Processor *destination = NULL; - uuid_t destinationUUID; - connection->getDestinationProcessorUUID(destinationUUID); - destination = this->findProcessor(destinationUUID); - if (destination && destination != source) - destination->addConnection(connection); - } -} - -void ProcessGroup::removeConnection(Connection *connection) { - std::lock_guard lock(mtx_); - - if (connections_.find(connection) != connections_.end()) { - // We do not have the same connection in this process group yet - connections_.erase(connection); - logger_->log_info("Remove connection %s into process group %s", - connection->getName().c_str(), name_.c_str()); - uuid_t sourceUUID; - Processor *source = NULL; - connection->getSourceProcessorUUID(sourceUUID); - source = this->findProcessor(sourceUUID); - if (source) - source->removeConnection(connection); - Processor *destination = NULL; - uuid_t destinationUUID; - connection->getDestinationProcessorUUID(destinationUUID); - destination = this->findProcessor(destinationUUID); - if (destination && destination != source) - destination->removeConnection(connection); - } -} diff --git a/libminifi/src/ProcessSession.cpp b/libminifi/src/ProcessSession.cpp deleted file mode 100644 index 3b3eb64842..0000000000 --- a/libminifi/src/ProcessSession.cpp +++ /dev/null @@ -1,790 +0,0 @@ -/** - * @file ProcessSession.cpp - * ProcessSession class implementation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "ProcessSession.h" -#include "FlowController.h" - -ProcessSession::ProcessSession(ProcessContext *processContext) : _processContext(processContext) { - logger_ = Logger::getLogger(); - logger_->log_trace("ProcessSession created for %s", _processContext->getProcessor()->getName().c_str()); - _provenanceReport = NULL; - if (FlowControllerFactory::getFlowController()->getProvenanceRepository()->isEnable()) - { - _provenanceReport = new ProvenanceReporter(_processContext->getProcessor()->getUUIDStr(), - _processContext->getProcessor()->getName()); - } -} - -FlowFileRecord* ProcessSession::create() -{ - std::map empty; - FlowFileRecord *record = new FlowFileRecord(empty); - - if (record) - { - _addedFlowFiles[record->getUUIDStr()] = record; - logger_->log_debug("Create FlowFile with UUID %s", record->getUUIDStr().c_str()); - std::string details = _processContext->getProcessor()->getName() + " creates flow record " + record->getUUIDStr(); - if (_provenanceReport) - _provenanceReport->create(record, details); - } - - return record; -} - -FlowFileRecord* ProcessSession::create(FlowFileRecord *parent) -{ - std::map empty; - FlowFileRecord *record = new FlowFileRecord(empty); - - if (record) - { - _addedFlowFiles[record->getUUIDStr()] = record; - logger_->log_debug("Create FlowFile with UUID %s", record->getUUIDStr().c_str()); - } - - if (record) - { - // Copy attributes - std::map parentAttributes = parent->getAttributes(); - std::map::iterator it; - for (it = parentAttributes.begin(); it!= parentAttributes.end(); it++) - { - if (it->first == FlowAttributeKey(ALTERNATE_IDENTIFIER) || - it->first == FlowAttributeKey(DISCARD_REASON) || - it->first == FlowAttributeKey(UUID)) - // Do not copy special attributes from parent - continue; - record->setAttribute(it->first, it->second); - } - record->_lineageStartDate = parent->_lineageStartDate; - record->_lineageIdentifiers = parent->_lineageIdentifiers; - record->_lineageIdentifiers.insert(parent->_uuidStr); - - } - return record; -} - -FlowFileRecord* ProcessSession::clone(FlowFileRecord *parent) -{ - FlowFileRecord *record = this->create(parent); - if (record) - { - // Copy Resource Claim - record->_claim = parent->_claim; - if (record->_claim) - { - record->_offset = parent->_offset; - record->_size = parent->_size; - record->_claim->increaseFlowFileRecordOwnedCount(); - } - if (_provenanceReport) - _provenanceReport->clone(parent, record); - } - return record; -} - -FlowFileRecord* ProcessSession::cloneDuringTransfer(FlowFileRecord *parent) -{ - std::map empty; - FlowFileRecord *record = new FlowFileRecord(empty); - - if (record) - { - this->_clonedFlowFiles[record->getUUIDStr()] = record; - logger_->log_debug("Clone FlowFile with UUID %s during transfer", record->getUUIDStr().c_str()); - // Copy attributes - std::map parentAttributes = parent->getAttributes(); - std::map::iterator it; - for (it = parentAttributes.begin(); it!= parentAttributes.end(); it++) - { - if (it->first == FlowAttributeKey(ALTERNATE_IDENTIFIER) || - it->first == FlowAttributeKey(DISCARD_REASON) || - it->first == FlowAttributeKey(UUID)) - // Do not copy special attributes from parent - continue; - record->setAttribute(it->first, it->second); - } - record->_lineageStartDate = parent->_lineageStartDate; - record->_lineageIdentifiers = parent->_lineageIdentifiers; - record->_lineageIdentifiers.insert(parent->_uuidStr); - - // Copy Resource Claim - record->_claim = parent->_claim; - if (record->_claim) - { - record->_offset = parent->_offset; - record->_size = parent->_size; - record->_claim->increaseFlowFileRecordOwnedCount(); - } - if (_provenanceReport) - _provenanceReport->clone(parent, record); - } - - return record; -} - -FlowFileRecord* ProcessSession::clone(FlowFileRecord *parent, long offset, long size) -{ - FlowFileRecord *record = this->create(parent); - if (record) - { - if (parent->_claim) - { - if ((offset + size) > (long) parent->_size) - { - // Set offset and size - logger_->log_error("clone offset %d and size %d exceed parent size %d", - offset, size, parent->_size); - // Remove the Add FlowFile for the session - std::map::iterator it = - this->_addedFlowFiles.find(record->getUUIDStr()); - if (it != this->_addedFlowFiles.end()) - this->_addedFlowFiles.erase(record->getUUIDStr()); - delete record; - return NULL; - } - record->_offset = parent->_offset + parent->_offset; - record->_size = size; - // Copy Resource Claim - record->_claim = parent->_claim; - record->_claim->increaseFlowFileRecordOwnedCount(); - } - if (_provenanceReport) - _provenanceReport->clone(parent, record); - } - return record; -} - -void ProcessSession::remove(FlowFileRecord *flow) -{ - flow->_markedDelete = true; - _deletedFlowFiles[flow->getUUIDStr()] = flow; - std::string reason = _processContext->getProcessor()->getName() + " drop flow record " + flow->getUUIDStr(); - if (_provenanceReport) - _provenanceReport->drop(flow, reason); -} - -void ProcessSession::putAttribute(FlowFileRecord *flow, std::string key, std::string value) -{ - flow->setAttribute(key, value); - std::string details = _processContext->getProcessor()->getName() + " modify flow record " + flow->getUUIDStr() + - " attribute " + key + ":" + value; - if (_provenanceReport) - _provenanceReport->modifyAttributes(flow, details); -} - -void ProcessSession::removeAttribute(FlowFileRecord *flow, std::string key) -{ - flow->removeAttribute(key); - std::string details = _processContext->getProcessor()->getName() + " remove flow record " + flow->getUUIDStr() + - " attribute " + key; - if (_provenanceReport) - _provenanceReport->modifyAttributes(flow, details); -} - -void ProcessSession::penalize(FlowFileRecord *flow) -{ - flow->_penaltyExpirationMs = getTimeMillis() + this->_processContext->getProcessor()->getPenalizationPeriodMsec(); -} - -void ProcessSession::transfer(FlowFileRecord *flow, Relationship relationship) -{ - _transferRelationship[flow->getUUIDStr()] = relationship; -} - -void ProcessSession::write(FlowFileRecord *flow, OutputStreamCallback *callback) -{ - ResourceClaim *claim = NULL; - - claim = new ResourceClaim(); - - try - { - std::ofstream fs; - uint64_t startTime = getTimeMillis(); - fs.open(claim->getContentFullPath().c_str(), std::fstream::out | std::fstream::binary | std::fstream::trunc); - if (fs.is_open()) - { - // Call the callback to write the content - callback->process(&fs); - if (fs.good() && fs.tellp() >= 0) - { - flow->_size = fs.tellp(); - flow->_offset = 0; - if (flow->_claim) - { - // Remove the old claim - flow->_claim->decreaseFlowFileRecordOwnedCount(); - flow->_claim = NULL; - } - flow->_claim = claim; - claim->increaseFlowFileRecordOwnedCount(); - /* - logger_->log_debug("Write offset %d length %d into content %s for FlowFile UUID %s", - flow->_offset, flow->_size, flow->_claim->getContentFullPath().c_str(), flow->getUUIDStr().c_str()); */ - fs.close(); - std::string details = _processContext->getProcessor()->getName() + " modify flow record content " + flow->getUUIDStr(); - uint64_t endTime = getTimeMillis(); - if (_provenanceReport) - _provenanceReport->modifyContent(flow, details, endTime - startTime); - } - else - { - fs.close(); - throw Exception(FILE_OPERATION_EXCEPTION, "File Write Error"); - } - } - else - { - throw Exception(FILE_OPERATION_EXCEPTION, "File Open Error"); - } - } - catch (std::exception &exception) - { - if (flow && flow->_claim == claim) - { - flow->_claim->decreaseFlowFileRecordOwnedCount(); - flow->_claim = NULL; - } - if (claim) - delete claim; - logger_->log_debug("Caught Exception %s", exception.what()); - throw; - } - catch (...) - { - if (flow && flow->_claim == claim) - { - flow->_claim->decreaseFlowFileRecordOwnedCount(); - flow->_claim = NULL; - } - if (claim) - delete claim; - logger_->log_debug("Caught Exception during process session write"); - throw; - } -} - -void ProcessSession::append(FlowFileRecord *flow, OutputStreamCallback *callback) -{ - ResourceClaim *claim = NULL; - - if (flow->_claim == NULL) - { - // No existed claim for append, we need to create new claim - return write(flow, callback); - } - - claim = flow->_claim; - - try - { - std::ofstream fs; - uint64_t startTime = getTimeMillis(); - fs.open(claim->getContentFullPath().c_str(), std::fstream::out | std::fstream::binary | std::fstream::app); - if (fs.is_open()) - { - // Call the callback to write the content - std::streampos oldPos = fs.tellp(); - callback->process(&fs); - if (fs.good() && fs.tellp() >= 0) - { - uint64_t appendSize = fs.tellp() - oldPos; - flow->_size += appendSize; - /* - logger_->log_debug("Append offset %d extra length %d to new size %d into content %s for FlowFile UUID %s", - flow->_offset, appendSize, flow->_size, claim->getContentFullPath().c_str(), flow->getUUIDStr().c_str()); */ - fs.close(); - std::string details = _processContext->getProcessor()->getName() + " modify flow record content " + flow->getUUIDStr(); - uint64_t endTime = getTimeMillis(); - if (_provenanceReport) - _provenanceReport->modifyContent(flow, details, endTime - startTime); - } - else - { - fs.close(); - throw Exception(FILE_OPERATION_EXCEPTION, "File Write Error"); - } - } - else - { - throw Exception(FILE_OPERATION_EXCEPTION, "File Open Error"); - } - } - catch (std::exception &exception) - { - logger_->log_debug("Caught Exception %s", exception.what()); - throw; - } - catch (...) - { - logger_->log_debug("Caught Exception during process session append"); - throw; - } -} - -void ProcessSession::read(FlowFileRecord *flow, InputStreamCallback *callback) -{ - try - { - ResourceClaim *claim = NULL; - if (flow->_claim == NULL) - { - // No existed claim for read, we throw exception - throw Exception(FILE_OPERATION_EXCEPTION, "No Content Claim existed for read"); - } - - claim = flow->_claim; - std::ifstream fs; - fs.open(claim->getContentFullPath().c_str(), std::fstream::in | std::fstream::binary); - if (fs.is_open()) - { - fs.seekg(flow->_offset, fs.beg); - - if (fs.good()) - { - callback->process(&fs); - /* - logger_->log_debug("Read offset %d size %d content %s for FlowFile UUID %s", - flow->_offset, flow->_size, claim->getContentFullPath().c_str(), flow->getUUIDStr().c_str()); */ - fs.close(); - } - else - { - fs.close(); - throw Exception(FILE_OPERATION_EXCEPTION, "File Read Error"); - } - } - else - { - throw Exception(FILE_OPERATION_EXCEPTION, "File Open Error"); - } - } - catch (std::exception &exception) - { - logger_->log_debug("Caught Exception %s", exception.what()); - throw; - } - catch (...) - { - logger_->log_debug("Caught Exception during process session read"); - throw; - } -} - -void ProcessSession::import(std::string source, FlowFileRecord *flow, bool keepSource, uint64_t offset) -{ - ResourceClaim *claim = NULL; - - claim = new ResourceClaim(); - char *buf = NULL; - int size = 4096; - buf = new char [size]; - - try - { - std::ofstream fs; - uint64_t startTime = getTimeMillis(); - fs.open(claim->getContentFullPath().c_str(), std::fstream::out | std::fstream::binary | std::fstream::trunc); - std::ifstream input; - input.open(source.c_str(), std::fstream::in | std::fstream::binary); - - if (fs.is_open() && input.is_open()) - { - // Open the source file and stream to the flow file - input.seekg(offset, fs.beg); - while (input.good()) - { - input.read(buf, size); - if (input) - fs.write(buf, size); - else - fs.write(buf, input.gcount()); - } - - if (fs.good() && fs.tellp() >= 0) - { - flow->_size = fs.tellp(); - flow->_offset = 0; - if (flow->_claim) - { - // Remove the old claim - flow->_claim->decreaseFlowFileRecordOwnedCount(); - flow->_claim = NULL; - } - flow->_claim = claim; - claim->increaseFlowFileRecordOwnedCount(); - - logger_->log_debug("Import offset %d length %d into content %s for FlowFile UUID %s", - flow->_offset, flow->_size, flow->_claim->getContentFullPath().c_str(), flow->getUUIDStr().c_str()); - - fs.close(); - input.close(); - if (!keepSource) - std::remove(source.c_str()); - std::string details = _processContext->getProcessor()->getName() + " modify flow record content " + flow->getUUIDStr(); - uint64_t endTime = getTimeMillis(); - if (_provenanceReport) - _provenanceReport->modifyContent(flow, details, endTime - startTime); - } - else - { - fs.close(); - input.close(); - throw Exception(FILE_OPERATION_EXCEPTION, "File Import Error"); - } - } - else - { - throw Exception(FILE_OPERATION_EXCEPTION, "File Import Error"); - } - - delete[] buf; - } - catch (std::exception &exception) - { - if (flow && flow->_claim == claim) - { - flow->_claim->decreaseFlowFileRecordOwnedCount(); - flow->_claim = NULL; - } - if (claim) - delete claim; - logger_->log_debug("Caught Exception %s", exception.what()); - delete[] buf; - throw; - } - catch (...) - { - if (flow && flow->_claim == claim) - { - flow->_claim->decreaseFlowFileRecordOwnedCount(); - flow->_claim = NULL; - } - if (claim) - delete claim; - logger_->log_debug("Caught Exception during process session write"); - delete[] buf; - throw; - } -} - -void ProcessSession::commit() -{ - try - { - // First we clone the flow record based on the transfered relationship for updated flow record - for (auto && it : _updatedFlowFiles) - { - FlowFileRecord *record = it.second; - if (record->_markedDelete) - continue; - std::map::iterator itRelationship = - this->_transferRelationship.find(record->getUUIDStr()); - if (itRelationship != _transferRelationship.end()) - { - Relationship relationship = itRelationship->second; - // Find the relationship, we need to find the connections for that relationship - std::set connections = - _processContext->getProcessor()->getOutGoingConnections(relationship.getName()); - if (connections.empty()) - { - // No connection - if (!_processContext->getProcessor()->isAutoTerminated(relationship)) - { - // Not autoterminate, we should have the connect - std::string message = "Connect empty for non auto terminated relationship" + relationship.getName(); - throw Exception(PROCESS_SESSION_EXCEPTION, message.c_str()); - } - else - { - // Autoterminated - remove(record); - } - } - else - { - // We connections, clone the flow and assign the connection accordingly - for (std::set::iterator itConnection = connections.begin(); itConnection != connections.end(); ++itConnection) - { - Connection *connection(*itConnection); - if (itConnection == connections.begin()) - { - // First connection which the flow need be routed to - record->_connection = connection; - } - else - { - // Clone the flow file and route to the connection - FlowFileRecord *cloneRecord; - cloneRecord = this->cloneDuringTransfer(record); - if (cloneRecord) - cloneRecord->_connection = connection; - else - throw Exception(PROCESS_SESSION_EXCEPTION, "Can not clone the flow for transfer"); - } - } - } - } - else - { - // Can not find relationship for the flow - throw Exception(PROCESS_SESSION_EXCEPTION, "Can not find the transfer relationship for the flow"); - } - } - // Do the samething for added flow file - for(const auto it : _addedFlowFiles) - { - FlowFileRecord *record = it.second; - if (record->_markedDelete) - continue; - std::map::iterator itRelationship = - this->_transferRelationship.find(record->getUUIDStr()); - if (itRelationship != _transferRelationship.end()) - { - Relationship relationship = itRelationship->second; - // Find the relationship, we need to find the connections for that relationship - std::set connections = - _processContext->getProcessor()->getOutGoingConnections(relationship.getName()); - if (connections.empty()) - { - // No connection - if (!_processContext->getProcessor()->isAutoTerminated(relationship)) - { - // Not autoterminate, we should have the connect - std::string message = "Connect empty for non auto terminated relationship " + relationship.getName(); - throw Exception(PROCESS_SESSION_EXCEPTION, message.c_str()); - } - else - { - // Autoterminated - remove(record); - } - } - else - { - // We connections, clone the flow and assign the connection accordingly - for (std::set::iterator itConnection = connections.begin(); itConnection != connections.end(); ++itConnection) - { - Connection *connection(*itConnection); - if (itConnection == connections.begin()) - { - // First connection which the flow need be routed to - record->_connection = connection; - } - else - { - // Clone the flow file and route to the connection - FlowFileRecord *cloneRecord; - cloneRecord = this->cloneDuringTransfer(record); - if (cloneRecord) - cloneRecord->_connection = connection; - else - throw Exception(PROCESS_SESSION_EXCEPTION, "Can not clone the flow for transfer"); - } - } - } - } - else - { - // Can not find relationship for the flow - throw Exception(PROCESS_SESSION_EXCEPTION, "Can not find the transfer relationship for the flow"); - } - } - // Complete process the added and update flow files for the session, send the flow file to its queue - for(const auto &it : _updatedFlowFiles) - { - FlowFileRecord *record = it.second; - if (record->_markedDelete) - { - continue; - } - if (record->_connection) - record->_connection->put(record); - else - delete record; - } - for(const auto &it : _addedFlowFiles) - { - FlowFileRecord *record = it.second; - if (record->_markedDelete) - { - continue; - } - if (record->_connection) - record->_connection->put(record); - else - delete record; - } - // Process the clone flow files - for(const auto &it : _clonedFlowFiles) - { - FlowFileRecord *record = it.second; - if (record->_markedDelete) - { - continue; - } - if (record->_connection) - record->_connection->put(record); - else - delete record; - } - // Delete the deleted flow files - for(const auto &it : _deletedFlowFiles) - { - FlowFileRecord *record = it.second; - delete record; - } - // Delete the snapshot - for(const auto &it : _originalFlowFiles) - { - FlowFileRecord *record = it.second; - delete record; - } - // All done - _updatedFlowFiles.clear(); - _addedFlowFiles.clear(); - _clonedFlowFiles.clear(); - _deletedFlowFiles.clear(); - _originalFlowFiles.clear(); - // persistent the provenance report - if (this->_provenanceReport) - this->_provenanceReport->commit(); - logger_->log_trace("ProcessSession committed for %s", _processContext->getProcessor()->getName().c_str()); - } - catch (std::exception &exception) - { - logger_->log_debug("Caught Exception %s", exception.what()); - throw; - } - catch (...) - { - logger_->log_debug("Caught Exception during process session commit"); - throw; - } -} - - -void ProcessSession::rollback() -{ - try - { - // Requeue the snapshot of the flowfile back - for(const auto &it : _originalFlowFiles) - { - FlowFileRecord *record = it.second; - if (record->_orginalConnection) - { - record->_snapshot = false; - record->_orginalConnection->put(record); - } - else - delete record; - } - _originalFlowFiles.clear(); - // Process the clone flow files - for(const auto &it : _clonedFlowFiles) - { - FlowFileRecord *record = it.second; - delete record; - } - _clonedFlowFiles.clear(); - for(const auto &it : _addedFlowFiles) - { - FlowFileRecord *record = it.second; - delete record; - } - _addedFlowFiles.clear(); - for(const auto &it : _updatedFlowFiles) - { - FlowFileRecord *record = it.second; - delete record; - } - _updatedFlowFiles.clear(); - _deletedFlowFiles.clear(); - logger_->log_trace("ProcessSession rollback for %s", _processContext->getProcessor()->getName().c_str()); - } - catch (std::exception &exception) - { - logger_->log_debug("Caught Exception %s", exception.what()); - throw; - } - catch (...) - { - logger_->log_debug("Caught Exception during process session roll back"); - throw; - } -} - -FlowFileRecord *ProcessSession::get() -{ - Connection *first = _processContext->getProcessor()->getNextIncomingConnection(); - - if (first == NULL) - return NULL; - - Connection *current = first; - - do - { - std::set expired; - FlowFileRecord *ret = current->poll(expired); - if (expired.size() > 0) - { - // Remove expired flow record - for (std::set::iterator it = expired.begin(); it != expired.end(); ++it) - { - FlowFileRecord *record = *it; - std::string details = _processContext->getProcessor()->getName() + " expire flow record " + record->getUUIDStr(); - if (_provenanceReport) - _provenanceReport->expire(record, details); - delete (record); - } - } - if (ret) - { - // add the flow record to the current process session update map - ret->_markedDelete = false; - _updatedFlowFiles[ret->getUUIDStr()] = ret; - std::map empty; - FlowFileRecord *snapshot = new FlowFileRecord(empty); - logger_->log_debug("Create Snapshot FlowFile with UUID %s", snapshot->getUUIDStr().c_str()); - snapshot->duplicate(ret); - // save a snapshot - _originalFlowFiles[snapshot->getUUIDStr()] = snapshot; - return ret; - } - current = _processContext->getProcessor()->getNextIncomingConnection(); - } - while (current != NULL && current != first); - - return NULL; -} - diff --git a/libminifi/src/Processor.cpp b/libminifi/src/Processor.cpp deleted file mode 100644 index 94aaa20711..0000000000 --- a/libminifi/src/Processor.cpp +++ /dev/null @@ -1,526 +0,0 @@ -/** - * @file Processor.cpp - * Processor class implementation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "Processor.h" -#include "ProcessContext.h" -#include "ProcessSession.h" -#include "ProcessSessionFactory.h" - -Processor::Processor(std::string name, uuid_t uuid) -: _name(name) -{ - if (!uuid) - // Generate the global UUID for the flow record - uuid_generate(_uuid); - else - uuid_copy(_uuid, uuid); - - char uuidStr[37]; - uuid_unparse_lower(_uuid, uuidStr); - _uuidStr = uuidStr; - _hasWork.store(false); - // Setup the default values - _state = DISABLED; - _strategy = TIMER_DRIVEN; - _lossTolerant = false; - _triggerWhenEmpty = false; - _schedulingPeriodNano = MINIMUM_SCHEDULING_NANOS; - _runDurantionNano = 0; - _yieldPeriodMsec = DEFAULT_YIELD_PERIOD_SECONDS * 1000; - _penalizationPeriodMsec = DEFAULT_PENALIZATION_PERIOD_SECONDS * 1000; - _maxConcurrentTasks = 1; - _activeTasks = 0; - _yieldExpiration = 0; - _incomingConnectionsIter = this->_incomingConnections.begin(); - logger_ = Logger::getLogger(); - logger_->log_info("Processor %s created UUID %s", _name.c_str(), _uuidStr.c_str()); -} - -Processor::~Processor() -{ - -} - -bool Processor::isRunning() -{ - return (_state == RUNNING && _activeTasks > 0); -} - -void Processor::setScheduledState(ScheduledState state) -{ - _state = state; -} - -bool Processor::setSupportedProperties(std::set properties) -{ - if (isRunning()) - { - logger_->log_info("Can not set processor property while the process %s is running", - _name.c_str()); - return false; - } - - std::lock_guard lock(_mtx); - - _properties.clear(); - for (auto item : properties) - { - _properties[item.getName()] = item; - logger_->log_info("Processor %s supported property name %s", _name.c_str(), item.getName().c_str()); - } - - return true; -} - -bool Processor::setSupportedRelationships(std::set relationships) -{ - if (isRunning()) - { - logger_->log_info("Can not set processor supported relationship while the process %s is running", - _name.c_str()); - return false; - } - - std::lock_guard lock(_mtx); - - _relationships.clear(); - for(auto item : relationships) - { - _relationships[item.getName()] = item; - logger_->log_info("Processor %s supported relationship name %s", _name.c_str(), item.getName().c_str()); - } - - return true; -} - -bool Processor::setAutoTerminatedRelationships(std::set relationships) -{ - if (isRunning()) - { - logger_->log_info("Can not set processor auto terminated relationship while the process %s is running", - _name.c_str()); - return false; - } - - std::lock_guard lock(_mtx); - - _autoTerminatedRelationships.clear(); - for(auto item : relationships) - { - _autoTerminatedRelationships[item.getName()] = item; - logger_->log_info("Processor %s auto terminated relationship name %s", _name.c_str(), item.getName().c_str()); - } - - return true; -} - -bool Processor::isAutoTerminated(Relationship relationship) -{ - bool isRun = isRunning(); - - auto conditionalLock = !isRun ? - std::unique_lock() - : std::unique_lock(_mtx); - - const auto &it = _autoTerminatedRelationships.find(relationship.getName()); - if (it != _autoTerminatedRelationships.end()) - { - return true; - } - else - { - return false; - } -} - -bool Processor::isSupportedRelationship(Relationship relationship) -{ - bool isRun = isRunning(); - - auto conditionalLock = !isRun ? - std::unique_lock() - : std::unique_lock(_mtx); - - const auto &it = _relationships.find(relationship.getName()); - if (it != _relationships.end()) - { - return true; - } - else - { - return false; - } -} - -bool Processor::getProperty(std::string name, std::string &value) -{ - bool isRun = isRunning(); - - - auto conditionalLock = !isRun ? - std::unique_lock() - : std::unique_lock(_mtx); - - const auto &it = _properties.find(name); - if (it != _properties.end()) - { - Property item = it->second; - value = item.getValue(); - return true; - } - else - { - return false; - } -} - -bool Processor::setProperty(std::string name, std::string value) -{ - - std::lock_guard lock(_mtx); - auto &&it = _properties.find(name); - - if (it != _properties.end()) - { - Property item = it->second; - item.setValue(value); - _properties[item.getName()] = item; - logger_->log_info("Processor %s property name %s value %s", _name.c_str(), item.getName().c_str(), value.c_str()); - return true; - } - else - { - return false; - } -} - -bool Processor::setProperty(Property prop, std::string value) { - - std::lock_guard lock(_mtx); - auto it = _properties.find( - prop.getName()); - - if (it != _properties.end()) { - Property item = it->second; - item.setValue(value); - _properties[item.getName()] = item; - logger_->log_info("Processor %s property name %s value %s", - _name.c_str(), item.getName().c_str(), value.c_str()); - return true; - } else { - Property newProp(prop); - newProp.setValue(value); - _properties.insert( - std::pair(prop.getName(), newProp)); - return true; - - return false; - } -} - -std::set Processor::getOutGoingConnections(std::string relationship) -{ - std::set empty; - - auto &&it = _outGoingConnections.find(relationship); - if (it != _outGoingConnections.end()) - { - return _outGoingConnections[relationship]; - } - else - { - return empty; - } -} - -bool Processor::addConnection(Connection *connection) -{ - bool ret = false; - - - if (isRunning()) - { - logger_->log_info("Can not add connection while the process %s is running", - _name.c_str()); - return false; - } - - - std::lock_guard lock(_mtx); - - uuid_t srcUUID; - uuid_t destUUID; - - connection->getSourceProcessorUUID(srcUUID); - connection->getDestinationProcessorUUID(destUUID); - char uuid_str[37]; - - - uuid_unparse_lower(_uuid, uuid_str); - std::string my_uuid = uuid_str; - uuid_unparse_lower(destUUID, uuid_str); - std::string destination_uuid = uuid_str; - if (my_uuid == destination_uuid) - { - // Connection is destination to the current processor - if (_incomingConnections.find(connection) == _incomingConnections.end()) - { - _incomingConnections.insert(connection); - connection->setDestinationProcessor(this); - logger_->log_info("Add connection %s into Processor %s incoming connection", - connection->getName().c_str(), _name.c_str()); - _incomingConnectionsIter = this->_incomingConnections.begin(); - ret = true; - } - } - uuid_unparse_lower(srcUUID, uuid_str); - std::string source_uuid = uuid_str; - if (my_uuid == source_uuid) - { - std::string relationship = connection->getRelationship().getName(); - // Connection is source from the current processor - auto &&it = - _outGoingConnections.find(relationship); - if (it != _outGoingConnections.end()) - { - // We already has connection for this relationship - std::set existedConnection = it->second; - if (existedConnection.find(connection) == existedConnection.end()) - { - // We do not have the same connection for this relationship yet - existedConnection.insert(connection); - connection->setSourceProcessor(this); - _outGoingConnections[relationship] = existedConnection; - logger_->log_info("Add connection %s into Processor %s outgoing connection for relationship %s", - connection->getName().c_str(), _name.c_str(), relationship.c_str()); - ret = true; - } - } - else - { - - // We do not have any outgoing connection for this relationship yet - std::set newConnection; - newConnection.insert(connection); - connection->setSourceProcessor(this); - _outGoingConnections[relationship] = newConnection; - logger_->log_info("Add connection %s into Processor %s outgoing connection for relationship %s", - connection->getName().c_str(), _name.c_str(), relationship.c_str()); - ret = true; - } - } - - - return ret; -} - -void Processor::removeConnection(Connection *connection) -{ - if (isRunning()) - { - logger_->log_info("Can not remove connection while the process %s is running", - _name.c_str()); - return; - } - - std::lock_guard lock(_mtx); - - uuid_t srcUUID; - uuid_t destUUID; - - connection->getSourceProcessorUUID(srcUUID); - connection->getDestinationProcessorUUID(destUUID); - - if (uuid_compare(_uuid, destUUID) == 0) - { - // Connection is destination to the current processor - if (_incomingConnections.find(connection) != _incomingConnections.end()) - { - _incomingConnections.erase(connection); - connection->setDestinationProcessor(NULL); - logger_->log_info("Remove connection %s into Processor %s incoming connection", - connection->getName().c_str(), _name.c_str()); - _incomingConnectionsIter = this->_incomingConnections.begin(); - } - } - - if (uuid_compare(_uuid, srcUUID) == 0) - { - std::string relationship = connection->getRelationship().getName(); - // Connection is source from the current processor - auto &&it = - _outGoingConnections.find(relationship); - if (it == _outGoingConnections.end()) - { - return; - } - else - { - if (_outGoingConnections[relationship].find(connection) != _outGoingConnections[relationship].end()) - { - _outGoingConnections[relationship].erase(connection); - connection->setSourceProcessor(NULL); - logger_->log_info("Remove connection %s into Processor %s outgoing connection for relationship %s", - connection->getName().c_str(), _name.c_str(), relationship.c_str()); - } - } - } -} - -Connection *Processor::getNextIncomingConnection() -{ - std::lock_guard lock(_mtx); - - if (_incomingConnections.size() == 0) - return NULL; - - if (_incomingConnectionsIter == _incomingConnections.end()) - _incomingConnectionsIter = _incomingConnections.begin(); - - Connection *ret = *_incomingConnectionsIter; - _incomingConnectionsIter++; - - if (_incomingConnectionsIter == _incomingConnections.end()) - _incomingConnectionsIter = _incomingConnections.begin(); - - return ret; -} - -bool Processor::flowFilesQueued() -{ - std::lock_guard lock(_mtx); - - if (_incomingConnections.size() == 0) - return false; - - for(auto &&connection : _incomingConnections) - { - if (connection->getQueueSize() > 0) - return true; - } - - return false; -} - -bool Processor::flowFilesOutGoingFull() -{ - std::lock_guard lock(_mtx); - - for(auto &&connection : _outGoingConnections) - { - // We already has connection for this relationship - std::set existedConnection = connection.second; - for(const auto connection : existedConnection) - { - if (connection->isFull()) - return true; - } - } - - return false; -} - -void Processor::onTrigger(ProcessContext *context, ProcessSessionFactory *sessionFactory) -{ - auto session = sessionFactory->createSession(); - - try { - // Call the virtual trigger function - onTrigger(context, session.get()); - session->commit(); - } - catch (std::exception &exception) - { - logger_->log_debug("Caught Exception %s", exception.what()); - session->rollback(); - throw; - } - catch (...) - { - logger_->log_debug("Caught Exception Processor::onTrigger"); - session->rollback(); - throw; - } -} - -void Processor::waitForWork(uint64_t timeoutMs) -{ - _hasWork.store( isWorkAvailable() ); - - if (!_hasWork.load()) - { - std::unique_lock lock(_workAvailableMtx); - _hasWorkCondition.wait_for(lock, std::chrono::milliseconds(timeoutMs), [&] { return _hasWork.load(); }); - } - -} - -void Processor::notifyWork() -{ - // Do nothing if we are not event-driven - if (_strategy != EVENT_DRIVEN) - { - return; - } - - { - _hasWork.store( isWorkAvailable() ); - - - if (_hasWork.load()) - { - _hasWorkCondition.notify_one(); - } - } -} - -bool Processor::isWorkAvailable() -{ - // We have work if any incoming connection has work - bool hasWork = false; - - try - { - for (const auto &conn : getIncomingConnections()) - { - if (conn->getQueueSize() > 0) - { - hasWork = true; - break; - } - } - } - catch (...) - { - logger_->log_error("Caught an exception while checking if work is available; unless it was positively determined that work is available, assuming NO work is available!"); - } - - return hasWork; -} diff --git a/libminifi/src/Provenance.cpp b/libminifi/src/Provenance.cpp deleted file mode 100644 index 58cf7308c5..0000000000 --- a/libminifi/src/Provenance.cpp +++ /dev/null @@ -1,566 +0,0 @@ -/** - * @file Provenance.cpp - * Provenance implemenatation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include "io/DataStream.h" -#include "io/Serializable.h" -#include "Provenance.h" -#include "Relationship.h" -#include "Logger.h" -#include "FlowController.h" - -//! DeSerialize -bool ProvenanceEventRecord::DeSerialize(ProvenanceRepository *repo, - std::string key) { - std::string value; - bool ret; - - ret = repo->Get(key, value); - - if (!ret) { - logger_->log_error("NiFi Provenance Store event %s can not found", - key.c_str()); - return false; - } else - logger_->log_debug("NiFi Provenance Read event %s length %d", - key.c_str(), value.length()); - - - DataStream stream((const uint8_t*)value.data(),value.length()); - - ret = DeSerialize(stream); - - if (ret) { - logger_->log_debug( - "NiFi Provenance retrieve event %s size %d eventType %d success", - _eventIdStr.c_str(), stream.getSize(), _eventType); - } else { - logger_->log_debug( - "NiFi Provenance retrieve event %s size %d eventType %d fail", - _eventIdStr.c_str(), stream.getSize(), _eventType); - } - - return ret; -} - -bool ProvenanceEventRecord::Serialize(ProvenanceRepository *repo) { - - DataStream outStream; - - int ret; - - ret = writeUTF(this->_eventIdStr,&outStream); - if (ret <= 0) { - - return false; - } - - uint32_t eventType = this->_eventType; - ret = write(eventType,&outStream); - if (ret != 4) { - - return false; - } - - ret = write(this->_eventTime,&outStream); - if (ret != 8) { - - return false; - } - - ret = write(this->_entryDate,&outStream); - if (ret != 8) { - return false; - } - - ret = write(this->_eventDuration,&outStream); - if (ret != 8) { - - return false; - } - - ret = write(this->_lineageStartDate,&outStream); - if (ret != 8) { - - return false; - } - - ret = writeUTF(this->_componentId,&outStream); - if (ret <= 0) { - - return false; - } - - ret = writeUTF(this->_componentType,&outStream); - if (ret <= 0) { - - return false; - } - - ret = writeUTF(this->_uuid,&outStream); - if (ret <= 0) { - - return false; - } - - ret = writeUTF(this->_details,&outStream); - if (ret <= 0) { - - return false; - } - - // write flow attributes - uint32_t numAttributes = this->_attributes.size(); - ret = write(numAttributes,&outStream); - if (ret != 4) { - - return false; - } - - for (auto itAttribute : _attributes) { - ret = writeUTF(itAttribute.first,&outStream, true); - if (ret <= 0) { - - return false; - } - ret = writeUTF(itAttribute.second,&outStream, true); - if (ret <= 0) { - - return false; - } - } - - ret = writeUTF(this->_contentFullPath,&outStream); - if (ret <= 0) { - - return false; - } - - ret = write(this->_size,&outStream); - if (ret != 8) { - - return false; - } - - ret = write(this->_offset,&outStream); - if (ret != 8) { - - return false; - } - - ret = writeUTF(this->_sourceQueueIdentifier,&outStream); - if (ret <= 0) { - - return false; - } - - if (this->_eventType == ProvenanceEventRecord::FORK - || this->_eventType == ProvenanceEventRecord::CLONE - || this->_eventType == ProvenanceEventRecord::JOIN) { - // write UUIDs - uint32_t number = this->_parentUuids.size(); - ret = write(number,&outStream); - if (ret != 4) { - - return false; - } - for (auto parentUUID : _parentUuids) { - ret = writeUTF(parentUUID,&outStream); - if (ret <= 0) { - - return false; - } - } - number = this->_childrenUuids.size(); - ret = write(number,&outStream); - if (ret != 4) { - return false; - } - for (auto childUUID : _childrenUuids) { - ret = writeUTF(childUUID,&outStream); - if (ret <= 0) { - - return false; - } - } - } else if (this->_eventType == ProvenanceEventRecord::SEND - || this->_eventType == ProvenanceEventRecord::FETCH) { - ret = writeUTF(this->_transitUri,&outStream); - if (ret <= 0) { - - return false; - } - } else if (this->_eventType == ProvenanceEventRecord::RECEIVE) { - ret = writeUTF(this->_transitUri,&outStream); - if (ret <= 0) { - - return false; - } - ret = writeUTF(this->_sourceSystemFlowFileIdentifier,&outStream); - if (ret <= 0) { - - return false; - } - } - - // Persistent to the DB - - if (repo->Put(_eventIdStr, const_cast(outStream.getBuffer()), outStream.getSize())) { - logger_->log_debug("NiFi Provenance Store event %s size %d success", - _eventIdStr.c_str(), outStream.getSize()); - return true; - } else { - logger_->log_error("NiFi Provenance Store event %s size %d fail", - _eventIdStr.c_str(), outStream.getSize()); - return false; - } - - // cleanup - - return true; -} - -bool ProvenanceEventRecord::DeSerialize(const uint8_t *buffer, const int bufferSize) { - - int ret; - - DataStream outStream(buffer,bufferSize); - - ret = readUTF(this->_eventIdStr,&outStream); - - if (ret <= 0) { - return false; - } - - uint32_t eventType; - ret = read(eventType,&outStream); - if (ret != 4) { - return false; - } - this->_eventType = (ProvenanceEventRecord::ProvenanceEventType) eventType; - - ret = read(this->_eventTime,&outStream); - if (ret != 8) { - return false; - } - - ret = read(this->_entryDate,&outStream); - if (ret != 8) { - return false; - } - - ret = read(this->_eventDuration,&outStream); - if (ret != 8) { - return false; - } - - ret = read(this->_lineageStartDate,&outStream); - if (ret != 8) { - return false; - } - - ret = readUTF(this->_componentId,&outStream); - if (ret <= 0) { - return false; - } - - ret = readUTF(this->_componentType,&outStream); - if (ret <= 0) { - return false; - } - - ret = readUTF(this->_uuid,&outStream); - if (ret <= 0) { - return false; - } - - ret = readUTF(this->_details,&outStream); - - if (ret <= 0) { - return false; - } - - // read flow attributes - uint32_t numAttributes = 0; - ret = read(numAttributes,&outStream); - if (ret != 4) { - return false; - } - - for (uint32_t i = 0; i < numAttributes; i++) { - std::string key; - ret = readUTF(key,&outStream, true); - if (ret <= 0) { - return false; - } - std::string value; - ret = readUTF(value,&outStream, true); - if (ret <= 0) { - return false; - } - this->_attributes[key] = value; - } - - ret = readUTF(this->_contentFullPath,&outStream); - if (ret <= 0) { - return false; - } - - ret = read(this->_size,&outStream); - if (ret != 8) { - return false; - } - - ret = read(this->_offset,&outStream); - if (ret != 8) { - return false; - } - - ret = readUTF(this->_sourceQueueIdentifier,&outStream); - if (ret <= 0) { - return false; - } - - if (this->_eventType == ProvenanceEventRecord::FORK - || this->_eventType == ProvenanceEventRecord::CLONE - || this->_eventType == ProvenanceEventRecord::JOIN) { - // read UUIDs - uint32_t number = 0; - ret = read(number,&outStream); - if (ret != 4) { - return false; - } - - - for (uint32_t i = 0; i < number; i++) { - std::string parentUUID; - ret = readUTF(parentUUID,&outStream); - if (ret <= 0) { - return false; - } - this->addParentUuid(parentUUID); - } - number = 0; - ret = read(number,&outStream); - if (ret != 4) { - return false; - } - for (uint32_t i = 0; i < number; i++) { - std::string childUUID; - ret = readUTF(childUUID,&outStream); - if (ret <= 0) { - return false; - } - this->addChildUuid(childUUID); - } - } else if (this->_eventType == ProvenanceEventRecord::SEND - || this->_eventType == ProvenanceEventRecord::FETCH) { - ret = readUTF(this->_transitUri,&outStream); - if (ret <= 0) { - return false; - } - } else if (this->_eventType == ProvenanceEventRecord::RECEIVE) { - ret = readUTF(this->_transitUri,&outStream); - if (ret <= 0) { - return false; - } - ret = readUTF(this->_sourceSystemFlowFileIdentifier,&outStream); - if (ret <= 0) { - return false; - } - } - - return true; -} - -void ProvenanceReporter::commit() { - if (!FlowControllerFactory::getFlowController()->getProvenanceRepository()->isEnable()) - return; - for (auto event : _events) { - if (!FlowControllerFactory::getFlowController()->getProvenanceRepository()->isFull()) { - event->Serialize( - FlowControllerFactory::getFlowController()->getProvenanceRepository()); - } else { - logger_->log_debug("Provenance Repository is full"); - } - } -} - -void ProvenanceReporter::create(FlowFileRecord *flow, std::string detail) { - ProvenanceEventRecord *event = allocate(ProvenanceEventRecord::CREATE, - flow); - - if (event) { - event->setDetails(detail); - add(event); - } -} - -void ProvenanceReporter::route(FlowFileRecord *flow, Relationship relation, - std::string detail, uint64_t processingDuration) { - ProvenanceEventRecord *event = allocate(ProvenanceEventRecord::ROUTE, flow); - - if (event) { - event->setDetails(detail); - event->setRelationship(relation.getName()); - event->setEventDuration(processingDuration); - add(event); - } -} - -void ProvenanceReporter::modifyAttributes(FlowFileRecord *flow, - std::string detail) { - ProvenanceEventRecord *event = allocate( - ProvenanceEventRecord::ATTRIBUTES_MODIFIED, flow); - - if (event) { - event->setDetails(detail); - add(event); - } -} - -void ProvenanceReporter::modifyContent(FlowFileRecord *flow, std::string detail, - uint64_t processingDuration) { - ProvenanceEventRecord *event = allocate( - ProvenanceEventRecord::CONTENT_MODIFIED, flow); - - if (event) { - event->setDetails(detail); - event->setEventDuration(processingDuration); - add(event); - } -} - -void ProvenanceReporter::clone(FlowFileRecord *parent, FlowFileRecord *child) { - ProvenanceEventRecord *event = allocate(ProvenanceEventRecord::CLONE, - parent); - - if (event) { - event->addChildFlowFile(child); - event->addParentFlowFile(parent); - add(event); - } -} - -void ProvenanceReporter::join(std::vector parents, - FlowFileRecord *child, std::string detail, - uint64_t processingDuration) { - ProvenanceEventRecord *event = allocate(ProvenanceEventRecord::JOIN, child); - - if (event) { - event->addChildFlowFile(child); - std::vector::iterator it; - for (it = parents.begin(); it != parents.end(); it++) { - FlowFileRecord *record = *it; - event->addParentFlowFile(record); - } - event->setDetails(detail); - event->setEventDuration(processingDuration); - add(event); - } -} - -void ProvenanceReporter::fork(std::vector child, - FlowFileRecord *parent, std::string detail, - uint64_t processingDuration) { - ProvenanceEventRecord *event = allocate(ProvenanceEventRecord::FORK, - parent); - - if (event) { - event->addParentFlowFile(parent); - std::vector::iterator it; - for (it = child.begin(); it != child.end(); it++) { - FlowFileRecord *record = *it; - event->addChildFlowFile(record); - } - event->setDetails(detail); - event->setEventDuration(processingDuration); - add(event); - } -} - -void ProvenanceReporter::expire(FlowFileRecord *flow, std::string detail) { - ProvenanceEventRecord *event = allocate(ProvenanceEventRecord::EXPIRE, - flow); - - if (event) { - event->setDetails(detail); - add(event); - } -} - -void ProvenanceReporter::drop(FlowFileRecord *flow, std::string reason) { - ProvenanceEventRecord *event = allocate(ProvenanceEventRecord::DROP, flow); - - if (event) { - std::string dropReason = "Discard reason: " + reason; - event->setDetails(dropReason); - add(event); - } -} - -void ProvenanceReporter::send(FlowFileRecord *flow, std::string transitUri, - std::string detail, uint64_t processingDuration, bool force) { - ProvenanceEventRecord *event = allocate(ProvenanceEventRecord::SEND, flow); - - if (event) { - event->setTransitUri(transitUri); - event->setDetails(detail); - event->setEventDuration(processingDuration); - if (!force) { - add(event); - } else { - if (!FlowControllerFactory::getFlowController()->getProvenanceRepository()->isFull()) - event->Serialize( - FlowControllerFactory::getFlowController()->getProvenanceRepository()); - delete event; - } - } -} - -void ProvenanceReporter::receive(FlowFileRecord *flow, std::string transitUri, - std::string sourceSystemFlowFileIdentifier, std::string detail, - uint64_t processingDuration) { - ProvenanceEventRecord *event = allocate(ProvenanceEventRecord::RECEIVE, - flow); - - if (event) { - event->setTransitUri(transitUri); - event->setDetails(detail); - event->setEventDuration(processingDuration); - event->setSourceSystemFlowFileIdentifier( - sourceSystemFlowFileIdentifier); - add(event); - } -} - -void ProvenanceReporter::fetch(FlowFileRecord *flow, std::string transitUri, - std::string detail, uint64_t processingDuration) { - ProvenanceEventRecord *event = allocate(ProvenanceEventRecord::FETCH, flow); - - if (event) { - event->setTransitUri(transitUri); - event->setDetails(detail); - event->setEventDuration(processingDuration); - add(event); - } -} - diff --git a/libminifi/src/PutFile.cpp b/libminifi/src/PutFile.cpp deleted file mode 100644 index d7cc83ad60..0000000000 --- a/libminifi/src/PutFile.cpp +++ /dev/null @@ -1,203 +0,0 @@ -/** - * @file PutFile.cpp - * PutFile class implementation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include - -#include "utils/StringUtils.h" -#include "utils/TimeUtil.h" -#include "PutFile.h" -#include "ProcessContext.h" -#include "ProcessSession.h" - -const std::string PutFile::CONFLICT_RESOLUTION_STRATEGY_REPLACE("replace"); -const std::string PutFile::CONFLICT_RESOLUTION_STRATEGY_IGNORE("ignore"); -const std::string PutFile::CONFLICT_RESOLUTION_STRATEGY_FAIL("fail"); - -const std::string PutFile::ProcessorName("PutFile"); - -Property PutFile::Directory("Output Directory", "The output directory to which to put files", "."); -Property PutFile::ConflictResolution("Conflict Resolution Strategy", "Indicates what should happen when a file with the same name already exists in the output directory", CONFLICT_RESOLUTION_STRATEGY_FAIL); - -Relationship PutFile::Success("success", "All files are routed to success"); -Relationship PutFile::Failure("failure", "Failed files (conflict, write failure, etc.) are transferred to failure"); - -void PutFile::initialize() -{ - //! Set the supported properties - std::set properties; - properties.insert(Directory); - properties.insert(ConflictResolution); - setSupportedProperties(properties); - //! Set the supported relationships - std::set relationships; - relationships.insert(Success); - relationships.insert(Failure); - setSupportedRelationships(relationships); -} - -void PutFile::onTrigger(ProcessContext *context, ProcessSession *session) -{ - std::string directory; - - if (!context->getProperty(Directory.getName(), directory)) - { - logger_->log_error("Directory attribute is missing or invalid"); - return; - } - - std::string conflictResolution; - - if (!context->getProperty(ConflictResolution.getName(), conflictResolution)) - { - logger_->log_error("Conflict Resolution Strategy attribute is missing or invalid"); - return; - } - - FlowFileRecord *flowFile = session->get(); - - // Do nothing if there are no incoming files - if (!flowFile) - { - return; - } - - std::string filename; - flowFile->getAttribute(FILENAME, filename); - - // Generate a safe (universally-unique) temporary filename on the same partition - char tmpFileUuidStr[37]; - uuid_t tmpFileUuid; - uuid_generate(tmpFileUuid); - uuid_unparse_lower(tmpFileUuid, tmpFileUuidStr); - std::stringstream tmpFileSs; - tmpFileSs << directory << "/." << filename << "." << tmpFileUuidStr; - std::string tmpFile = tmpFileSs.str(); - logger_->log_info("PutFile using temporary file %s", tmpFile.c_str()); - - // Determine dest full file paths - std::stringstream destFileSs; - destFileSs << directory << "/" << filename; - std::string destFile = destFileSs.str(); - - logger_->log_info("PutFile writing file %s into directory %s", filename.c_str(), directory.c_str()); - - // If file exists, apply conflict resolution strategy - struct stat statResult; - - if (stat(destFile.c_str(), &statResult) == 0) - { - logger_->log_info("Destination file %s exists; applying Conflict Resolution Strategy: %s", destFile.c_str(), conflictResolution.c_str()); - - if (conflictResolution == CONFLICT_RESOLUTION_STRATEGY_REPLACE) - { - putFile(session, flowFile, tmpFile, destFile); - } - else if (conflictResolution == CONFLICT_RESOLUTION_STRATEGY_IGNORE) - { - session->transfer(flowFile, Success); - } - else - { - session->transfer(flowFile, Failure); - } - } - else - { - putFile(session, flowFile, tmpFile, destFile); - } -} - -bool PutFile::putFile(ProcessSession *session, FlowFileRecord *flowFile, const std::string &tmpFile, const std::string &destFile) -{ - - ReadCallback cb(tmpFile, destFile); - session->read(flowFile, &cb); - - if (cb.commit()) - { - session->transfer(flowFile, Success); - return true; - } - else - { - session->transfer(flowFile, Failure); - } - return false; -} - -PutFile::ReadCallback::ReadCallback(const std::string &tmpFile, const std::string &destFile) -: _tmpFile(tmpFile) -, _tmpFileOs(tmpFile) -, _destFile(destFile) -{ - logger_ = Logger::getLogger(); -} - -// Copy the entire file contents to the temporary file -void PutFile::ReadCallback::process(std::ifstream *stream) -{ - // Copy file contents into tmp file - _writeSucceeded = false; - _tmpFileOs << stream->rdbuf(); - _writeSucceeded = true; -} - -// Renames tmp file to final destination -// Returns true if commit succeeded -bool PutFile::ReadCallback::commit() -{ - bool success = false; - - logger_->log_info("PutFile committing put file operation to %s", _destFile.c_str()); - - if (_writeSucceeded) - { - _tmpFileOs.close(); - - if (rename(_tmpFile.c_str(), _destFile.c_str())) - { - logger_->log_info("PutFile commit put file operation to %s failed because rename() call failed", _destFile.c_str()); - } - else - { - success = true; - logger_->log_info("PutFile commit put file operation to %s succeeded", _destFile.c_str()); - } - } - else - { - logger_->log_error("PutFile commit put file operation to %s failed because write failed", _destFile.c_str()); - } - - return success; -} - -// Clean up resources -PutFile::ReadCallback::~ReadCallback() { - // Close tmp file - _tmpFileOs.close(); - - // Clean up tmp file, if necessary - unlink(_tmpFile.c_str()); -} diff --git a/libminifi/src/RealTimeDataCollector.cpp b/libminifi/src/RealTimeDataCollector.cpp deleted file mode 100644 index 7dd64693a0..0000000000 --- a/libminifi/src/RealTimeDataCollector.cpp +++ /dev/null @@ -1,481 +0,0 @@ -/** - * @file RealTimeDataCollector.cpp - * RealTimeDataCollector class implementation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "utils/StringUtils.h" -#include "RealTimeDataCollector.h" -#include "ProcessContext.h" -#include "ProcessSession.h" - -const std::string RealTimeDataCollector::ProcessorName("RealTimeDataCollector"); -Property RealTimeDataCollector::FILENAME("File Name", "File Name for the real time processor to process", "data.osp"); -Property RealTimeDataCollector::REALTIMESERVERNAME("Real Time Server Name", "Real Time Server Name", "localhost"); -Property RealTimeDataCollector::REALTIMESERVERPORT("Real Time Server Port", "Real Time Server Port", "10000"); -Property RealTimeDataCollector::BATCHSERVERNAME("Batch Server Name", "Batch Server Name", "localhost"); -Property RealTimeDataCollector::BATCHSERVERPORT("Batch Server Port", "Batch Server Port", "10001"); -Property RealTimeDataCollector::ITERATION("Iteration", - "If true, sample osp file will be iterated", "true"); -Property RealTimeDataCollector::REALTIMEMSGID("Real Time Message ID", "Real Time Message ID", "41"); -Property RealTimeDataCollector::BATCHMSGID("Batch Message ID", "Batch Message ID", "172, 30, 48"); -Property RealTimeDataCollector::REALTIMEINTERVAL("Real Time Interval", "Real Time Data Collection Interval in msec", "10 ms"); -Property RealTimeDataCollector::BATCHINTERVAL("Batch Time Interval", "Batch Processing Interval in msec", "100 ms"); -Property RealTimeDataCollector::BATCHMAXBUFFERSIZE("Batch Max Buffer Size", "Batch Buffer Maximum size in bytes", "262144"); -Relationship RealTimeDataCollector::Success("success", "success operational on the flow record"); - -void RealTimeDataCollector::initialize() -{ - //! Set the supported properties - std::set properties; - properties.insert(FILENAME); - properties.insert(REALTIMESERVERNAME); - properties.insert(REALTIMESERVERPORT); - properties.insert(BATCHSERVERNAME); - properties.insert(BATCHSERVERPORT); - properties.insert(ITERATION); - properties.insert(REALTIMEMSGID); - properties.insert(BATCHMSGID); - properties.insert(REALTIMEINTERVAL); - properties.insert(BATCHINTERVAL); - properties.insert(BATCHMAXBUFFERSIZE); - - setSupportedProperties(properties); - //! Set the supported relationships - std::set relationships; - relationships.insert(Success); - setSupportedRelationships(relationships); - -} - -int RealTimeDataCollector::connectServer(const char *host, uint16_t port) -{ - in_addr_t addr; - int sock = 0; - struct hostent *h; -#ifdef __MACH__ - h = gethostbyname(host); -#else - char buf[1024]; - struct hostent he; - int hh_errno; - gethostbyname_r(host, &he, buf, sizeof(buf), &h, &hh_errno); -#endif - memcpy((char *) &addr, h->h_addr_list[0], h->h_length); - sock = socket(AF_INET, SOCK_STREAM, 0); - if (sock < 0) - { - logger_->log_error("Could not create socket to hostName %s", host); - return 0; - } - -#ifndef __MACH__ - int opt = 1; - bool nagle_off = true; - - if (nagle_off) - { - if (setsockopt(sock, SOL_TCP, TCP_NODELAY, (void *)&opt, sizeof(opt)) < 0) - { - logger_->log_error("setsockopt() TCP_NODELAY failed"); - close(sock); - return 0; - } - if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, - (char *)&opt, sizeof(opt)) < 0) - { - logger_->log_error("setsockopt() SO_REUSEADDR failed"); - close(sock); - return 0; - } - } - - int sndsize = 256*1024; - if (setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (char *)&sndsize, (int)sizeof(sndsize)) < 0) - { - logger_->log_error("setsockopt() SO_SNDBUF failed"); - close(sock); - return 0; - } -#endif - - struct sockaddr_in sa; - socklen_t socklen; - int status; - - //TODO bind socket to the interface - memset(&sa, 0, sizeof(sa)); - sa.sin_family = AF_INET; - sa.sin_addr.s_addr = htonl(INADDR_ANY); - sa.sin_port = htons(0); - socklen = sizeof(sa); - if (bind(sock, (struct sockaddr *)&sa, socklen) < 0) - { - logger_->log_error("socket bind failed"); - close(sock); - return 0; - } - - memset(&sa, 0, sizeof(sa)); - sa.sin_family = AF_INET; - sa.sin_addr.s_addr = addr; - sa.sin_port = htons(port); - socklen = sizeof(sa); - - status = connect(sock, (struct sockaddr *)&sa, socklen); - - if (status < 0) - { - logger_->log_error("socket connect failed to %s %d", host, port); - close(sock); - return 0; - } - - logger_->log_info("socket %d connect to server %s port %d success", sock, host, port); - - return sock; -} - -int RealTimeDataCollector::sendData(int socket, const char *buf, int buflen) -{ - int ret = 0, bytes = 0; - - while (bytes < buflen) - { - ret = send(socket, buf+bytes, buflen-bytes, 0); - //check for errors - if (ret == -1) - { - return ret; - } - bytes+=ret; - } - - if (ret) - logger_->log_debug("Send data size %d over socket %d", buflen, socket); - - return ret; -} - -void RealTimeDataCollector::onTriggerRealTime(ProcessContext *context, ProcessSession *session) -{ - if (_realTimeAccumulated >= this->_realTimeInterval) - { - std::string value; - if (this->getProperty(REALTIMEMSGID.getName(), value)) - { - this->_realTimeMsgID.clear(); - this->logger_->log_info("Real Time Msg IDs %s", value.c_str()); - std::stringstream lineStream(value); - std::string cell; - - while(std::getline(lineStream, cell, ',')) - { - this->_realTimeMsgID.push_back(cell); - // this->logger_->log_debug("Real Time Msg ID %s", cell.c_str()); - } - } - if (this->getProperty(BATCHMSGID.getName(), value)) - { - this->_batchMsgID.clear(); - this->logger_->log_info("Batch Msg IDs %s", value.c_str()); - std::stringstream lineStream(value); - std::string cell; - - while(std::getline(lineStream, cell, ',')) - { - cell = StringUtils::trim(cell); - this->_batchMsgID.push_back(cell); - } - } - // Open the file - if (!this->_fileStream.is_open()) - { - _fileStream.open(this->_fileName.c_str(), std::ifstream::in); - if (this->_fileStream.is_open()) - logger_->log_debug("open %s", _fileName.c_str()); - } - if (!_fileStream.good()) - { - logger_->log_error("load data file failed %s", _fileName.c_str()); - return; - } - if (this->_fileStream.is_open()) - { - std::string line; - - while (std::getline(_fileStream, line)) - { - line += "\n"; - std::stringstream lineStream(line); - std::string cell; - if (std::getline(lineStream, cell, ',')) - { - cell = StringUtils::trim(cell); - // Check whether it match to the batch traffic - for (std::vector::iterator it = _batchMsgID.begin(); it != _batchMsgID.end(); ++it) - { - if (cell == *it) - { - // push the batch data to the queue - std::lock_guard lock(_mtx); - while ((_queuedDataSize + line.size()) > _batchMaxBufferSize) - { - std::string item = _queue.front(); - _queuedDataSize -= item.size(); - logger_->log_debug("Pop item size %d from batch queue, queue buffer size %d", item.size(), _queuedDataSize); - _queue.pop(); - } - _queue.push(line); - _queuedDataSize += line.size(); - logger_->log_debug("Push batch msg ID %s into batch queue, queue buffer size %d", cell.c_str(), _queuedDataSize); - } - } - bool findRealTime = false; - // Check whether it match to the real time traffic - for (std::vector::iterator it = _realTimeMsgID.begin(); it != _realTimeMsgID.end(); ++it) - { - if (cell == *it) - { - int status = 0; - if (this->_realTimeSocket <= 0) - { - // Connect the LTE socket - uint16_t port = _realTimeServerPort; - this->_realTimeSocket = connectServer(_realTimeServerName.c_str(), port); - } - if (this->_realTimeSocket) - { - // try to send the data - status = sendData(_realTimeSocket, line.data(), line.size()); - if (status < 0) - { - close(_realTimeSocket); - _realTimeSocket = 0; - } - } - if (this->_realTimeSocket <= 0 || status < 0) - { - // push the batch data to the queue - std::lock_guard lock(_mtx); - while ((_queuedDataSize + line.size()) > _batchMaxBufferSize) - { - std::string item = _queue.front(); - _queuedDataSize -= item.size(); - logger_->log_debug("Pop item size %d from batch queue, queue buffer size %d", item.size(), _queuedDataSize); - _queue.pop(); - } - _queue.push(line); - _queuedDataSize += line.size(); - logger_->log_debug("Push real time msg ID %s into batch queue, queue buffer size %d", cell.c_str(), _queuedDataSize); - } - // find real time - findRealTime = true; - } // cell - } // for real time pattern - if (findRealTime) - // we break the while once we find the first real time - break; - } // if get line - } // while - if (_fileStream.eof()) - { - _fileStream.close(); - } - } // if open - _realTimeAccumulated = 0; - } - _realTimeAccumulated += context->getProcessor()->getSchedulingPeriodNano(); -} - -void RealTimeDataCollector::onTriggerBatch(ProcessContext *context, ProcessSession *session) -{ - if (_batchAcccumulated >= this->_batchInterval) - { - // logger_->log_info("onTriggerBatch"); - // dequeue the batch and send over WIFI - int status = 0; - if (this->_batchSocket <= 0) - { - // Connect the WIFI socket - uint16_t port = _batchServerPort; - this->_batchSocket = connectServer(_batchServerName.c_str(), port); - } - if (this->_batchSocket) - { - std::lock_guard lock(_mtx); - - while (!_queue.empty()) - { - std::string line = _queue.front(); - status = sendData(_batchSocket, line.data(), line.size()); - _queue.pop(); - _queuedDataSize -= line.size(); - if (status < 0) - { - close(_batchSocket); - _batchSocket = 0; - break; - } - } - } - _batchAcccumulated = 0; - } - _batchAcccumulated += context->getProcessor()->getSchedulingPeriodNano(); -} - -void RealTimeDataCollector::onTrigger(ProcessContext *context, ProcessSession *session) -{ - std::thread::id id = std::this_thread::get_id(); - - if (id == _realTimeThreadId) - return onTriggerRealTime(context, session); - else if (id == _batchThreadId) - return onTriggerBatch(context, session); - else - { - std::lock_guard lock(_mtx); - if (!this->_firstInvoking) - { - this->_fileName = "data.osp"; - std::string value; - if (this->getProperty(FILENAME.getName(), value)) - { - this->_fileName = value; - this->logger_->log_info("Data Collector File Name %s", _fileName.c_str()); - } - this->_realTimeServerName = "localhost"; - if (this->getProperty(REALTIMESERVERNAME.getName(), value)) - { - this->_realTimeServerName = value; - this->logger_->log_info("Real Time Server Name %s", this->_realTimeServerName.c_str()); - } - this->_realTimeServerPort = 10000; - if (this->getProperty(REALTIMESERVERPORT.getName(), value)) - { - Property::StringToInt(value, _realTimeServerPort); - this->logger_->log_info("Real Time Server Port %d", _realTimeServerPort); - } - if (this->getProperty(BATCHSERVERNAME.getName(), value)) - { - this->_batchServerName = value; - this->logger_->log_info("Batch Server Name %s", this->_batchServerName.c_str()); - } - this->_batchServerPort = 10001; - if (this->getProperty(BATCHSERVERPORT.getName(), value)) - { - Property::StringToInt(value, _batchServerPort); - this->logger_->log_info("Batch Server Port %d", _batchServerPort); - } - if (this->getProperty(ITERATION.getName(), value)) - { - StringUtils::StringToBool(value, this->_iteration); - logger_->log_info("Iteration %d", _iteration); - } - this->_realTimeInterval = 10000000; //10 msec - if (this->getProperty(REALTIMEINTERVAL.getName(), value)) - { - TimeUnit unit; - if (Property::StringToTime(value, _realTimeInterval, unit) && - Property::ConvertTimeUnitToNS(_realTimeInterval, unit, _realTimeInterval)) - { - logger_->log_info("Real Time Interval: [%d] ns", _realTimeInterval); - } - } - this->_batchInterval = 100000000; //100 msec - if (this->getProperty(BATCHINTERVAL.getName(), value)) - { - TimeUnit unit; - if (Property::StringToTime(value, _batchInterval, unit) && - Property::ConvertTimeUnitToNS(_batchInterval, unit, _batchInterval)) - { - logger_->log_info("Batch Time Interval: [%d] ns", _batchInterval); - } - } - this->_batchMaxBufferSize = 256*1024; - if (this->getProperty(BATCHMAXBUFFERSIZE.getName(), value)) - { - Property::StringToInt(value, _batchMaxBufferSize); - this->logger_->log_info("Batch Max Buffer Size %d", _batchMaxBufferSize); - } - if (this->getProperty(REALTIMEMSGID.getName(), value)) - { - this->logger_->log_info("Real Time Msg IDs %s", value.c_str()); - std::stringstream lineStream(value); - std::string cell; - - while(std::getline(lineStream, cell, ',')) - { - this->_realTimeMsgID.push_back(cell); - this->logger_->log_info("Real Time Msg ID %s", cell.c_str()); - } - } - if (this->getProperty(BATCHMSGID.getName(), value)) - { - this->logger_->log_info("Batch Msg IDs %s", value.c_str()); - std::stringstream lineStream(value); - std::string cell; - - while(std::getline(lineStream, cell, ',')) - { - cell = StringUtils::trim(cell); - this->_batchMsgID.push_back(cell); - this->logger_->log_info("Batch Msg ID %s", cell.c_str()); - } - } - // Connect the LTE socket - uint16_t port = _realTimeServerPort; - - this->_realTimeSocket = connectServer(_realTimeServerName.c_str(), port); - - // Connect the WIFI socket - port = _batchServerPort; - - this->_batchSocket = connectServer(_batchServerName.c_str(), port); - - // Open the file - _fileStream.open(this->_fileName.c_str(), std::ifstream::in); - if (!_fileStream.good()) - { - logger_->log_error("load data file failed %s", _fileName.c_str()); - return; - } - else - { - logger_->log_debug("open %s", _fileName.c_str()); - } - _realTimeThreadId = id; - this->_firstInvoking = true; - } - else - { - if (id != _realTimeThreadId) - _batchThreadId = id; - this->_firstInvoking = false; - } - } -} diff --git a/libminifi/src/RemoteProcessorGroupPort.cpp b/libminifi/src/RemoteProcessorGroupPort.cpp index dd1d035fbc..a74aa28a54 100644 --- a/libminifi/src/RemoteProcessorGroupPort.cpp +++ b/libminifi/src/RemoteProcessorGroupPort.cpp @@ -33,89 +33,103 @@ #include "io/SocketFactory.h" #include "utils/TimeUtil.h" -#include "ProcessContext.h" -#include "ProcessSession.h" - -const std::string RemoteProcessorGroupPort::ProcessorName("RemoteProcessorGroupPort"); -Property RemoteProcessorGroupPort::hostName("Host Name", "Remote Host Name.", "localhost"); -Property RemoteProcessorGroupPort::port("Port", "Remote Port", "9999"); -Relationship RemoteProcessorGroupPort::relation; - -void RemoteProcessorGroupPort::initialize() -{ - - //! Set the supported properties - std::set properties; - properties.insert(hostName); - properties.insert(port); - setSupportedProperties(properties); - //! Set the supported relationships - std::set relationships; - relationships.insert(relation); - setSupportedRelationships(relationships); +#include "core/ProcessContext.h" +#include "core/ProcessSession.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { + +const std::string RemoteProcessorGroupPort::ProcessorName( + "RemoteProcessorGroupPort"); +core::Property RemoteProcessorGroupPort::hostName( + "Host Name", "Remote Host Name.", "localhost"); +core::Property RemoteProcessorGroupPort::port( + "Port", "Remote Port", "9999"); +core::Relationship RemoteProcessorGroupPort::relation; + +void RemoteProcessorGroupPort::initialize() { + + // Set the supported properties + std::set properties; + properties.insert(hostName); + properties.insert(port); + setSupportedProperties(properties); + // Set the supported relationships + std::set relationships; + relationships.insert(relation); + setSupportedRelationships(relationships); } -void RemoteProcessorGroupPort::onTrigger(ProcessContext *context, ProcessSession *session) -{ - std::string value; - - if (!transmitting_) - return; - - std::string host = peer_.getHostName(); - uint16_t sport = peer_.getPort(); - int64_t lvalue; - - if (context->getProperty(hostName.getName(), value)) - { - host = value; - } - if (context->getProperty(port.getName(), value) && Property::StringToInt(value, lvalue)) - { - sport = (uint16_t) lvalue; - } - - if (host != peer_.getHostName() || sport != peer_.getPort()) - - { - - std::unique_ptr str = std::unique_ptr(SocketFactory::getInstance()->createSocket(host,sport)); - peer_ = std::move(Site2SitePeer (std::move(str), host, sport)); - protocol_->setPeer(&peer_); - - } - - - - bool needReset = false; - - - if (host != peer_.getHostName()) - { - peer_.setHostName(host); - needReset= true; - } - if (sport != peer_.getPort()) - { - peer_.setPort(sport); - needReset = true; - } - if (needReset) - protocol_->tearDown(); - - if (!protocol_->bootstrap()) - { - // bootstrap the client protocol if needeed - context->yield(); - logger_->log_error("Site2Site bootstrap failed yield period %d peer ", context->getProcessor()->getYieldPeriodMsec()); - return; - } - - if (direction_ == RECEIVE) - protocol_->receiveFlowFiles(context, session); - else - protocol_->transferFlowFiles(context, session); - - return; +void RemoteProcessorGroupPort::onTrigger( + core::ProcessContext *context, + core::ProcessSession *session) { + std::string value; + + if (!transmitting_) + return; + + std::string host = peer_.getHostName(); + uint16_t sport = peer_.getPort(); + int64_t lvalue; + + if (context->getProperty(hostName.getName(), value)) { + host = value; + } + if (context->getProperty(port.getName(), value) + && core::Property::StringToInt(value, + lvalue)) { + sport = (uint16_t) lvalue; + } + + if (host != peer_.getHostName() || sport != peer_.getPort()) + + { + + std::unique_ptr str = + std::unique_ptr( + org::apache::nifi::minifi::io::SocketFactory::getInstance() + ->createSocket(host, sport)); + peer_ = std::move(Site2SitePeer(std::move(str), host, sport)); + protocol_->setPeer(&peer_); + + } + + bool needReset = false; + + if (host != peer_.getHostName()) { + peer_.setHostName(host); + needReset = true; + } + if (sport != peer_.getPort()) { + peer_.setPort(sport); + needReset = true; + } + if (needReset) + protocol_->tearDown(); + + if (!protocol_->bootstrap()) { + // bootstrap the client protocol if needeed + context->yield(); + std::shared_ptr processor = std::static_pointer_cast( + context->getProcessorNode().getProcessor()); + logger_->log_error("Site2Site bootstrap failed yield period %d peer ", + processor->getYieldPeriodMsec()); + return; + } + + if (direction_ == RECEIVE) + protocol_->receiveFlowFiles(context, session); + else + protocol_->transferFlowFiles(context, session); + + return; } + + +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/Repository.cpp b/libminifi/src/Repository.cpp deleted file mode 100644 index 1a3c7b03cd..0000000000 --- a/libminifi/src/Repository.cpp +++ /dev/null @@ -1,140 +0,0 @@ -/** - * @file Repository.cpp - * Repository implemenatation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include "io/DataStream.h" -#include "io/Serializable.h" -#include "Relationship.h" -#include "Logger.h" -#include "FlowController.h" -#include "Repository.h" -#include "Provenance.h" -#include "FlowFileRepository.h" - -const char *Repository::RepositoryTypeStr[MAX_REPO_TYPE] = {"Provenace Repository", "FlowFile Repository"}; -uint64_t Repository::_repoSize[MAX_REPO_TYPE] = {0, 0}; - -void Repository::start() { - if (!_enable) - return; - if (this->_purgePeriod <= 0) - return; - if (_running) - return; - _running = true; - logger_->log_info("%s Repository Monitor Thread Start", RepositoryTypeStr[_type]); - _thread = new std::thread(run, this); - _thread->detach(); -} - -void Repository::stop() { - if (!_running) - return; - _running = false; - logger_->log_info("%s Repository Monitor Thread Stop", RepositoryTypeStr[_type]); -} - -void Repository::run(Repository *repo) { -#ifdef LEVELDB_SUPPORT - // threshold for purge - uint64_t purgeThreshold = repo->_maxPartitionBytes * 3 / 4; - while (repo->_running) { - std::this_thread::sleep_for( - std::chrono::milliseconds(repo->_purgePeriod)); - uint64_t curTime = getTimeMillis(); - uint64_t size = repo->repoSize(); - if (size >= purgeThreshold) { - std::vector purgeList; - leveldb::Iterator* it = repo->_db->NewIterator( - leveldb::ReadOptions()); - if (repo->_type == PROVENANCE) - { - for (it->SeekToFirst(); it->Valid(); it->Next()) { - ProvenanceEventRecord eventRead; - std::string key = it->key().ToString(); - if (eventRead.DeSerialize((uint8_t *) it->value().data(), - (int) it->value().size())) { - if ((curTime - eventRead.getEventTime()) - > repo->_maxPartitionMillis) - purgeList.push_back(key); - } else { - repo->logger_->log_debug( - "NiFi %s retrieve event %s fail", - RepositoryTypeStr[repo->_type], - key.c_str()); - purgeList.push_back(key); - } - } - } - if (repo->_type == FLOWFILE) - { - for (it->SeekToFirst(); it->Valid(); it->Next()) { - FlowFileEventRecord eventRead; - std::string key = it->key().ToString(); - if (eventRead.DeSerialize((uint8_t *) it->value().data(), - (int) it->value().size())) { - if ((curTime - eventRead.getEventTime()) - > repo->_maxPartitionMillis) - purgeList.push_back(key); - } else { - repo->logger_->log_debug( - "NiFi %s retrieve event %s fail", - RepositoryTypeStr[repo->_type], - key.c_str()); - purgeList.push_back(key); - } - } - } - delete it; - for (auto eventId : purgeList) - { - repo->logger_->log_info("Repository Repo %s Purge %s", - RepositoryTypeStr[repo->_type], - eventId.c_str()); - repo->Delete(eventId); - } - } - if (size > repo->_maxPartitionBytes) - repo->_repoFull = true; - else - repo->_repoFull = false; - } -#endif - return; -} - -//! repoSize -uint64_t Repository::repoSize() -{ - _repoSize[_type] = 0; - if (_type == PROVENANCE) - { - if (ftw(_directory.c_str(), repoSumProvenance, 1) != 0) - _repoSize[_type] = 0; - } - if (_type == FLOWFILE) - { - if (ftw(_directory.c_str(), repoSumFlowFile, 1) != 0) - _repoSize[_type] = 0; - } - return _repoSize[_type]; -} - diff --git a/libminifi/src/ResourceClaim.cpp b/libminifi/src/ResourceClaim.cpp index be52b490bf..826ca1dd42 100644 --- a/libminifi/src/ResourceClaim.cpp +++ b/libminifi/src/ResourceClaim.cpp @@ -23,27 +23,36 @@ #include "ResourceClaim.h" -std::atomic ResourceClaim::_localResourceClaimNumber(0); +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +std::atomic ResourceClaim::_localResourceClaimNumber(0); -std::string ResourceClaim::default_directory_path=DEFAULT_CONTENT_DIRECTORY; +std::string ResourceClaim::default_directory_path = DEFAULT_CONTENT_DIRECTORY; ResourceClaim::ResourceClaim(const std::string contentDirectory) -: _id(_localResourceClaimNumber.load()), - _flowFileRecordOwnedCount(0) -{ - - char uuidStr[37]; - - // Generate the global UUID for the resource claim - uuid_generate(_uuid); - // Increase the local ID for the resource claim - ++_localResourceClaimNumber; - uuid_unparse_lower(_uuid, uuidStr); - // Create the full content path for the content - _contentFullPath = contentDirectory + "/" + uuidStr; - - configure_ = Configure::getConfigure(); - logger_ = Logger::getLogger(); - logger_->log_debug("Resource Claim created %s", uuidStr); + : _id(_localResourceClaimNumber.load()), + _flowFileRecordOwnedCount(0) { + + char uuidStr[37]; + + // Generate the global UUID for the resource claim + uuid_generate(_uuid); + // Increase the local ID for the resource claim + ++_localResourceClaimNumber; + uuid_unparse_lower(_uuid, uuidStr); + // Create the full content path for the content + _contentFullPath = contentDirectory + "/" + uuidStr; + + configure_ = Configure::getConfigure(); + + logger_ = logging::Logger::getLogger(); + logger_->log_debug("Resource Claim created %s", _contentFullPath.c_str()); } + +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/SchedulingAgent.cpp b/libminifi/src/SchedulingAgent.cpp index 984abdcc45..8cb88e00a3 100644 --- a/libminifi/src/SchedulingAgent.cpp +++ b/libminifi/src/SchedulingAgent.cpp @@ -22,64 +22,71 @@ #include #include "Exception.h" #include "SchedulingAgent.h" +#include "core/Processor.h" -bool SchedulingAgent::hasWorkToDo(Processor *processor) -{ - // Whether it has work to do - if (processor->getTriggerWhenEmpty() || !processor->hasIncomingConnections() || - processor->flowFilesQueued()) - return true; - else - return false; +namespace org { +namespace apache { +namespace nifi { +namespace minifi { + +bool SchedulingAgent::hasWorkToDo( + std::shared_ptr processor) { + // Whether it has work to do + if (processor->getTriggerWhenEmpty() || !processor->hasIncomingConnections() + || processor->flowFilesQueued()) + return true; + else + return false; } -bool SchedulingAgent::hasTooMuchOutGoing(Processor *processor) -{ - return processor->flowFilesOutGoingFull(); +bool SchedulingAgent::hasTooMuchOutGoing( + std::shared_ptr processor) { + return processor->flowFilesOutGoingFull(); } -bool SchedulingAgent::onTrigger(Processor *processor, ProcessContext *processContext, ProcessSessionFactory *sessionFactory) -{ - if (processor->isYield()) - return false; +bool SchedulingAgent::onTrigger( + std::shared_ptr processor, + core::ProcessContext *processContext, + core::ProcessSessionFactory *sessionFactory) { + if (processor->isYield()) + return false; + + // No need to yield, reset yield expiration to 0 + processor->clearYield(); - // No need to yield, reset yield expiration to 0 - processor->clearYield(); + if (!hasWorkToDo(processor)) + // No work to do, yield + return true; - if (!hasWorkToDo(processor)) - // No work to do, yield - return true; + if (hasTooMuchOutGoing(processor)) + // need to apply backpressure + return true; - if(hasTooMuchOutGoing(processor)) - // need to apply backpressure - return true; + //TODO runDuration - //TODO runDuration + processor->incrementActiveTasks(); + try { + processor->onTrigger(processContext, sessionFactory); + processor->decrementActiveTask(); + } catch (Exception &exception) { + // Normal exception + logger_->log_debug("Caught Exception %s", exception.what()); + processor->decrementActiveTask(); + } catch (std::exception &exception) { + logger_->log_debug("Caught Exception %s", exception.what()); + processor->yield(_administrativeYieldDuration); + processor->decrementActiveTask(); + } catch (...) { + logger_->log_debug("Caught Exception during SchedulingAgent::onTrigger"); + processor->yield(_administrativeYieldDuration); + processor->decrementActiveTask(); + } + + return false; +} - processor->incrementActiveTasks(); - try - { - processor->onTrigger(processContext, sessionFactory); - processor->decrementActiveTask(); - } - catch (Exception &exception) - { - // Normal exception - logger_->log_debug("Caught Exception %s", exception.what()); - processor->decrementActiveTask(); - } - catch (std::exception &exception) - { - logger_->log_debug("Caught Exception %s", exception.what()); - processor->yield(_administrativeYieldDuration); - processor->decrementActiveTask(); - } - catch (...) - { - logger_->log_debug("Caught Exception during SchedulingAgent::onTrigger"); - processor->yield(_administrativeYieldDuration); - processor->decrementActiveTask(); - } - return false; -} \ No newline at end of file +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/Site2SiteClientProtocol.cpp b/libminifi/src/Site2SiteClientProtocol.cpp index bd4de975ef..3c93866cd2 100644 --- a/libminifi/src/Site2SiteClientProtocol.cpp +++ b/libminifi/src/Site2SiteClientProtocol.cpp @@ -29,1308 +29,1219 @@ #include "Site2SitePeer.h" #include "Site2SiteClientProtocol.h" -bool Site2SiteClientProtocol::establish() -{ - if (_peerState != IDLE) - { - logger_->log_error("Site2Site peer state is not idle while try to establish"); - return false; - } - - bool ret = peer_->Open(); - - if (!ret) - { - logger_->log_error("Site2Site peer socket open failed"); - return false; - } - - // Negotiate the version - ret = initiateResourceNegotiation(); - - if (!ret) - { - logger_->log_error("Site2Site Protocol Version Negotiation failed"); - /* - peer_->yield(); - tearDown(); */ - return false; - } - - logger_->log_info("Site2Site socket established"); - _peerState = ESTABLISHED; - - return true; +namespace org { +namespace apache { +namespace nifi { +namespace minifi { + +bool Site2SiteClientProtocol::establish() { + if (_peerState != IDLE) { + logger_->log_error( + "Site2Site peer state is not idle while try to establish"); + return false; + } + + bool ret = peer_->Open(); + + if (!ret) { + logger_->log_error("Site2Site peer socket open failed"); + return false; + } + + // Negotiate the version + ret = initiateResourceNegotiation(); + + if (!ret) { + logger_->log_error("Site2Site Protocol Version Negotiation failed"); + /* + peer_->yield(); + tearDown(); */ + return false; + } + + logger_->log_info("Site2Site socket established"); + _peerState = ESTABLISHED; + + return true; } -bool Site2SiteClientProtocol::initiateResourceNegotiation() -{ - // Negotiate the version - if (_peerState != IDLE) - { - logger_->log_error("Site2Site peer state is not idle while initiateResourceNegotiation"); - return false; - } - - logger_->log_info("Negotiate protocol version with destination port %s current version %d", _portIdStr.c_str(), _currentVersion); - - int ret = peer_->writeUTF(this->getResourceName()); - - logger_->log_info("result of writing resource name is %i",ret); - if (ret <= 0) - { - logger_->log_debug("result of writing resource name is %i",ret); - // tearDown(); - return false; - } - - ret = peer_->write(_currentVersion); - - if (ret <= 0) - { - logger_->log_info("result of writing version is %i",ret); - // tearDown(); - return false; - } - - uint8_t statusCode; - ret = peer_->read(statusCode); - - if (ret <= 0) - { - logger_->log_info("result of writing version status code %i",ret); - // tearDown(); - return false; - } - logger_->log_info("status code is %i",statusCode); - switch (statusCode) - { - case RESOURCE_OK: - logger_->log_info("Site2Site Protocol Negotiate protocol version OK"); - return true; - case DIFFERENT_RESOURCE_VERSION: - uint32_t serverVersion; - ret = peer_->read(serverVersion); - if (ret <= 0) - { - // tearDown(); - return false; - } - logger_->log_info("Site2Site Server Response asked for a different protocol version %d", serverVersion); - for (unsigned int i = (_currentVersionIndex + 1); i < sizeof(_supportedVersion)/sizeof(uint32_t); i++) - { - if (serverVersion >= _supportedVersion[i]) - { - _currentVersion = _supportedVersion[i]; - _currentVersionIndex = i; - return initiateResourceNegotiation(); - } - } - ret = -1; - // tearDown(); - return false; - case NEGOTIATED_ABORT: - logger_->log_info("Site2Site Negotiate protocol response ABORT"); - ret = -1; - // tearDown(); - return false; - default: - logger_->log_info("Negotiate protocol response unknown code %d", statusCode); - return true; - } - - return true; +bool Site2SiteClientProtocol::initiateResourceNegotiation() { + // Negotiate the version + if (_peerState != IDLE) { + logger_->log_error( + "Site2Site peer state is not idle while initiateResourceNegotiation"); + return false; + } + + logger_->log_info( + "Negotiate protocol version with destination port %s current version %d", + _portIdStr.c_str(), _currentVersion); + + int ret = peer_->writeUTF(this->getResourceName()); + + logger_->log_info("result of writing resource name is %i", ret); + if (ret <= 0) { + logger_->log_debug("result of writing resource name is %i", ret); + // tearDown(); + return false; + } + + ret = peer_->write(_currentVersion); + + if (ret <= 0) { + logger_->log_info("result of writing version is %i", ret); + // tearDown(); + return false; + } + + uint8_t statusCode; + ret = peer_->read(statusCode); + + if (ret <= 0) { + logger_->log_info("result of writing version status code %i", ret); + // tearDown(); + return false; + } + logger_->log_info("status code is %i", statusCode); + switch (statusCode) { + case RESOURCE_OK: + logger_->log_info("Site2Site Protocol Negotiate protocol version OK"); + return true; + case DIFFERENT_RESOURCE_VERSION: + uint32_t serverVersion; + ret = peer_->read(serverVersion); + if (ret <= 0) { + // tearDown(); + return false; + } + logger_->log_info( + "Site2Site Server Response asked for a different protocol version %d", + serverVersion); + for (unsigned int i = (_currentVersionIndex + 1); + i < sizeof(_supportedVersion) / sizeof(uint32_t); i++) { + if (serverVersion >= _supportedVersion[i]) { + _currentVersion = _supportedVersion[i]; + _currentVersionIndex = i; + return initiateResourceNegotiation(); + } + } + ret = -1; + // tearDown(); + return false; + case NEGOTIATED_ABORT: + logger_->log_info("Site2Site Negotiate protocol response ABORT"); + ret = -1; + // tearDown(); + return false; + default: + logger_->log_info("Negotiate protocol response unknown code %d", + statusCode); + return true; + } + + return true; } -bool Site2SiteClientProtocol::initiateCodecResourceNegotiation() -{ - // Negotiate the version - if (_peerState != HANDSHAKED) - { - logger_->log_error("Site2Site peer state is not handshaked while initiateCodecResourceNegotiation"); - return false; - } - - logger_->log_info("Negotiate Codec version with destination port %s current version %d", _portIdStr.c_str(), _currentCodecVersion); - - int ret = peer_->writeUTF(this->getCodecResourceName()); - - if (ret <= 0) - { - logger_->log_debug("result of getCodecResourceName is %i",ret); - // tearDown(); - return false; - } - - ret = peer_->write(_currentCodecVersion); - - if (ret <= 0) - { - logger_->log_debug("result of _currentCodecVersion is %i",ret); - // tearDown(); - return false; - } - - uint8_t statusCode; - ret = peer_->read(statusCode); - - if (ret <= 0) - { - // tearDown(); - return false; - } - - switch (statusCode) - { - case RESOURCE_OK: - logger_->log_info("Site2Site Codec Negotiate version OK"); - return true; - case DIFFERENT_RESOURCE_VERSION: - uint32_t serverVersion; - ret = peer_->read(serverVersion); - if (ret <= 0) - { - // tearDown(); - return false; - } - logger_->log_info("Site2Site Server Response asked for a different codec version %d", serverVersion); - for (unsigned int i = (_currentCodecVersionIndex + 1); i < sizeof(_supportedCodecVersion)/sizeof(uint32_t); i++) - { - if (serverVersion >= _supportedCodecVersion[i]) - { - _currentCodecVersion = _supportedCodecVersion[i]; - _currentCodecVersionIndex = i; - return initiateCodecResourceNegotiation(); - } - } - ret = -1; - // tearDown(); - return false; - case NEGOTIATED_ABORT: - logger_->log_info("Site2Site Codec Negotiate response ABORT"); - ret = -1; - // tearDown(); - return false; - default: - logger_->log_info("Negotiate Codec response unknown code %d", statusCode); - return true; - } - - return true; +bool Site2SiteClientProtocol::initiateCodecResourceNegotiation() { + // Negotiate the version + if (_peerState != HANDSHAKED) { + logger_->log_error( + "Site2Site peer state is not handshaked while initiateCodecResourceNegotiation"); + return false; + } + + logger_->log_info( + "Negotiate Codec version with destination port %s current version %d", + _portIdStr.c_str(), _currentCodecVersion); + + int ret = peer_->writeUTF(this->getCodecResourceName()); + + if (ret <= 0) { + logger_->log_debug("result of getCodecResourceName is %i", ret); + // tearDown(); + return false; + } + + ret = peer_->write(_currentCodecVersion); + + if (ret <= 0) { + logger_->log_debug("result of _currentCodecVersion is %i", ret); + // tearDown(); + return false; + } + + uint8_t statusCode; + ret = peer_->read(statusCode); + + if (ret <= 0) { + // tearDown(); + return false; + } + + switch (statusCode) { + case RESOURCE_OK: + logger_->log_info("Site2Site Codec Negotiate version OK"); + return true; + case DIFFERENT_RESOURCE_VERSION: + uint32_t serverVersion; + ret = peer_->read(serverVersion); + if (ret <= 0) { + // tearDown(); + return false; + } + logger_->log_info( + "Site2Site Server Response asked for a different codec version %d", + serverVersion); + for (unsigned int i = (_currentCodecVersionIndex + 1); + i < sizeof(_supportedCodecVersion) / sizeof(uint32_t); i++) { + if (serverVersion >= _supportedCodecVersion[i]) { + _currentCodecVersion = _supportedCodecVersion[i]; + _currentCodecVersionIndex = i; + return initiateCodecResourceNegotiation(); + } + } + ret = -1; + // tearDown(); + return false; + case NEGOTIATED_ABORT: + logger_->log_info("Site2Site Codec Negotiate response ABORT"); + ret = -1; + // tearDown(); + return false; + default: + logger_->log_info("Negotiate Codec response unknown code %d", statusCode); + return true; + } + + return true; } -bool Site2SiteClientProtocol::handShake() -{ - if (_peerState != ESTABLISHED) - { - logger_->log_error("Site2Site peer state is not established while handshake"); - return false; - } - logger_->log_info("Site2Site Protocol Perform hand shake with destination port %s", _portIdStr.c_str()); - uuid_t uuid; - // Generate the global UUID for the com identify - uuid_generate(uuid); - char uuidStr[37]; - uuid_unparse_lower(uuid, uuidStr); - _commsIdentifier = uuidStr; - - int ret = peer_->writeUTF(_commsIdentifier); - - if (ret <= 0) - { - // tearDown(); - return false; - } - - std::map properties; - properties[HandShakePropertyStr[GZIP]] = "false"; - properties[HandShakePropertyStr[PORT_IDENTIFIER]] = _portIdStr; - properties[HandShakePropertyStr[REQUEST_EXPIRATION_MILLIS]] = std::to_string(this->_timeOut); - if (this->_currentVersion >= 5) - { - if (this->_batchCount > 0) - properties[HandShakePropertyStr[BATCH_COUNT]] = std::to_string(this->_batchCount); - if (this->_batchSize > 0) - properties[HandShakePropertyStr[BATCH_SIZE]] = std::to_string(this->_batchSize); - if (this->_batchDuration > 0) - properties[HandShakePropertyStr[BATCH_DURATION]] = std::to_string(this->_batchDuration); - } - - if (_currentVersion >= 3) - { - ret = peer_->writeUTF(peer_->getURL()); - if (ret <= 0) - { - // tearDown(); - return false; - } - } - - uint32_t size = properties.size(); - ret = peer_->write(size); - if (ret <= 0) - { - // tearDown(); - return false; - } - - std::map::iterator it; - for (it = properties.begin(); it!= properties.end(); it++) - { - ret = peer_->writeUTF(it->first); - if (ret <= 0) - { - // tearDown(); - return false; - } - ret = peer_->writeUTF(it->second); - if (ret <= 0) - { - // tearDown(); - return false; - } - logger_->log_info("Site2Site Protocol Send handshake properties %s %s", it->first.c_str(), it->second.c_str()); - } - - RespondCode code; - std::string message; - - ret = this->readRespond(code, message); - - if (ret <= 0) - { - // tearDown(); - return false; - } - - switch (code) - { - case PROPERTIES_OK: - logger_->log_info("Site2Site HandShake Completed"); - _peerState = HANDSHAKED; - return true; - case PORT_NOT_IN_VALID_STATE: +bool Site2SiteClientProtocol::handShake() { + if (_peerState != ESTABLISHED) { + logger_->log_error( + "Site2Site peer state is not established while handshake"); + return false; + } + logger_->log_info( + "Site2Site Protocol Perform hand shake with destination port %s", + _portIdStr.c_str()); + uuid_t uuid; + // Generate the global UUID for the com identify + uuid_generate(uuid); + char uuidStr[37]; + uuid_unparse_lower(uuid, uuidStr); + _commsIdentifier = uuidStr; + + int ret = peer_->writeUTF(_commsIdentifier); + + if (ret <= 0) { + // tearDown(); + return false; + } + + std::map properties; + properties[HandShakePropertyStr[GZIP]] = "false"; + properties[HandShakePropertyStr[PORT_IDENTIFIER]] = _portIdStr; + properties[HandShakePropertyStr[REQUEST_EXPIRATION_MILLIS]] = std::to_string( + this->_timeOut); + if (this->_currentVersion >= 5) { + if (this->_batchCount > 0) + properties[HandShakePropertyStr[BATCH_COUNT]] = std::to_string( + this->_batchCount); + if (this->_batchSize > 0) + properties[HandShakePropertyStr[BATCH_SIZE]] = std::to_string( + this->_batchSize); + if (this->_batchDuration > 0) + properties[HandShakePropertyStr[BATCH_DURATION]] = std::to_string( + this->_batchDuration); + } + + if (_currentVersion >= 3) { + ret = peer_->writeUTF(peer_->getURL()); + if (ret <= 0) { + // tearDown(); + return false; + } + } + + uint32_t size = properties.size(); + ret = peer_->write(size); + if (ret <= 0) { + // tearDown(); + return false; + } + + std::map::iterator it; + for (it = properties.begin(); it != properties.end(); it++) { + ret = peer_->writeUTF(it->first); + if (ret <= 0) { + // tearDown(); + return false; + } + ret = peer_->writeUTF(it->second); + if (ret <= 0) { + // tearDown(); + return false; + } + logger_->log_info("Site2Site Protocol Send handshake properties %s %s", + it->first.c_str(), it->second.c_str()); + } + + RespondCode code; + std::string message; + + ret = this->readRespond(code, message); + + if (ret <= 0) { + // tearDown(); + return false; + } + + switch (code) { + case PROPERTIES_OK: + logger_->log_info("Site2Site HandShake Completed"); + _peerState = HANDSHAKED; + return true; + case PORT_NOT_IN_VALID_STATE: case UNKNOWN_PORT: case PORTS_DESTINATION_FULL: - logger_->log_error("Site2Site HandShake Failed because destination port is either invalid or full"); - ret = -1; - /* - peer_->yield(); - tearDown(); */ - return false; - default: - logger_->log_info("HandShake Failed because of unknown respond code %d", code); - ret = -1; - /* - peer_->yield(); - tearDown(); */ - return false; - } - - return false; + logger_->log_error( + "Site2Site HandShake Failed because destination port is either invalid or full"); + ret = -1; + /* + peer_->yield(); + tearDown(); */ + return false; + default: + logger_->log_info("HandShake Failed because of unknown respond code %d", + code); + ret = -1; + /* + peer_->yield(); + tearDown(); */ + return false; + } + + return false; } -void Site2SiteClientProtocol::tearDown() -{ - if (_peerState >= ESTABLISHED) - { - logger_->log_info("Site2Site Protocol tearDown"); - // need to write shutdown request - writeRequestType(SHUTDOWN); - } - - std::map::iterator it; - for (it = _transactionMap.begin(); it!= _transactionMap.end(); it++) - { - delete it->second; - } - _transactionMap.clear(); - peer_->Close(); - _peerState = IDLE; +void Site2SiteClientProtocol::tearDown() { + if (_peerState >= ESTABLISHED) { + logger_->log_info("Site2Site Protocol tearDown"); + // need to write shutdown request + writeRequestType(SHUTDOWN); + } + + std::map::iterator it; + for (it = _transactionMap.begin(); it != _transactionMap.end(); it++) { + delete it->second; + } + _transactionMap.clear(); + peer_->Close(); + _peerState = IDLE; } -int Site2SiteClientProtocol::writeRequestType(RequestType type) -{ - if (type >= MAX_REQUEST_TYPE) - return -1; +int Site2SiteClientProtocol::writeRequestType(RequestType type) { + if (type >= MAX_REQUEST_TYPE) + return -1; - return peer_->writeUTF(RequestTypeStr[type]); + return peer_->writeUTF(RequestTypeStr[type]); } -int Site2SiteClientProtocol::readRequestType(RequestType &type) -{ - std::string requestTypeStr; +int Site2SiteClientProtocol::readRequestType(RequestType &type) { + std::string requestTypeStr; - int ret = peer_->readUTF(requestTypeStr); + int ret = peer_->readUTF(requestTypeStr); - if (ret <= 0) - return ret; + if (ret <= 0) + return ret; - for (int i = (int) NEGOTIATE_FLOWFILE_CODEC; i <= (int) SHUTDOWN; i++) - { - if (RequestTypeStr[i] == requestTypeStr) - { - type = (RequestType) i; - return ret; - } - } + for (int i = (int) NEGOTIATE_FLOWFILE_CODEC; i <= (int) SHUTDOWN; i++) { + if (RequestTypeStr[i] == requestTypeStr) { + type = (RequestType) i; + return ret; + } + } - return -1; + return -1; } -int Site2SiteClientProtocol::readRespond(RespondCode &code, std::string &message) -{ - uint8_t firstByte; +int Site2SiteClientProtocol::readRespond(RespondCode &code, + std::string &message) { + uint8_t firstByte; - int ret = peer_->read(firstByte); + int ret = peer_->read(firstByte); - if (ret <= 0 || firstByte != CODE_SEQUENCE_VALUE_1) - return -1; + if (ret <= 0 || firstByte != CODE_SEQUENCE_VALUE_1) + return -1; - uint8_t secondByte; + uint8_t secondByte; - ret = peer_->read(secondByte); + ret = peer_->read(secondByte); - if (ret <= 0 || secondByte != CODE_SEQUENCE_VALUE_2) - return -1; + if (ret <= 0 || secondByte != CODE_SEQUENCE_VALUE_2) + return -1; - uint8_t thirdByte; + uint8_t thirdByte; - ret = peer_->read(thirdByte); + ret = peer_->read(thirdByte); - if (ret <= 0) - return ret; + if (ret <= 0) + return ret; - code = (RespondCode) thirdByte; + code = (RespondCode) thirdByte; - RespondCodeContext *resCode = this->getRespondCodeContext(code); + RespondCodeContext *resCode = this->getRespondCodeContext(code); - if ( resCode == NULL) - { - // Not a valid respond code - return -1; - } - if (resCode->hasDescription) - { - ret = peer_->readUTF(message); - if (ret <= 0) - return -1; - } - return 3 + message.size(); + if (resCode == NULL) { + // Not a valid respond code + return -1; + } + if (resCode->hasDescription) { + ret = peer_->readUTF(message); + if (ret <= 0) + return -1; + } + return 3 + message.size(); } -int Site2SiteClientProtocol::writeRespond(RespondCode code, std::string message) -{ - RespondCodeContext *resCode = this->getRespondCodeContext(code); - - if (resCode == NULL) - { - // Not a valid respond code - return -1; - } - - uint8_t codeSeq[3]; - codeSeq[0] = CODE_SEQUENCE_VALUE_1; - codeSeq[1] = CODE_SEQUENCE_VALUE_2; - codeSeq[2] = (uint8_t) code; - - int ret = peer_->write(codeSeq, 3); - - if (ret != 3) - return -1; - - if (resCode->hasDescription) - { - ret = peer_->writeUTF(message); - if (ret > 0) - return (3 + ret); - else - return ret; - } - else - return 3; +int Site2SiteClientProtocol::writeRespond(RespondCode code, + std::string message) { + RespondCodeContext *resCode = this->getRespondCodeContext(code); + + if (resCode == NULL) { + // Not a valid respond code + return -1; + } + + uint8_t codeSeq[3]; + codeSeq[0] = CODE_SEQUENCE_VALUE_1; + codeSeq[1] = CODE_SEQUENCE_VALUE_2; + codeSeq[2] = (uint8_t) code; + + int ret = peer_->write(codeSeq, 3); + + if (ret != 3) + return -1; + + if (resCode->hasDescription) { + ret = peer_->writeUTF(message); + if (ret > 0) + return (3 + ret); + else + return ret; + } else + return 3; } -bool Site2SiteClientProtocol::negotiateCodec() -{ - if (_peerState != HANDSHAKED) - { - logger_->log_error("Site2Site peer state is not handshaked while negotiate codec"); - return false; - } +bool Site2SiteClientProtocol::negotiateCodec() { + if (_peerState != HANDSHAKED) { + logger_->log_error( + "Site2Site peer state is not handshaked while negotiate codec"); + return false; + } - logger_->log_info("Site2Site Protocol Negotiate Codec with destination port %s", _portIdStr.c_str()); + logger_->log_info( + "Site2Site Protocol Negotiate Codec with destination port %s", + _portIdStr.c_str()); - int status = this->writeRequestType(NEGOTIATE_FLOWFILE_CODEC); + int status = this->writeRequestType(NEGOTIATE_FLOWFILE_CODEC); - if (status <= 0) - { - // tearDown(); - return false; - } + if (status <= 0) { + // tearDown(); + return false; + } - // Negotiate the codec version - bool ret = initiateCodecResourceNegotiation(); + // Negotiate the codec version + bool ret = initiateCodecResourceNegotiation(); - if (!ret) - { - logger_->log_error("Site2Site Codec Version Negotiation failed"); - /* - peer_->yield(); - tearDown(); */ - return false; - } + if (!ret) { + logger_->log_error("Site2Site Codec Version Negotiation failed"); + /* + peer_->yield(); + tearDown(); */ + return false; + } - logger_->log_info("Site2Site Codec Completed and move to READY state for data transfer"); - _peerState = READY; + logger_->log_info( + "Site2Site Codec Completed and move to READY state for data transfer"); + _peerState = READY; - return true; + return true; } -bool Site2SiteClientProtocol::bootstrap() -{ - if (_peerState == READY) - return true; - - tearDown(); - - if (establish() && handShake() && negotiateCodec()) - { - logger_->log_info("Site2Site Ready For data transaction"); - return true; - } - else - { - peer_->yield(); - tearDown(); - return false; - } +bool Site2SiteClientProtocol::bootstrap() { + if (_peerState == READY) + return true; + + tearDown(); + + if (establish() && handShake() && negotiateCodec()) { + logger_->log_info("Site2Site Ready For data transaction"); + return true; + } else { + peer_->yield(); + tearDown(); + return false; + } } -Transaction* Site2SiteClientProtocol::createTransaction(std::string &transactionID, TransferDirection direction) -{ - int ret; - bool dataAvailable; - Transaction *transaction = NULL; - - if (_peerState != READY) - { - bootstrap(); - } - - if (_peerState != READY) - { - return NULL; - } - - if (direction == RECEIVE) - { - ret = writeRequestType(RECEIVE_FLOWFILES); - - if (ret <= 0) - { - // tearDown(); - return NULL; - } - - RespondCode code; - std::string message; - - ret = readRespond(code, message); - - if (ret <= 0) - { - // tearDown(); - return NULL; - } - - CRCStream crcstream(peer_); - switch (code) - { - case MORE_DATA: - dataAvailable = true; - logger_->log_info("Site2Site peer indicates that data is available"); - transaction = new Transaction(direction,crcstream); - _transactionMap[transaction->getUUIDStr()] = transaction; - transactionID = transaction->getUUIDStr(); - transaction->setDataAvailable(dataAvailable); - logger_->log_info("Site2Site create transaction %s", transaction->getUUIDStr().c_str()); - return transaction; - case NO_MORE_DATA: - dataAvailable = false; - logger_->log_info("Site2Site peer indicates that no data is available"); - transaction = new Transaction(direction,crcstream); - _transactionMap[transaction->getUUIDStr()] = transaction; - transactionID = transaction->getUUIDStr(); - transaction->setDataAvailable(dataAvailable); - logger_->log_info("Site2Site create transaction %s", transaction->getUUIDStr().c_str()); - return transaction; - default: - logger_->log_info("Site2Site got unexpected response %d when asking for data", code); - // tearDown(); - return NULL; - } - } - else - { - ret = writeRequestType(SEND_FLOWFILES); - - if (ret <= 0) - { - // tearDown(); - return NULL; - } - else - { - CRCStream crcstream(peer_); - transaction = new Transaction(direction,crcstream); - _transactionMap[transaction->getUUIDStr()] = transaction; - transactionID = transaction->getUUIDStr(); - logger_->log_info("Site2Site create transaction %s", transaction->getUUIDStr().c_str()); - return transaction; - } - } +Transaction* Site2SiteClientProtocol::createTransaction( + std::string &transactionID, TransferDirection direction) { + int ret; + bool dataAvailable; + Transaction *transaction = NULL; + + if (_peerState != READY) { + bootstrap(); + } + + if (_peerState != READY) { + return NULL; + } + + if (direction == RECEIVE) { + ret = writeRequestType(RECEIVE_FLOWFILES); + + if (ret <= 0) { + // tearDown(); + return NULL; + } + + RespondCode code; + std::string message; + + ret = readRespond(code, message); + + if (ret <= 0) { + // tearDown(); + return NULL; + } + + org::apache::nifi::minifi::io::CRCStream crcstream(peer_); + switch (code) { + case MORE_DATA: + dataAvailable = true; + logger_->log_info("Site2Site peer indicates that data is available"); + transaction = new Transaction(direction, crcstream); + _transactionMap[transaction->getUUIDStr()] = transaction; + transactionID = transaction->getUUIDStr(); + transaction->setDataAvailable(dataAvailable); + logger_->log_info("Site2Site create transaction %s", + transaction->getUUIDStr().c_str()); + return transaction; + case NO_MORE_DATA: + dataAvailable = false; + logger_->log_info("Site2Site peer indicates that no data is available"); + transaction = new Transaction(direction, crcstream); + _transactionMap[transaction->getUUIDStr()] = transaction; + transactionID = transaction->getUUIDStr(); + transaction->setDataAvailable(dataAvailable); + logger_->log_info("Site2Site create transaction %s", + transaction->getUUIDStr().c_str()); + return transaction; + default: + logger_->log_info( + "Site2Site got unexpected response %d when asking for data", code); + // tearDown(); + return NULL; + } + } else { + ret = writeRequestType(SEND_FLOWFILES); + + if (ret <= 0) { + // tearDown(); + return NULL; + } else { + org::apache::nifi::minifi::io::CRCStream crcstream(peer_); + transaction = new Transaction(direction, crcstream); + _transactionMap[transaction->getUUIDStr()] = transaction; + transactionID = transaction->getUUIDStr(); + logger_->log_info("Site2Site create transaction %s", + transaction->getUUIDStr().c_str()); + return transaction; + } + } } -bool Site2SiteClientProtocol::receive(std::string transactionID, DataPacket *packet, bool &eof) -{ - int ret; - Transaction *transaction = NULL; - - if (_peerState != READY) - { - bootstrap(); - } - - if (_peerState != READY) - { - return false; - } - - std::map::iterator it = this->_transactionMap.find(transactionID); - - if (it == _transactionMap.end()) - { - return false; - } - else - { - transaction = it->second; - } - - if (transaction->getState() != TRANSACTION_STARTED && transaction->getState() != DATA_EXCHANGED) - { - logger_->log_info("Site2Site transaction %s is not at started or exchanged state", transactionID.c_str()); - return false; - } - - if (transaction->getDirection() != RECEIVE) - { - logger_->log_info("Site2Site transaction %s direction is wrong", transactionID.c_str()); - return false; - } - - if (!transaction->isDataAvailable()) - { - eof = true; - return true; - } - - if (transaction->_transfers > 0) - { - // if we already has transfer before, check to see whether another one is available - RespondCode code; - std::string message; - - ret = readRespond(code, message); - - if (ret <= 0) - { - return false; - } - if (code == CONTINUE_TRANSACTION) - { - logger_->log_info("Site2Site transaction %s peer indicate continue transaction", transactionID.c_str()); - transaction->_dataAvailable = true; - } - else if (code == FINISH_TRANSACTION) - { - logger_->log_info("Site2Site transaction %s peer indicate finish transaction", transactionID.c_str()); - transaction->_dataAvailable = false; - } - else - { - logger_->log_info("Site2Site transaction %s peer indicate wrong respond code %d", transactionID.c_str(), code); - return false; - } - } - - if (!transaction->isDataAvailable()) - { - eof = true; - return true; - } - - // start to read the packet - uint32_t numAttributes; - ret = transaction->getStream().read(numAttributes); - if (ret <= 0 || numAttributes > MAX_NUM_ATTRIBUTES) - { - return false; - } - - // read the attributes - for (unsigned int i = 0; i < numAttributes; i++) - { - std::string key; - std::string value; - ret = transaction->getStream().readUTF(key,true); - if (ret <= 0) - { - return false; - } - ret = transaction->getStream().readUTF(value,true); - if (ret <= 0) - { - return false; - } - packet->_attributes[key] = value; - logger_->log_info("Site2Site transaction %s receives attribute key %s value %s", transactionID.c_str(), key.c_str(), value.c_str()); - } - - uint64_t len; - ret = transaction->getStream().read(len); - if (ret <= 0) - { - return false; - } - - packet->_size = len; - transaction->_transfers++; - transaction->_state = DATA_EXCHANGED; - transaction->_bytes += len; - logger_->log_info("Site2Site transaction %s receives flow record %d, total length %d", transactionID.c_str(), - transaction->_transfers, transaction->_bytes); - - return true; +bool Site2SiteClientProtocol::receive(std::string transactionID, + DataPacket *packet, bool &eof) { + int ret; + Transaction *transaction = NULL; + + if (_peerState != READY) { + bootstrap(); + } + + if (_peerState != READY) { + return false; + } + + std::map::iterator it = + this->_transactionMap.find(transactionID); + + if (it == _transactionMap.end()) { + return false; + } else { + transaction = it->second; + } + + if (transaction->getState() != TRANSACTION_STARTED + && transaction->getState() != DATA_EXCHANGED) { + logger_->log_info( + "Site2Site transaction %s is not at started or exchanged state", + transactionID.c_str()); + return false; + } + + if (transaction->getDirection() != RECEIVE) { + logger_->log_info("Site2Site transaction %s direction is wrong", + transactionID.c_str()); + return false; + } + + if (!transaction->isDataAvailable()) { + eof = true; + return true; + } + + if (transaction->_transfers > 0) { + // if we already has transfer before, check to see whether another one is available + RespondCode code; + std::string message; + + ret = readRespond(code, message); + + if (ret <= 0) { + return false; + } + if (code == CONTINUE_TRANSACTION) { + logger_->log_info( + "Site2Site transaction %s peer indicate continue transaction", + transactionID.c_str()); + transaction->_dataAvailable = true; + } else if (code == FINISH_TRANSACTION) { + logger_->log_info( + "Site2Site transaction %s peer indicate finish transaction", + transactionID.c_str()); + transaction->_dataAvailable = false; + } else { + logger_->log_info( + "Site2Site transaction %s peer indicate wrong respond code %d", + transactionID.c_str(), code); + return false; + } + } + + if (!transaction->isDataAvailable()) { + eof = true; + return true; + } + + // start to read the packet + uint32_t numAttributes; + ret = transaction->getStream().read(numAttributes); + if (ret <= 0 || numAttributes > MAX_NUM_ATTRIBUTES) { + return false; + } + + // read the attributes + for (unsigned int i = 0; i < numAttributes; i++) { + std::string key; + std::string value; + ret = transaction->getStream().readUTF(key, true); + if (ret <= 0) { + return false; + } + ret = transaction->getStream().readUTF(value, true); + if (ret <= 0) { + return false; + } + packet->_attributes[key] = value; + logger_->log_info( + "Site2Site transaction %s receives attribute key %s value %s", + transactionID.c_str(), key.c_str(), value.c_str()); + } + + uint64_t len; + ret = transaction->getStream().read(len); + if (ret <= 0) { + return false; + } + + packet->_size = len; + transaction->_transfers++; + transaction->_state = DATA_EXCHANGED; + transaction->_bytes += len; + logger_->log_info( + "Site2Site transaction %s receives flow record %d, total length %d", + transactionID.c_str(), transaction->_transfers, transaction->_bytes); + + return true; } -bool Site2SiteClientProtocol::send(std::string transactionID, DataPacket *packet, FlowFileRecord *flowFile, ProcessSession *session) -{ - int ret; - Transaction *transaction = NULL; - - if (_peerState != READY) - { - bootstrap(); - } - - if (_peerState != READY) - { - return false; - } - - std::map::iterator it = this->_transactionMap.find(transactionID); - - if (it == _transactionMap.end()) - { - return false; - } - else - { - transaction = it->second; - } - - if (transaction->getState() != TRANSACTION_STARTED && transaction->getState() != DATA_EXCHANGED) - { - logger_->log_info("Site2Site transaction %s is not at started or exchanged state", transactionID.c_str()); - return false; - } - - if (transaction->getDirection() != SEND) - { - logger_->log_info("Site2Site transaction %s direction is wrong", transactionID.c_str()); - return false; - } - - if (transaction->_transfers > 0) - { - ret = writeRespond(CONTINUE_TRANSACTION, "CONTINUE_TRANSACTION"); - if (ret <= 0) - { - return false; - } - } - - // start to read the packet - uint32_t numAttributes = packet->_attributes.size(); - ret = transaction->getStream().write(numAttributes); - if (ret != 4) - { - return false; - } - - std::map::iterator itAttribute; - for (itAttribute = packet->_attributes.begin(); itAttribute!= packet->_attributes.end(); itAttribute++) - { - ret = transaction->getStream().writeUTF(itAttribute->first, true); - - if (ret <= 0) - { - return false; - } - ret = transaction->getStream().writeUTF(itAttribute->second, true); - if (ret <= 0) - { - return false; - } - logger_->log_info("Site2Site transaction %s send attribute key %s value %s", transactionID.c_str(), - itAttribute->first.c_str(), itAttribute->second.c_str()); - } - - uint64_t len = flowFile->getSize() ; - ret = transaction->getStream().write(len); - if (ret != 8) - { - return false; - } - - if (flowFile->getSize()) - { - Site2SiteClientProtocol::ReadCallback callback(packet); - session->read(flowFile, &callback); - if (flowFile->getSize() != packet->_size) - { - return false; - } - } - - transaction->_transfers++; - transaction->_state = DATA_EXCHANGED; - transaction->_bytes += len; - logger_->log_info("Site2Site transaction %s send flow record %d, total length %d", transactionID.c_str(), - transaction->_transfers, transaction->_bytes); - - return true; +bool Site2SiteClientProtocol::send( + std::string transactionID, DataPacket *packet, std::shared_ptr flowFile, + core::ProcessSession *session) { + int ret; + Transaction *transaction = NULL; + + if (_peerState != READY) { + bootstrap(); + } + + if (_peerState != READY) { + return false; + } + + std::map::iterator it = + this->_transactionMap.find(transactionID); + + if (it == _transactionMap.end()) { + return false; + } else { + transaction = it->second; + } + + if (transaction->getState() != TRANSACTION_STARTED + && transaction->getState() != DATA_EXCHANGED) { + logger_->log_info( + "Site2Site transaction %s is not at started or exchanged state", + transactionID.c_str()); + return false; + } + + if (transaction->getDirection() != SEND) { + logger_->log_info("Site2Site transaction %s direction is wrong", + transactionID.c_str()); + return false; + } + + if (transaction->_transfers > 0) { + ret = writeRespond(CONTINUE_TRANSACTION, "CONTINUE_TRANSACTION"); + if (ret <= 0) { + return false; + } + } + + // start to read the packet + uint32_t numAttributes = packet->_attributes.size(); + ret = transaction->getStream().write(numAttributes); + if (ret != 4) { + return false; + } + + std::map::iterator itAttribute; + for (itAttribute = packet->_attributes.begin(); + itAttribute != packet->_attributes.end(); itAttribute++) { + ret = transaction->getStream().writeUTF(itAttribute->first, true); + + if (ret <= 0) { + return false; + } + ret = transaction->getStream().writeUTF(itAttribute->second, true); + if (ret <= 0) { + return false; + } + logger_->log_info("Site2Site transaction %s send attribute key %s value %s", + transactionID.c_str(), itAttribute->first.c_str(), + itAttribute->second.c_str()); + } + + uint64_t len = flowFile->getSize(); + ret = transaction->getStream().write(len); + if (ret != 8) { + return false; + } + + if (flowFile->getSize()) { + Site2SiteClientProtocol::ReadCallback callback(packet); + session->read(flowFile, &callback); + if (flowFile->getSize() != packet->_size) { + return false; + } + } + + transaction->_transfers++; + transaction->_state = DATA_EXCHANGED; + transaction->_bytes += len; + logger_->log_info( + "Site2Site transaction %s send flow record %d, total length %d", + transactionID.c_str(), transaction->_transfers, transaction->_bytes); + + return true; } -void Site2SiteClientProtocol::receiveFlowFiles(ProcessContext *context, ProcessSession *session) -{ - uint64_t bytes = 0; - int transfers = 0; - Transaction *transaction = NULL; - - if (_peerState != READY) - { - bootstrap(); - } - - if (_peerState != READY) - { - context->yield(); - tearDown(); - throw Exception(SITE2SITE_EXCEPTION, "Can not establish handshake with peer"); - return; - } - - // Create the transaction - std::string transactionID; - transaction = createTransaction(transactionID, RECEIVE); - - if (transaction == NULL) - { - context->yield(); - tearDown(); - throw Exception(SITE2SITE_EXCEPTION, "Can not create transaction"); - return; - } - - try - { - while (true) - { - std::map empty; - uint64_t startTime = getTimeMillis(); - DataPacket packet(this, transaction, empty); - bool eof = false; - - if (!receive(transactionID, &packet, eof)) - { - throw Exception(SITE2SITE_EXCEPTION, "Receive Failed"); - return; - } - if (eof) - { - // transaction done - break; - } - FlowFileRecord *flowFile = session->create(); - if (!flowFile) - { - throw Exception(SITE2SITE_EXCEPTION, "Flow File Creation Failed"); - return; - } - std::map::iterator it; - std::string sourceIdentifier; - for (it = packet._attributes.begin(); it!= packet._attributes.end(); it++) - { - if (it->first == FlowAttributeKey(UUID)) - sourceIdentifier = it->second; - flowFile->addAttribute(it->first, it->second); - } - - if (packet._size > 0) - { - Site2SiteClientProtocol::WriteCallback callback(&packet); - session->write(flowFile, &callback); - if (flowFile->getSize() != packet._size) - { - throw Exception(SITE2SITE_EXCEPTION, "Receive Size Not Right"); - return; - } - } - Relationship relation; // undefined relationship - uint64_t endTime = getTimeMillis(); - std::string transitUri = peer_->getURL() + "/" + sourceIdentifier; - std::string details = "urn:nifi:" + sourceIdentifier + "Remote Host=" + peer_->getHostName(); - session->getProvenanceReporter()->receive(flowFile, transitUri, sourceIdentifier, details, endTime - startTime); - session->transfer(flowFile, relation); - // receive the transfer for the flow record - bytes += packet._size; - transfers++; - } // while true - - if (!confirm(transactionID)) - { - throw Exception(SITE2SITE_EXCEPTION, "Confirm Transaction Failed"); - return; - } - if (!complete(transactionID)) - { - throw Exception(SITE2SITE_EXCEPTION, "Complete Transaction Failed"); - return; - } - logger_->log_info("Site2Site transaction %s successfully receive flow record %d, content bytes %d", - transactionID.c_str(), transfers, bytes); - // we yield the receive if we did not get anything - if (transfers == 0) - context->yield(); - } - catch (std::exception &exception) - { - if (transaction) - deleteTransaction(transactionID); - context->yield(); - tearDown(); - logger_->log_debug("Caught Exception %s", exception.what()); - throw; - } - catch (...) - { - if (transaction) - deleteTransaction(transactionID); - context->yield(); - tearDown(); - logger_->log_debug("Caught Exception during Site2SiteClientProtocol::receiveFlowFiles"); - throw; - } - - deleteTransaction(transactionID); - - return; +void Site2SiteClientProtocol::receiveFlowFiles( + core::ProcessContext *context, + core::ProcessSession *session) { + uint64_t bytes = 0; + int transfers = 0; + Transaction *transaction = NULL; + + if (_peerState != READY) { + bootstrap(); + } + + if (_peerState != READY) { + context->yield(); + tearDown(); + throw Exception(SITE2SITE_EXCEPTION, + "Can not establish handshake with peer"); + return; + } + + // Create the transaction + std::string transactionID; + transaction = createTransaction(transactionID, RECEIVE); + + if (transaction == NULL) { + context->yield(); + tearDown(); + throw Exception(SITE2SITE_EXCEPTION, "Can not create transaction"); + return; + } + + try { + while (true) { + std::map empty; + uint64_t startTime = getTimeMillis(); + DataPacket packet(this, transaction, empty); + bool eof = false; + + if (!receive(transactionID, &packet, eof)) { + throw Exception(SITE2SITE_EXCEPTION, "Receive Failed"); + return; + } + if (eof) { + // transaction done + break; + } + std::shared_ptr flowFile = std::static_pointer_cast(session->create());; + if (!flowFile) { + throw Exception(SITE2SITE_EXCEPTION, "Flow File Creation Failed"); + return; + } + std::map::iterator it; + std::string sourceIdentifier; + for (it = packet._attributes.begin(); it != packet._attributes.end(); + it++) { + if (it->first == FlowAttributeKey(UUID)) + sourceIdentifier = it->second; + flowFile->addAttribute(it->first, it->second); + } + + if (packet._size > 0) { + Site2SiteClientProtocol::WriteCallback callback(&packet); + session->write(flowFile, &callback); + if (flowFile->getSize() != packet._size) { + throw Exception(SITE2SITE_EXCEPTION, "Receive Size Not Right"); + return; + } + } + core::Relationship relation; // undefined relationship + uint64_t endTime = getTimeMillis(); + std::string transitUri = peer_->getURL() + "/" + sourceIdentifier; + std::string details = "urn:nifi:" + sourceIdentifier + "Remote Host=" + + peer_->getHostName(); + session->getProvenanceReporter()->receive(flowFile, transitUri, + sourceIdentifier, details, + endTime - startTime); + session->transfer(flowFile, relation); + // receive the transfer for the flow record + bytes += packet._size; + transfers++; + } // while true + + if (!confirm(transactionID)) { + throw Exception(SITE2SITE_EXCEPTION, "Confirm Transaction Failed"); + return; + } + if (!complete(transactionID)) { + throw Exception(SITE2SITE_EXCEPTION, "Complete Transaction Failed"); + return; + } + logger_->log_info( + "Site2Site transaction %s successfully receive flow record %d, content bytes %d", + transactionID.c_str(), transfers, bytes); + // we yield the receive if we did not get anything + if (transfers == 0) + context->yield(); + } catch (std::exception &exception) { + if (transaction) + deleteTransaction(transactionID); + context->yield(); + tearDown(); + logger_->log_debug("Caught Exception %s", exception.what()); + throw; + } catch (...) { + if (transaction) + deleteTransaction(transactionID); + context->yield(); + tearDown(); + logger_->log_debug( + "Caught Exception during Site2SiteClientProtocol::receiveFlowFiles"); + throw; + } + + deleteTransaction(transactionID); + + return; } -bool Site2SiteClientProtocol::confirm(std::string transactionID) -{ - int ret; - Transaction *transaction = NULL; - - if (_peerState != READY) - { - bootstrap(); - } - - if (_peerState != READY) - { - return false; - } - - std::map::iterator it = this->_transactionMap.find(transactionID); - - if (it == _transactionMap.end()) - { - return false; - } - else - { - transaction = it->second; - } - - if (transaction->getState() == TRANSACTION_STARTED && !transaction->isDataAvailable() && - transaction->getDirection() == RECEIVE) - { - transaction->_state = TRANSACTION_CONFIRMED; - return true; - } - - if (transaction->getState() != DATA_EXCHANGED) - return false; - - if (transaction->getDirection() == RECEIVE) - { - if (transaction->isDataAvailable()) - return false; - // we received a FINISH_TRANSACTION indicator. Send back a CONFIRM_TRANSACTION message - // to peer so that we can verify that the connection is still open. This is a two-phase commit, - // which helps to prevent the chances of data duplication. Without doing this, we may commit the - // session and then when we send the response back to the peer, the peer may have timed out and may not - // be listening. As a result, it will re-send the data. By doing this two-phase commit, we narrow the - // Critical Section involved in this transaction so that rather than the Critical Section being the - // time window involved in the entire transaction, it is reduced to a simple round-trip conversation. - long crcValue = transaction->getCRC(); - std::string crc = std::to_string(crcValue); - logger_->log_info("Site2Site Send confirm with CRC %d to transaction %s", transaction->getCRC(), - transactionID.c_str()); - ret = writeRespond(CONFIRM_TRANSACTION, crc); - if (ret <= 0) - return false; - RespondCode code; - std::string message; - readRespond(code, message); - if (ret <= 0) - return false; - - if (code == CONFIRM_TRANSACTION) - { - logger_->log_info("Site2Site transaction %s peer confirm transaction", transactionID.c_str()); - transaction->_state = TRANSACTION_CONFIRMED; - return true; - } - else if (code == BAD_CHECKSUM) - { - logger_->log_info("Site2Site transaction %s peer indicate bad checksum", transactionID.c_str()); - /* - transaction->_state = TRANSACTION_CONFIRMED; - return true; */ - return false; - } - else - { - logger_->log_info("Site2Site transaction %s peer unknown respond code %d", - transactionID.c_str(), code); - return false; - } - } - else - { - logger_->log_info("Site2Site Send FINISH TRANSACTION for transaction %s", - transactionID.c_str()); - ret = writeRespond(FINISH_TRANSACTION, "FINISH_TRANSACTION"); - if (ret <= 0) - return false; - RespondCode code; - std::string message; - readRespond(code, message); - if (ret <= 0) - return false; - - // we've sent a FINISH_TRANSACTION. Now we'll wait for the peer to send a 'Confirm Transaction' response - if (code == CONFIRM_TRANSACTION) - { - logger_->log_info("Site2Site transaction %s peer confirm transaction with CRC %s", transactionID.c_str(), message.c_str()); - if (this->_currentVersion > 3) - { - long crcValue = transaction->getCRC(); - std::string crc = std::to_string(crcValue); - if (message == crc) - { - logger_->log_info("Site2Site transaction %s CRC matched", transactionID.c_str()); - ret = writeRespond(CONFIRM_TRANSACTION, "CONFIRM_TRANSACTION"); - if (ret <= 0) - return false; - transaction->_state = TRANSACTION_CONFIRMED; - return true; - } - else - { - logger_->log_info("Site2Site transaction %s CRC not matched %s", transactionID.c_str(), crc.c_str()); - ret = writeRespond(BAD_CHECKSUM, "BAD_CHECKSUM"); - /* - ret = writeRespond(CONFIRM_TRANSACTION, "CONFIRM_TRANSACTION"); - if (ret <= 0) - return false; - transaction->_state = TRANSACTION_CONFIRMED; - return true; */ - return false; - } - } - ret = writeRespond(CONFIRM_TRANSACTION, "CONFIRM_TRANSACTION"); - if (ret <= 0) - return false; - transaction->_state = TRANSACTION_CONFIRMED; - return true; - } - else - { - logger_->log_info("Site2Site transaction %s peer unknown respond code %d", - transactionID.c_str(), code); - return false; - } - return false; - } +bool Site2SiteClientProtocol::confirm(std::string transactionID) { + int ret; + Transaction *transaction = NULL; + + if (_peerState != READY) { + bootstrap(); + } + + if (_peerState != READY) { + return false; + } + + std::map::iterator it = + this->_transactionMap.find(transactionID); + + if (it == _transactionMap.end()) { + return false; + } else { + transaction = it->second; + } + + if (transaction->getState() == TRANSACTION_STARTED + && !transaction->isDataAvailable() + && transaction->getDirection() == RECEIVE) { + transaction->_state = TRANSACTION_CONFIRMED; + return true; + } + + if (transaction->getState() != DATA_EXCHANGED) + return false; + + if (transaction->getDirection() == RECEIVE) { + if (transaction->isDataAvailable()) + return false; + // we received a FINISH_TRANSACTION indicator. Send back a CONFIRM_TRANSACTION message + // to peer so that we can verify that the connection is still open. This is a two-phase commit, + // which helps to prevent the chances of data duplication. Without doing this, we may commit the + // session and then when we send the response back to the peer, the peer may have timed out and may not + // be listening. As a result, it will re-send the data. By doing this two-phase commit, we narrow the + // Critical Section involved in this transaction so that rather than the Critical Section being the + // time window involved in the entire transaction, it is reduced to a simple round-trip conversation. + long crcValue = transaction->getCRC(); + std::string crc = std::to_string(crcValue); + logger_->log_info("Site2Site Send confirm with CRC %d to transaction %s", + transaction->getCRC(), transactionID.c_str()); + ret = writeRespond(CONFIRM_TRANSACTION, crc); + if (ret <= 0) + return false; + RespondCode code; + std::string message; + readRespond(code, message); + if (ret <= 0) + return false; + + if (code == CONFIRM_TRANSACTION) { + logger_->log_info("Site2Site transaction %s peer confirm transaction", + transactionID.c_str()); + transaction->_state = TRANSACTION_CONFIRMED; + return true; + } else if (code == BAD_CHECKSUM) { + logger_->log_info("Site2Site transaction %s peer indicate bad checksum", + transactionID.c_str()); + /* + transaction->_state = TRANSACTION_CONFIRMED; + return true; */ + return false; + } else { + logger_->log_info("Site2Site transaction %s peer unknown respond code %d", + transactionID.c_str(), code); + return false; + } + } else { + logger_->log_info("Site2Site Send FINISH TRANSACTION for transaction %s", + transactionID.c_str()); + ret = writeRespond(FINISH_TRANSACTION, "FINISH_TRANSACTION"); + if (ret <= 0) + return false; + RespondCode code; + std::string message; + readRespond(code, message); + if (ret <= 0) + return false; + + // we've sent a FINISH_TRANSACTION. Now we'll wait for the peer to send a 'Confirm Transaction' response + if (code == CONFIRM_TRANSACTION) { + logger_->log_info( + "Site2Site transaction %s peer confirm transaction with CRC %s", + transactionID.c_str(), message.c_str()); + if (this->_currentVersion > 3) { + long crcValue = transaction->getCRC(); + std::string crc = std::to_string(crcValue); + if (message == crc) { + logger_->log_info("Site2Site transaction %s CRC matched", + transactionID.c_str()); + ret = writeRespond(CONFIRM_TRANSACTION, "CONFIRM_TRANSACTION"); + if (ret <= 0) + return false; + transaction->_state = TRANSACTION_CONFIRMED; + return true; + } else { + logger_->log_info("Site2Site transaction %s CRC not matched %s", + transactionID.c_str(), crc.c_str()); + ret = writeRespond(BAD_CHECKSUM, "BAD_CHECKSUM"); + /* + ret = writeRespond(CONFIRM_TRANSACTION, "CONFIRM_TRANSACTION"); + if (ret <= 0) + return false; + transaction->_state = TRANSACTION_CONFIRMED; + return true; */ + return false; + } + } + ret = writeRespond(CONFIRM_TRANSACTION, "CONFIRM_TRANSACTION"); + if (ret <= 0) + return false; + transaction->_state = TRANSACTION_CONFIRMED; + return true; + } else { + logger_->log_info("Site2Site transaction %s peer unknown respond code %d", + transactionID.c_str(), code); + return false; + } + return false; + } } -void Site2SiteClientProtocol::cancel(std::string transactionID) -{ - Transaction *transaction = NULL; - - if (_peerState != READY) - { - return; - } - - std::map::iterator it = this->_transactionMap.find(transactionID); - - if (it == _transactionMap.end()) - { - return; - } - else - { - transaction = it->second; - } - - if (transaction->getState() == TRANSACTION_CANCELED || transaction->getState() == TRANSACTION_COMPLETED - || transaction->getState() == TRANSACTION_ERROR) - { - return; - } - - this->writeRespond(CANCEL_TRANSACTION, "Cancel"); - transaction->_state = TRANSACTION_CANCELED; - - tearDown(); - return; +void Site2SiteClientProtocol::cancel(std::string transactionID) { + Transaction *transaction = NULL; + + if (_peerState != READY) { + return; + } + + std::map::iterator it = + this->_transactionMap.find(transactionID); + + if (it == _transactionMap.end()) { + return; + } else { + transaction = it->second; + } + + if (transaction->getState() == TRANSACTION_CANCELED + || transaction->getState() == TRANSACTION_COMPLETED + || transaction->getState() == TRANSACTION_ERROR) { + return; + } + + this->writeRespond(CANCEL_TRANSACTION, "Cancel"); + transaction->_state = TRANSACTION_CANCELED; + + tearDown(); + return; } -void Site2SiteClientProtocol::deleteTransaction(std::string transactionID) -{ - Transaction *transaction = NULL; +void Site2SiteClientProtocol::deleteTransaction(std::string transactionID) { + Transaction *transaction = NULL; - std::map::iterator it = this->_transactionMap.find(transactionID); + std::map::iterator it = + this->_transactionMap.find(transactionID); - if (it == _transactionMap.end()) - { - return; - } - else - { - transaction = it->second; - } + if (it == _transactionMap.end()) { + return; + } else { + transaction = it->second; + } - logger_->log_info("Site2Site delete transaction %s", transaction->getUUIDStr().c_str()); - delete transaction; - _transactionMap.erase(transactionID); + logger_->log_info("Site2Site delete transaction %s", + transaction->getUUIDStr().c_str()); + delete transaction; + _transactionMap.erase(transactionID); } -void Site2SiteClientProtocol::error(std::string transactionID) -{ - Transaction *transaction = NULL; +void Site2SiteClientProtocol::error(std::string transactionID) { + Transaction *transaction = NULL; - std::map::iterator it = this->_transactionMap.find(transactionID); + std::map::iterator it = + this->_transactionMap.find(transactionID); - if (it == _transactionMap.end()) - { - return; - } - else - { - transaction = it->second; - } + if (it == _transactionMap.end()) { + return; + } else { + transaction = it->second; + } - transaction->_state = TRANSACTION_ERROR; - tearDown(); - return; + transaction->_state = TRANSACTION_ERROR; + tearDown(); + return; } -//! Complete the transaction -bool Site2SiteClientProtocol::complete(std::string transactionID) -{ - int ret; - Transaction *transaction = NULL; - - if (_peerState != READY) - { - bootstrap(); - } - - if (_peerState != READY) - { - return false; - } - - std::map::iterator it = this->_transactionMap.find(transactionID); - - if (it == _transactionMap.end()) - { - return false; - } - else - { - transaction = it->second; - } - - if (transaction->getState() != TRANSACTION_CONFIRMED) - { - return false; - } - - if (transaction->getDirection() == RECEIVE) - { - if (transaction->_transfers == 0) - { - transaction->_state = TRANSACTION_COMPLETED; - return true; - } - else - { - logger_->log_info("Site2Site transaction %s send finished", transactionID.c_str()); - ret = this->writeRespond(TRANSACTION_FINISHED, "Finished"); - if (ret <= 0) - return false; - else - { - transaction->_state = TRANSACTION_COMPLETED; - return true; - } - } - } - else - { - RespondCode code; - std::string message; - int ret; - - ret = readRespond(code, message); - - if (ret <= 0) - return false; - - if (code == TRANSACTION_FINISHED) - { - logger_->log_info("Site2Site transaction %s peer finished transaction", transactionID.c_str()); - transaction->_state = TRANSACTION_COMPLETED; - return true; - } - else - { - logger_->log_info("Site2Site transaction %s peer unknown respond code %d", - transactionID.c_str(), code); - return false; - } - } +// Complete the transaction +bool Site2SiteClientProtocol::complete(std::string transactionID) { + int ret; + Transaction *transaction = NULL; + + if (_peerState != READY) { + bootstrap(); + } + + if (_peerState != READY) { + return false; + } + + std::map::iterator it = + this->_transactionMap.find(transactionID); + + if (it == _transactionMap.end()) { + return false; + } else { + transaction = it->second; + } + + if (transaction->getState() != TRANSACTION_CONFIRMED) { + return false; + } + + if (transaction->getDirection() == RECEIVE) { + if (transaction->_transfers == 0) { + transaction->_state = TRANSACTION_COMPLETED; + return true; + } else { + logger_->log_info("Site2Site transaction %s send finished", + transactionID.c_str()); + ret = this->writeRespond(TRANSACTION_FINISHED, "Finished"); + if (ret <= 0) + return false; + else { + transaction->_state = TRANSACTION_COMPLETED; + return true; + } + } + } else { + RespondCode code; + std::string message; + int ret; + + ret = readRespond(code, message); + + if (ret <= 0) + return false; + + if (code == TRANSACTION_FINISHED) { + logger_->log_info("Site2Site transaction %s peer finished transaction", + transactionID.c_str()); + transaction->_state = TRANSACTION_COMPLETED; + return true; + } else { + logger_->log_info("Site2Site transaction %s peer unknown respond code %d", + transactionID.c_str(), code); + return false; + } + } } -void Site2SiteClientProtocol::transferFlowFiles(ProcessContext *context, ProcessSession *session) -{ - FlowFileRecord *flow = session->get(); - Transaction *transaction = NULL; - - if (!flow) - return; - - if (_peerState != READY) - { - bootstrap(); - } - - if (_peerState != READY) - { - context->yield(); - tearDown(); - throw Exception(SITE2SITE_EXCEPTION, "Can not establish handshake with peer"); - return; - } - - // Create the transaction - std::string transactionID; - transaction = createTransaction(transactionID, SEND); - - if (transaction == NULL) - { - context->yield(); - tearDown(); - throw Exception(SITE2SITE_EXCEPTION, "Can not create transaction"); - return; - } - - bool continueTransaction = true; - uint64_t startSendingNanos = getTimeNano(); - - try - { - while (continueTransaction) - { - uint64_t startTime = getTimeMillis(); - DataPacket packet(this, transaction, flow->getAttributes()); - - if (!send(transactionID, &packet, flow, session)) - { - throw Exception(SITE2SITE_EXCEPTION, "Send Failed"); - return; - } - logger_->log_info("Site2Site transaction %s send flow record %s", - transactionID.c_str(), flow->getUUIDStr().c_str()); - uint64_t endTime = getTimeMillis(); - std::string transitUri = peer_->getURL() + "/" + flow->getUUIDStr(); - std::string details = "urn:nifi:" + flow->getUUIDStr() + "Remote Host=" + peer_->getHostName(); - session->getProvenanceReporter()->send(flow, transitUri, details, endTime - startTime, false); - session->remove(flow); - - uint64_t transferNanos = getTimeNano() - startSendingNanos; - if (transferNanos > _batchSendNanos) - break; - - flow = session->get(); - if (!flow) - { - continueTransaction = false; - } - } // while true - - if (!confirm(transactionID)) - { - throw Exception(SITE2SITE_EXCEPTION, "Confirm Failed"); - return; - } - if (!complete(transactionID)) - { - throw Exception(SITE2SITE_EXCEPTION, "Complete Failed"); - return; - } - logger_->log_info("Site2Site transaction %s successfully send flow record %d, content bytes %d", - transactionID.c_str(), transaction->_transfers, transaction->_bytes); - } - catch (std::exception &exception) - { - if (transaction) - deleteTransaction(transactionID); - context->yield(); - tearDown(); - logger_->log_debug("Caught Exception %s", exception.what()); - throw; - } - catch (...) - { - if (transaction) - deleteTransaction(transactionID); - context->yield(); - tearDown(); - logger_->log_debug("Caught Exception during Site2SiteClientProtocol::transferFlowFiles"); - throw; - } - - deleteTransaction(transactionID); - - return; +void Site2SiteClientProtocol::transferFlowFiles( + core::ProcessContext *context, + core::ProcessSession *session) { + std::shared_ptr flow = std::static_pointer_cast(session->get());; + Transaction *transaction = NULL; + + if (!flow) + return; + + if (_peerState != READY) { + bootstrap(); + } + + if (_peerState != READY) { + context->yield(); + tearDown(); + throw Exception(SITE2SITE_EXCEPTION, + "Can not establish handshake with peer"); + return; + } + + // Create the transaction + std::string transactionID; + transaction = createTransaction(transactionID, SEND); + + if (transaction == NULL) { + context->yield(); + tearDown(); + throw Exception(SITE2SITE_EXCEPTION, "Can not create transaction"); + return; + } + + bool continueTransaction = true; + uint64_t startSendingNanos = getTimeNano(); + + try { + while (continueTransaction) { + uint64_t startTime = getTimeMillis(); + DataPacket packet(this, transaction, flow->getAttributes()); + + if (!send(transactionID, &packet, flow, session)) { + throw Exception(SITE2SITE_EXCEPTION, "Send Failed"); + return; + } + logger_->log_info("Site2Site transaction %s send flow record %s", + transactionID.c_str(), flow->getUUIDStr().c_str()); + uint64_t endTime = getTimeMillis(); + std::string transitUri = peer_->getURL() + "/" + flow->getUUIDStr(); + std::string details = "urn:nifi:" + flow->getUUIDStr() + "Remote Host=" + + peer_->getHostName(); + session->getProvenanceReporter()->send(flow, transitUri, details, + endTime - startTime, false); + session->remove(flow); + + uint64_t transferNanos = getTimeNano() - startSendingNanos; + if (transferNanos > _batchSendNanos) + break; + + flow = std::static_pointer_cast(session->get());; + if (!flow) { + continueTransaction = false; + } + } // while true + + if (!confirm(transactionID)) { + throw Exception(SITE2SITE_EXCEPTION, "Confirm Failed"); + return; + } + if (!complete(transactionID)) { + throw Exception(SITE2SITE_EXCEPTION, "Complete Failed"); + return; + } + logger_->log_info( + "Site2Site transaction %s successfully send flow record %d, content bytes %d", + transactionID.c_str(), transaction->_transfers, transaction->_bytes); + } catch (std::exception &exception) { + if (transaction) + deleteTransaction(transactionID); + context->yield(); + tearDown(); + logger_->log_debug("Caught Exception %s", exception.what()); + throw; + } catch (...) { + if (transaction) + deleteTransaction(transactionID); + context->yield(); + tearDown(); + logger_->log_debug( + "Caught Exception during Site2SiteClientProtocol::transferFlowFiles"); + throw; + } + + deleteTransaction(transactionID); + + return; } + + +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/Site2SitePeer.cpp b/libminifi/src/Site2SitePeer.cpp index ae3cc2da93..64732ac757 100644 --- a/libminifi/src/Site2SitePeer.cpp +++ b/libminifi/src/Site2SitePeer.cpp @@ -31,6 +31,12 @@ #include "Site2SitePeer.h" #include "FlowController.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { + bool Site2SitePeer::Open() { if (IsNullOrEmpty (host_)) @@ -53,3 +59,9 @@ void Site2SitePeer::Close() { stream_->closeStream(); } + + +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/TailFile.cpp b/libminifi/src/TailFile.cpp deleted file mode 100644 index 36b5e53925..0000000000 --- a/libminifi/src/TailFile.cpp +++ /dev/null @@ -1,269 +0,0 @@ -/** - * @file TailFile.cpp - * TailFile class implementation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "utils/TimeUtil.h" -#include "utils/StringUtils.h" -#include "TailFile.h" -#include "ProcessContext.h" -#include "ProcessSession.h" - -const std::string TailFile::ProcessorName("TailFile"); -Property TailFile::FileName("File to Tail", "Fully-qualified filename of the file that should be tailed", ""); -Property TailFile::StateFile("State File", - "Specifies the file that should be used for storing state about what data has been ingested so that upon restart NiFi can resume from where it left off", "TailFileState"); -Relationship TailFile::Success("success", "All files are routed to success"); - -void TailFile::initialize() -{ - //! Set the supported properties - std::set properties; - properties.insert(FileName); - properties.insert(StateFile); - setSupportedProperties(properties); - //! Set the supported relationships - std::set relationships; - relationships.insert(Success); - setSupportedRelationships(relationships); -} - -std::string TailFile::trimLeft(const std::string& s) -{ - return StringUtils::trimLeft(s); -} - -std::string TailFile::trimRight(const std::string& s) -{ - return StringUtils::trimRight(s); -} - -void TailFile::parseStateFileLine(char *buf) -{ - char *line = buf; - - while ((line[0] == ' ') || (line[0] =='\t')) - ++line; - - char first = line[0]; - if ((first == '\0') || (first == '#') || (first == '\r') || (first == '\n') || (first == '=')) - { - return; - } - - char *equal = strchr(line, '='); - if (equal == NULL) - { - return; - } - - equal[0] = '\0'; - std::string key = line; - - equal++; - while ((equal[0] == ' ') || (equal[0] == '\t')) - ++equal; - - first = equal[0]; - if ((first == '\0') || (first == '\r') || (first== '\n')) - { - return; - } - - std::string value = equal; - key = trimRight(key); - value = trimRight(value); - - if (key == "FILENAME") - this->_currentTailFileName = value; - if (key == "POSITION") - this->_currentTailFilePosition = std::stoi(value); - - return; -} - -void TailFile::recoverState() -{ - std::ifstream file(_stateFile.c_str(), std::ifstream::in); - if (!file.good()) - { - logger_->log_error("load state file failed %s", _stateFile.c_str()); - return; - } - const unsigned int bufSize = 512; - char buf[bufSize]; - for (file.getline(buf,bufSize); file.good(); file.getline(buf,bufSize)) - { - parseStateFileLine(buf); - } -} - -void TailFile::storeState() -{ - std::ofstream file(_stateFile.c_str()); - if (!file.is_open()) - { - logger_->log_error("store state file failed %s", _stateFile.c_str()); - return; - } - file << "FILENAME=" << this->_currentTailFileName << "\n"; - file << "POSITION=" << this->_currentTailFilePosition << "\n"; - file.close(); -} - -static bool sortTailMatchedFileItem(TailMatchedFileItem i, TailMatchedFileItem j) -{ - return (i.modifiedTime < j.modifiedTime); -} -void TailFile::checkRollOver() -{ - struct stat statbuf; - std::vector matchedFiles; - std::string fullPath = this->_fileLocation + "/" + _currentTailFileName; - - if (stat(fullPath.c_str(), &statbuf) == 0) - { - if (statbuf.st_size > this->_currentTailFilePosition) - // there are new input for the current tail file - return; - - uint64_t modifiedTimeCurrentTailFile = ((uint64_t) (statbuf.st_mtime) * 1000); - std::string pattern = _fileName; - std::size_t found = _fileName.find_last_of("."); - if (found != std::string::npos) - pattern = _fileName.substr(0,found); - DIR *d; - d = opendir(this->_fileLocation.c_str()); - if (!d) - return; - while (1) - { - struct dirent *entry; - entry = readdir(d); - if (!entry) - break; - std::string d_name = entry->d_name; - if (!(entry->d_type & DT_DIR)) - { - std::string fileName = d_name; - std::string fileFullName = this->_fileLocation + "/" + d_name; - if (fileFullName.find(pattern) != std::string::npos && stat(fileFullName.c_str(), &statbuf) == 0) - { - if (((uint64_t) (statbuf.st_mtime) * 1000) >= modifiedTimeCurrentTailFile) - { - TailMatchedFileItem item; - item.fileName = fileName; - item.modifiedTime = ((uint64_t) (statbuf.st_mtime) * 1000); - matchedFiles.push_back(item); - } - } - } - } - closedir(d); - - // Sort the list based on modified time - std::sort(matchedFiles.begin(), matchedFiles.end(), sortTailMatchedFileItem); - for (std::vector::iterator it = matchedFiles.begin(); it!=matchedFiles.end(); ++it) - { - TailMatchedFileItem item = *it; - if (item.fileName == _currentTailFileName) - { - ++it; - if (it!=matchedFiles.end()) - { - TailMatchedFileItem nextItem = *it; - logger_->log_info("TailFile File Roll Over from %s to %s", _currentTailFileName.c_str(), nextItem.fileName.c_str()); - _currentTailFileName = nextItem.fileName; - _currentTailFilePosition = 0; - storeState(); - } - break; - } - } - } - else - return; -} - - -void TailFile::onTrigger(ProcessContext *context, ProcessSession *session) -{ - std::string value; - if (context->getProperty(FileName.getName(), value)) - { - std::size_t found = value.find_last_of("/\\"); - this->_fileLocation = value.substr(0,found); - this->_fileName = value.substr(found+1); - } - if (context->getProperty(StateFile.getName(), value)) - { - _stateFile = value + "." + getUUIDStr(); - } - if (!this->_stateRecovered) - { - _stateRecovered = true; - this->_currentTailFileName = _fileName; - this->_currentTailFilePosition = 0; - // recover the state if we have not done so - this->recoverState(); - } - checkRollOver(); - std::string fullPath = this->_fileLocation + "/" + _currentTailFileName; - struct stat statbuf; - if (stat(fullPath.c_str(), &statbuf) == 0) - { - if (statbuf.st_size <= this->_currentTailFilePosition) - // there are no new input for the current tail file - { - context->yield(); - return; - } - FlowFileRecord *flowFile = session->create(); - if (!flowFile) - return; - std::size_t found = _currentTailFileName.find_last_of("."); - std::string baseName = _currentTailFileName.substr(0,found); - std::string extension = _currentTailFileName.substr(found+1); - flowFile->updateAttribute(PATH, _fileLocation); - flowFile->addAttribute(ABSOLUTE_PATH, fullPath); - session->import(fullPath, flowFile, true, this->_currentTailFilePosition); - session->transfer(flowFile, Success); - logger_->log_info("TailFile %s for %d bytes", _currentTailFileName.c_str(), flowFile->getSize()); - std::string logName = baseName + "." + std::to_string(_currentTailFilePosition) + "-" + - std::to_string(_currentTailFilePosition + flowFile->getSize()) + "." + extension; - flowFile->updateAttribute(FILENAME, logName); - this->_currentTailFilePosition += flowFile->getSize(); - storeState(); - } -} - diff --git a/libminifi/src/ThreadedSchedulingAgent.cpp b/libminifi/src/ThreadedSchedulingAgent.cpp index 2008fecede..6c042814ca 100644 --- a/libminifi/src/ThreadedSchedulingAgent.cpp +++ b/libminifi/src/ThreadedSchedulingAgent.cpp @@ -22,103 +22,125 @@ #include "ThreadedSchedulingAgent.h" -#include "ProcessContext.h" -#include "ProcessSession.h" -#include "ProcessSessionFactory.h" - -void ThreadedSchedulingAgent::schedule(Processor *processor) -{ - std::lock_guard lock(_mtx); - - _administrativeYieldDuration = 0; - std::string yieldValue; - - if (configure_->get(Configure::nifi_administrative_yield_duration, yieldValue)) - { - TimeUnit unit; - if (Property::StringToTime(yieldValue, _administrativeYieldDuration, unit) && - Property::ConvertTimeUnitToMS(_administrativeYieldDuration, unit, _administrativeYieldDuration)) - { - logger_->log_debug("nifi_administrative_yield_duration: [%d] ms", _administrativeYieldDuration); - } - } - - _boredYieldDuration = 0; - if (configure_->get(Configure::nifi_bored_yield_duration, yieldValue)) - { - TimeUnit unit; - if (Property::StringToTime(yieldValue, _boredYieldDuration, unit) && - Property::ConvertTimeUnitToMS(_boredYieldDuration, unit, _boredYieldDuration)) - { - logger_->log_debug("nifi_bored_yield_duration: [%d] ms", _boredYieldDuration); - } - } - - if (processor->getScheduledState() != RUNNING) - { - logger_->log_info("Can not schedule threads for processor %s because it is not running", processor->getName().c_str()); - return; - } - - std::map>::iterator it = - _threads.find(processor->getUUIDStr()); - if (it != _threads.end()) - { - logger_->log_info("Can not schedule threads for processor %s because there are existing threads running"); - return; - } - - auto processContext = std::make_shared(processor); - auto sessionFactory = std::make_shared(processContext.get()); - - processor->onSchedule(processContext.get(), sessionFactory.get()); - - std::vector threads; - for (int i = 0; i < processor->getMaxConcurrentTasks(); i++) - { - ThreadedSchedulingAgent *agent = this; - std::thread *thread = new std::thread([agent, processor, processContext, sessionFactory] () { - agent->run(processor, processContext.get(), sessionFactory.get()); - }); - thread->detach(); - threads.push_back(thread); - logger_->log_info("Scheduled thread %d running for process %s", thread->get_id(), - processor->getName().c_str()); - } - _threads[processor->getUUIDStr().c_str()] = threads; - - return; +#include "core/Connectable.h" +#include "core/ProcessorNode.h" +#include "core/ProcessContext.h" +#include "core/ProcessSession.h" +#include "core/ProcessSessionFactory.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { + +void ThreadedSchedulingAgent::schedule( + std::shared_ptr processor) { + std::lock_guard < std::mutex > lock(mutex_); + + _administrativeYieldDuration = 0; + std::string yieldValue; + + if (configure_->get(Configure::nifi_administrative_yield_duration, + yieldValue)) { + core::TimeUnit unit; + if (core::Property::StringToTime( + yieldValue, _administrativeYieldDuration, unit) + && core::Property::ConvertTimeUnitToMS( + _administrativeYieldDuration, unit, _administrativeYieldDuration)) { + logger_->log_debug("nifi_administrative_yield_duration: [%d] ms", + _administrativeYieldDuration); + } + } + + _boredYieldDuration = 0; + if (configure_->get(Configure::nifi_bored_yield_duration, yieldValue)) { + core::TimeUnit unit; + if (core::Property::StringToTime( + yieldValue, _boredYieldDuration, unit) + && core::Property::ConvertTimeUnitToMS( + _boredYieldDuration, unit, _boredYieldDuration)) { + logger_->log_debug("nifi_bored_yield_duration: [%d] ms", + _boredYieldDuration); + } + } + + if (processor->getScheduledState() != core::RUNNING) { + logger_->log_info( + "Can not schedule threads for processor %s because it is not running", + processor->getName().c_str()); + return; + } + + std::map>::iterator it = + _threads.find(processor->getUUIDStr()); + if (it != _threads.end()) { + logger_->log_info( + "Can not schedule threads for processor %s because there are existing threads running"); + return; + } + + core::ProcessorNode processor_node(processor); + auto processContext = std::make_shared + < core::ProcessContext > (processor_node,repo_); + auto sessionFactory = std::make_shared + < core::ProcessSessionFactory + > (processContext.get()); + + processor->onSchedule(processContext.get(), sessionFactory.get()); + + std::vector threads; + for (int i = 0; i < processor->getMaxConcurrentTasks(); i++) { + ThreadedSchedulingAgent *agent = this; + std::thread *thread = new std::thread( + [agent, processor, processContext, sessionFactory] () { + agent->run(processor, processContext.get(), sessionFactory.get()); + }); + thread->detach(); + threads.push_back(thread); + logger_->log_info("Scheduled thread %d running for process %s", + thread->get_id(), processor->getName().c_str()); + } + _threads[processor->getUUIDStr().c_str()] = threads; + + return; } -void ThreadedSchedulingAgent::unschedule(Processor *processor) -{ - std::lock_guard lock(_mtx); - - logger_->log_info("Shutting down threads for processor %s/%s", - processor->getName().c_str(), - processor->getUUIDStr().c_str()); - - if (processor->getScheduledState() != RUNNING) - { - logger_->log_info("Cannot unschedule threads for processor %s because it is not running", processor->getName().c_str()); - return; - } - - std::map>::iterator it = - _threads.find(processor->getUUIDStr()); - - if (it == _threads.end()) - { - logger_->log_info("Cannot unschedule threads for processor %s because there are no existing threads running", processor->getName().c_str()); - return; - } - for (std::vector::iterator itThread = it->second.begin(); itThread != it->second.end(); ++itThread) - { - std::thread *thread = *itThread; - logger_->log_info("Scheduled thread %d deleted for process %s", thread->get_id(), - processor->getName().c_str()); - delete thread; - } - _threads.erase(processor->getUUIDStr()); - processor->clearActiveTask(); +void ThreadedSchedulingAgent::unschedule(std::shared_ptr processor) { + std::lock_guard < std::mutex > lock(mutex_); + + logger_->log_info("Shutting down threads for processor %s/%s", + processor->getName().c_str(), + processor->getUUIDStr().c_str()); + + if (processor->getScheduledState() != core::RUNNING) { + logger_->log_info( + "Cannot unschedule threads for processor %s because it is not running", + processor->getName().c_str()); + return; + } + + std::map>::iterator it = + _threads.find(processor->getUUIDStr()); + + if (it == _threads.end()) { + logger_->log_info( + "Cannot unschedule threads for processor %s because there are no existing threads running", + processor->getName().c_str()); + return; + } + for (std::vector::iterator itThread = it->second.begin(); + itThread != it->second.end(); ++itThread) { + std::thread *thread = *itThread; + logger_->log_info("Scheduled thread %d deleted for process %s", + thread->get_id(), processor->getName().c_str()); + delete thread; + } + _threads.erase(processor->getUUIDStr()); + processor->clearActiveTask(); } + + +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/TimerDrivenSchedulingAgent.cpp b/libminifi/src/TimerDrivenSchedulingAgent.cpp index 30dc96c7bd..3895e81198 100644 --- a/libminifi/src/TimerDrivenSchedulingAgent.cpp +++ b/libminifi/src/TimerDrivenSchedulingAgent.cpp @@ -20,26 +20,38 @@ #include #include #include -#include "Property.h" #include "TimerDrivenSchedulingAgent.h" +#include "core/Property.h" -void TimerDrivenSchedulingAgent::run(Processor *processor, ProcessContext *processContext, ProcessSessionFactory *sessionFactory) -{ - while (this->_running) - { - bool shouldYield = this->onTrigger(processor, processContext, sessionFactory); +namespace org { +namespace apache { +namespace nifi { +namespace minifi { - if (processor->isYield()) - { - // Honor the yield - std::this_thread::sleep_for(std::chrono::milliseconds(processor->getYieldTime())); - } - else if (shouldYield && this->_boredYieldDuration > 0) - { - // No work to do or need to apply back pressure - std::this_thread::sleep_for(std::chrono::milliseconds(this->_boredYieldDuration)); - } - std::this_thread::sleep_for(std::chrono::nanoseconds(processor->getSchedulingPeriodNano())); - } - return; +void TimerDrivenSchedulingAgent::run( + std::shared_ptr processor, + core::ProcessContext *processContext, + core::ProcessSessionFactory *sessionFactory) { + while (this->running_) { + bool shouldYield = this->onTrigger(processor, processContext, + sessionFactory); + + if (processor->isYield()) { + // Honor the yield + std::this_thread::sleep_for( + std::chrono::milliseconds(processor->getYieldTime())); + } else if (shouldYield && this->_boredYieldDuration > 0) { + // No work to do or need to apply back pressure + std::this_thread::sleep_for( + std::chrono::milliseconds(this->_boredYieldDuration)); + } + std::this_thread::sleep_for( + std::chrono::nanoseconds(processor->getSchedulingPeriodNano())); + } + return; } + +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/core/ConfigurableComponent.cpp b/libminifi/src/core/ConfigurableComponent.cpp new file mode 100644 index 0000000000..e5703d116e --- /dev/null +++ b/libminifi/src/core/ConfigurableComponent.cpp @@ -0,0 +1,143 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "core/ConfigurableComponent.h" + +#include "core/Property.h" +#include "core/logging/Logger.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +ConfigurableComponent::ConfigurableComponent(std::shared_ptr logger) + : logger_(logger) { + +} + +ConfigurableComponent::ConfigurableComponent( + const ConfigurableComponent &&other) + : properties_(std::move(other.properties_)), + logger_(std::move(other.logger_)) { + +} +ConfigurableComponent::~ConfigurableComponent() { + +} + +/** + * Get property using the provided name. + * @param name property name. + * @param value value passed in by reference + * @return result of getting property. + */ +bool ConfigurableComponent::getProperty(const std::string name, + std::string &value) { + std::lock_guard lock(configuration_mutex_); + + auto &&it = properties_.find(name); + + if (it != properties_.end()) { + Property item = it->second; + value = item.getValue(); + logger_->log_info("Processor %s property name %s value %s", name.c_str(), + item.getName().c_str(), value.c_str()); + return true; + } else { + return false; + } +} +/** + * Sets the property using the provided name + * @param property name + * @param value property value. + * @return result of setting property. + */ +bool ConfigurableComponent::setProperty(const std::string name, + std::string value) { + std::lock_guard lock(configuration_mutex_); + auto &&it = properties_.find(name); + + if (it != properties_.end()) { + Property item = it->second; + item.setValue(value); + properties_[item.getName()] = item; + logger_->log_info("Component %s property name %s value %s", name.c_str(), + item.getName().c_str(), value.c_str()); + return true; + } else { + return false; + } +} + +/** + * Sets the property using the provided name + * @param property name + * @param value property value. + * @return whether property was set or not + */ +bool ConfigurableComponent::setProperty(Property &prop, std::string value) { + std::lock_guard lock(configuration_mutex_); + auto it = properties_.find(prop.getName()); + + if (it != properties_.end()) { + Property item = it->second; + item.setValue(value); + properties_[item.getName()] = item; + logger_->log_info("property name %s value %s", prop.getName().c_str(), + item.getName().c_str(), value.c_str()); + return true; + } else { + Property newProp(prop); + newProp.setValue(value); + properties_.insert( + std::pair(prop.getName(), newProp)); + return true; + + } + return false; +} + +/** + * Sets supported properties for the ConfigurableComponent + * @param supported properties + * @return result of set operation. + */ +bool ConfigurableComponent::setSupportedProperties( + std::set properties) { + if (!canEdit()) { + return false; + } + + std::lock_guard lock(configuration_mutex_); + + properties_.clear(); + for (auto item : properties) { + properties_[item.getName()] = item; + } + + return true; +} + +} /* namespace components */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/core/Connectable.cpp b/libminifi/src/core/Connectable.cpp new file mode 100644 index 0000000000..ac61568d00 --- /dev/null +++ b/libminifi/src/core/Connectable.cpp @@ -0,0 +1,174 @@ +/* + * Connectable.cpp + * + * Created on: Feb 27, 2017 + * Author: mparisi + */ + +#include "../../include/core/Connectable.h" + +#include +#include "core/logging/Logger.h" +#include "core/Relationship.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +Connectable::Connectable(std::string name, uuid_t uuid) + : CoreComponent(name, uuid), + max_concurrent_tasks_(1) { + +} + +Connectable::Connectable(const Connectable &&other) + : CoreComponent(std::move(other)), + max_concurrent_tasks_(std::move(other.max_concurrent_tasks_)) { + has_work_ = other.has_work_.load(); + strategy_ = other.strategy_.load(); +} + +Connectable::~Connectable() { + +} + +bool Connectable::setSupportedRelationships( + std::set relationships) { + if (isRunning()) { + logger_->log_info( + "Can not set processor supported relationship while the process %s is running", + name_.c_str()); + return false; + } + + std::lock_guard lock(relationship_mutex_); + + relationships_.clear(); + for (auto item : relationships) { + relationships_[item.getName()] = item; + logger_->log_info("Processor %s supported relationship name %s", + name_.c_str(), item.getName().c_str()); + } + + return true; +} + +// Whether the relationship is supported +bool Connectable::isSupportedRelationship(core::Relationship relationship) { + const bool requiresLock = isRunning(); + + const auto conditionalLock = + !requiresLock ? + std::unique_lock() : + std::unique_lock(relationship_mutex_); + + const auto &it = relationships_.find(relationship.getName()); + if (it != relationships_.end()) { + return true; + } else { + return false; + } +} + +bool Connectable::setAutoTerminatedRelationships( + std::set relationships) { + if (isRunning()) { + logger_->log_info( + "Can not set processor auto terminated relationship while the process %s is running", + name_.c_str()); + return false; + } + + std::lock_guard lock(relationship_mutex_); + + auto_terminated_relationships_.clear(); + for (auto item : relationships) { + auto_terminated_relationships_[item.getName()] = item; + logger_->log_info("Processor %s auto terminated relationship name %s", + name_.c_str(), item.getName().c_str()); + } + + return true; +} + +// Check whether the relationship is auto terminated +bool Connectable::isAutoTerminated(core::Relationship relationship) { + const bool requiresLock = isRunning(); + + const auto conditionalLock = + !requiresLock ? + std::unique_lock() : + std::unique_lock(relationship_mutex_); + + const auto &it = auto_terminated_relationships_.find(relationship.getName()); + if (it != auto_terminated_relationships_.end()) { + return true; + } else { + return false; + } +} + +void Connectable::waitForWork(uint64_t timeoutMs) { + has_work_.store(isWorkAvailable()); + + if (!has_work_.load()) { + std::unique_lock lock(work_available_mutex_); + work_condition_.wait_for(lock, std::chrono::milliseconds(timeoutMs), + [&] {return has_work_.load();}); + } + +} + +void Connectable::notifyWork() { + // Do nothing if we are not event-driven + if (strategy_ != EVENT_DRIVEN) { + return; + } + + { + has_work_.store(isWorkAvailable()); + + if (has_work_.load()) { + work_condition_.notify_one(); + } + } + +} + +std::set> Connectable::getOutGoingConnections( + std::string relationship) { + std::set> empty; + + auto &&it = _outGoingConnections.find(relationship); + if (it != _outGoingConnections.end()) { + return _outGoingConnections[relationship]; + } else { + return empty; + } +} + +std::shared_ptr Connectable::getNextIncomingConnection() { + std::lock_guard lock(relationship_mutex_); + + if (_incomingConnections.size() == 0) + return NULL; + + if (incoming_connections_Iter == _incomingConnections.end()) + incoming_connections_Iter = _incomingConnections.begin(); + + std::shared_ptr ret = *incoming_connections_Iter; + incoming_connections_Iter++; + + if (incoming_connections_Iter == _incomingConnections.end()) + incoming_connections_Iter = _incomingConnections.begin(); + + return ret; +} + +} /* namespace components */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/core/Core.cpp b/libminifi/src/core/Core.cpp new file mode 100644 index 0000000000..6582783476 --- /dev/null +++ b/libminifi/src/core/Core.cpp @@ -0,0 +1,51 @@ +/* + * Core.cpp + * + * Created on: Mar 10, 2017 + * Author: mparisi + */ + +#include "core/core.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +// Set UUID +void CoreComponent::setUUID(uuid_t uuid) { + uuid_copy(uuid_, uuid); + char uuidStr[37]; + uuid_unparse_lower(uuid_, uuidStr); + uuidStr_ = uuidStr; +} +// Get UUID +bool CoreComponent::getUUID(uuid_t uuid) const { + if (uuid) { + uuid_copy(uuid, uuid_); + return true; + } else { + return false; + } +} + +// Get UUID +unsigned const char *CoreComponent::getUUID() const { + return uuid_; +} + +// Set Processor Name +void CoreComponent::setName(const std::string name) { + name_ = name; + +} +// Get Process Name +std::string CoreComponent::getName() const { + return name_; +} +} +} +} +} +} diff --git a/libminifi/src/core/ProcessGroup.cpp b/libminifi/src/core/ProcessGroup.cpp new file mode 100644 index 0000000000..ef5ffd97a9 --- /dev/null +++ b/libminifi/src/core/ProcessGroup.cpp @@ -0,0 +1,314 @@ +/** + * @file ProcessGroup.cpp + * ProcessGroup class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/ProcessGroup.h" +#include "core/Processor.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +ProcessGroup::ProcessGroup(ProcessGroupType type, std::string name, uuid_t uuid, + ProcessGroup *parent) + : name_(name), + type_(type), + parent_process_group_(parent) { + if (!uuid) + // Generate the global UUID for the flow record + uuid_generate(uuid_); + else + uuid_copy(uuid_, uuid); + + yield_period_msec_ = 0; + transmitting_ = false; + + logger_ = logging::Logger::getLogger(); + logger_->log_info("ProcessGroup %s created", name_.c_str()); +} + +ProcessGroup::~ProcessGroup() { + for (auto &&connection : connections_) { + connection->drain(); + } + + for (std::set::iterator it = child_process_groups_.begin(); + it != child_process_groups_.end(); ++it) { + ProcessGroup *processGroup(*it); + delete processGroup; + } + +} + +bool ProcessGroup::isRootProcessGroup() { + std::lock_guard lock(mutex_); + return (type_ == ROOT_PROCESS_GROUP); +} + +void ProcessGroup::addProcessor(std::shared_ptr processor) { + std::lock_guard lock(mutex_); + + if (processors_.find(processor) == processors_.end()) { + // We do not have the same processor in this process group yet + processors_.insert(processor); + logger_->log_info("Add processor %s into process group %s", + processor->getName().c_str(), name_.c_str()); + } +} + +void ProcessGroup::removeProcessor(std::shared_ptr processor) { + std::lock_guard lock(mutex_); + + if (processors_.find(processor) != processors_.end()) { + // We do have the same processor in this process group yet + processors_.erase(processor); + logger_->log_info("Remove processor %s from process group %s", + processor->getName().c_str(), name_.c_str()); + } +} + +void ProcessGroup::addProcessGroup(ProcessGroup *child) { + std::lock_guard lock(mutex_); + + if (child_process_groups_.find(child) == child_process_groups_.end()) { + // We do not have the same child process group in this process group yet + child_process_groups_.insert(child); + logger_->log_info("Add child process group %s into process group %s", + child->getName().c_str(), name_.c_str()); + } +} + +void ProcessGroup::removeProcessGroup(ProcessGroup *child) { + std::lock_guard lock(mutex_); + + if (child_process_groups_.find(child) != child_process_groups_.end()) { + // We do have the same child process group in this process group yet + child_process_groups_.erase(child); + logger_->log_info("Remove child process group %s from process group %s", + child->getName().c_str(), name_.c_str()); + } +} + +void ProcessGroup::startProcessing(TimerDrivenSchedulingAgent *timeScheduler, + EventDrivenSchedulingAgent *eventScheduler) { + std::lock_guard lock(mutex_); + + try { + // Start all the processor node, input and output ports + for (auto processor : processors_) { + logger_->log_debug("Starting %s", processor->getName().c_str()); + + if (!processor->isRunning() + && processor->getScheduledState() != DISABLED) { + if (processor->getSchedulingStrategy() == TIMER_DRIVEN) + timeScheduler->schedule(processor); + else if (processor->getSchedulingStrategy() == EVENT_DRIVEN) + eventScheduler->schedule(processor); + } + } + // Start processing the group + for (auto processGroup : child_process_groups_) { + processGroup->startProcessing(timeScheduler, eventScheduler); + } + } catch (std::exception &exception) { + logger_->log_debug("Caught Exception %s", exception.what()); + throw; + } catch (...) { + logger_->log_debug( + "Caught Exception during process group start processing"); + throw; + } +} + +void ProcessGroup::stopProcessing(TimerDrivenSchedulingAgent *timeScheduler, + EventDrivenSchedulingAgent *eventScheduler) { + std::lock_guard lock(mutex_); + + try { + // Stop all the processor node, input and output ports + for (std::set >::iterator it = + processors_.begin(); it != processors_.end(); ++it) { + std::shared_ptr processor(*it); + if (processor->getSchedulingStrategy() == TIMER_DRIVEN) + timeScheduler->unschedule(processor); + else if (processor->getSchedulingStrategy() == EVENT_DRIVEN) + eventScheduler->unschedule(processor); + } + + for (std::set::iterator it = child_process_groups_.begin(); + it != child_process_groups_.end(); ++it) { + ProcessGroup *processGroup(*it); + processGroup->stopProcessing(timeScheduler, eventScheduler); + } + } catch (std::exception &exception) { + logger_->log_debug("Caught Exception %s", exception.what()); + throw; + } catch (...) { + logger_->log_debug("Caught Exception during process group stop processing"); + throw; + } +} + +std::shared_ptr ProcessGroup::findProcessor(uuid_t uuid) { + + std::shared_ptr ret = NULL; + // std::lock_guard lock(mutex_); + + for (auto processor : processors_) { + logger_->log_info("find processor %s", processor->getName().c_str()); + uuid_t processorUUID; + + if (processor->getUUID(processorUUID)) { + + char uuid_str[37]; // ex. "1b4e28ba-2fa1-11d2-883f-0016d3cca427" + "\0" + uuid_unparse_lower(processorUUID, uuid_str); + std::string processorUUIDstr = uuid_str; + uuid_unparse_lower(uuid, uuid_str); + std::string uuidStr = uuid_str; + if (processorUUIDstr == uuidStr) { + return processor; + } + } + + } + for (auto processGroup : child_process_groups_) { + + logger_->log_info("find processor child %s", + processGroup->getName().c_str()); + std::shared_ptr processor = processGroup->findProcessor(uuid); + if (processor) + return processor; + } + + return ret; +} + +std::shared_ptr ProcessGroup::findProcessor( + const std::string &processorName) { + std::shared_ptr ret = NULL; + + for (auto processor : processors_) { + logger_->log_debug("Current processor is %s", processor->getName().c_str()); + if (processor->getName() == processorName) + return processor; + } + + for (auto processGroup : child_process_groups_) { + std::shared_ptr processor = processGroup->findProcessor( + processorName); + if (processor) + return processor; + } + + return ret; +} + +void ProcessGroup::updatePropertyValue(std::string processorName, + std::string propertyName, + std::string propertyValue) { + std::lock_guard lock(mutex_); + + for (auto processor : processors_) { + if (processor->getName() == processorName) { + processor->setProperty(propertyName, propertyValue); + } + } + + for (auto processGroup : child_process_groups_) { + processGroup->updatePropertyValue(processorName, propertyName, + propertyValue); + } + + return; +} + +void ProcessGroup::getConnections(std::map &connectionMap) + { + for (auto connection : connections_) + { + connectionMap[connection->getUUIDStr()] = connection; + } + + for (auto processGroup: child_process_groups_) { + processGroup->getConnections(connectionMap); + } + } + + +void ProcessGroup::addConnection(std::shared_ptr connection) { + std::lock_guard lock(mutex_); + + if (connections_.find(connection) == connections_.end()) { + // We do not have the same connection in this process group yet + connections_.insert(connection); + logger_->log_info("Add connection %s into process group %s", + connection->getName().c_str(), name_.c_str()); + uuid_t sourceUUID; + std::shared_ptr source = NULL; + connection->getSourceUUID(sourceUUID); + source = this->findProcessor(sourceUUID); + if (source) + source->addConnection(connection); + std::shared_ptr destination = NULL; + uuid_t destinationUUID; + connection->getDestinationUUID(destinationUUID); + destination = this->findProcessor(destinationUUID); + if (destination && destination != source) + destination->addConnection(connection); + } +} + +void ProcessGroup::removeConnection(std::shared_ptr connection) { + std::lock_guard lock(mutex_); + + if (connections_.find(connection) != connections_.end()) { + // We do not have the same connection in this process group yet + connections_.erase(connection); + logger_->log_info("Remove connection %s into process group %s", + connection->getName().c_str(), name_.c_str()); + uuid_t sourceUUID; + std::shared_ptr source = NULL; + connection->getSourceUUID(sourceUUID); + source = this->findProcessor(sourceUUID); + if (source) + source->removeConnection(connection); + std::shared_ptr destination = NULL; + uuid_t destinationUUID; + connection->getDestinationUUID(destinationUUID); + destination = this->findProcessor(destinationUUID); + if (destination && destination != source) + destination->removeConnection(connection); + } +} + +} /* namespace processor */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/core/ProcessSession.cpp b/libminifi/src/core/ProcessSession.cpp new file mode 100644 index 0000000000..a948e8a52c --- /dev/null +++ b/libminifi/src/core/ProcessSession.cpp @@ -0,0 +1,974 @@ +/** + * @file ProcessSession.cpp + * ProcessSession class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/ProcessSession.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +std::shared_ptr ProcessSession::create() { + std::map empty; + std::shared_ptr record = + std::make_shared(empty); + + _addedFlowFiles[record->getUUIDStr()] = record; + logger_->log_debug("Create FlowFile with UUID %s", + record->getUUIDStr().c_str()); + std::string details = process_context_->getProcessorNode().getName() + + " creates flow record " + record->getUUIDStr(); + provenance_report_->create(record, details); + + return record; +} + +std::shared_ptr ProcessSession::create( + std::shared_ptr &parent) { + std::map empty; + std::shared_ptr record = + std::make_shared(empty); + + if (record) { + _addedFlowFiles[record->getUUIDStr()] = record; + logger_->log_debug("Create FlowFile with UUID %s", + record->getUUIDStr().c_str()); + } + + if (record) { + // Copy attributes + std::map parentAttributes = + parent->getAttributes(); + std::map::iterator it; + for (it = parentAttributes.begin(); it != parentAttributes.end(); it++) { + if (it->first == FlowAttributeKey(ALTERNATE_IDENTIFIER) + || it->first == FlowAttributeKey(DISCARD_REASON) + || it->first == FlowAttributeKey(UUID)) + // Do not copy special attributes from parent + continue; + record->setAttribute(it->first, it->second); + } + record->setLineageStartDate(parent->getlineageStartDate()); + record->setLineageIdentifiers(parent->getlineageIdentifiers()); + parent->getlineageIdentifiers().insert(parent->getUUIDStr()); + + } + return record; +} + +std::shared_ptr ProcessSession::clone( + std::shared_ptr &parent) { + std::shared_ptr record = + this->create(parent); + if (record) { + // Copy Resource Claim + std::shared_ptr parent_claim = parent->getResourceClaim(); + record->setResourceClaim(parent_claim); + if (parent_claim != nullptr) { + record->setOffset(parent->getOffset()); + record->setSize(parent->getSize()); + record->getResourceClaim()->increaseFlowFileRecordOwnedCount(); + ; + } + provenance_report_->clone(parent, record); + } + return record; +} + +std::shared_ptr ProcessSession::cloneDuringTransfer( + std::shared_ptr &parent) { + std::map empty; + std::shared_ptr record = + std::make_shared(empty); + + if (record) { + this->_clonedFlowFiles[record->getUUIDStr()] = record; + logger_->log_debug("Clone FlowFile with UUID %s during transfer", + record->getUUIDStr().c_str()); + // Copy attributes + std::map parentAttributes = + parent->getAttributes(); + std::map::iterator it; + for (it = parentAttributes.begin(); it != parentAttributes.end(); it++) { + if (it->first == FlowAttributeKey(ALTERNATE_IDENTIFIER) + || it->first == FlowAttributeKey(DISCARD_REASON) + || it->first == FlowAttributeKey(UUID)) + // Do not copy special attributes from parent + continue; + record->setAttribute(it->first, it->second); + } + record->setLineageStartDate(parent->getlineageStartDate()); + + record->setLineageIdentifiers(parent->getlineageIdentifiers()); + record->getlineageIdentifiers().insert(parent->getUUIDStr()); + + // Copy Resource Claim + std::shared_ptr parent_claim = parent->getResourceClaim(); + record->setResourceClaim(parent_claim); + if (parent_claim != nullptr) { + record->setOffset(parent->getOffset()); + record->setSize(parent->getSize()); + record->getResourceClaim()->increaseFlowFileRecordOwnedCount(); + ; + } + provenance_report_->clone(parent, record); + } + + return record; +} + +std::shared_ptr ProcessSession::clone( + std::shared_ptr &parent, + long offset, long size) { + std::shared_ptr record = + this->create(parent); + if (record) { + + if (parent->getResourceClaim()) { + if ((offset + size) > (long) parent->getSize()) { + // Set offset and size + logger_->log_error("clone offset %d and size %d exceed parent size %d", + offset, size, parent->getSize()); + // Remove the Add FlowFile for the session + std::map >::iterator it = + this->_addedFlowFiles.find(record->getUUIDStr()); + if (it != this->_addedFlowFiles.end()) + this->_addedFlowFiles.erase(record->getUUIDStr()); + return nullptr; + } + record->setOffset(parent->getOffset() + parent->getOffset()); + record->setSize(size); + // Copy Resource Claim + std::shared_ptr parent_claim = parent->getResourceClaim(); + record->setResourceClaim(parent_claim); + if (parent_claim != nullptr) { + + record->getResourceClaim()->increaseFlowFileRecordOwnedCount(); + } + } + provenance_report_->clone(parent, record); + } + return record; +} + +void ProcessSession::remove( + std::shared_ptr &flow) { + flow->setDeleted(true); + _deletedFlowFiles[flow->getUUIDStr()] = flow; + std::string reason = process_context_->getProcessorNode().getName() + + " drop flow record " + flow->getUUIDStr(); + provenance_report_->drop(flow, reason); +} + +void ProcessSession::remove( + std::shared_ptr &&flow) { + flow->setDeleted(true); + _deletedFlowFiles[flow->getUUIDStr()] = flow; + std::string reason = process_context_->getProcessorNode().getName() + + " drop flow record " + flow->getUUIDStr(); + provenance_report_->drop(flow, reason); +} + +void ProcessSession::putAttribute( + std::shared_ptr &flow, + std::string key, std::string value) { + flow->setAttribute(key, value); + std::string details = process_context_->getProcessorNode().getName() + + " modify flow record " + flow->getUUIDStr() + " attribute " + key + ":" + + value; + provenance_report_->modifyAttributes(flow, details); +} + +void ProcessSession::removeAttribute( + std::shared_ptr &flow, + std::string key) { + flow->removeAttribute(key); + std::string details = process_context_->getProcessorNode().getName() + + " remove flow record " + flow->getUUIDStr() + " attribute " + key; + provenance_report_->modifyAttributes(flow, details); +} + +void ProcessSession::putAttribute( + std::shared_ptr &&flow, + std::string key, std::string value) { + flow->setAttribute(key, value); + std::string details = process_context_->getProcessorNode().getName() + + " modify flow record " + flow->getUUIDStr() + " attribute " + key + ":" + + value; + provenance_report_->modifyAttributes(flow, details); +} + +void ProcessSession::removeAttribute( + std::shared_ptr &&flow, + std::string key) { + flow->removeAttribute(key); + std::string details = process_context_->getProcessorNode().getName() + + " remove flow record " + flow->getUUIDStr() + " attribute " + key; + provenance_report_->modifyAttributes(flow, details); +} + +void ProcessSession::penalize( + std::shared_ptr &flow) { + flow->setPenaltyExpiration( + getTimeMillis() + + process_context_->getProcessorNode().getPenalizationPeriodMsec()); +} + +void ProcessSession::penalize( + std::shared_ptr &&flow) { + flow->setPenaltyExpiration( + getTimeMillis() + + process_context_->getProcessorNode().getPenalizationPeriodMsec()); +} + +void ProcessSession::transfer( + std::shared_ptr &flow, + Relationship relationship) { + _transferRelationship[flow->getUUIDStr()] = relationship; +} + +void ProcessSession::transfer( + std::shared_ptr &&flow, + Relationship relationship) { + _transferRelationship[flow->getUUIDStr()] = relationship; +} + +void ProcessSession::write( + std::shared_ptr &flow, + OutputStreamCallback *callback) { + std::shared_ptr claim = std::make_shared( + DEFAULT_CONTENT_DIRECTORY); + + try { + std::ofstream fs; + uint64_t startTime = getTimeMillis(); + fs.open(claim->getContentFullPath().c_str(), + std::fstream::out | std::fstream::binary | std::fstream::trunc); + if (fs.is_open()) { + // Call the callback to write the content + callback->process(&fs); + if (fs.good() && fs.tellp() >= 0) { + flow->setSize(fs.tellp()); + flow->setOffset(0); + std::shared_ptr flow_claim = flow->getResourceClaim(); + if (flow_claim != nullptr) { + // Remove the old claim + flow_claim->decreaseFlowFileRecordOwnedCount(); + flow->clearResourceClaim(); + } + flow->setResourceClaim(claim); + claim->increaseFlowFileRecordOwnedCount(); + /* + logger_->log_debug("Write offset %d length %d into content %s for FlowFile UUID %s", + flow->_offset, flow->_size, flow->_claim->getContentFullPath().c_str(), flow->getUUIDStr().c_str()); */ + fs.close(); + std::string details = process_context_->getProcessorNode().getName() + + " modify flow record content " + flow->getUUIDStr(); + uint64_t endTime = getTimeMillis(); + provenance_report_->modifyContent(flow, details, endTime - startTime); + } else { + fs.close(); + throw Exception(FILE_OPERATION_EXCEPTION, "File Write Error"); + } + } else { + throw Exception(FILE_OPERATION_EXCEPTION, "File Open Error"); + } + } catch (std::exception &exception) { + if (flow && flow->getResourceClaim() == claim) { + flow->getResourceClaim()->decreaseFlowFileRecordOwnedCount(); + flow->clearResourceClaim(); + } + logger_->log_debug("Caught Exception %s", exception.what()); + throw; + } catch (...) { + if (flow && flow->getResourceClaim() == claim) { + flow->getResourceClaim()->decreaseFlowFileRecordOwnedCount(); + flow->clearResourceClaim(); + } + logger_->log_debug("Caught Exception during process session write"); + throw; + } +} + +void ProcessSession::write( + std::shared_ptr &&flow, + OutputStreamCallback *callback) { + std::shared_ptr claim = std::make_shared(); + try { + std::ofstream fs; + uint64_t startTime = getTimeMillis(); + fs.open(claim->getContentFullPath().c_str(), + std::fstream::out | std::fstream::binary | std::fstream::trunc); + if (fs.is_open()) { + // Call the callback to write the content + callback->process(&fs); + if (fs.good() && fs.tellp() >= 0) { + flow->setSize(fs.tellp()); + flow->setOffset(0); + std::shared_ptr flow_claim = flow->getResourceClaim(); + if (flow_claim != nullptr) { + // Remove the old claim + flow_claim->decreaseFlowFileRecordOwnedCount(); + flow->clearResourceClaim(); + } + flow->setResourceClaim(claim); + claim->increaseFlowFileRecordOwnedCount(); + /* + logger_->log_debug("Write offset %d length %d into content %s for FlowFile UUID %s", + flow->_offset, flow->_size, flow->_claim->getContentFullPath().c_str(), flow->getUUIDStr().c_str()); */ + fs.close(); + std::string details = process_context_->getProcessorNode().getName() + + " modify flow record content " + flow->getUUIDStr(); + uint64_t endTime = getTimeMillis(); + provenance_report_->modifyContent(flow, details, endTime - startTime); + } else { + fs.close(); + throw Exception(FILE_OPERATION_EXCEPTION, "File Write Error"); + } + } else { + throw Exception(FILE_OPERATION_EXCEPTION, "File Open Error"); + } + } catch (std::exception &exception) { + if (flow && flow->getResourceClaim() == claim) { + flow->getResourceClaim()->decreaseFlowFileRecordOwnedCount(); + flow->clearResourceClaim(); + } + logger_->log_debug("Caught Exception %s", exception.what()); + throw; + } catch (...) { + if (flow && flow->getResourceClaim() == claim) { + flow->getResourceClaim()->decreaseFlowFileRecordOwnedCount(); + flow->clearResourceClaim(); + } + logger_->log_debug("Caught Exception during process session write"); + throw; + } +} + +void ProcessSession::append( + std::shared_ptr &&flow, + OutputStreamCallback *callback) { + std::shared_ptr claim = nullptr; + + if (flow->getResourceClaim() == nullptr) { + // No existed claim for append, we need to create new claim + return write(flow, callback); + } + + claim = flow->getResourceClaim(); + + try { + std::ofstream fs; + uint64_t startTime = getTimeMillis(); + fs.open(claim->getContentFullPath().c_str(), + std::fstream::out | std::fstream::binary | std::fstream::app); + if (fs.is_open()) { + // Call the callback to write the content + std::streampos oldPos = fs.tellp(); + callback->process(&fs); + if (fs.good() && fs.tellp() >= 0) { + uint64_t appendSize = fs.tellp() - oldPos; + flow->setSize(flow->getSize() + appendSize); + /* + logger_->log_debug("Append offset %d extra length %d to new size %d into content %s for FlowFile UUID %s", + flow->_offset, appendSize, flow->_size, claim->getContentFullPath().c_str(), flow->getUUIDStr().c_str()); */ + fs.close(); + std::string details = process_context_->getProcessorNode().getName() + + " modify flow record content " + flow->getUUIDStr(); + uint64_t endTime = getTimeMillis(); + provenance_report_->modifyContent(flow, details, endTime - startTime); + } else { + fs.close(); + throw Exception(FILE_OPERATION_EXCEPTION, "File Write Error"); + } + } else { + throw Exception(FILE_OPERATION_EXCEPTION, "File Open Error"); + } + } catch (std::exception &exception) { + logger_->log_debug("Caught Exception %s", exception.what()); + throw; + } catch (...) { + logger_->log_debug("Caught Exception during process session append"); + throw; + } +} + +void ProcessSession::append( + std::shared_ptr &flow, + OutputStreamCallback *callback) { + std::shared_ptr claim = nullptr; + + if (flow->getResourceClaim() == nullptr) { + // No existed claim for append, we need to create new claim + return write(flow, callback); + } + + claim = flow->getResourceClaim(); + + try { + std::ofstream fs; + uint64_t startTime = getTimeMillis(); + fs.open(claim->getContentFullPath().c_str(), + std::fstream::out | std::fstream::binary | std::fstream::app); + if (fs.is_open()) { + // Call the callback to write the content + std::streampos oldPos = fs.tellp(); + callback->process(&fs); + if (fs.good() && fs.tellp() >= 0) { + uint64_t appendSize = fs.tellp() - oldPos; + flow->setSize(flow->getSize() + appendSize); + /* + logger_->log_debug("Append offset %d extra length %d to new size %d into content %s for FlowFile UUID %s", + flow->_offset, appendSize, flow->_size, claim->getContentFullPath().c_str(), flow->getUUIDStr().c_str()); */ + fs.close(); + std::string details = process_context_->getProcessorNode().getName() + + " modify flow record content " + flow->getUUIDStr(); + uint64_t endTime = getTimeMillis(); + provenance_report_->modifyContent(flow, details, endTime - startTime); + } else { + fs.close(); + throw Exception(FILE_OPERATION_EXCEPTION, "File Write Error"); + } + } else { + throw Exception(FILE_OPERATION_EXCEPTION, "File Open Error"); + } + } catch (std::exception &exception) { + logger_->log_debug("Caught Exception %s", exception.what()); + throw; + } catch (...) { + logger_->log_debug("Caught Exception during process session append"); + throw; + } +} + +void ProcessSession::read( + std::shared_ptr &flow, + InputStreamCallback *callback) { + try { + std::shared_ptr claim = nullptr; + + if (flow->getResourceClaim() == nullptr) { + // No existed claim for read, we throw exception + throw Exception(FILE_OPERATION_EXCEPTION, + "No Content Claim existed for read"); + } + + claim = flow->getResourceClaim(); + std::ifstream fs; + fs.open(claim->getContentFullPath().c_str(), + std::fstream::in | std::fstream::binary); + if (fs.is_open()) { + fs.seekg(flow->getOffset(), fs.beg); + + if (fs.good()) { + callback->process(&fs); + /* + logger_->log_debug("Read offset %d size %d content %s for FlowFile UUID %s", + flow->_offset, flow->_size, claim->getContentFullPath().c_str(), flow->getUUIDStr().c_str()); */ + fs.close(); + } else { + fs.close(); + throw Exception(FILE_OPERATION_EXCEPTION, "File Read Error"); + } + } else { + throw Exception(FILE_OPERATION_EXCEPTION, "File Open Error"); + } + } catch (std::exception &exception) { + logger_->log_debug("Caught Exception %s", exception.what()); + throw; + } catch (...) { + logger_->log_debug("Caught Exception during process session read"); + throw; + } +} + +void ProcessSession::read( + std::shared_ptr &&flow, + InputStreamCallback *callback) { + try { + std::shared_ptr claim = nullptr; + + if (flow->getResourceClaim() == nullptr) { + // No existed claim for read, we throw exception + throw Exception(FILE_OPERATION_EXCEPTION, + "No Content Claim existed for read"); + } + + claim = flow->getResourceClaim(); + std::ifstream fs; + fs.open(claim->getContentFullPath().c_str(), + std::fstream::in | std::fstream::binary); + if (fs.is_open()) { + fs.seekg(flow->getOffset(), fs.beg); + + if (fs.good()) { + callback->process(&fs); + /* + logger_->log_debug("Read offset %d size %d content %s for FlowFile UUID %s", + flow->_offset, flow->_size, claim->getContentFullPath().c_str(), flow->getUUIDStr().c_str()); */ + fs.close(); + } else { + fs.close(); + throw Exception(FILE_OPERATION_EXCEPTION, "File Read Error"); + } + } else { + throw Exception(FILE_OPERATION_EXCEPTION, "File Open Error"); + } + } catch (std::exception &exception) { + logger_->log_debug("Caught Exception %s", exception.what()); + throw; + } catch (...) { + logger_->log_debug("Caught Exception during process session read"); + throw; + } +} + +void ProcessSession::import( + std::string source, + std::shared_ptr &flow, + bool keepSource, uint64_t offset) { + std::shared_ptr claim = std::make_shared(); + char *buf = NULL; + int size = 4096; + buf = new char[size]; + + try { + std::ofstream fs; + uint64_t startTime = getTimeMillis(); + fs.open(claim->getContentFullPath().c_str(), + std::fstream::out | std::fstream::binary | std::fstream::trunc); + std::ifstream input; + input.open(source.c_str(), std::fstream::in | std::fstream::binary); + + if (fs.is_open() && input.is_open()) { + // Open the source file and stream to the flow file + input.seekg(offset, fs.beg); + while (input.good()) { + input.read(buf, size); + if (input) + fs.write(buf, size); + else + fs.write(buf, input.gcount()); + } + + if (fs.good() && fs.tellp() >= 0) { + flow->setSize(fs.tellp()); + flow->setOffset(0); + if (flow->getResourceClaim() != nullptr) { + // Remove the old claim + flow->getResourceClaim()->decreaseFlowFileRecordOwnedCount(); + flow->clearResourceClaim(); + } + flow->setResourceClaim(claim); + claim->increaseFlowFileRecordOwnedCount(); + + logger_->log_debug( + "Import offset %d length %d into content %s for FlowFile UUID %s", + flow->getOffset(), flow->getSize(), + flow->getResourceClaim()->getContentFullPath().c_str(), + flow->getUUIDStr().c_str()); + + fs.close(); + input.close(); + if (!keepSource) + std::remove(source.c_str()); + std::string details = process_context_->getProcessorNode().getName() + + " modify flow record content " + flow->getUUIDStr(); + uint64_t endTime = getTimeMillis(); + provenance_report_->modifyContent(flow, details, endTime - startTime); + } else { + fs.close(); + input.close(); + throw Exception(FILE_OPERATION_EXCEPTION, "File Import Error"); + } + } else { + throw Exception(FILE_OPERATION_EXCEPTION, "File Import Error"); + } + + delete[] buf; + } catch (std::exception &exception) { + if (flow && flow->getResourceClaim() == claim) { + flow->getResourceClaim()->decreaseFlowFileRecordOwnedCount(); + flow->clearResourceClaim(); + } + logger_->log_debug("Caught Exception %s", exception.what()); + delete[] buf; + throw; + } catch (...) { + if (flow && flow->getResourceClaim() == claim) { + flow->getResourceClaim()->decreaseFlowFileRecordOwnedCount(); + flow->clearResourceClaim(); + } + logger_->log_debug("Caught Exception during process session write"); + delete[] buf; + throw; + } +} + +void ProcessSession::import( + std::string source, + std::shared_ptr &&flow, + bool keepSource, uint64_t offset) { + std::shared_ptr claim = std::make_shared(); + + + + char *buf = NULL; + int size = 4096; + buf = new char[size]; + + try { + std::ofstream fs; + uint64_t startTime = getTimeMillis(); + fs.open(claim->getContentFullPath().c_str(), + std::fstream::out | std::fstream::binary | std::fstream::trunc); + std::ifstream input; + input.open(source.c_str(), std::fstream::in | std::fstream::binary); + + if (fs.is_open() && input.is_open()) { + // Open the source file and stream to the flow file + input.seekg(offset, fs.beg); + while (input.good()) { + input.read(buf, size); + if (input) + fs.write(buf, size); + else + fs.write(buf, input.gcount()); + } + + if (fs.good() && fs.tellp() >= 0) { + flow->setSize(fs.tellp()); + flow->setOffset(0); + if (flow->getResourceClaim() != nullptr) { + // Remove the old claim + flow->getResourceClaim()->decreaseFlowFileRecordOwnedCount(); + flow->clearResourceClaim(); + } + flow->setResourceClaim(claim); + claim->increaseFlowFileRecordOwnedCount(); + + logger_->log_debug( + "Import offset %d length %d into content %s for FlowFile UUID %s", + flow->getOffset(), flow->getSize(), + flow->getResourceClaim()->getContentFullPath().c_str(), + flow->getUUIDStr().c_str()); + + fs.close(); + input.close(); + if (!keepSource) + std::remove(source.c_str()); + std::string details = process_context_->getProcessorNode().getName() + + " modify flow record content " + flow->getUUIDStr(); + uint64_t endTime = getTimeMillis(); + provenance_report_->modifyContent(flow, details, endTime - startTime); + } else { + fs.close(); + input.close(); + throw Exception(FILE_OPERATION_EXCEPTION, "File Import Error"); + } + } else { + throw Exception(FILE_OPERATION_EXCEPTION, "File Import Error"); + } + + delete[] buf; + } catch (std::exception &exception) { + if (flow && flow->getResourceClaim() == claim) { + flow->getResourceClaim()->decreaseFlowFileRecordOwnedCount(); + flow->clearResourceClaim(); + } + logger_->log_debug("Caught Exception %s", exception.what()); + delete[] buf; + throw; + } catch (...) { + if (flow && flow->getResourceClaim() == claim) { + flow->getResourceClaim()->decreaseFlowFileRecordOwnedCount(); + flow->clearResourceClaim(); + } + logger_->log_debug("Caught Exception during process session write"); + delete[] buf; + throw; + } +} + +void ProcessSession::commit() { + + try { + // First we clone the flow record based on the transfered relationship for updated flow record + for (auto && it : _updatedFlowFiles) { + std::shared_ptr record = it + .second; + if (record->isDeleted()) + continue; + std::map::iterator itRelationship = this + ->_transferRelationship.find(record->getUUIDStr()); + if (itRelationship != _transferRelationship.end()) { + Relationship relationship = itRelationship->second; + // Find the relationship, we need to find the connections for that relationship + std::set> connections = process_context_ + ->getProcessorNode().getOutGoingConnections(relationship.getName()); + if (connections.empty()) { + // No connection + if (!process_context_->getProcessorNode().isAutoTerminated( + relationship)) { + // Not autoterminate, we should have the connect + std::string message = + "Connect empty for non auto terminated relationship" + + relationship.getName(); + throw Exception(PROCESS_SESSION_EXCEPTION, message.c_str()); + } else { + // Autoterminated + remove(record); + } + } else { + // We connections, clone the flow and assign the connection accordingly + for (std::set>::iterator itConnection = + connections.begin(); itConnection != connections.end(); + ++itConnection) { + std::shared_ptr connection = *itConnection; + if (itConnection == connections.begin()) { + // First connection which the flow need be routed to + record->setConnection(connection); + } else { + // Clone the flow file and route to the connection + std::shared_ptr cloneRecord; + cloneRecord = this->cloneDuringTransfer(record); + if (cloneRecord) + cloneRecord->setConnection(connection); + else + throw Exception(PROCESS_SESSION_EXCEPTION, + "Can not clone the flow for transfer"); + } + } + } + } else { + // Can not find relationship for the flow + throw Exception(PROCESS_SESSION_EXCEPTION, + "Can not find the transfer relationship for the flow"); + } + } + + // Do the samething for added flow file + for (const auto it : _addedFlowFiles) { + std::shared_ptr record = it + .second; + if (record->isDeleted()) + continue; + std::map::iterator itRelationship = this + ->_transferRelationship.find(record->getUUIDStr()); + if (itRelationship != _transferRelationship.end()) { + Relationship relationship = itRelationship->second; + // Find the relationship, we need to find the connections for that relationship + std::set> connections = process_context_ + ->getProcessorNode().getOutGoingConnections(relationship.getName()); + if (connections.empty()) { + // No connection + if (!process_context_->getProcessorNode().isAutoTerminated( + relationship)) { + // Not autoterminate, we should have the connect + std::string message = + "Connect empty for non auto terminated relationship " + + relationship.getName(); + throw Exception(PROCESS_SESSION_EXCEPTION, message.c_str()); + } else { + // Autoterminated + remove(record); + } + } else { + // We connections, clone the flow and assign the connection accordingly + for (std::set>::iterator itConnection = + connections.begin(); itConnection != connections.end(); + ++itConnection) { + std::shared_ptr connection(*itConnection); + if (itConnection == connections.begin()) { + // First connection which the flow need be routed to + record->setConnection(connection); + } else { + // Clone the flow file and route to the connection + std::shared_ptr cloneRecord; + cloneRecord = this->cloneDuringTransfer(record); + if (cloneRecord) + cloneRecord->setConnection(connection); + else + throw Exception(PROCESS_SESSION_EXCEPTION, + "Can not clone the flow for transfer"); + } + } + } + } else { + // Can not find relationship for the flow + throw Exception(PROCESS_SESSION_EXCEPTION, + "Can not find the transfer relationship for the flow"); + } + } + + + std::shared_ptr connection = nullptr; + // Complete process the added and update flow files for the session, send the flow file to its queue + for (const auto &it : _updatedFlowFiles) { + std::shared_ptr record = it + .second; + if (record->isDeleted()) { + continue; + } + + connection = std::static_pointer_cast( + record->getConnection()); + if ((connection) != nullptr) + connection->put(record); + } + for (const auto &it : _addedFlowFiles) { + std::shared_ptr record = it + .second; + if (record->isDeleted()) { + continue; + } + connection = std::static_pointer_cast( + record->getConnection()); + if ((connection) != nullptr) + connection->put(record); + } + // Process the clone flow files + for (const auto &it : _clonedFlowFiles) { + std::shared_ptr record = it + .second; + if (record->isDeleted()) { + continue; + } + connection = std::static_pointer_cast( + record->getConnection()); + if ((connection) != nullptr) + connection->put(record); + } + + // All done + _updatedFlowFiles.clear(); + _addedFlowFiles.clear(); + _clonedFlowFiles.clear(); + _deletedFlowFiles.clear(); + _originalFlowFiles.clear(); + // persistent the provenance report + this->provenance_report_->commit(); + logger_->log_trace("ProcessSession committed for %s", + process_context_->getProcessorNode().getName().c_str()); + } catch (std::exception &exception) { + logger_->log_debug("Caught Exception %s", exception.what()); + throw; + } catch (...) { + logger_->log_debug("Caught Exception during process session commit"); + throw; + } +} + +void ProcessSession::rollback() { + try { + std::shared_ptr connection = nullptr; + // Requeue the snapshot of the flowfile back + for (const auto &it : _originalFlowFiles) { + std::shared_ptr record = it + .second; + connection = std::static_pointer_cast( + record->getOriginalConnection()); + if ((connection) != nullptr) { + std::shared_ptr flowf = std::static_pointer_cast< + FlowFileRecord>(record); + flowf->setSnapShot(false); + connection->put(record); + } + + } + _originalFlowFiles.clear(); + + _clonedFlowFiles.clear(); + _addedFlowFiles.clear(); + _updatedFlowFiles.clear(); + _deletedFlowFiles.clear(); + logger_->log_trace("ProcessSession rollback for %s", + process_context_->getProcessorNode().getName().c_str()); + } catch (std::exception &exception) { + logger_->log_debug("Caught Exception %s", exception.what()); + throw; + } catch (...) { + logger_->log_debug("Caught Exception during process session roll back"); + throw; + } +} + +std::shared_ptr ProcessSession::get() { + std::shared_ptr first = process_context_->getProcessorNode() + .getNextIncomingConnection(); + + if (first == NULL) + return NULL; + + std::shared_ptr current = std::static_pointer_cast( + first); + + do { + std::set > expired; + std::shared_ptr ret = + current->poll(expired); + if (expired.size() > 0) { + // Remove expired flow record + for (std::set >::iterator it = + expired.begin(); it != expired.end(); ++it) { + std::shared_ptr record = *it; + std::string details = process_context_->getProcessorNode().getName() + + " expire flow record " + record->getUUIDStr(); + provenance_report_->expire(record, details); + } + } + if (ret) { + // add the flow record to the current process session update map + ret->setDeleted(false); + _updatedFlowFiles[ret->getUUIDStr()] = ret; + std::map empty; + std::shared_ptr snapshot = + std::make_shared(empty); + logger_->log_debug("Create Snapshot FlowFile with UUID %s", + snapshot->getUUIDStr().c_str()); + snapshot = ret; +// snapshot->duplicate(ret); + // save a snapshot + _originalFlowFiles[snapshot->getUUIDStr()] = snapshot; + return ret; + } + current = std::static_pointer_cast( + process_context_->getProcessorNode().getNextIncomingConnection()); + } while (current != NULL && current != first); + + return NULL; +} + +} /* namespace processor */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/ProcessSessionFactory.cpp b/libminifi/src/core/ProcessSessionFactory.cpp similarity index 74% rename from libminifi/src/ProcessSessionFactory.cpp rename to libminifi/src/core/ProcessSessionFactory.cpp index a105b1c97c..445ca58cad 100644 --- a/libminifi/src/ProcessSessionFactory.cpp +++ b/libminifi/src/core/ProcessSessionFactory.cpp @@ -18,11 +18,25 @@ * limitations under the License. */ -#include "ProcessSessionFactory.h" +#include "core/ProcessSessionFactory.h" #include + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + std::unique_ptr ProcessSessionFactory::createSession() { - return std::unique_ptr(new ProcessSession(_processContext)); + return std::unique_ptr(new ProcessSession(process_context_)); } + + +} /* namespace processor */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/core/Processor.cpp b/libminifi/src/core/Processor.cpp new file mode 100644 index 0000000000..ba52c28a12 --- /dev/null +++ b/libminifi/src/core/Processor.cpp @@ -0,0 +1,272 @@ +/** + * @file Processor.cpp + * Processor class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/Processor.h" + +#include "Connection.h" +#include "core/Connectable.h" +#include "core/ProcessContext.h" +#include "core/ProcessSession.h" +#include "core/ProcessSessionFactory.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +Processor::Processor(std::string name, uuid_t uuid) + : Connectable(name, uuid), + ConfigurableComponent(logging::Logger::getLogger()) { + + has_work_.store(false); + // Setup the default values + state_ = DISABLED; + strategy_ = TIMER_DRIVEN; + loss_tolerant_ = false; + _triggerWhenEmpty = false; + scheduling_period_nano_ = MINIMUM_SCHEDULING_NANOS; + run_durantion_nano_ = 0; + yield_period_msec_ = DEFAULT_YIELD_PERIOD_SECONDS * 1000; + _penalizationPeriodMsec = DEFAULT_PENALIZATION_PERIOD_SECONDS * 1000; + max_concurrent_tasks_ = 1; + active_tasks_ = 0; + yield_expiration_ = 0; + incoming_connections_Iter = this->_incomingConnections.begin(); + logger_ = logging::Logger::getLogger(); + logger_->log_info("Processor %s created UUID %s", name_.c_str(), + uuidStr_.c_str()); +} + +bool Processor::isRunning() { + return (state_ == RUNNING && active_tasks_ > 0); +} + +void Processor::setScheduledState(ScheduledState state) { + state_ = state; +} + +bool Processor::addConnection(std::shared_ptr conn) { + + bool ret = false; + + if (isRunning()) { + logger_->log_info("Can not add connection while the process %s is running", + name_.c_str()); + return false; + } + std::shared_ptr connection = std::static_pointer_cast(conn); + std::lock_guard lock(mutex_); + + uuid_t srcUUID; + uuid_t destUUID; + + connection->getSourceUUID(srcUUID); + connection->getDestinationUUID(destUUID); + char uuid_str[37]; + + uuid_unparse_lower(uuid_, uuid_str); + std::string my_uuid = uuid_str; + uuid_unparse_lower(destUUID, uuid_str); + std::string destination_uuid = uuid_str; + if (my_uuid == destination_uuid) { + // Connection is destination to the current processor + if (_incomingConnections.find(connection) == _incomingConnections.end()) { + _incomingConnections.insert(connection); + connection->setDestination(shared_from_this()); + logger_->log_info( + "Add connection %s into Processor %s incoming connection", + connection->getName().c_str(), name_.c_str()); + incoming_connections_Iter = this->_incomingConnections.begin(); + ret = true; + } + } + uuid_unparse_lower(srcUUID, uuid_str); + std::string source_uuid = uuid_str; + if (my_uuid == source_uuid) { + std::string relationship = connection->getRelationship().getName(); + // Connection is source from the current processor + auto &&it = _outGoingConnections.find(relationship); + if (it != _outGoingConnections.end()) { + // We already has connection for this relationship + std::set> existedConnection = it->second; + if (existedConnection.find(connection) == existedConnection.end()) { + // We do not have the same connection for this relationship yet + existedConnection.insert(connection); + connection->setSource(shared_from_this()); + _outGoingConnections[relationship] = existedConnection; + logger_->log_info( + "Add connection %s into Processor %s outgoing connection for relationship %s", + connection->getName().c_str(), name_.c_str(), relationship.c_str()); + ret = true; + } + } else { + + // We do not have any outgoing connection for this relationship yet + std::set> newConnection; + newConnection.insert(connection); + connection->setSource(shared_from_this()); + _outGoingConnections[relationship] = newConnection; + logger_->log_info( + "Add connection %s into Processor %s outgoing connection for relationship %s", + connection->getName().c_str(), name_.c_str(), relationship.c_str()); + ret = true; + } + } + + return ret; +} + +void Processor::removeConnection(std::shared_ptr conn) { + if (isRunning()) { + logger_->log_info( + "Can not remove connection while the process %s is running", + name_.c_str()); + return; + } + + std::lock_guard lock(mutex_); + + uuid_t srcUUID; + uuid_t destUUID; + + std::shared_ptr connection = std::static_pointer_cast(conn); + + connection->getSourceUUID(srcUUID); + connection->getDestinationUUID(destUUID); + + if (uuid_compare(uuid_, destUUID) == 0) { + // Connection is destination to the current processor + if (_incomingConnections.find(connection) != _incomingConnections.end()) { + _incomingConnections.erase(connection); + connection->setDestination(NULL); + logger_->log_info( + "Remove connection %s into Processor %s incoming connection", + connection->getName().c_str(), name_.c_str()); + incoming_connections_Iter = this->_incomingConnections.begin(); + } + } + + if (uuid_compare(uuid_, srcUUID) == 0) { + std::string relationship = connection->getRelationship().getName(); + // Connection is source from the current processor + auto &&it = _outGoingConnections.find(relationship); + if (it == _outGoingConnections.end()) { + return; + } else { + if (_outGoingConnections[relationship].find(connection) + != _outGoingConnections[relationship].end()) { + _outGoingConnections[relationship].erase(connection); + connection->setSource(NULL); + logger_->log_info( + "Remove connection %s into Processor %s outgoing connection for relationship %s", + connection->getName().c_str(), name_.c_str(), relationship.c_str()); + } + } + } +} + + + +bool Processor::flowFilesQueued() { + std::lock_guard lock(mutex_); + + if (_incomingConnections.size() == 0) + return false; + + for (auto &&conn : _incomingConnections) { + std::shared_ptr connection = std::static_pointer_cast(conn); + if (connection->getQueueSize() > 0) + return true; + } + + return false; +} + +bool Processor::flowFilesOutGoingFull() { + std::lock_guard lock(mutex_); + + for (auto &&connection : _outGoingConnections) { + // We already has connection for this relationship + std::set> existedConnection = connection.second; + for (const auto conn : existedConnection) { + std::shared_ptr connection = std::static_pointer_cast(conn); + if (connection->isFull()) + return true; + } + } + + return false; +} + +void Processor::onTrigger(ProcessContext *context, + ProcessSessionFactory *sessionFactory) { + auto session = sessionFactory->createSession(); + + try { + // Call the virtual trigger function + onTrigger(context, session.get()); + session->commit(); + } catch (std::exception &exception) { + logger_->log_debug("Caught Exception %s", exception.what()); + session->rollback(); + throw; + } catch (...) { + logger_->log_debug("Caught Exception Processor::onTrigger"); + session->rollback(); + throw; + } +} + +bool Processor::isWorkAvailable() { + // We have work if any incoming connection has work + bool hasWork = false; + + try { + for (const auto &conn : _incomingConnections) { + std::shared_ptr connection = std::static_pointer_cast(conn); + if (connection->getQueueSize() > 0) { + hasWork = true; + break; + } + } + } catch (...) { + logger_->log_error( + "Caught an exception while checking if work is available; unless it was positively determined that work is available, assuming NO work is available!"); + } + + return hasWork; +} + +} /* namespace processor */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/core/ProcessorNode.cpp b/libminifi/src/core/ProcessorNode.cpp new file mode 100644 index 0000000000..44491d35bc --- /dev/null +++ b/libminifi/src/core/ProcessorNode.cpp @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "core/ProcessorNode.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +ProcessorNode::ProcessorNode(const std::shared_ptr processor) + : processor_(processor), + Connectable(processor->getName(),0), + ConfigurableComponent(logging::Logger::getLogger()) { + + uuid_t copy; + processor->getUUID(copy); + setUUID( copy ); + + +} + +ProcessorNode::ProcessorNode(const ProcessorNode &other) + : processor_(other.processor_), + Connectable(other.getName(), 0), + ConfigurableComponent(logging::Logger::getLogger()) { + + uuid_t copy; + processor_->getUUID(copy); + setUUID( copy ); + +} + +ProcessorNode::~ProcessorNode() { + +} + +bool ProcessorNode::isWorkAvailable() { + return processor_->isWorkAvailable(); +} + +bool ProcessorNode::isRunning() { + return processor_->isRunning(); +} + +} /* namespace processor */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/core/Property.cpp b/libminifi/src/core/Property.cpp new file mode 100644 index 0000000000..13028a4b89 --- /dev/null +++ b/libminifi/src/core/Property.cpp @@ -0,0 +1,58 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "core/Property.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +// Get Name for the property +std::string Property::getName() const { + return name_; +} +// Get Description for the property +std::string Property::getDescription() { + return description_; +} +// Get value for the property +std::string Property::getValue() const { + return value_; +} +// Set value for the property +void Property::setValue(std::string value) { + value_ = value; +} +// Compare +bool Property::operator <(const Property & right) const { + return name_ < right.name_; +} + +const Property &Property::operator=(const Property &other) { + name_ = other.name_; + value_ = other.value_; + return *this; +} + +} /* namespace components */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/core/Record.cpp b/libminifi/src/core/Record.cpp new file mode 100644 index 0000000000..a911b2f61e --- /dev/null +++ b/libminifi/src/core/Record.cpp @@ -0,0 +1,219 @@ +/* + * Copyright 2017 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/FlowFile.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +FlowFile::FlowFile() + : size_(0), + id_(0), + stored(false), + offset_(0), + last_queue_date_(0), + penaltyExpiration_ms_(0), + claim_(nullptr), + marked_delete_(false), + connection_(nullptr), + original_connection_() { + entry_date_ = getTimeMillis(); + lineage_start_date_ = entry_date_; + + char uuidStr[37]; + + // Generate the global UUID for the flow record + uuid_generate(uuid_); + + uuid_unparse_lower(uuid_, uuidStr); + uuid_str_ = uuidStr; + +} + +FlowFile::~FlowFile() { + +} + +FlowFile& FlowFile::operator=(const FlowFile& other) { + + uuid_copy(uuid_, other.uuid_); + stored = other.stored; + marked_delete_ = other.marked_delete_; + entry_date_ = other.entry_date_; + lineage_start_date_ = other.lineage_start_date_; + lineage_Identifiers_ = other.lineage_Identifiers_; + last_queue_date_ = other.last_queue_date_; + size_ = other.size_; + penaltyExpiration_ms_ = other.penaltyExpiration_ms_; + attributes_ = other.attributes_; + claim_ = other.claim_; + if (claim_ != nullptr) + this->claim_->increaseFlowFileRecordOwnedCount(); + uuid_str_ = other.uuid_str_; + connection_ = other.connection_; + original_connection_ = other.original_connection_; + + return *this; +} + +/** + * Returns whether or not this flow file record + * is marked as deleted. + * @return marked deleted + */ +bool FlowFile::isDeleted() { + return marked_delete_; +} + +/** + * Sets whether to mark this flow file record + * as deleted + * @param deleted deleted flag + */ +void FlowFile::setDeleted(const bool deleted) { + marked_delete_ = deleted; +} + +std::shared_ptr FlowFile::getResourceClaim() { + return claim_; +} + +void FlowFile::clearResourceClaim() { + claim_ = nullptr; +} +void FlowFile::setResourceClaim(std::shared_ptr &claim) { + claim_ = claim; +} + +// ! Get Entry Date +uint64_t FlowFile::getEntryDate() { + return entry_date_; +} +// ! Get Lineage Start Date +uint64_t FlowFile::getlineageStartDate() { + return lineage_start_date_; +} + +std::set &FlowFile::getlineageIdentifiers() { + return lineage_Identifiers_; +} + +bool FlowFile::getAttribute(std::string key, std::string &value) { + auto it = attributes_.find(key); + if (it != attributes_.end()) { + value = it->second; + return true; + } else { + return false; + } +} + +// Get Size +uint64_t FlowFile::getSize() { + return size_; +} +// ! Get Offset +uint64_t FlowFile::getOffset() { + return offset_; +} + +bool FlowFile::removeAttribute(const std::string key) { + auto it = attributes_.find(key); + if (it != attributes_.end()) { + attributes_.erase(key); + return true; + } else { + return false; + } +} + +bool FlowFile::updateAttribute(const std::string key, const std::string value) { + auto it = attributes_.find(key); + if (it != attributes_.end()) { + attributes_[key] = value; + return true; + } else { + return false; + } +} + +bool FlowFile::addAttribute(const std::string &key, const std::string &value) { + auto it = attributes_.find(key); + if (it != attributes_.end()) { + // attribute already there in the map + return false; + } else { + attributes_[key] = value; + return true; + } +} + +void FlowFile::setLineageStartDate(const uint64_t date) { + lineage_start_date_ = date; +} + +/** + * Sets the original connection with a shared pointer. + * @param connection shared connection. + */ +void FlowFile::setOriginalConnection( + std::shared_ptr &connection) { + original_connection_ = connection; +} + +/** + * Sets the connection with a shared pointer. + * @param connection shared connection. + */ +void FlowFile::setConnection( + std::shared_ptr &connection) { + connection_ = connection; +} + +/** + * Sets the connection with a shared pointer. + * @param connection shared connection. + */ +void FlowFile::setConnection( + std::shared_ptr &&connection) { + connection_ = connection; +} + +/** + * Returns the connection referenced by this record. + * @return shared connection pointer. + */ +std::shared_ptr FlowFile::getConnection() { + return connection_; +} + +/** + * Returns the original connection referenced by this record. + * @return shared original connection pointer. + */ +std::shared_ptr FlowFile::getOriginalConnection() { + return original_connection_; +} + +} +} +} +} +} diff --git a/libminifi/src/core/Repository.cpp b/libminifi/src/core/Repository.cpp new file mode 100644 index 0000000000..967ecc9baf --- /dev/null +++ b/libminifi/src/core/Repository.cpp @@ -0,0 +1,65 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include "io/DataStream.h" +#include "io/Serializable.h" +#include "Relationship.h" +#include "Logger.h" +#include "FlowController.h" +#include "core/Repository.h" +#include "provenance/Provenance.h" +#include "FlowFileRepository.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +void Repository::start() { + if (this->purge_period_ <= 0) + return; + if (running_) + return; + thread_ = std::thread(&Repository::threadExecutor, this); + thread_.detach(); + running_ = true; + logger_->log_info("%s Repository Monitor Thread Start", name_); +} + +void Repository::stop() { + if (!running_) + return; + running_ = false; + if (thread_.joinable()) + thread_.join(); + logger_->log_info("%s Repository Monitor Thread Stop", name_); +} + +// repoSize +uint64_t Repository::repoSize() { + return repo_size_; +} + +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/BaseLogger.cpp b/libminifi/src/core/logging/BaseLogger.cpp similarity index 50% rename from libminifi/src/BaseLogger.cpp rename to libminifi/src/core/logging/BaseLogger.cpp index 1b3b2fd491..a6b43a8056 100644 --- a/libminifi/src/BaseLogger.cpp +++ b/libminifi/src/core/logging/BaseLogger.cpp @@ -16,7 +16,14 @@ * limitations under the License. */ -#include "BaseLogger.h" +#include "core/logging/BaseLogger.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { +namespace logging { // Logger related configuration items. const char *BaseLogger::nifi_log_level = "nifi.log.level"; @@ -27,61 +34,57 @@ const char *BaseLogger::nifi_log_appender = "nifi.log.appender"; * @param format format string ('man printf' for syntax) * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match */ - void BaseLogger::log_error(const char * const format, ...) { - if (logger_ == NULL || !logger_->should_log(spdlog::level::level_enum::err)) - return; - FILL_BUFFER - log_str(err,buffer); +void BaseLogger::log_error(const char * const format, ...) { + if (logger_ == NULL || !logger_->should_log(spdlog::level::level_enum::err)) + return; + FILL_BUFFER + log_str(err, buffer); } /** * @brief Log warn message * @param format format string ('man printf' for syntax) * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match */ - void BaseLogger::log_warn(const char * const format, ...) { - if (logger_ == NULL - || !logger_->should_log(spdlog::level::level_enum::warn)) - return; - FILL_BUFFER - log_str(warn,buffer); +void BaseLogger::log_warn(const char * const format, ...) { + if (logger_ == NULL || !logger_->should_log(spdlog::level::level_enum::warn)) + return; + FILL_BUFFER + log_str(warn, buffer); } /** * @brief Log info message * @param format format string ('man printf' for syntax) * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match */ - void BaseLogger::log_info(const char * const format, ...) { - if (logger_ == NULL - || !logger_->should_log(spdlog::level::level_enum::info)) - return; - FILL_BUFFER - log_str(info,buffer); +void BaseLogger::log_info(const char * const format, ...) { + if (logger_ == NULL || !logger_->should_log(spdlog::level::level_enum::info)) + return; + FILL_BUFFER + log_str(info, buffer); } /** * @brief Log debug message * @param format format string ('man printf' for syntax) * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match */ - void BaseLogger::log_debug(const char * const format, ...) { +void BaseLogger::log_debug(const char * const format, ...) { - if (logger_ == NULL - || !logger_->should_log(spdlog::level::level_enum::debug)) - return; - FILL_BUFFER - log_str(debug,buffer); + if (logger_ == NULL || !logger_->should_log(spdlog::level::level_enum::debug)) + return; + FILL_BUFFER + log_str(debug, buffer); } /** * @brief Log trace message * @param format format string ('man printf' for syntax) * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match */ - void BaseLogger::log_trace(const char * const format, ...) { +void BaseLogger::log_trace(const char * const format, ...) { - if (logger_ == NULL - || !logger_->should_log(spdlog::level::level_enum::trace)) - return; - FILL_BUFFER - log_str(debug,buffer); + if (logger_ == NULL || !logger_->should_log(spdlog::level::level_enum::trace)) + return; + FILL_BUFFER + log_str(debug, buffer); } // overridables @@ -92,62 +95,67 @@ const char *BaseLogger::nifi_log_appender = "nifi.log.appender"; * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match */ void BaseLogger::log_str(LOG_LEVEL_E level, const std::string &buffer) { - switch (level) { - case err: - case critical: - if (stderr_ != nullptr) { - stderr_->error(buffer); - } else { - logger_->error(buffer); - } - break; - case warn: - logger_->warn(buffer); - break; - case info: - logger_->info(buffer); - break; - case debug: - logger_->debug(buffer); - break; - case trace: - logger_->trace(buffer); - break; - case off: - break; - default: - logger_->info(buffer); - break; - } + switch (level) { + case err: + case critical: + if (stderr_ != nullptr) { + stderr_->error(buffer); + } else { + logger_->error(buffer); + } + break; + case warn: + logger_->warn(buffer); + break; + case info: + logger_->info(buffer); + break; + case debug: + logger_->debug(buffer); + break; + case trace: + logger_->trace(buffer); + break; + case off: + break; + default: + logger_->info(buffer); + break; + } } void BaseLogger::setLogLevel(const std::string &level, - LOG_LEVEL_E defaultLevel) { - std::string logLevel = level; - std::transform(logLevel.begin(), logLevel.end(), logLevel.begin(), - ::tolower); + LOG_LEVEL_E defaultLevel) { + std::string logLevel = level; + std::transform(logLevel.begin(), logLevel.end(), logLevel.begin(), ::tolower); - if (logLevel == "trace") { - setLogLevel(trace); - } else if (logLevel == "debug") { - setLogLevel(debug); - } else if (logLevel == "info") { - setLogLevel(info); - } else if (logLevel == "warn") { - setLogLevel(warn); - } else if (logLevel == "error") { - setLogLevel(err); - } else if (logLevel == "critical") { - setLogLevel(critical); - } else if (logLevel == "off") { - setLogLevel(off); - } else { - setLogLevel(defaultLevel); - } + if (logLevel == "trace") { + setLogLevel(trace); + } else if (logLevel == "debug") { + setLogLevel(debug); + } else if (logLevel == "info") { + setLogLevel(info); + } else if (logLevel == "warn") { + setLogLevel(warn); + } else if (logLevel == "error") { + setLogLevel(err); + } else if (logLevel == "critical") { + setLogLevel(critical); + } else if (logLevel == "off") { + setLogLevel(off); + } else { + setLogLevel(defaultLevel); + } } void BaseLogger::set_error_logger(std::shared_ptr other) { - stderr_ = std::move(other); + stderr_ = std::move(other); } +} /* namespace logging */ +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/LogAppenders.cpp b/libminifi/src/core/logging/LogAppenders.cpp similarity index 80% rename from libminifi/src/LogAppenders.cpp rename to libminifi/src/core/logging/LogAppenders.cpp index c90588d2e4..5d92334570 100644 --- a/libminifi/src/LogAppenders.cpp +++ b/libminifi/src/core/logging/LogAppenders.cpp @@ -16,10 +16,24 @@ * limitations under the License. */ -#include "../include/LogAppenders.h" +#include "core/logging/LogAppenders.h" +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { +namespace logging { const char *OutputStreamAppender::nifi_log_output_stream_error_stderr="nifi.log.outputstream.appender.error.stderr"; const char *RollingAppender::nifi_log_rolling_apender_file = "nifi.log.rolling.appender.file"; const char *RollingAppender::nifi_log_rolling_appender_max_files = "nifi.log.rolling.appender.max.files"; const char *RollingAppender::nifi_log_rolling_appender_max_file_size = "nifi.log.rolling.appender.max.file_size"; + + +} /* namespace logging */ +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/Logger.cpp b/libminifi/src/core/logging/Logger.cpp similarity index 77% rename from libminifi/src/Logger.cpp rename to libminifi/src/core/logging/Logger.cpp index e90667d216..d8cadf3dee 100644 --- a/libminifi/src/Logger.cpp +++ b/libminifi/src/core/logging/Logger.cpp @@ -17,12 +17,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "../include/Logger.h" +#include "core/logging/Logger.h" #include #include #include +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { +namespace logging { std::shared_ptr Logger::singleton_logger_(nullptr); +} /* namespace logging */ +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/core/repository/FlowFileRepository.cpp b/libminifi/src/core/repository/FlowFileRepository.cpp new file mode 100644 index 0000000000..f27af10bf3 --- /dev/null +++ b/libminifi/src/core/repository/FlowFileRepository.cpp @@ -0,0 +1,107 @@ +#include "core/repository/FlowFileRepository.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { +namespace repository { + +void FlowFileRepository::run() { + // threshold for purge + uint64_t purgeThreshold = max_partition_bytes_ * 3 / 4; + while (running_) { + std::this_thread::sleep_for(std::chrono::milliseconds(purge_period_)); + uint64_t curTime = getTimeMillis(); + uint64_t size = repoSize(); + if (size >= purgeThreshold) { + std::vector purgeList; + leveldb::Iterator* it = db_->NewIterator(leveldb::ReadOptions()); + + for (it->SeekToFirst(); it->Valid(); it->Next()) { + FlowFileEventRecord eventRead; + std::string key = it->key().ToString(); + if (eventRead.DeSerialize((uint8_t *) it->value().data(), + (int) it->value().size())) { + if ((curTime - eventRead.getEventTime()) > max_partition_millis_) + purgeList.push_back(key); + } else { + logger_->log_debug("NiFi %s retrieve event %s fail", name_, + key.c_str()); + purgeList.push_back(key); + } + } + delete it; + for (auto eventId : purgeList) { + logger_->log_info("Repository Repo %s Purge %s", name_, + eventId.c_str()); + Delete(eventId); + } + } + if (size > max_partition_bytes_) + repo_full_ = true; + else + repo_full_ = false; + } + return; +} + +void FlowFileRepository::loadFlowFileToConnections(std::map &connectionMap) + { + + std::vector purgeList; + leveldb::Iterator* it = db_->NewIterator( + leveldb::ReadOptions()); + + for (it->SeekToFirst(); it->Valid(); it->Next()) + { + FlowFileEventRecord eventRead; + std::string key = it->key().ToString(); + if (eventRead.DeSerialize((uint8_t *) it->value().data(), + (int) it->value().size())) + { + auto search = connectionMap->find(eventRead.getConnectionUuid()); + if (search != connectionMap->end()) + { + // we find the connection for the persistent flowfile, create the flowfile and enqueue that + FlowFileRecord *record = new FlowFileRecord(&eventRead); + // set store to repo to true so that we do need to persistent again in enqueue + record->setStoredToRepository(true); + search->second->put(record); + } + else + { + if (eventRead.getContentFullPath().length() > 0) + { + std::remove(eventRead.getContentFullPath().c_str()); + } + purgeList.push_back(key); + } + } + else + { + purgeList.push_back(key); + } + } + + delete it; + std::vector::iterator itPurge; + for (itPurge = purgeList.begin(); itPurge != purgeList.end(); + itPurge++) + { + std::string eventId = *itPurge; + logger_->log_info("Repository Repo %s Purge %s", + RepositoryTypeStr[_type], + eventId.c_str()); + Delete(eventId); + } + + + return; + +} + +} /* namespace repository */ +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/io/BaseStream.cpp b/libminifi/src/io/BaseStream.cpp index cf3fe45504..1400a1d1d7 100644 --- a/libminifi/src/io/BaseStream.cpp +++ b/libminifi/src/io/BaseStream.cpp @@ -18,6 +18,12 @@ #include "io/BaseStream.h" #include "io/Serializable.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace io { /** * write 4 bytes to stream * @param base_value non encoded value @@ -26,7 +32,7 @@ * @return resulting write size **/ int BaseStream::write(uint32_t base_value, bool is_little_endian) { - return ::Serializable::write(base_value, (DataStream*) this, is_little_endian); + return Serializable::write(base_value, (DataStream*) this, is_little_endian); } /** @@ -37,7 +43,7 @@ int BaseStream::write(uint32_t base_value, bool is_little_endian) { * @return resulting write size **/ int BaseStream::write(uint16_t base_value, bool is_little_endian) { - return ::Serializable::write(base_value, (DataStream*) this, is_little_endian); + return Serializable::write(base_value, (DataStream*) this, is_little_endian); } /** @@ -48,7 +54,7 @@ int BaseStream::write(uint16_t base_value, bool is_little_endian) { * @return resulting write size **/ int BaseStream::write(uint8_t *value, int len) { - return ::Serializable::write(value, len, (DataStream*) this); + return Serializable::write(value, len, (DataStream*) this); } /** @@ -59,7 +65,7 @@ int BaseStream::write(uint8_t *value, int len) { * @return resulting write size **/ int BaseStream::write(uint64_t base_value, bool is_little_endian) { - return ::Serializable::write(base_value, (DataStream*) this, is_little_endian); + return Serializable::write(base_value, (DataStream*) this, is_little_endian); } /** @@ -69,7 +75,7 @@ int BaseStream::write(uint64_t base_value, bool is_little_endian) { **/ int BaseStream::write(bool value) { uint8_t v = value; - return ::Serializable::write(v); + return Serializable::write(v); } /** @@ -78,7 +84,7 @@ int BaseStream::write(bool value) { * @return resulting write size **/ int BaseStream::writeUTF(std::string str, bool widen) { - return ::Serializable::writeUTF(str, (DataStream*) this, widen); + return Serializable::writeUTF(str, (DataStream*) this, widen); } /** @@ -88,7 +94,7 @@ int BaseStream::writeUTF(std::string str, bool widen) { * @return resulting read size **/ int BaseStream::read(uint8_t &value) { - return ::Serializable::read(value, (DataStream*) this); + return Serializable::read(value, (DataStream*) this); } /** @@ -98,7 +104,7 @@ int BaseStream::read(uint8_t &value) { * @return resulting read size **/ int BaseStream::read(uint16_t &base_value, bool is_little_endian) { - return ::Serializable::read(base_value, (DataStream*) this); + return Serializable::read(base_value, (DataStream*) this); } /** @@ -108,7 +114,7 @@ int BaseStream::read(uint16_t &base_value, bool is_little_endian) { * @return resulting read size **/ int BaseStream::read(char &value) { - return ::Serializable::read(value, (DataStream*) this); + return Serializable::read(value, (DataStream*) this); } /** @@ -119,7 +125,7 @@ int BaseStream::read(char &value) { * @return resulting read size **/ int BaseStream::read(uint8_t *value, int len) { - return ::Serializable::read(value, len, (DataStream*) this); + return Serializable::read(value, len, (DataStream*) this); } /** @@ -129,7 +135,7 @@ int BaseStream::read(uint8_t *value, int len) { * @return resulting read size **/ int BaseStream::read(uint32_t &value, bool is_little_endian) { - return ::Serializable::read(value, (DataStream*) this, is_little_endian); + return Serializable::read(value, (DataStream*) this, is_little_endian); } /** @@ -139,7 +145,7 @@ int BaseStream::read(uint32_t &value, bool is_little_endian) { * @return resulting read size **/ int BaseStream::read(uint64_t &value, bool is_little_endian) { - return ::Serializable::read(value, (DataStream*) this, is_little_endian); + return Serializable::read(value, (DataStream*) this, is_little_endian); } /** @@ -149,5 +155,12 @@ int BaseStream::read(uint64_t &value, bool is_little_endian) { * @return resulting read size **/ int BaseStream::readUTF(std::string &str, bool widen) { - return ::Serializable::readUTF(str, (DataStream*) this, widen); + return Serializable::readUTF(str, (DataStream*) this, widen); } + + +} /* namespace io */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/io/ClientSocket.cpp b/libminifi/src/io/ClientSocket.cpp index 39b71b4941..e726f0ff86 100644 --- a/libminifi/src/io/ClientSocket.cpp +++ b/libminifi/src/io/ClientSocket.cpp @@ -30,54 +30,65 @@ #include "io/validation.h" #include "io/ClientSocket.h" +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace io { std::string Socket::HOSTNAME = Socket::getMyHostName(0); - Socket::Socket(const std::string &hostname, const uint16_t port, - const uint16_t listeners = -1) : - requested_hostname_(hostname), port_(port), addr_info_(0), socket_file_descriptor_( - -1), socket_max_(0), listeners_(listeners), canonical_hostname_("") { - logger_ = Logger::getLogger(); - FD_ZERO(&total_list_); - FD_ZERO(&read_fds_); + const uint16_t listeners = -1) + : requested_hostname_(hostname), + port_(port), + addr_info_(0), + socket_file_descriptor_(-1), + socket_max_(0), + listeners_(listeners), + canonical_hostname_("") { + logger_ = logging::Logger::getLogger(); + FD_ZERO(&total_list_); + FD_ZERO(&read_fds_); } -Socket::Socket(const std::string &hostname, const uint16_t port) : - ::Socket(hostname, port, 0) { +Socket::Socket(const std::string &hostname, const uint16_t port) + : Socket(hostname, port, 0) { } -Socket::Socket(const Socket &&other) : - requested_hostname_(std::move(other.requested_hostname_)), port_( - std::move(other.port_)), addr_info_(std::move(other.addr_info_)), socket_file_descriptor_( - other.socket_file_descriptor_), socket_max_( - other.socket_max_.load()), listeners_(other.listeners_), total_list_( - other.total_list_), read_fds_(other.read_fds_), canonical_hostname_( - std::move(other.canonical_hostname_)) { - logger_ = Logger::getLogger(); +Socket::Socket(const Socket &&other) + : requested_hostname_(std::move(other.requested_hostname_)), + port_(std::move(other.port_)), + addr_info_(std::move(other.addr_info_)), + socket_file_descriptor_(other.socket_file_descriptor_), + socket_max_(other.socket_max_.load()), + listeners_(other.listeners_), + total_list_(other.total_list_), + read_fds_(other.read_fds_), + canonical_hostname_(std::move(other.canonical_hostname_)) { + logger_ = logging::Logger::getLogger(); } Socket::~Socket() { - closeStream(); + closeStream(); } -void Socket::closeStream() -{ - if (0 != addr_info_) { - freeaddrinfo(addr_info_); - addr_info_=0; - } +void Socket::closeStream() { + if (0 != addr_info_) { + freeaddrinfo(addr_info_); + addr_info_ = 0; + } - if (socket_file_descriptor_ >= 0) - { - close(socket_file_descriptor_); - socket_file_descriptor_ = -1; - } + if (socket_file_descriptor_ >= 0) { + close(socket_file_descriptor_); + socket_file_descriptor_ = -1; + } } +<<<<<<< HEAD int8_t Socket::createConnection(const addrinfo *p,in_addr_t &addr) { if ((socket_file_descriptor_ = socket(p->ai_family, p->ai_socktype, p->ai_protocol)) == -1) { @@ -138,12 +149,66 @@ int8_t Socket::createConnection(const addrinfo *p,in_addr_t &addr) { FD_SET(socket_file_descriptor_, &total_list_); socket_max_ = socket_file_descriptor_; return 0; +======= +int8_t Socket::createConnection(const addrinfo *p) { + if ((socket_file_descriptor_ = socket(p->ai_family, p->ai_socktype, + p->ai_protocol)) == -1) { + logger_->log_error("error while connecting to server socket"); + return -1; + } + + setSocketOptions(socket_file_descriptor_); + + if (listeners_ > 0) { + + struct sockaddr_in *sa_loc = (struct sockaddr_in*) p->ai_addr; + sa_loc->sin_family = AF_INET; + sa_loc->sin_port = htons(port_); + sa_loc->sin_addr.s_addr = htonl(INADDR_ANY); + + if (bind(socket_file_descriptor_, p->ai_addr, p->ai_addrlen) == -1) { + logger_->log_error("Could not bind to socket", strerror(errno)); + return -1; + } + } + { + if (listeners_ <= 0) { + struct sockaddr_in *sa_loc = (struct sockaddr_in*) p->ai_addr; + sa_loc->sin_family = AF_INET; + //sa_loc->sin_port = htons(port); + sa_loc->sin_port = htons(port_); + sa_loc->sin_addr.s_addr = htonl(INADDR_ANY); // inet_addr(requested_hostname.c_str()); + if (connect(socket_file_descriptor_, p->ai_addr, p->ai_addrlen) == -1) { + close(socket_file_descriptor_); + socket_file_descriptor_ = -1; + logger_->log_warn("Could not connect to socket, error:%s", + strerror(errno)); + return -1; + + } + } + } + + // listen + if (listeners_ > 0) { + if (listen(socket_file_descriptor_, listeners_) == -1) { + logger_->log_warn("attempted connection, saw %s", strerror(errno)); + return -1; + } + + } + // add the listener to the total set + FD_SET(socket_file_descriptor_, &total_list_); + socket_max_ = socket_file_descriptor_; + return 0; +>>>>>>> MINIFI-217: First commit. Updates namespaces and removes } short Socket::initialize() { - struct sockaddr_in servAddr; + struct sockaddr_in servAddr; +<<<<<<< HEAD addrinfo hints = { sizeof(addrinfo) }; memset(&hints, 0, sizeof hints); // make sure the struct is empty hints.ai_family = AF_UNSPEC; @@ -153,16 +218,28 @@ short Socket::initialize() { hints.ai_flags |= AI_PASSIVE; hints.ai_protocol = 0; /* any protocol */ - - int errcode = getaddrinfo(requested_hostname_.c_str(), 0, &hints, &addr_info_); - - if (errcode != 0) { - logger_->log_error("Saw error during getaddrinfo, error: %s",strerror(errno)); - return -1; - } - - socket_file_descriptor_ = -1; - +======= + addrinfo hints = { sizeof(addrinfo) }; + memset(&hints, 0, sizeof hints); // make sure the struct is empty + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_CANONNAME; + if (listeners_ > 0) + hints.ai_flags |= AI_PASSIVE; +>>>>>>> MINIFI-217: First commit. Updates namespaces and removes + + hints.ai_protocol = 0; /* any protocol */ + + int errcode = getaddrinfo(requested_hostname_.c_str(), 0, &hints, + &addr_info_); + + if (errcode != 0) { + logger_->log_error("Saw error during getaddrinfo, error: %s", + strerror(errno)); + return -1; + } + +<<<<<<< HEAD in_addr_t addr; struct hostent *h; #ifdef __MACH__ @@ -197,12 +274,30 @@ short Socket::initialize() { break; } } +======= + socket_file_descriptor_ = -1; + + auto p = addr_info_; + for (; p != NULL; p = p->ai_next) { + if (IsNullOrEmpty(canonical_hostname_)) { + if (!IsNullOrEmpty(p) && !IsNullOrEmpty(p->ai_canonname)) + canonical_hostname_ = p->ai_canonname; + } - return -1; + //we've successfully connected + if (port_ > 0 && createConnection(p) >= 0) { + return 0; + break; + } + } +>>>>>>> MINIFI-217: First commit. Updates namespaces and removes + + return -1; } short Socket::select_descriptor(const uint16_t msec) { +<<<<<<< HEAD struct timeval tv; int retval; @@ -255,69 +350,130 @@ short Socket::select_descriptor(const uint16_t msec) { } return -1; +======= + struct timeval tv; + int retval; + + read_fds_ = total_list_; + + tv.tv_sec = msec / 1000; + tv.tv_usec = (msec % 1000) * 1000; + + std::lock_guard guard(selection_mutex_); + + if (msec > 0) + retval = select(socket_max_ + 1, &read_fds_, NULL, NULL, &tv); + else + retval = select(socket_max_ + 1, &read_fds_, NULL, NULL, NULL); + + if (retval < 0) { + logger_->log_error("Saw error during selection, error:%i %s", retval, + strerror(errno)); + return retval; + } + + for (int i = 0; i <= socket_max_; i++) { + if (FD_ISSET(i, &read_fds_)) { + + if (i == socket_file_descriptor_) { + if (listeners_ > 0) { + struct sockaddr_storage remoteaddr; // client address + socklen_t addrlen = sizeof remoteaddr; + int newfd = accept(socket_file_descriptor_, + (struct sockaddr *) &remoteaddr, &addrlen); + FD_SET(newfd, &total_list_); // add to master set + if (newfd > socket_max_) { // keep track of the max + socket_max_ = newfd; + } + return newfd; + + } else { + return socket_file_descriptor_; + } + // we have a new connection + } else { + // data to be received on i + return i; + } + } + + } + + return -1; +>>>>>>> MINIFI-217: First commit. Updates namespaces and removes } short Socket::setSocketOptions(const int sock) { - int opt = 1; - bool nagle_off = true; + int opt = 1; + bool nagle_off = true; #ifndef __MACH__ - if (nagle_off) { - if (setsockopt(sock, SOL_TCP, TCP_NODELAY, (void *) &opt, sizeof(opt)) - < 0) { - logger_->log_error("setsockopt() TCP_NODELAY failed"); - close(sock); - return -1; - } - if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *) &opt, - sizeof(opt)) < 0) { - logger_->log_error("setsockopt() SO_REUSEADDR failed"); - close(sock); - return -1; - } - } + if (nagle_off) { + if (setsockopt(sock, SOL_TCP, TCP_NODELAY, (void *) &opt, sizeof(opt)) + < 0) { + logger_->log_error("setsockopt() TCP_NODELAY failed"); + close(sock); + return -1; + } + if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *) &opt, + sizeof(opt)) < 0) { + logger_->log_error("setsockopt() SO_REUSEADDR failed"); + close(sock); + return -1; + } + } - int sndsize = 256 * 1024; - if (setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (char *) &sndsize, - (int) sizeof(sndsize)) < 0) { - logger_->log_error("setsockopt() SO_SNDBUF failed"); - close(sock); - return -1; - } + int sndsize = 256 * 1024; + if (setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (char *) &sndsize, + (int) sizeof(sndsize)) < 0) { + logger_->log_error("setsockopt() SO_SNDBUF failed"); + close(sock); + return -1; + } #else - if (listeners_ > 0) { - // lose the pesky "address already in use" error message - if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *) &opt, - sizeof(opt)) < 0) { - logger_->log_error("setsockopt() SO_REUSEADDR failed"); - close(sock); - return -1; - } - } + if (listeners_ > 0) { + // lose the pesky "address already in use" error message + if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *) &opt, sizeof(opt)) + < 0) { + logger_->log_error("setsockopt() SO_REUSEADDR failed"); + close(sock); + return -1; + } + } #endif - return 0; + return 0; } std::string Socket::getHostname() const { - return canonical_hostname_; + return canonical_hostname_; } int Socket::writeData(std::vector &buf, int buflen) { - if (buf.capacity() < buflen) - return -1; - return writeData((uint8_t*) &buf[0], buflen); + if (buf.capacity() < buflen) + return -1; + return writeData((uint8_t*) &buf[0], buflen); } // data stream overrides int Socket::writeData(uint8_t *value, int size) { - int ret = 0, bytes = 0; + int ret = 0, bytes = 0; - while (bytes < size) { + while (bytes < size) { + ret = send(socket_file_descriptor_, value + bytes, size - bytes, 0); + //check for errors + if (ret <= 0) { + close(socket_file_descriptor_); + logger_->log_error("Could not send to %d, error: %s", + socket_file_descriptor_, strerror(errno)); + return ret; + } + bytes += ret; +<<<<<<< HEAD ret = send(socket_file_descriptor_, value + bytes, size - bytes, 0); //check for errors @@ -333,122 +489,131 @@ int Socket::writeData(uint8_t *value, int size) { if (ret) logger_->log_trace("Send data size %d over socket %d", size, socket_file_descriptor_); +======= + } - return bytes; + if (ret) + logger_->log_debug("Send data size %d over socket %d", size, + socket_file_descriptor_); +>>>>>>> MINIFI-217: First commit. Updates namespaces and removes + + return bytes; } template inline std::vector Socket::readBuffer(const T& t) { - std::vector buf; - buf.resize(sizeof t); - readData((uint8_t*) &buf[0], sizeof(t)); - return buf; + std::vector buf; + buf.resize(sizeof t); + readData((uint8_t*) &buf[0], sizeof(t)); + return buf; } +int Socket::write(uint64_t base_value, bool is_little_endian) { +<<<<<<< HEAD int Socket::write(uint64_t base_value, bool is_little_endian){ return Serializable::write(base_value,this,is_little_endian); +======= + return Serializable::write(base_value, this, is_little_endian); +>>>>>>> MINIFI-217: First commit. Updates namespaces and removes } - -int Socket::write(uint32_t base_value, bool is_little_endian){ - return Serializable::write(base_value,this,is_little_endian); +int Socket::write(uint32_t base_value, bool is_little_endian) { + return Serializable::write(base_value, this, is_little_endian); } -int Socket::write(uint16_t base_value, bool is_little_endian){ - return Serializable::write(base_value,this,is_little_endian); +int Socket::write(uint16_t base_value, bool is_little_endian) { + return Serializable::write(base_value, this, is_little_endian); } - int Socket::read(uint64_t &value, bool is_little_endian) { - auto buf = readBuffer(value); - - if (is_little_endian) { - value = ((uint64_t) buf[0] << 56) | ((uint64_t) (buf[1] & 255) << 48) - | ((uint64_t) (buf[2] & 255) << 40) - | ((uint64_t) (buf[3] & 255) << 32) - | ((uint64_t) (buf[4] & 255) << 24) - | ((uint64_t) (buf[5] & 255) << 16) - | ((uint64_t) (buf[6] & 255) << 8) - | ((uint64_t) (buf[7] & 255) << 0); - } else { - value = ((uint64_t) buf[0] << 0) | ((uint64_t) (buf[1] & 255) << 8) - | ((uint64_t) (buf[2] & 255) << 16) - | ((uint64_t) (buf[3] & 255) << 24) - | ((uint64_t) (buf[4] & 255) << 32) - | ((uint64_t) (buf[5] & 255) << 40) - | ((uint64_t) (buf[6] & 255) << 48) - | ((uint64_t) (buf[7] & 255) << 56); - } - return sizeof(value); + auto buf = readBuffer(value); + + if (is_little_endian) { + value = ((uint64_t) buf[0] << 56) | ((uint64_t) (buf[1] & 255) << 48) + | ((uint64_t) (buf[2] & 255) << 40) | ((uint64_t) (buf[3] & 255) << 32) + | ((uint64_t) (buf[4] & 255) << 24) | ((uint64_t) (buf[5] & 255) << 16) + | ((uint64_t) (buf[6] & 255) << 8) | ((uint64_t) (buf[7] & 255) << 0); + } else { + value = ((uint64_t) buf[0] << 0) | ((uint64_t) (buf[1] & 255) << 8) + | ((uint64_t) (buf[2] & 255) << 16) | ((uint64_t) (buf[3] & 255) << 24) + | ((uint64_t) (buf[4] & 255) << 32) | ((uint64_t) (buf[5] & 255) << 40) + | ((uint64_t) (buf[6] & 255) << 48) | ((uint64_t) (buf[7] & 255) << 56); + } + return sizeof(value); } int Socket::read(uint32_t &value, bool is_little_endian) { - auto buf = readBuffer(value); + auto buf = readBuffer(value); - if (is_little_endian) { - value = (buf[0] << 24) | (buf[1] << 16) | (buf[2] << 8) | buf[3]; - } else { - value = buf[0] | buf[1] << 8 | buf[2] << 16 | buf[3] << 24; + if (is_little_endian) { + value = (buf[0] << 24) | (buf[1] << 16) | (buf[2] << 8) | buf[3]; + } else { + value = buf[0] | buf[1] << 8 | buf[2] << 16 | buf[3] << 24; - } + } - return sizeof(value); + return sizeof(value); } int Socket::read(uint16_t &value, bool is_little_endian) { - auto buf = readBuffer(value); + auto buf = readBuffer(value); - if (is_little_endian) { - value = (buf[0] << 8) | buf[1]; - } else { - value = buf[0] | buf[1] << 8; + if (is_little_endian) { + value = (buf[0] << 8) | buf[1]; + } else { + value = buf[0] | buf[1] << 8; - } - return sizeof(value); + } + return sizeof(value); } int Socket::readData(std::vector &buf, int buflen) { - if (buf.capacity() < buflen) - { - buf.resize(buflen); - } - return readData((uint8_t*) &buf[0], buflen); + if (buf.capacity() < buflen) { + buf.resize(buflen); + } + return readData((uint8_t*) &buf[0], buflen); } int Socket::readData(uint8_t *buf, int buflen) { - int total_read = 0; - while (buflen) { - short fd = select_descriptor(1000); - if (fd < 0) { - - logger_->log_info("fd close %i",buflen); - close(socket_file_descriptor_); - return -1; - } + int total_read = 0; + while (buflen) { + short fd = select_descriptor(1000); + if (fd < 0) { - int bytes_read = recv(fd, buf, buflen, 0); - if (bytes_read <= 0) { - if (bytes_read == 0) - logger_->log_info("Other side hung up on %d", fd); - else { - logger_->log_error("Could not recv on %d, error: %s",fd, strerror(errno)); - } - return -1; - } + logger_->log_info("fd close %i", buflen); + close(socket_file_descriptor_); + return -1; + } + int bytes_read = recv(fd, buf, buflen, 0); + if (bytes_read <= 0) { + if (bytes_read == 0) + logger_->log_info("Other side hung up on %d", fd); + else { + logger_->log_error("Could not recv on %d, error: %s", fd, + strerror(errno)); + } + return -1; + } - buflen -= bytes_read; - buf += bytes_read; - total_read += bytes_read; - } + buflen -= bytes_read; + buf += bytes_read; + total_read += bytes_read; + } - return total_read; + return total_read; } + +} /* namespace io */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/io/DataStream.cpp b/libminifi/src/io/DataStream.cpp index 3ff0a57383..7a10bd99cd 100644 --- a/libminifi/src/io/DataStream.cpp +++ b/libminifi/src/io/DataStream.cpp @@ -27,6 +27,11 @@ #include #include "io/DataStream.h" +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace io { int DataStream::writeData(uint8_t *value, int size) { @@ -126,3 +131,10 @@ int DataStream::readData(uint8_t *buf,int buflen) { readBuffer += buflen; return buflen; } + + +} /* namespace io */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/io/EndianCheck.cpp b/libminifi/src/io/EndianCheck.cpp index f1bc98b6e5..1b5020d0a9 100644 --- a/libminifi/src/io/EndianCheck.cpp +++ b/libminifi/src/io/EndianCheck.cpp @@ -18,5 +18,16 @@ #include "io/EndianCheck.h" - +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace io { bool EndiannessCheck::IS_LITTLE = EndiannessCheck::is_little_endian(); + + +} /* namespace io */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/io/Serializable.cpp b/libminifi/src/io/Serializable.cpp index 78dc63c81a..f8c623ab66 100644 --- a/libminifi/src/io/Serializable.cpp +++ b/libminifi/src/io/Serializable.cpp @@ -22,7 +22,11 @@ #include #include "io/DataStream.h" #include "io/Serializable.h" - +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace io { #define htonll_r(x) ((((uint64_t)htonl(x)) << 32) + htonl((x) >> 32)) @@ -216,3 +220,10 @@ int Serializable::writeUTF(std::string str,DataStream *stream, bool widen) { } return ret; } + + +} /* namespace io */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/io/SocketFactory.cpp b/libminifi/src/io/SocketFactory.cpp index cbfdf965e3..34833d833f 100644 --- a/libminifi/src/io/SocketFactory.cpp +++ b/libminifi/src/io/SocketFactory.cpp @@ -20,5 +20,18 @@ #include #include +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace io { + std::atomic SocketFactory::context_instance_; std::mutex SocketFactory::context_mutex_; + + +} /* namespace io */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/io/TLSSocket.cpp b/libminifi/src/io/TLSSocket.cpp index 1c81f6c25e..e7fc02f0c8 100644 --- a/libminifi/src/io/TLSSocket.cpp +++ b/libminifi/src/io/TLSSocket.cpp @@ -15,20 +15,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifdef OPENSSL_SUPPORT -#include "Property.h" #include "Configure.h" #include "io/TLSSocket.h" #include "utils/StringUtils.h" + +#include "core/Property.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace io { + #include #include - std::atomic TLSContext::context_instance; std::mutex TLSContext::context_mutex; TLSContext::TLSContext() : - error_value(0), ctx(0), logger_(Logger::getLogger()), configuration( + error_value(0), ctx(0), logger_(logging::Logger::getLogger()), configuration( Configure::getConfigure()) { } @@ -44,7 +50,7 @@ short TLSContext::initialize() { bool needClientCert = true; if (!(configuration->get(Configure::nifi_security_need_ClientAuth, clientAuthStr) - && StringUtils::StringToBool(clientAuthStr, needClientCert))) { + && org::apache::nifi::minifi::utils::StringUtils::StringToBool(clientAuthStr, needClientCert))) { needClientCert = true; } @@ -141,15 +147,15 @@ TLSSocket::~TLSSocket() */ TLSSocket::TLSSocket(const std::string &hostname, const uint16_t port, const uint16_t listeners) : - ::Socket(hostname, port, listeners), ssl(0) { + Socket(hostname, port, listeners), ssl(0) { } TLSSocket::TLSSocket(const std::string &hostname, const uint16_t port) : - ::Socket(hostname, port, 0), ssl(0) { + Socket(hostname, port, 0), ssl(0) { } TLSSocket::TLSSocket(const TLSSocket &&d) : - ::Socket(std::move(d)), ssl(0) { + Socket(std::move(d)), ssl(0) { } short TLSSocket::initialize() { @@ -234,4 +240,10 @@ int TLSSocket::readData(uint8_t *buf, int buflen) { return total_read; } -#endif // #ifdef OPENSSL_SUPPORT + + +} /* namespace io */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/processors/AppendHostInfo.cpp b/libminifi/src/processors/AppendHostInfo.cpp new file mode 100644 index 0000000000..24ccc9a26d --- /dev/null +++ b/libminifi/src/processors/AppendHostInfo.cpp @@ -0,0 +1,124 @@ +/** + * @file AppendHostInfo.cpp + * AppendHostInfo class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include "processors/AppendHostInfo.h" +#include "core/ProcessContext.h" +#include "core/Property.h" +#include "core/ProcessSession.h" + +#include +#include +#include +#include +#include +#include + +#include "../../include/core/FlowFile.h" +#include "io/ClientSocket.h" + +#define __USE_POSIX +#include + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +#ifndef HOST_NAME_MAX +#define HOST_NAME_MAX 255 +#endif + +const std::string AppendHostInfo::ProcessorName("AppendHostInfo"); +core::Property AppendHostInfo::InterfaceName( + "Network Interface Name", + "Network interface from which to read an IP v4 address", "eth0"); +core::Property AppendHostInfo::HostAttribute( + "Hostname Attribute", + "Flowfile attribute to used to record the agent's hostname", + "source.hostname"); +core::Property AppendHostInfo::IPAttribute( + "IP Attribute", + "Flowfile attribute to used to record the agent's IP address", + "source.ipv4"); +core::Relationship AppendHostInfo::Success( + "success", "success operational on the flow record"); + +void AppendHostInfo::initialize() { + // Set the supported properties + std::set properties; + properties.insert(InterfaceName); + properties.insert(HostAttribute); + properties.insert(IPAttribute); + setSupportedProperties(properties); + + // Set the supported relationships + std::set relationships; + relationships.insert(Success); + setSupportedRelationships(relationships); +} + +void AppendHostInfo::onTrigger( + core::ProcessContext *context, + core::ProcessSession *session) { + std::shared_ptr flow = + session->get(); + if (!flow) + return; + + //Get Hostname + + std::string hostAttribute = ""; + context->getProperty(HostAttribute.getName(), hostAttribute); + flow->addAttribute(hostAttribute.c_str(), + org::apache::nifi::minifi::io::Socket::getMyHostName()); + + //Get IP address for the specified interface + std::string iface; + context->getProperty(InterfaceName.getName(), iface); + //Confirm the specified interface name exists on this device + if (if_nametoindex(iface.c_str()) != 0) { + struct ifreq ifr; + int fd = socket(AF_INET, SOCK_DGRAM, 0); + //Type of address to retrieve - IPv4 IP address + ifr.ifr_addr.sa_family = AF_INET; + //Copy the interface name in the ifreq structure + strncpy(ifr.ifr_name, iface.c_str(), IFNAMSIZ - 1); + ioctl(fd, SIOCGIFADDR, &ifr); + close(fd); + + std::string ipAttribute; + context->getProperty(IPAttribute.getName(), ipAttribute); + flow->addAttribute( + ipAttribute.c_str(), + inet_ntoa(((struct sockaddr_in *) &ifr.ifr_addr)->sin_addr)); + } + + // Transfer to the relationship + session->transfer(flow, Success); +} + +} /* namespace processors */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/processors/ExecuteProcess.cpp b/libminifi/src/processors/ExecuteProcess.cpp new file mode 100644 index 0000000000..3cbbc1ba49 --- /dev/null +++ b/libminifi/src/processors/ExecuteProcess.cpp @@ -0,0 +1,255 @@ +/** + * @file ExecuteProcess.cpp + * ExecuteProcess class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "processors/ExecuteProcess.h" +#include "core/ProcessContext.h" +#include "core/ProcessSession.h" +#include +#include "utils/StringUtils.h" +#include "utils/TimeUtil.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +const std::string ExecuteProcess::ProcessorName("ExecuteProcess"); +core::Property ExecuteProcess::Command( + "Command", + "Specifies the command to be executed; if just the name of an executable is provided, it must be in the user's environment PATH.", + ""); +core::Property ExecuteProcess::CommandArguments( + "Command Arguments", + "The arguments to supply to the executable delimited by white space. White space can be escaped by enclosing it in double-quotes.", + ""); +core::Property ExecuteProcess::WorkingDir( + "Working Directory", + "The directory to use as the current working directory when executing the command", + ""); +core::Property ExecuteProcess::BatchDuration( + "Batch Duration", + "If the process is expected to be long-running and produce textual output, a batch duration can be specified.", + "0"); +core::Property ExecuteProcess::RedirectErrorStream( + "Redirect Error Stream", + "If true will redirect any error stream output of the process to the output stream.", + "false"); +core::Relationship ExecuteProcess::Success( + "success", "All created FlowFiles are routed to this relationship."); + +void ExecuteProcess::initialize() { + // Set the supported properties + std::set properties; + properties.insert(Command); + properties.insert(CommandArguments); + properties.insert(WorkingDir); + properties.insert(BatchDuration); + properties.insert(RedirectErrorStream); + setSupportedProperties(properties); + // Set the supported relationships + std::set relationships; + relationships.insert(Success); + setSupportedRelationships(relationships); +} + +void ExecuteProcess::onTrigger( + core::ProcessContext *context, + core::ProcessSession *session) { + std::string value; + if (context->getProperty(Command.getName(), value)) { + this->_command = value; + } + if (context->getProperty(CommandArguments.getName(), value)) { + this->_commandArgument = value; + } + if (context->getProperty(WorkingDir.getName(), value)) { + this->_workingDir = value; + } + if (context->getProperty(BatchDuration.getName(), value)) { + core::TimeUnit unit; + if (core::Property::StringToTime(value, + _batchDuration, + unit) + && core::Property::ConvertTimeUnitToMS( + _batchDuration, unit, _batchDuration)) { + + } + } + if (context->getProperty(RedirectErrorStream.getName(), value)) { + org::apache::nifi::minifi::utils::StringUtils::StringToBool( + value, _redirectErrorStream); + } + this->_fullCommand = _command + " " + _commandArgument; + if (_fullCommand.length() == 0) { + yield(); + return; + } + if (_workingDir.length() > 0 && _workingDir != ".") { + // change to working directory + if (chdir(_workingDir.c_str()) != 0) { + logger_->log_error("Execute Command can not chdir %s", + _workingDir.c_str()); + yield(); + return; + } + } + logger_->log_info("Execute Command %s", _fullCommand.c_str()); + // split the command into array + char cstr[_fullCommand.length() + 1]; + std::strcpy(cstr, _fullCommand.c_str()); + char *p = std::strtok(cstr, " "); + int argc = 0; + char *argv[64]; + while (p != 0 && argc < 64) { + argv[argc] = p; + p = std::strtok(NULL, " "); + argc++; + } + argv[argc] = NULL; + int status, died; + if (!_processRunning) { + _processRunning = true; + // if the process has not launched yet + // create the pipe + if (pipe(_pipefd) == -1) { + _processRunning = false; + yield(); + return; + } + switch (_pid = fork()) { + case -1: + logger_->log_error("Execute Process fork failed"); + _processRunning = false; + close(_pipefd[0]); + close(_pipefd[1]); + yield(); + break; + case 0: // this is the code the child runs + close(1); // close stdout + dup(_pipefd[1]); // points pipefd at file descriptor + if (_redirectErrorStream) + // redirect stderr + dup2(_pipefd[1], 2); + close(_pipefd[0]); + execvp(argv[0], argv); + exit(1); + break; + default: // this is the code the parent runs + // the parent isn't going to write to the pipe + close(_pipefd[1]); + if (_batchDuration > 0) { + while (1) { + std::this_thread::sleep_for( + std::chrono::milliseconds(_batchDuration)); + char buffer[4096]; + int numRead = read(_pipefd[0], buffer, sizeof(buffer)); + if (numRead <= 0) + break; + logger_->log_info("Execute Command Respond %d", numRead); + ExecuteProcess::WriteCallback callback(buffer, numRead); + std::shared_ptr flowFile = std::static_pointer_cast< + FlowFileRecord>(session->create()); + if (!flowFile) + continue; + flowFile->addAttribute("command", _command.c_str()); + flowFile->addAttribute("command.arguments", + _commandArgument.c_str()); + session->write(flowFile, &callback); + session->transfer(flowFile, Success); + session->commit(); + } + } else { + char buffer[4096]; + char *bufPtr = buffer; + int totalRead = 0; + std::shared_ptr flowFile = nullptr; + while (1) { + int numRead = read(_pipefd[0], bufPtr, + (sizeof(buffer) - totalRead)); + if (numRead <= 0) { + if (totalRead > 0) { + logger_->log_info("Execute Command Respond %d", totalRead); + // child exits and close the pipe + ExecuteProcess::WriteCallback callback(buffer, totalRead); + if (!flowFile) { + flowFile = std::static_pointer_cast( + session->create()); + if (!flowFile) + break; + flowFile->addAttribute("command", _command.c_str()); + flowFile->addAttribute("command.arguments", + _commandArgument.c_str()); + session->write(flowFile, &callback); + } else { + session->append(flowFile, &callback); + } + session->transfer(flowFile, Success); + } + break; + } else { + if (numRead == (sizeof(buffer) - totalRead)) { + // we reach the max buffer size + logger_->log_info("Execute Command Max Respond %d", + sizeof(buffer)); + ExecuteProcess::WriteCallback callback(buffer, sizeof(buffer)); + if (!flowFile) { + flowFile = std::static_pointer_cast( + session->create()); + if (!flowFile) + continue; + flowFile->addAttribute("command", _command.c_str()); + flowFile->addAttribute("command.arguments", + _commandArgument.c_str()); + session->write(flowFile, &callback); + } else { + session->append(flowFile, &callback); + } + // Rewind + totalRead = 0; + bufPtr = buffer; + } else { + totalRead += numRead; + bufPtr += numRead; + } + } + } + } + + died = wait(&status); + if (WIFEXITED(status)) { + logger_->log_info("Execute Command Complete %s status %d pid %d", + _fullCommand.c_str(), WEXITSTATUS(status), _pid); + } else { + logger_->log_info("Execute Command Complete %s status %d pid %d", + _fullCommand.c_str(), WTERMSIG(status), _pid); + } + + close(_pipefd[0]); + _processRunning = false; + break; + } + } +} + +} /* namespace processors */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/GenerateFlowFile.cpp b/libminifi/src/processors/GenerateFlowFile.cpp similarity index 67% rename from libminifi/src/GenerateFlowFile.cpp rename to libminifi/src/processors/GenerateFlowFile.cpp index 12d7f7095e..ebdaaa3b51 100644 --- a/libminifi/src/GenerateFlowFile.cpp +++ b/libminifi/src/processors/GenerateFlowFile.cpp @@ -28,36 +28,41 @@ #include #include "utils/StringUtils.h" -#include "GenerateFlowFile.h" -#include "ProcessContext.h" -#include "ProcessSession.h" +#include "processors/GenerateFlowFile.h" +#include "core/ProcessContext.h" +#include "core/ProcessSession.h" +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { const char *GenerateFlowFile::DATA_FORMAT_BINARY = "Binary"; const char *GenerateFlowFile::DATA_FORMAT_TEXT = "Text"; const std::string GenerateFlowFile::ProcessorName("GenerateFlowFile"); -Property GenerateFlowFile::FileSize("File Size", "The size of the file that will be used", "1 kB"); -Property GenerateFlowFile::BatchSize("Batch Size", "The number of FlowFiles to be transferred in each invocation", "1"); -Property GenerateFlowFile::DataFormat("Data Format", "Specifies whether the data should be Text or Binary", GenerateFlowFile::DATA_FORMAT_BINARY); -Property GenerateFlowFile::UniqueFlowFiles("Unique FlowFiles", +core::Property GenerateFlowFile::FileSize("File Size", "The size of the file that will be used", "1 kB"); +core::Property GenerateFlowFile::BatchSize("Batch Size", "The number of FlowFiles to be transferred in each invocation", "1"); +core::Property GenerateFlowFile::DataFormat("Data Format", "Specifies whether the data should be Text or Binary", GenerateFlowFile::DATA_FORMAT_BINARY); +core::Property GenerateFlowFile::UniqueFlowFiles("Unique FlowFiles", "If true, each FlowFile that is generated will be unique. If false, a random value will be generated and all FlowFiles", "true"); -Relationship GenerateFlowFile::Success("success", "success operational on the flow record"); +core::Relationship GenerateFlowFile::Success("success", "success operational on the flow record"); void GenerateFlowFile::initialize() { - //! Set the supported properties - std::set properties; + // Set the supported properties + std::set properties; properties.insert(FileSize); properties.insert(BatchSize); properties.insert(DataFormat); properties.insert(UniqueFlowFiles); setSupportedProperties(properties); - //! Set the supported relationships - std::set relationships; + // Set the supported relationships + std::set relationships; relationships.insert(Success); setSupportedRelationships(relationships); } -void GenerateFlowFile::onTrigger(ProcessContext *context, ProcessSession *session) +void GenerateFlowFile::onTrigger(core::ProcessContext *context, core::ProcessSession *session) { int64_t batchSize = 1; bool uniqueFlowFile = true; @@ -66,15 +71,15 @@ void GenerateFlowFile::onTrigger(ProcessContext *context, ProcessSession *sessio std::string value; if (context->getProperty(FileSize.getName(), value)) { - Property::StringToInt(value, fileSize); + core::Property::StringToInt(value, fileSize); } if (context->getProperty(BatchSize.getName(), value)) { - Property::StringToInt(value, batchSize); + core::Property::StringToInt(value, batchSize); } if (context->getProperty(UniqueFlowFiles.getName(), value)) { - StringUtils::StringToBool(value, uniqueFlowFile); + org::apache::nifi::minifi::utils::StringUtils::StringToBool(value, uniqueFlowFile); } if (!uniqueFlowFile) @@ -95,7 +100,7 @@ void GenerateFlowFile::onTrigger(ProcessContext *context, ProcessSession *sessio for (int i = 0; i < batchSize; i++) { // For each batch - FlowFileRecord *flowFile = session->create(); + std::shared_ptr flowFile = std::static_pointer_cast(session->create()); if (!flowFile) return; if (fileSize > 0) @@ -124,7 +129,7 @@ void GenerateFlowFile::onTrigger(ProcessContext *context, ProcessSession *sessio for (int i = 0; i < batchSize; i++) { // For each batch - FlowFileRecord *flowFile = session->create(); + std::shared_ptr flowFile = std::static_pointer_cast(session->create()); if (!flowFile) return; if (fileSize > 0) @@ -133,3 +138,8 @@ void GenerateFlowFile::onTrigger(ProcessContext *context, ProcessSession *sessio } } } +} /* namespace processors */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/processors/GetFile.cpp b/libminifi/src/processors/GetFile.cpp new file mode 100644 index 0000000000..cf05657d65 --- /dev/null +++ b/libminifi/src/processors/GetFile.cpp @@ -0,0 +1,340 @@ +/** + * @file GetFile.cpp + * GetFile class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if (__GNUC__ >= 4) +#if (__GNUC_MINOR__ < 9) +#include +#endif +#endif +#include "utils/StringUtils.h" +#include +#include "utils/TimeUtil.h" +#include "processors/GetFile.h" +#include "core/ProcessContext.h" +#include "core/ProcessSession.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { +const std::string GetFile::ProcessorName("GetFile"); +core::Property GetFile::BatchSize( + "Batch Size", "The maximum number of files to pull in each iteration", + "10"); +core::Property GetFile::Directory( + "Input Directory", "The input directory from which to pull files", "."); +core::Property GetFile::IgnoreHiddenFile( + "Ignore Hidden Files", + "Indicates whether or not hidden files should be ignored", "true"); +core::Property GetFile::KeepSourceFile( + "Keep Source File", + "If true, the file is not deleted after it has been copied to the Content Repository", + "false"); +core::Property GetFile::MaxAge( + "Maximum File Age", + "The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored", + "0 sec"); +core::Property GetFile::MinAge( + "Minimum File Age", + "The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored", + "0 sec"); +core::Property GetFile::MaxSize( + "Maximum File Size", + "The maximum size that a file can be in order to be pulled", "0 B"); +core::Property GetFile::MinSize( + "Minimum File Size", + "The minimum size that a file must be in order to be pulled", "0 B"); +core::Property GetFile::PollInterval( + "Polling Interval", + "Indicates how long to wait before performing a directory listing", + "0 sec"); +core::Property GetFile::Recurse( + "Recurse Subdirectories", + "Indicates whether or not to pull files from subdirectories", "true"); +core::Property GetFile::FileFilter( + "File Filter", + "Only files whose names match the given regular expression will be picked up", + "[^\\.].*"); +core::Relationship GetFile::Success( + "success", "All files are routed to success"); + +void GetFile::initialize() { + // Set the supported properties + std::set properties; + properties.insert(BatchSize); + properties.insert(Directory); + properties.insert(IgnoreHiddenFile); + properties.insert(KeepSourceFile); + properties.insert(MaxAge); + properties.insert(MinAge); + properties.insert(MaxSize); + properties.insert(MinSize); + properties.insert(PollInterval); + properties.insert(Recurse); + properties.insert(FileFilter); + setSupportedProperties(properties); + // Set the supported relationships + std::set relationships; + relationships.insert(Success); + setSupportedRelationships(relationships); +} + +void GetFile::onTrigger( + core::ProcessContext *context, + core::ProcessSession *session) { + std::string value; + + logger_->log_info("onTrigger GetFile"); + if (context->getProperty(Directory.getName(), value)) { + _directory = value; + } + if (context->getProperty(BatchSize.getName(), value)) { + core::Property::StringToInt(value, _batchSize); + } + if (context->getProperty(IgnoreHiddenFile.getName(), value)) { + org::apache::nifi::minifi::utils::StringUtils::StringToBool( + value, _ignoreHiddenFile); + } + if (context->getProperty(KeepSourceFile.getName(), value)) { + org::apache::nifi::minifi::utils::StringUtils::StringToBool( + value, _keepSourceFile); + } + + logger_->log_info("onTrigger GetFile"); + if (context->getProperty(MaxAge.getName(), value)) { + core::TimeUnit unit; + if (core::Property::StringToTime(value, _maxAge, + unit) + && core::Property::ConvertTimeUnitToMS( + _maxAge, unit, _maxAge)) { + + } + } + if (context->getProperty(MinAge.getName(), value)) { + core::TimeUnit unit; + if (core::Property::StringToTime(value, _minAge, + unit) + && core::Property::ConvertTimeUnitToMS( + _minAge, unit, _minAge)) { + + } + } + if (context->getProperty(MaxSize.getName(), value)) { + core::Property::StringToInt(value, _maxSize); + } + if (context->getProperty(MinSize.getName(), value)) { + core::Property::StringToInt(value, _minSize); + } + if (context->getProperty(PollInterval.getName(), value)) { + core::TimeUnit unit; + if (core::Property::StringToTime(value, + _pollInterval, + unit) + && core::Property::ConvertTimeUnitToMS( + _pollInterval, unit, _pollInterval)) { + + } + } + if (context->getProperty(Recurse.getName(), value)) { + org::apache::nifi::minifi::utils::StringUtils::StringToBool(value, + _recursive); + } + + if (context->getProperty(FileFilter.getName(), value)) { + _fileFilter = value; + } + + // Perform directory list + logger_->log_info("Is listing empty %i", isListingEmpty()); + if (isListingEmpty()) { + if (_pollInterval == 0 + || (getTimeMillis() - _lastDirectoryListingTime) > _pollInterval) { + performListing(_directory); + } + } + logger_->log_info("Is listing empty %i", isListingEmpty()); + + if (!isListingEmpty()) { + try { + std::queue list; + pollListing(list, _batchSize); + while (!list.empty()) { + + std::string fileName = list.front(); + list.pop(); + logger_->log_info("GetFile process %s", fileName.c_str()); + std::shared_ptr flowFile = std::static_pointer_cast< + FlowFileRecord>(session->create()); + if (flowFile == nullptr) + return; + std::size_t found = fileName.find_last_of("/\\"); + std::string path = fileName.substr(0, found); + std::string name = fileName.substr(found + 1); + flowFile->updateKeyedAttribute(FILENAME, name); + flowFile->updateKeyedAttribute(PATH, path); + flowFile->addKeyedAttribute(ABSOLUTE_PATH, fileName); + session->import(fileName, flowFile, _keepSourceFile); + session->transfer(flowFile, Success); + } + } catch (std::exception &exception) { + logger_->log_debug("GetFile Caught Exception %s", exception.what()); + throw; + } catch (...) { + throw; + } + } + +} + +bool GetFile::isListingEmpty() { + std::lock_guard lock(mutex_); + + return _dirList.empty(); +} + +void GetFile::putListing(std::string fileName) { + std::lock_guard lock(mutex_); + + _dirList.push(fileName); +} + +void GetFile::pollListing(std::queue &list, int maxSize) { + std::lock_guard lock(mutex_); + + while (!_dirList.empty() && (maxSize == 0 || list.size() < maxSize)) { + std::string fileName = _dirList.front(); + _dirList.pop(); + list.push(fileName); + } + + return; +} + +bool GetFile::acceptFile(std::string fullName, std::string name) { + struct stat statbuf; + + if (stat(fullName.c_str(), &statbuf) == 0) { + if (_minSize > 0 && statbuf.st_size < _minSize) + return false; + + if (_maxSize > 0 && statbuf.st_size > _maxSize) + return false; + + uint64_t modifiedTime = ((uint64_t) (statbuf.st_mtime) * 1000); + uint64_t fileAge = getTimeMillis() - modifiedTime; + if (_minAge > 0 && fileAge < _minAge) + return false; + if (_maxAge > 0 && fileAge > _maxAge) + return false; + + if (_ignoreHiddenFile && fullName.c_str()[0] == '.') + return false; + + if (access(fullName.c_str(), R_OK) != 0) + return false; + + if (_keepSourceFile == false && access(fullName.c_str(), W_OK) != 0) + return false; + +#ifdef __GNUC__ +#if (__GNUC__ >= 4) +#if (__GNUC_MINOR__ < 9) + regex_t regex; + int ret = regcomp(®ex, _fileFilter.c_str(), 0); + if (ret) + return false; + ret = regexec(®ex, name.c_str(), (size_t) 0, NULL, 0); + regfree(®ex); + if (ret) + return false; +#else + try { + std::regex re(_fileFilter); + + if (!std::regex_match(name, re)) { + return false; + } + } catch (std::regex_error e) { + logger_->log_error("Invalid File Filter regex: %s.", e.what()); + return false; + } +#endif +#endif +#else + logger_->log_info("Cannot support regex filtering"); +#endif + return true; + } + + return false; +} + +void GetFile::performListing(std::string dir) { + logger_->log_info("Performing file listing against %s", dir.c_str()); + DIR *d; + d = opendir(dir.c_str()); + if (!d) + return; + // only perform a listing while we are not empty + logger_->log_info("Performing file listing against %s", dir.c_str()); + while (isRunning()) { + struct dirent *entry; + entry = readdir(d); + if (!entry) + break; + std::string d_name = entry->d_name; + if ((entry->d_type & DT_DIR)) { + // if this is a directory + if (_recursive && strcmp(d_name.c_str(), "..") != 0 + && strcmp(d_name.c_str(), ".") != 0) { + std::string path = dir + "/" + d_name; + performListing(path); + } + } else { + std::string fileName = dir + "/" + d_name; + if (acceptFile(fileName, d_name)) { + // check whether we can take this file + putListing(fileName); + } + } + } + closedir(d); +} + +} /* namespace processors */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/processors/ListenHTTP.cpp b/libminifi/src/processors/ListenHTTP.cpp new file mode 100644 index 0000000000..36c743eb91 --- /dev/null +++ b/libminifi/src/processors/ListenHTTP.cpp @@ -0,0 +1,380 @@ +/** + * @file ListenHTTP.cpp + * ListenHTTP class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include + +#include + +#include "processors/ListenHTTP.h" + +#include "utils/TimeUtil.h" +#include "core/ProcessContext.h" +#include "core/ProcessSession.h" +#include "core/ProcessSessionFactory.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +const std::string ListenHTTP::ProcessorName("ListenHTTP"); + +core::Property ListenHTTP::BasePath( + "Base Path", "Base path for incoming connections", "contentListener"); +core::Property ListenHTTP::Port( + "Listening Port", "The Port to listen on for incoming connections", ""); +core::Property ListenHTTP::AuthorizedDNPattern( + "Authorized DN Pattern", + "A Regular Expression to apply against the Distinguished Name of incoming connections. If the Pattern does not match the DN, the connection will be refused.", + ".*"); +core::Property ListenHTTP::SSLCertificate( + "SSL Certificate", + "File containing PEM-formatted file including TLS/SSL certificate and key", + ""); +core::Property ListenHTTP::SSLCertificateAuthority( + "SSL Certificate Authority", + "File containing trusted PEM-formatted certificates", ""); +core::Property ListenHTTP::SSLVerifyPeer( + "SSL Verify Peer", + "Whether or not to verify the client's certificate (yes/no)", "no"); +core::Property ListenHTTP::SSLMinimumVersion( + "SSL Minimum Version", + "Minimum TLS/SSL version allowed (SSL2, SSL3, TLS1.0, TLS1.1, TLS1.2)", + "SSL2"); +core::Property ListenHTTP::HeadersAsAttributesRegex( + "HTTP Headers to receive as Attributes (Regex)", + "Specifies the Regular Expression that determines the names of HTTP Headers that should be passed along as FlowFile attributes", + ""); + +core::Relationship ListenHTTP::Success( + "success", "All files are routed to success"); + +void ListenHTTP::initialize() { + _logger->log_info("Initializing ListenHTTP"); + + // Set the supported properties + std::set < core::Property > properties; + properties.insert(BasePath); + properties.insert(Port); + properties.insert(AuthorizedDNPattern); + properties.insert(SSLCertificate); + properties.insert(SSLCertificateAuthority); + properties.insert(SSLVerifyPeer); + properties.insert(SSLMinimumVersion); + properties.insert(HeadersAsAttributesRegex); + setSupportedProperties (properties); + // Set the supported relationships + std::set < core::Relationship > relationships; + relationships.insert(Success); + setSupportedRelationships (relationships); +} + +void ListenHTTP::onSchedule( + core::ProcessContext *context, + core::ProcessSessionFactory *sessionFactory) { + + std::string basePath; + + if (!context->getProperty(BasePath.getName(), basePath)) { + _logger->log_info( + "%s attribute is missing, so default value of %s will be used", + BasePath.getName().c_str(), BasePath.getValue().c_str()); + basePath = BasePath.getValue(); + } + + basePath.insert(0, "/"); + + std::string listeningPort; + + if (!context->getProperty(Port.getName(), listeningPort)) { + _logger->log_error("%s attribute is missing or invalid", + Port.getName().c_str()); + return; + } + + std::string authDNPattern; + + if (context->getProperty(AuthorizedDNPattern.getName(), authDNPattern) + && !authDNPattern.empty()) { + _logger->log_info("ListenHTTP using %s: %s", + AuthorizedDNPattern.getName().c_str(), + authDNPattern.c_str()); + } + + std::string sslCertFile; + + if (context->getProperty(SSLCertificate.getName(), sslCertFile) + && !sslCertFile.empty()) { + _logger->log_info("ListenHTTP using %s: %s", + SSLCertificate.getName().c_str(), sslCertFile.c_str()); + } + + // Read further TLS/SSL options only if TLS/SSL usage is implied by virtue of certificate value being set + std::string sslCertAuthorityFile; + std::string sslVerifyPeer; + std::string sslMinVer; + + if (!sslCertFile.empty()) { + if (context->getProperty(SSLCertificateAuthority.getName(), + sslCertAuthorityFile) + && !sslCertAuthorityFile.empty()) { + _logger->log_info("ListenHTTP using %s: %s", + SSLCertificateAuthority.getName().c_str(), + sslCertAuthorityFile.c_str()); + } + + if (context->getProperty(SSLVerifyPeer.getName(), sslVerifyPeer)) { + if (sslVerifyPeer.empty() || sslVerifyPeer.compare("no") == 0) { + _logger->log_info("ListenHTTP will not verify peers"); + } else { + _logger->log_info("ListenHTTP will verify peers"); + } + } else { + _logger->log_info("ListenHTTP will not verify peers"); + } + + if (context->getProperty(SSLMinimumVersion.getName(), sslMinVer)) { + _logger->log_info("ListenHTTP using %s: %s", + SSLMinimumVersion.getName().c_str(), sslMinVer.c_str()); + } + } + + std::string headersAsAttributesPattern; + + if (context->getProperty(HeadersAsAttributesRegex.getName(), + headersAsAttributesPattern) + && !headersAsAttributesPattern.empty()) { + _logger->log_info("ListenHTTP using %s: %s", + HeadersAsAttributesRegex.getName().c_str(), + headersAsAttributesPattern.c_str()); + } + + auto numThreads = getMaxConcurrentTasks(); + + _logger->log_info( + "ListenHTTP starting HTTP server on port %s and path %s with %d threads", + listeningPort.c_str(), basePath.c_str(), numThreads); + + // Initialize web server + std::vector < std::string > options; + options.push_back("enable_keep_alive"); + options.push_back("yes"); + options.push_back("keep_alive_timeout_ms"); + options.push_back("15000"); + options.push_back("num_threads"); + options.push_back(std::to_string(numThreads)); + + if (sslCertFile.empty()) { + options.push_back("listening_ports"); + options.push_back(listeningPort); + } else { + listeningPort += "s"; + options.push_back("listening_ports"); + options.push_back(listeningPort); + + options.push_back("ssl_certificate"); + options.push_back(sslCertFile); + + if (!sslCertAuthorityFile.empty()) { + options.push_back("ssl_ca_file"); + options.push_back(sslCertAuthorityFile); + } + + if (sslVerifyPeer.empty() || sslVerifyPeer.compare("no") == 0) { + options.push_back("ssl_verify_peer"); + options.push_back("no"); + } else { + options.push_back("ssl_verify_peer"); + options.push_back("yes"); + } + + if (sslMinVer.compare("SSL2") == 0) { + options.push_back("ssl_protocol_version"); + options.push_back(std::to_string(0)); + } else if (sslMinVer.compare("SSL3") == 0) { + options.push_back("ssl_protocol_version"); + options.push_back(std::to_string(1)); + } else if (sslMinVer.compare("TLS1.0") == 0) { + options.push_back("ssl_protocol_version"); + options.push_back(std::to_string(2)); + } else if (sslMinVer.compare("TLS1.1") == 0) { + options.push_back("ssl_protocol_version"); + options.push_back(std::to_string(3)); + } else { + options.push_back("ssl_protocol_version"); + options.push_back(std::to_string(4)); + } + } + + _server.reset(new CivetServer(options)); + _handler.reset( + new Handler(context, sessionFactory, std::move(authDNPattern), + std::move(headersAsAttributesPattern))); + _server->addHandler(basePath, _handler.get()); +} + +ListenHTTP::~ListenHTTP() { +} + +void ListenHTTP::onTrigger( + core::ProcessContext *context, + core::ProcessSession *session) { + + std::shared_ptr < FlowFileRecord > flowFile = std::static_pointer_cast + < FlowFileRecord > (session->get()); + + // Do nothing if there are no incoming files + if (!flowFile) { + return; + } +} + +ListenHTTP::Handler::Handler( + core::ProcessContext *context, + core::ProcessSessionFactory *sessionFactory, + std::string &&authDNPattern, std::string &&headersAsAttributesPattern) + : _authDNRegex(std::move(authDNPattern)), + _headersAsAttributesRegex(std::move(headersAsAttributesPattern)) { + _processContext = context; + _processSessionFactory = sessionFactory; +} + +void ListenHTTP::Handler::sendErrorResponse(struct mg_connection *conn) { + mg_printf(conn, "HTTP/1.1 500 Internal Server Error\r\n" + "Content-Type: text/html\r\n" + "Content-Length: 0\r\n\r\n"); +} + +bool ListenHTTP::Handler::handlePost(CivetServer *server, + struct mg_connection *conn) { + _logger = logging::Logger::getLogger(); + + auto req_info = mg_get_request_info(conn); + _logger->log_info("ListenHTTP handling POST request of length %d", + req_info->content_length); + + // If this is a two-way TLS connection, authorize the peer against the configured pattern + if (req_info->is_ssl && req_info->client_cert != nullptr) { + if (!std::regex_match(req_info->client_cert->subject, _authDNRegex)) { + mg_printf(conn, "HTTP/1.1 403 Forbidden\r\n" + "Content-Type: text/html\r\n" + "Content-Length: 0\r\n\r\n"); + _logger->log_warn("ListenHTTP client DN not authorized: %s", + req_info->client_cert->subject); + return true; + } + } + + // Always send 100 Continue, as allowed per standard to minimize client delay (https://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html) + mg_printf(conn, "HTTP/1.1 100 Continue\r\n\r\n"); + + auto session = _processSessionFactory->createSession(); + ListenHTTP::WriteCallback callback(conn, req_info); + auto flowFile = std::static_pointer_cast < FlowFileRecord + > (session->create()); + + if (!flowFile) { + sendErrorResponse(conn); + return true; + } + + try { + session->write(flowFile, &callback); + + // Add filename from "filename" header value (and pattern headers) + for (int i = 0; i < req_info->num_headers; i++) { + auto header = &req_info->http_headers[i]; + + if (strcmp("filename", header->name) == 0) { + if (!flowFile->updateAttribute("filename", header->value)) { + flowFile->addAttribute("filename", header->value); + } + } else if (std::regex_match(header->name, _headersAsAttributesRegex)) { + if (!flowFile->updateAttribute(header->name, header->value)) { + flowFile->addAttribute(header->name, header->value); + } + } + } + + session->transfer(flowFile, Success); + session->commit(); + } catch (std::exception &exception) { + _logger->log_debug("ListenHTTP Caught Exception %s", exception.what()); + sendErrorResponse(conn); + session->rollback(); + throw; + } catch (...) { + _logger->log_debug("ListenHTTP Caught Exception Processor::onTrigger"); + sendErrorResponse(conn); + session->rollback(); + throw; + } + + mg_printf(conn, "HTTP/1.1 200 OK\r\n" + "Content-Type: text/html\r\n" + "Content-Length: 0\r\n\r\n"); + + return true; +} + +ListenHTTP::WriteCallback::WriteCallback( + struct mg_connection *conn, const struct mg_request_info *reqInfo) { + _logger = logging::Logger::getLogger(); + _conn = conn; + _reqInfo = reqInfo; +} + +void ListenHTTP::WriteCallback::process(std::ofstream *stream) { + long long rlen; + long long nlen = 0; + long long tlen = _reqInfo->content_length; + char buf[16384]; + + while (nlen < tlen) { + rlen = tlen - nlen; + + if (rlen > sizeof(buf)) { + rlen = sizeof(buf); + } + + // Read a buffer of data from client + rlen = mg_read(_conn, &buf[0], (size_t) rlen); + + if (rlen <= 0) { + break; + } + + // Transfer buffer data to the output stream + stream->write(&buf[0], rlen); + + nlen += rlen; + } +} + +} /* namespace processors */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/processors/ListenSyslog.cpp b/libminifi/src/processors/ListenSyslog.cpp new file mode 100644 index 0000000000..2dd223cfea --- /dev/null +++ b/libminifi/src/processors/ListenSyslog.cpp @@ -0,0 +1,331 @@ +/** + * @file ListenSyslog.cpp + * ListenSyslog class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include "utils/TimeUtil.h" +#include "utils/StringUtils.h" +#include "processors/ListenSyslog.h" +#include "core/ProcessContext.h" +#include "core/ProcessSession.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +const std::string ListenSyslog::ProcessorName("ListenSyslog"); +core::Property ListenSyslog::RecvBufSize( + "Receive Buffer Size", + "The size of each buffer used to receive Syslog messages.", "65507 B"); +core::Property ListenSyslog::MaxSocketBufSize( + "Max Size of Socket Buffer", + "The maximum size of the socket buffer that should be used.", "1 MB"); +core::Property ListenSyslog::MaxConnections( + "Max Number of TCP Connections", + "The maximum number of concurrent connections to accept Syslog messages in TCP mode.", + "2"); +core::Property ListenSyslog::MaxBatchSize( + "Max Batch Size", + "The maximum number of Syslog events to add to a single FlowFile.", "1"); +core::Property ListenSyslog::MessageDelimiter( + "Message Delimiter", + "Specifies the delimiter to place between Syslog messages when multiple messages are bundled together (see core::Property).", + "\n"); +core::Property ListenSyslog::ParseMessages( + "Parse Messages", + "Indicates if the processor should parse the Syslog messages. If set to false, each outgoing FlowFile will only.", + "false"); +core::Property ListenSyslog::Protocol( + "Protocol", "The protocol for Syslog communication.", "UDP"); +core::Property ListenSyslog::Port( + "Port", "The port for Syslog communication.", "514"); +core::Relationship ListenSyslog::Success( + "success", "All files are routed to success"); +core::Relationship ListenSyslog::Invalid( + "invalid", "SysLog message format invalid"); + +void ListenSyslog::initialize() { + // Set the supported properties + std::set properties; + properties.insert(RecvBufSize); + properties.insert(MaxSocketBufSize); + properties.insert(MaxConnections); + properties.insert(MaxBatchSize); + properties.insert(MessageDelimiter); + properties.insert(ParseMessages); + properties.insert(Protocol); + properties.insert(Port); + setSupportedProperties(properties); + // Set the supported relationships + std::set relationships; + relationships.insert(Success); + relationships.insert(Invalid); + setSupportedRelationships(relationships); +} + +void ListenSyslog::startSocketThread() { + if (_thread != NULL) + return; + + logger_->log_info("ListenSysLog Socket Thread Start"); + _serverTheadRunning = true; + _thread = new std::thread(run, this); + _thread->detach(); +} + +void ListenSyslog::run(ListenSyslog *process) { + process->runThread(); +} + +void ListenSyslog::runThread() { + while (_serverTheadRunning) { + if (_resetServerSocket) { + _resetServerSocket = false; + // need to reset the socket + std::vector::iterator it; + for (it = _clientSockets.begin(); it != _clientSockets.end(); ++it) { + int clientSocket = *it; + close(clientSocket); + } + _clientSockets.clear(); + if (_serverSocket > 0) { + close(_serverSocket); + _serverSocket = 0; + } + } + + if (_serverSocket <= 0) { + uint16_t portno = _port; + struct sockaddr_in serv_addr; + int sockfd; + if (_protocol == "TCP") + sockfd = socket(AF_INET, SOCK_STREAM, 0); + else + sockfd = socket(AF_INET, SOCK_DGRAM, 0); + if (sockfd < 0) { + logger_->log_info("ListenSysLog Server socket creation failed"); + break; + } + bzero((char *) &serv_addr, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + serv_addr.sin_addr.s_addr = INADDR_ANY; + serv_addr.sin_port = htons(portno); + if (bind(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) { + logger_->log_error("ListenSysLog Server socket bind failed"); + break; + } + if (_protocol == "TCP") + listen(sockfd, 5); + _serverSocket = sockfd; + logger_->log_error("ListenSysLog Server socket %d bind OK to port %d", + _serverSocket, portno); + } + FD_ZERO(&_readfds); + FD_SET(_serverSocket, &_readfds); + _maxFds = _serverSocket; + std::vector::iterator it; + for (it = _clientSockets.begin(); it != _clientSockets.end(); ++it) { + int clientSocket = *it; + if (clientSocket >= _maxFds) + _maxFds = clientSocket; + FD_SET(clientSocket, &_readfds); + } + fd_set fds; + struct timeval tv; + int retval; + fds = _readfds; + tv.tv_sec = 0; + // 100 msec + tv.tv_usec = 100000; + retval = select(_maxFds + 1, &fds, NULL, NULL, &tv); + if (retval < 0) + break; + if (retval == 0) + continue; + if (FD_ISSET(_serverSocket, &fds)) { + // server socket, either we have UDP datagram or TCP connection request + if (_protocol == "TCP") { + socklen_t clilen; + struct sockaddr_in cli_addr; + clilen = sizeof(cli_addr); + int newsockfd = accept(_serverSocket, (struct sockaddr *) &cli_addr, + &clilen); + if (newsockfd > 0) { + if (_clientSockets.size() < _maxConnections) { + _clientSockets.push_back(newsockfd); + logger_->log_info("ListenSysLog new client socket %d connection", + newsockfd); + continue; + } else { + close(newsockfd); + } + } + } else { + socklen_t clilen; + struct sockaddr_in cli_addr; + clilen = sizeof(cli_addr); + int recvlen = recvfrom(_serverSocket, _buffer, sizeof(_buffer), 0, + (struct sockaddr *) &cli_addr, &clilen); + if (recvlen > 0 + && (recvlen + getEventQueueByteSize()) <= _recvBufSize) { + uint8_t *payload = new uint8_t[recvlen]; + memcpy(payload, _buffer, recvlen); + putEvent(payload, recvlen); + } + } + } + it = _clientSockets.begin(); + while (it != _clientSockets.end()) { + int clientSocket = *it; + if (FD_ISSET(clientSocket, &fds)) { + int recvlen = readline(clientSocket, (char *) _buffer, sizeof(_buffer)); + if (recvlen <= 0) { + close(clientSocket); + logger_->log_info("ListenSysLog client socket %d close", + clientSocket); + it = _clientSockets.erase(it); + } else { + if ((recvlen + getEventQueueByteSize()) <= _recvBufSize) { + uint8_t *payload = new uint8_t[recvlen]; + memcpy(payload, _buffer, recvlen); + putEvent(payload, recvlen); + } + ++it; + } + } + } + } + return; +} + +int ListenSyslog::readline(int fd, char *bufptr, size_t len) { + char *bufx = bufptr; + static char *bp; + static int cnt = 0; + static char b[2048]; + char c; + + while (--len > 0) { + if (--cnt <= 0) { + cnt = recv(fd, b, sizeof(b), 0); + if (cnt < 0) { + if ( errno == EINTR) { + len++; /* the while will decrement */ + continue; + } + return -1; + } + if (cnt == 0) + return 0; + bp = b; + } + c = *bp++; + *bufptr++ = c; + if (c == '\n') { + *bufptr = '\n'; + return bufptr - bufx + 1; + } + } + return -1; +} + +void ListenSyslog::onTrigger( + core::ProcessContext *context, + core::ProcessSession *session) { + std::string value; + bool needResetServerSocket = false; + if (context->getProperty(Protocol.getName(), value)) { + if (_protocol != value) + needResetServerSocket = true; + _protocol = value; + } + if (context->getProperty(RecvBufSize.getName(), value)) { + core::Property::StringToInt(value, _recvBufSize); + } + if (context->getProperty(MaxSocketBufSize.getName(), value)) { + core::Property::StringToInt(value, + _maxSocketBufSize); + } + if (context->getProperty(MaxConnections.getName(), value)) { + core::Property::StringToInt(value, + _maxConnections); + } + if (context->getProperty(MessageDelimiter.getName(), value)) { + _messageDelimiter = value; + } + if (context->getProperty(ParseMessages.getName(), value)) { + org::apache::nifi::minifi::utils::StringUtils::StringToBool(value, + _parseMessages); + } + if (context->getProperty(Port.getName(), value)) { + int64_t oldPort = _port; + core::Property::StringToInt(value, _port); + if (_port != oldPort) + needResetServerSocket = true; + } + if (context->getProperty(MaxBatchSize.getName(), value)) { + core::Property::StringToInt(value, + _maxBatchSize); + } + + if (needResetServerSocket) + _resetServerSocket = true; + + startSocketThread(); + + // read from the event queue + if (isEventQueueEmpty()) { + context->yield(); + return; + } + + std::queue eventQueue; + pollEvent(eventQueue, _maxBatchSize); + bool firstEvent = true; + std::shared_ptr flowFile = NULL; + while (!eventQueue.empty()) { + SysLogEvent event = eventQueue.front(); + eventQueue.pop(); + if (firstEvent) { + flowFile = std::static_pointer_cast(session->create()); + if (!flowFile) + return; + ListenSyslog::WriteCallback callback((char *) event.payload, event.len); + session->write(flowFile, &callback); + delete[] event.payload; + firstEvent = false; + } else { + ListenSyslog::WriteCallback callback((char *) event.payload, event.len); + session->append(flowFile, &callback); + delete[] event.payload; + } + } + flowFile->addAttribute("syslog.protocol", _protocol); + flowFile->addAttribute("syslog.port", std::to_string(_port)); + session->transfer(flowFile, Success); +} + +} /* namespace processors */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/processors/LogAttribute.cpp b/libminifi/src/processors/LogAttribute.cpp new file mode 100644 index 0000000000..e2cf16c73e --- /dev/null +++ b/libminifi/src/processors/LogAttribute.cpp @@ -0,0 +1,176 @@ +/** + * @file LogAttribute.cpp + * LogAttribute class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utils/TimeUtil.h" +#include "utils/StringUtils.h" +#include "processors/LogAttribute.h" +#include "core/ProcessContext.h" +#include "core/ProcessSession.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { +const std::string LogAttribute::ProcessorName("LogAttribute"); +core::Property LogAttribute::LogLevel( + "Log Level", "The Log Level to use when logging the Attributes", "info"); +core::Property LogAttribute::AttributesToLog( + "Attributes to Log", + "A comma-separated list of Attributes to Log. If not specified, all attributes will be logged.", + ""); +core::Property LogAttribute::AttributesToIgnore( + "Attributes to Ignore", + "A comma-separated list of Attributes to ignore. If not specified, no attributes will be ignored.", + ""); +core::Property LogAttribute::LogPayload( + "Log Payload", + "If true, the FlowFile's payload will be logged, in addition to its attributes; otherwise, just the Attributes will be logged.", + "false"); +core::Property LogAttribute::LogPrefix( + "Log prefix", + "Log prefix appended to the log lines. It helps to distinguish the output of multiple LogAttribute processors.", + ""); +core::Relationship LogAttribute::Success( + "success", "success operational on the flow record"); + +void LogAttribute::initialize() { + // Set the supported properties + std::set properties; + properties.insert(LogLevel); + properties.insert(AttributesToLog); + properties.insert(AttributesToIgnore); + properties.insert(LogPayload); + properties.insert(LogPrefix); + setSupportedProperties(properties); + // Set the supported relationships + std::set relationships; + relationships.insert(Success); + setSupportedRelationships(relationships); +} + +void LogAttribute::onTrigger( + core::ProcessContext *context, + core::ProcessSession *session) { + std::string dashLine = "--------------------------------------------------"; + LogAttrLevel level = LogAttrLevelInfo; + bool logPayload = false; + std::ostringstream message; + + std::shared_ptr flow = + session->get(); + + if (!flow) + return; + + std::string value; + if (context->getProperty(LogLevel.getName(), value)) { + logLevelStringToEnum(value, level); + } + if (context->getProperty(LogPrefix.getName(), value)) { + dashLine = "-----" + value + "-----"; + } + if (context->getProperty(LogPayload.getName(), value)) { + org::apache::nifi::minifi::utils::StringUtils::StringToBool(value, logPayload); + } + + message << "Logging for flow file " << "\n"; + message << dashLine; + message << "\nStandard FlowFile Attributes"; + message << "\n" << "UUID:" << flow->getUUIDStr(); + message << "\n" << "EntryDate:" << getTimeStr(flow->getEntryDate()); + message << "\n" << "lineageStartDate:" + << getTimeStr(flow->getlineageStartDate()); + message << "\n" << "Size:" << flow->getSize() << " Offset:" + << flow->getOffset(); + message << "\nFlowFile Attributes Map Content"; + std::map attrs = flow->getAttributes(); + std::map::iterator it; + for (it = attrs.begin(); it != attrs.end(); it++) { + message << "\n" << "key:" << it->first << " value:" << it->second; + } + message << "\nFlowFile Resource Claim Content"; + std::shared_ptr claim = flow->getResourceClaim(); + if (claim) { + message << "\n" << "Content Claim:" << claim->getContentFullPath(); + } + if (logPayload && flow->getSize() <= 1024 * 1024) { + message << "\n" << "Payload:" << "\n"; + ReadCallback callback(flow->getSize()); + session->read(flow, &callback); + for (unsigned int i = 0, j = 0; i < callback._readSize; i++) { + char temp[8]; + sprintf(temp, "%02x ", (unsigned char) (callback._buffer[i])); + message << temp; + j++; + if (j == 16) { + message << '\n'; + j = 0; + } + } + } + message << "\n" << dashLine << std::ends; + std::string output = message.str(); + + switch (level) { + case LogAttrLevelInfo: + logger_->log_info("%s", output.c_str()); + break; + case LogAttrLevelDebug: + logger_->log_debug("%s", output.c_str()); + break; + case LogAttrLevelError: + logger_->log_error("%s", output.c_str()); + break; + case LogAttrLevelTrace: + logger_->log_trace("%s", output.c_str()); + break; + case LogAttrLevelWarn: + logger_->log_warn("%s", output.c_str()); + break; + default: + break; + } + + // Test Import + /* + std::shared_ptr importRecord = session->create(); + session->import(claim->getContentFullPath(), importRecord); + session->transfer(importRecord, Success); */ + + // Transfer to the relationship + session->transfer(flow, Success); +} + + +} /* namespace processors */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/processors/PutFile.cpp b/libminifi/src/processors/PutFile.cpp new file mode 100644 index 0000000000..85cf09b420 --- /dev/null +++ b/libminifi/src/processors/PutFile.cpp @@ -0,0 +1,213 @@ +/** + * @file PutFile.cpp + * PutFile class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include + +#include "utils/StringUtils.h" +#include "utils/TimeUtil.h" +#include "processors/PutFile.h" +#include "core/ProcessContext.h" +#include "core/ProcessSession.h" + + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +const std::string PutFile::CONFLICT_RESOLUTION_STRATEGY_REPLACE("replace"); +const std::string PutFile::CONFLICT_RESOLUTION_STRATEGY_IGNORE("ignore"); +const std::string PutFile::CONFLICT_RESOLUTION_STRATEGY_FAIL("fail"); + +const std::string PutFile::ProcessorName("PutFile"); + +core::Property PutFile::Directory( + "Output Directory", "The output directory to which to put files", "."); +core::Property PutFile::ConflictResolution( + "Conflict Resolution Strategy", + "Indicates what should happen when a file with the same name already exists in the output directory", + CONFLICT_RESOLUTION_STRATEGY_FAIL); + +core::Relationship PutFile::Success( + "success", "All files are routed to success"); +core::Relationship PutFile::Failure( + "failure", + "Failed files (conflict, write failure, etc.) are transferred to failure"); + +void PutFile::initialize() { + // Set the supported properties + std::set properties; + properties.insert(Directory); + properties.insert(ConflictResolution); + setSupportedProperties(properties); + // Set the supported relationships + std::set relationships; + relationships.insert(Success); + relationships.insert(Failure); + setSupportedRelationships(relationships); +} + +void PutFile::onTrigger( + core::ProcessContext *context, + core::ProcessSession *session) { + std::string directory; + + if (!context->getProperty(Directory.getName(), directory)) { + logger_->log_error("Directory attribute is missing or invalid"); + return; + } + + std::string conflictResolution; + + if (!context->getProperty(ConflictResolution.getName(), conflictResolution)) { + logger_->log_error( + "Conflict Resolution Strategy attribute is missing or invalid"); + return; + } + + std::shared_ptr flowFile = std::static_pointer_cast(session->get()); + + // Do nothing if there are no incoming files + if (!flowFile) { + return; + } + + std::string filename; + flowFile->getKeyedAttribute(FILENAME, filename); + + // Generate a safe (universally-unique) temporary filename on the same partition + char tmpFileUuidStr[37]; + uuid_t tmpFileUuid; + uuid_generate(tmpFileUuid); + uuid_unparse_lower(tmpFileUuid, tmpFileUuidStr); + std::stringstream tmpFileSs; + tmpFileSs << directory << "/." << filename << "." << tmpFileUuidStr; + std::string tmpFile = tmpFileSs.str(); + logger_->log_info("PutFile using temporary file %s", tmpFile.c_str()); + + // Determine dest full file paths + std::stringstream destFileSs; + destFileSs << directory << "/" << filename; + std::string destFile = destFileSs.str(); + + logger_->log_info("PutFile writing file %s into directory %s", + filename.c_str(), directory.c_str()); + + // If file exists, apply conflict resolution strategy + struct stat statResult; + + if (stat(destFile.c_str(), &statResult) == 0) { + logger_->log_info( + "Destination file %s exists; applying Conflict Resolution Strategy: %s", + destFile.c_str(), conflictResolution.c_str()); + + if (conflictResolution == CONFLICT_RESOLUTION_STRATEGY_REPLACE) { + putFile(session, flowFile, tmpFile, destFile); + } else if (conflictResolution == CONFLICT_RESOLUTION_STRATEGY_IGNORE) { + session->transfer(flowFile, Success); + } else { + session->transfer(flowFile, Failure); + } + } else { + putFile(session, flowFile, tmpFile, destFile); + } +} + +bool PutFile::putFile(core::ProcessSession *session, + std::shared_ptr flowFile, const std::string &tmpFile, + const std::string &destFile) { + + ReadCallback cb(tmpFile, destFile); + session->read(flowFile, &cb); + + if (cb.commit()) { + session->transfer(flowFile, Success); + return true; + } else { + session->transfer(flowFile, Failure); + } + return false; +} + +PutFile::ReadCallback::ReadCallback(const std::string &tmpFile, + const std::string &destFile) + : _tmpFile(tmpFile), + _tmpFileOs(tmpFile), + _destFile(destFile) { + logger_ = logging::Logger::getLogger(); +} + +// Copy the entire file contents to the temporary file +void PutFile::ReadCallback::process(std::ifstream *stream) { + // Copy file contents into tmp file + _writeSucceeded = false; + _tmpFileOs << stream->rdbuf(); + _writeSucceeded = true; +} + +// Renames tmp file to final destination +// Returns true if commit succeeded +bool PutFile::ReadCallback::commit() { + bool success = false; + + logger_->log_info("PutFile committing put file operation to %s", + _destFile.c_str()); + + if (_writeSucceeded) { + _tmpFileOs.close(); + + if (rename(_tmpFile.c_str(), _destFile.c_str())) { + logger_->log_info( + "PutFile commit put file operation to %s failed because rename() call failed", + _destFile.c_str()); + } else { + success = true; + logger_->log_info("PutFile commit put file operation to %s succeeded", + _destFile.c_str()); + } + } else { + logger_->log_error( + "PutFile commit put file operation to %s failed because write failed", + _destFile.c_str()); + } + + return success; +} + +// Clean up resources +PutFile::ReadCallback::~ReadCallback() { + // Close tmp file + _tmpFileOs.close(); + + // Clean up tmp file, if necessary + unlink(_tmpFile.c_str()); +} + + +} /* namespace processors */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/processors/RealTimeDataCollector.cpp b/libminifi/src/processors/RealTimeDataCollector.cpp new file mode 100644 index 0000000000..922835d4b4 --- /dev/null +++ b/libminifi/src/processors/RealTimeDataCollector.cpp @@ -0,0 +1,480 @@ +/** + * @file RealTimeDataCollector.cpp + * RealTimeDataCollector class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utils/StringUtils.h" +#include "processors/RealTimeDataCollector.h" +#include "core/ProcessContext.h" +#include "core/ProcessSession.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { +const std::string RealTimeDataCollector::ProcessorName("RealTimeDataCollector"); +core::Property RealTimeDataCollector::FILENAME( + "File Name", "File Name for the real time processor to process", + "data.osp"); +core::Property RealTimeDataCollector::REALTIMESERVERNAME( + "Real Time Server Name", "Real Time Server Name", "localhost"); +core::Property RealTimeDataCollector::REALTIMESERVERPORT( + "Real Time Server Port", "Real Time Server Port", "10000"); +core::Property RealTimeDataCollector::BATCHSERVERNAME( + "Batch Server Name", "Batch Server Name", "localhost"); +core::Property RealTimeDataCollector::BATCHSERVERPORT( + "Batch Server Port", "Batch Server Port", "10001"); +core::Property RealTimeDataCollector::ITERATION( + "Iteration", "If true, sample osp file will be iterated", "true"); +core::Property RealTimeDataCollector::REALTIMEMSGID( + "Real Time Message ID", "Real Time Message ID", "41"); +core::Property RealTimeDataCollector::BATCHMSGID( + "Batch Message ID", "Batch Message ID", "172, 30, 48"); +core::Property RealTimeDataCollector::REALTIMEINTERVAL( + "Real Time Interval", "Real Time Data Collection Interval in msec", + "10 ms"); +core::Property RealTimeDataCollector::BATCHINTERVAL( + "Batch Time Interval", "Batch Processing Interval in msec", "100 ms"); +core::Property RealTimeDataCollector::BATCHMAXBUFFERSIZE( + "Batch Max Buffer Size", "Batch Buffer Maximum size in bytes", "262144"); +core::Relationship RealTimeDataCollector::Success( + "success", "success operational on the flow record"); + +void RealTimeDataCollector::initialize() { + // Set the supported properties + std::set properties; + properties.insert(FILENAME); + properties.insert(REALTIMESERVERNAME); + properties.insert(REALTIMESERVERPORT); + properties.insert(BATCHSERVERNAME); + properties.insert(BATCHSERVERPORT); + properties.insert(ITERATION); + properties.insert(REALTIMEMSGID); + properties.insert(BATCHMSGID); + properties.insert(REALTIMEINTERVAL); + properties.insert(BATCHINTERVAL); + properties.insert(BATCHMAXBUFFERSIZE); + + setSupportedProperties(properties); + // Set the supported relationships + std::set relationships; + relationships.insert(Success); + setSupportedRelationships(relationships); + +} + +int RealTimeDataCollector::connectServer(const char *host, uint16_t port) { + in_addr_t addr; + int sock = 0; + struct hostent *h; +#ifdef __MACH__ + h = gethostbyname(host); +#else + char buf[1024]; + struct hostent he; + int hh_errno; + gethostbyname_r(host, &he, buf, sizeof(buf), &h, &hh_errno); +#endif + memcpy((char *) &addr, h->h_addr_list[0], h->h_length); + sock = socket(AF_INET, SOCK_STREAM, 0); + if (sock < 0) { + logger_->log_error("Could not create socket to hostName %s", host); + return 0; + } + +#ifndef __MACH__ + int opt = 1; + bool nagle_off = true; + + if (nagle_off) + { + if (setsockopt(sock, SOL_TCP, TCP_NODELAY, (void *)&opt, sizeof(opt)) < 0) + { + logger_->log_error("setsockopt() TCP_NODELAY failed"); + close(sock); + return 0; + } + if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, + (char *)&opt, sizeof(opt)) < 0) + { + logger_->log_error("setsockopt() SO_REUSEADDR failed"); + close(sock); + return 0; + } + } + + int sndsize = 256*1024; + if (setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (char *)&sndsize, (int)sizeof(sndsize)) < 0) + { + logger_->log_error("setsockopt() SO_SNDBUF failed"); + close(sock); + return 0; + } +#endif + + struct sockaddr_in sa; + socklen_t socklen; + int status; + + //TODO bind socket to the interface + memset(&sa, 0, sizeof(sa)); + sa.sin_family = AF_INET; + sa.sin_addr.s_addr = htonl(INADDR_ANY); + sa.sin_port = htons(0); + socklen = sizeof(sa); + if (bind(sock, (struct sockaddr *) &sa, socklen) < 0) { + logger_->log_error("socket bind failed"); + close(sock); + return 0; + } + + memset(&sa, 0, sizeof(sa)); + sa.sin_family = AF_INET; + sa.sin_addr.s_addr = addr; + sa.sin_port = htons(port); + socklen = sizeof(sa); + + status = connect(sock, (struct sockaddr *) &sa, socklen); + + if (status < 0) { + logger_->log_error("socket connect failed to %s %d", host, port); + close(sock); + return 0; + } + + logger_->log_info("socket %d connect to server %s port %d success", sock, + host, port); + + return sock; +} + +int RealTimeDataCollector::sendData(int socket, const char *buf, int buflen) { + int ret = 0, bytes = 0; + + while (bytes < buflen) { + ret = send(socket, buf + bytes, buflen - bytes, 0); + //check for errors + if (ret == -1) { + return ret; + } + bytes += ret; + } + + if (ret) + logger_->log_debug("Send data size %d over socket %d", buflen, socket); + + return ret; +} + +void RealTimeDataCollector::onTriggerRealTime( + core::ProcessContext *context, + core::ProcessSession *session) { + if (_realTimeAccumulated >= this->_realTimeInterval) { + std::string value; + if (this->getProperty(REALTIMEMSGID.getName(), value)) { + this->_realTimeMsgID.clear(); + this->logger_->log_info("Real Time Msg IDs %s", value.c_str()); + std::stringstream lineStream(value); + std::string cell; + + while (std::getline(lineStream, cell, ',')) { + this->_realTimeMsgID.push_back(cell); + // this->logger_->log_debug("Real Time Msg ID %s", cell.c_str()); + } + } + if (this->getProperty(BATCHMSGID.getName(), value)) { + this->_batchMsgID.clear(); + this->logger_->log_info("Batch Msg IDs %s", value.c_str()); + std::stringstream lineStream(value); + std::string cell; + + while (std::getline(lineStream, cell, ',')) { + cell = org::apache::nifi::minifi::utils::StringUtils::trim(cell); + this->_batchMsgID.push_back(cell); + } + } + // Open the file + if (!this->_fileStream.is_open()) { + _fileStream.open(this->_fileName.c_str(), std::ifstream::in); + if (this->_fileStream.is_open()) + logger_->log_debug("open %s", _fileName.c_str()); + } + if (!_fileStream.good()) { + logger_->log_error("load data file failed %s", _fileName.c_str()); + return; + } + if (this->_fileStream.is_open()) { + std::string line; + + while (std::getline(_fileStream, line)) { + line += "\n"; + std::stringstream lineStream(line); + std::string cell; + if (std::getline(lineStream, cell, ',')) { + cell = org::apache::nifi::minifi::utils::StringUtils::trim(cell); + // Check whether it match to the batch traffic + for (std::vector::iterator it = _batchMsgID.begin(); + it != _batchMsgID.end(); ++it) { + if (cell == *it) { + // push the batch data to the queue + std::lock_guard lock(mutex_); + while ((_queuedDataSize + line.size()) > _batchMaxBufferSize) { + std::string item = _queue.front(); + _queuedDataSize -= item.size(); + logger_->log_debug( + "Pop item size %d from batch queue, queue buffer size %d", + item.size(), _queuedDataSize); + _queue.pop(); + } + _queue.push(line); + _queuedDataSize += line.size(); + logger_->log_debug( + "Push batch msg ID %s into batch queue, queue buffer size %d", + cell.c_str(), _queuedDataSize); + } + } + bool findRealTime = false; + // Check whether it match to the real time traffic + for (std::vector::iterator it = _realTimeMsgID.begin(); + it != _realTimeMsgID.end(); ++it) { + if (cell == *it) { + int status = 0; + if (this->_realTimeSocket <= 0) { + // Connect the LTE socket + uint16_t port = _realTimeServerPort; + this->_realTimeSocket = connectServer( + _realTimeServerName.c_str(), port); + } + if (this->_realTimeSocket) { + // try to send the data + status = sendData(_realTimeSocket, line.data(), line.size()); + if (status < 0) { + close(_realTimeSocket); + _realTimeSocket = 0; + } + } + if (this->_realTimeSocket <= 0 || status < 0) { + // push the batch data to the queue + std::lock_guard lock(mutex_); + while ((_queuedDataSize + line.size()) > _batchMaxBufferSize) { + std::string item = _queue.front(); + _queuedDataSize -= item.size(); + logger_->log_debug( + "Pop item size %d from batch queue, queue buffer size %d", + item.size(), _queuedDataSize); + _queue.pop(); + } + _queue.push(line); + _queuedDataSize += line.size(); + logger_->log_debug( + "Push real time msg ID %s into batch queue, queue buffer size %d", + cell.c_str(), _queuedDataSize); + } + // find real time + findRealTime = true; + } // cell + } // for real time pattern + if (findRealTime) + // we break the while once we find the first real time + break; + } // if get line + } // while + if (_fileStream.eof()) { + _fileStream.close(); + } + } // if open + _realTimeAccumulated = 0; + } + std::shared_ptr processor = std::static_pointer_cast( + context->getProcessorNode().getProcessor()); + _realTimeAccumulated += processor->getSchedulingPeriodNano(); +} + +void RealTimeDataCollector::onTriggerBatch( + core::ProcessContext *context, + core::ProcessSession *session) { + if (_batchAcccumulated >= this->_batchInterval) { + // logger_->log_info("onTriggerBatch"); + // dequeue the batch and send over WIFI + int status = 0; + if (this->_batchSocket <= 0) { + // Connect the WIFI socket + uint16_t port = _batchServerPort; + this->_batchSocket = connectServer(_batchServerName.c_str(), port); + } + if (this->_batchSocket) { + std::lock_guard lock(mutex_); + + while (!_queue.empty()) { + std::string line = _queue.front(); + status = sendData(_batchSocket, line.data(), line.size()); + _queue.pop(); + _queuedDataSize -= line.size(); + if (status < 0) { + close(_batchSocket); + _batchSocket = 0; + break; + } + } + } + _batchAcccumulated = 0; + } + std::shared_ptr processor = std::static_pointer_cast( + context->getProcessorNode().getProcessor()); + _batchAcccumulated += processor->getSchedulingPeriodNano(); +} + +void RealTimeDataCollector::onTrigger( + core::ProcessContext *context, + core::ProcessSession *session) { + std::thread::id id = std::this_thread::get_id(); + + if (id == _realTimeThreadId) + return onTriggerRealTime(context, session); + else if (id == _batchThreadId) + return onTriggerBatch(context, session); + else { + std::lock_guard lock(mutex_); + if (!this->_firstInvoking) { + this->_fileName = "data.osp"; + std::string value; + if (this->getProperty(FILENAME.getName(), value)) { + this->_fileName = value; + this->logger_->log_info("Data Collector File Name %s", + _fileName.c_str()); + } + this->_realTimeServerName = "localhost"; + if (this->getProperty(REALTIMESERVERNAME.getName(), value)) { + this->_realTimeServerName = value; + this->logger_->log_info("Real Time Server Name %s", + this->_realTimeServerName.c_str()); + } + this->_realTimeServerPort = 10000; + if (this->getProperty(REALTIMESERVERPORT.getName(), value)) { + core::Property::StringToInt( + value, _realTimeServerPort); + this->logger_->log_info("Real Time Server Port %d", + _realTimeServerPort); + } + if (this->getProperty(BATCHSERVERNAME.getName(), value)) { + this->_batchServerName = value; + this->logger_->log_info("Batch Server Name %s", + this->_batchServerName.c_str()); + } + this->_batchServerPort = 10001; + if (this->getProperty(BATCHSERVERPORT.getName(), value)) { + core::Property::StringToInt( + value, _batchServerPort); + this->logger_->log_info("Batch Server Port %d", _batchServerPort); + } + if (this->getProperty(ITERATION.getName(), value)) { + org::apache::nifi::minifi::utils::StringUtils::StringToBool(value, this->_iteration); + logger_->log_info("Iteration %d", _iteration); + } + this->_realTimeInterval = 10000000; //10 msec + if (this->getProperty(REALTIMEINTERVAL.getName(), value)) { + core::TimeUnit unit; + if (core::Property::StringToTime( + value, _realTimeInterval, unit) + && core::Property::ConvertTimeUnitToNS( + _realTimeInterval, unit, _realTimeInterval)) { + logger_->log_info("Real Time Interval: [%d] ns", _realTimeInterval); + } + } + this->_batchInterval = 100000000; //100 msec + if (this->getProperty(BATCHINTERVAL.getName(), value)) { + core::TimeUnit unit; + if (core::Property::StringToTime( + value, _batchInterval, unit) + && core::Property::ConvertTimeUnitToNS( + _batchInterval, unit, _batchInterval)) { + logger_->log_info("Batch Time Interval: [%d] ns", _batchInterval); + } + } + this->_batchMaxBufferSize = 256 * 1024; + if (this->getProperty(BATCHMAXBUFFERSIZE.getName(), value)) { + core::Property::StringToInt( + value, _batchMaxBufferSize); + this->logger_->log_info("Batch Max Buffer Size %d", + _batchMaxBufferSize); + } + if (this->getProperty(REALTIMEMSGID.getName(), value)) { + this->logger_->log_info("Real Time Msg IDs %s", value.c_str()); + std::stringstream lineStream(value); + std::string cell; + + while (std::getline(lineStream, cell, ',')) { + this->_realTimeMsgID.push_back(cell); + this->logger_->log_info("Real Time Msg ID %s", cell.c_str()); + } + } + if (this->getProperty(BATCHMSGID.getName(), value)) { + this->logger_->log_info("Batch Msg IDs %s", value.c_str()); + std::stringstream lineStream(value); + std::string cell; + + while (std::getline(lineStream, cell, ',')) { + cell = org::apache::nifi::minifi::utils::StringUtils::trim(cell); + this->_batchMsgID.push_back(cell); + this->logger_->log_info("Batch Msg ID %s", cell.c_str()); + } + } + // Connect the LTE socket + uint16_t port = _realTimeServerPort; + + this->_realTimeSocket = connectServer(_realTimeServerName.c_str(), port); + + // Connect the WIFI socket + port = _batchServerPort; + + this->_batchSocket = connectServer(_batchServerName.c_str(), port); + + // Open the file + _fileStream.open(this->_fileName.c_str(), std::ifstream::in); + if (!_fileStream.good()) { + logger_->log_error("load data file failed %s", _fileName.c_str()); + return; + } else { + logger_->log_debug("open %s", _fileName.c_str()); + } + _realTimeThreadId = id; + this->_firstInvoking = true; + } else { + if (id != _realTimeThreadId) + _batchThreadId = id; + this->_firstInvoking = false; + } + } +} +} /* namespace processors */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/processors/TailFile.cpp b/libminifi/src/processors/TailFile.cpp new file mode 100644 index 0000000000..859daa6bdf --- /dev/null +++ b/libminifi/src/processors/TailFile.cpp @@ -0,0 +1,271 @@ +/** + * @file TailFile.cpp + * TailFile class implementation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utils/TimeUtil.h" +#include "utils/StringUtils.h" +#include "processors/TailFile.h" +#include "core/ProcessContext.h" +#include "core/ProcessSession.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace processors { + +const std::string TailFile::ProcessorName("TailFile"); +core::Property TailFile::FileName( + "File to Tail", + "Fully-qualified filename of the file that should be tailed", ""); +core::Property TailFile::StateFile( + "State File", + "Specifies the file that should be used for storing state about what data has been ingested so that upon restart NiFi can resume from where it left off", + "TailFileState"); +core::Relationship TailFile::Success( + "success", "All files are routed to success"); + +void TailFile::initialize() { + // Set the supported properties + std::set properties; + properties.insert(FileName); + properties.insert(StateFile); + setSupportedProperties(properties); + // Set the supported relationships + std::set relationships; + relationships.insert(Success); + setSupportedRelationships(relationships); +} + +std::string TailFile::trimLeft(const std::string& s) { + return org::apache::nifi::minifi::utils::StringUtils::trimLeft(s); +} + +std::string TailFile::trimRight(const std::string& s) { + return org::apache::nifi::minifi::utils::StringUtils::trimRight(s); +} + +void TailFile::parseStateFileLine(char *buf) { + char *line = buf; + + while ((line[0] == ' ') || (line[0] == '\t')) + ++line; + + char first = line[0]; + if ((first == '\0') || (first == '#') || (first == '\r') || (first == '\n') + || (first == '=')) { + return; + } + + char *equal = strchr(line, '='); + if (equal == NULL) { + return; + } + + equal[0] = '\0'; + std::string key = line; + + equal++; + while ((equal[0] == ' ') || (equal[0] == '\t')) + ++equal; + + first = equal[0]; + if ((first == '\0') || (first == '\r') || (first == '\n')) { + return; + } + + std::string value = equal; + key = trimRight(key); + value = trimRight(value); + + if (key == "FILENAME") + this->_currentTailFileName = value; + if (key == "POSITION") + this->_currentTailFilePosition = std::stoi(value); + + return; +} + +void TailFile::recoverState() { + std::ifstream file(_stateFile.c_str(), std::ifstream::in); + if (!file.good()) { + logger_->log_error("load state file failed %s", _stateFile.c_str()); + return; + } + const unsigned int bufSize = 512; + char buf[bufSize]; + for (file.getline(buf, bufSize); file.good(); file.getline(buf, bufSize)) { + parseStateFileLine(buf); + } +} + +void TailFile::storeState() { + std::ofstream file(_stateFile.c_str()); + if (!file.is_open()) { + logger_->log_error("store state file failed %s", _stateFile.c_str()); + return; + } + file << "FILENAME=" << this->_currentTailFileName << "\n"; + file << "POSITION=" << this->_currentTailFilePosition << "\n"; + file.close(); +} + +static bool sortTailMatchedFileItem(TailMatchedFileItem i, + TailMatchedFileItem j) { + return (i.modifiedTime < j.modifiedTime); +} +void TailFile::checkRollOver() { + struct stat statbuf; + std::vector matchedFiles; + std::string fullPath = this->_fileLocation + "/" + _currentTailFileName; + + if (stat(fullPath.c_str(), &statbuf) == 0) { + if (statbuf.st_size > this->_currentTailFilePosition) + // there are new input for the current tail file + return; + + uint64_t modifiedTimeCurrentTailFile = + ((uint64_t) (statbuf.st_mtime) * 1000); + std::string pattern = _fileName; + std::size_t found = _fileName.find_last_of("."); + if (found != std::string::npos) + pattern = _fileName.substr(0, found); + DIR *d; + d = opendir(this->_fileLocation.c_str()); + if (!d) + return; + while (1) { + struct dirent *entry; + entry = readdir(d); + if (!entry) + break; + std::string d_name = entry->d_name; + if (!(entry->d_type & DT_DIR)) { + std::string fileName = d_name; + std::string fileFullName = this->_fileLocation + "/" + d_name; + if (fileFullName.find(pattern) != std::string::npos + && stat(fileFullName.c_str(), &statbuf) == 0) { + if (((uint64_t) (statbuf.st_mtime) * 1000) + >= modifiedTimeCurrentTailFile) { + TailMatchedFileItem item; + item.fileName = fileName; + item.modifiedTime = ((uint64_t) (statbuf.st_mtime) * 1000); + matchedFiles.push_back(item); + } + } + } + } + closedir(d); + + // Sort the list based on modified time + std::sort(matchedFiles.begin(), matchedFiles.end(), + sortTailMatchedFileItem); + for (std::vector::iterator it = matchedFiles.begin(); + it != matchedFiles.end(); ++it) { + TailMatchedFileItem item = *it; + if (item.fileName == _currentTailFileName) { + ++it; + if (it != matchedFiles.end()) { + TailMatchedFileItem nextItem = *it; + logger_->log_info("TailFile File Roll Over from %s to %s", + _currentTailFileName.c_str(), + nextItem.fileName.c_str()); + _currentTailFileName = nextItem.fileName; + _currentTailFilePosition = 0; + storeState(); + } + break; + } + } + } else + return; +} + +void TailFile::onTrigger( + core::ProcessContext *context, + core::ProcessSession *session) { + std::string value; + if (context->getProperty(FileName.getName(), value)) { + std::size_t found = value.find_last_of("/\\"); + this->_fileLocation = value.substr(0, found); + this->_fileName = value.substr(found + 1); + } + if (context->getProperty(StateFile.getName(), value)) { + _stateFile = value + "." + getUUIDStr(); + } + if (!this->_stateRecovered) { + _stateRecovered = true; + this->_currentTailFileName = _fileName; + this->_currentTailFilePosition = 0; + // recover the state if we have not done so + this->recoverState(); + } + checkRollOver(); + std::string fullPath = this->_fileLocation + "/" + _currentTailFileName; + struct stat statbuf; + if (stat(fullPath.c_str(), &statbuf) == 0) { + if (statbuf.st_size <= this->_currentTailFilePosition) + // there are no new input for the current tail file + { + context->yield(); + return; + } + std::shared_ptr flowFile = std::static_pointer_cast(session->create());; + if (!flowFile) + return; + std::size_t found = _currentTailFileName.find_last_of("."); + std::string baseName = _currentTailFileName.substr(0, found); + std::string extension = _currentTailFileName.substr(found + 1); + flowFile->updateKeyedAttribute(PATH, _fileLocation); + flowFile->addKeyedAttribute(ABSOLUTE_PATH, fullPath); + session->import(fullPath, flowFile, true, this->_currentTailFilePosition); + session->transfer(flowFile, Success); + logger_->log_info("TailFile %s for %d bytes", _currentTailFileName.c_str(), + flowFile->getSize()); + std::string logName = baseName + "." + + std::to_string(_currentTailFilePosition) + "-" + + std::to_string(_currentTailFilePosition + flowFile->getSize()) + "." + + extension; + flowFile->updateKeyedAttribute(FILENAME, logName); + this->_currentTailFilePosition += flowFile->getSize(); + storeState(); + } +} + + +} /* namespace processors */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/provenance/Provenance.cpp b/libminifi/src/provenance/Provenance.cpp new file mode 100644 index 0000000000..a90e182b35 --- /dev/null +++ b/libminifi/src/provenance/Provenance.cpp @@ -0,0 +1,578 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include "io/DataStream.h" +#include "io/Serializable.h" +#include "provenance/Provenance.h" + +#include "core/logging/Logger.h" +#include "core/Relationship.h" +#include "FlowController.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace provenance { + +// DeSerialize +bool ProvenanceEventRecord::DeSerialize( + const std::shared_ptr &repo, std::string key) { + std::string value; + bool ret; + + ret = repo->Get(key, value); + + if (!ret) { + logger_->log_error("NiFi Provenance Store event %s can not found", + key.c_str()); + return false; + } else + logger_->log_debug("NiFi Provenance Read event %s length %d", key.c_str(), + value.length()); + + org::apache::nifi::minifi::io::DataStream stream( + (const uint8_t*) value.data(), value.length()); + + ret = DeSerialize(stream); + + if (ret) { + logger_->log_debug( + "NiFi Provenance retrieve event %s size %d eventType %d success", + _eventIdStr.c_str(), stream.getSize(), _eventType); + } else { + logger_->log_debug( + "NiFi Provenance retrieve event %s size %d eventType %d fail", + _eventIdStr.c_str(), stream.getSize(), _eventType); + } + + return ret; +} + +bool ProvenanceEventRecord::Serialize( + const std::shared_ptr &repo) { + + org::apache::nifi::minifi::io::DataStream outStream; + + int ret; + + ret = writeUTF(this->_eventIdStr, &outStream); + if (ret <= 0) { + + return false; + } + + uint32_t eventType = this->_eventType; + ret = write(eventType, &outStream); + if (ret != 4) { + + return false; + } + + ret = write(this->_eventTime, &outStream); + if (ret != 8) { + + return false; + } + + ret = write(this->_entryDate, &outStream); + if (ret != 8) { + return false; + } + + ret = write(this->_eventDuration, &outStream); + if (ret != 8) { + + return false; + } + + ret = write(this->_lineageStartDate, &outStream); + if (ret != 8) { + + return false; + } + + ret = writeUTF(this->_componentId, &outStream); + if (ret <= 0) { + + return false; + } + + ret = writeUTF(this->_componentType, &outStream); + if (ret <= 0) { + + return false; + } + + ret = writeUTF(this->uuid_, &outStream); + if (ret <= 0) { + + return false; + } + + ret = writeUTF(this->_details, &outStream); + if (ret <= 0) { + + return false; + } + + // write flow attributes + uint32_t numAttributes = this->_attributes.size(); + ret = write(numAttributes, &outStream); + if (ret != 4) { + + return false; + } + + for (auto itAttribute : _attributes) { + ret = writeUTF(itAttribute.first, &outStream, true); + if (ret <= 0) { + + return false; + } + ret = writeUTF(itAttribute.second, &outStream, true); + if (ret <= 0) { + + return false; + } + } + + ret = writeUTF(this->_contentFullPath, &outStream); + if (ret <= 0) { + + return false; + } + + ret = write(this->_size, &outStream); + if (ret != 8) { + + return false; + } + + ret = write(this->_offset, &outStream); + if (ret != 8) { + + return false; + } + + ret = writeUTF(this->_sourceQueueIdentifier, &outStream); + if (ret <= 0) { + + return false; + } + + if (this->_eventType == ProvenanceEventRecord::FORK + || this->_eventType == ProvenanceEventRecord::CLONE + || this->_eventType == ProvenanceEventRecord::JOIN) { + // write UUIDs + uint32_t number = this->_parentUuids.size(); + ret = write(number, &outStream); + if (ret != 4) { + + return false; + } + for (auto parentUUID : _parentUuids) { + ret = writeUTF(parentUUID, &outStream); + if (ret <= 0) { + + return false; + } + } + number = this->_childrenUuids.size(); + ret = write(number, &outStream); + if (ret != 4) { + return false; + } + for (auto childUUID : _childrenUuids) { + ret = writeUTF(childUUID, &outStream); + if (ret <= 0) { + + return false; + } + } + } else if (this->_eventType == ProvenanceEventRecord::SEND + || this->_eventType == ProvenanceEventRecord::FETCH) { + ret = writeUTF(this->_transitUri, &outStream); + if (ret <= 0) { + + return false; + } + } else if (this->_eventType == ProvenanceEventRecord::RECEIVE) { + ret = writeUTF(this->_transitUri, &outStream); + if (ret <= 0) { + + return false; + } + ret = writeUTF(this->_sourceSystemFlowFileIdentifier, &outStream); + if (ret <= 0) { + + return false; + } + } + + // Persistent to the DB + + if (repo->Put(_eventIdStr, const_cast(outStream.getBuffer()), + outStream.getSize())) { + logger_->log_debug("NiFi Provenance Store event %s size %d success", + _eventIdStr.c_str(), outStream.getSize()); + } else { + logger_->log_error("NiFi Provenance Store event %s size %d fail", + _eventIdStr.c_str(), outStream.getSize()); + } + + // cleanup + + return true; +} + +bool ProvenanceEventRecord::DeSerialize(const uint8_t *buffer, + const int bufferSize) { + + int ret; + + org::apache::nifi::minifi::io::DataStream outStream(buffer, bufferSize); + + ret = readUTF(this->_eventIdStr, &outStream); + + if (ret <= 0) { + return false; + } + + uint32_t eventType; + ret = read(eventType, &outStream); + if (ret != 4) { + return false; + } + this->_eventType = (ProvenanceEventRecord::ProvenanceEventType) eventType; + + ret = read(this->_eventTime, &outStream); + if (ret != 8) { + return false; + } + + ret = read(this->_entryDate, &outStream); + if (ret != 8) { + return false; + } + + ret = read(this->_eventDuration, &outStream); + if (ret != 8) { + return false; + } + + ret = read(this->_lineageStartDate, &outStream); + if (ret != 8) { + return false; + } + + ret = readUTF(this->_componentId, &outStream); + if (ret <= 0) { + return false; + } + + ret = readUTF(this->_componentType, &outStream); + if (ret <= 0) { + return false; + } + + ret = readUTF(this->uuid_, &outStream); + if (ret <= 0) { + return false; + } + + ret = readUTF(this->_details, &outStream); + + if (ret <= 0) { + return false; + } + + // read flow attributes + uint32_t numAttributes = 0; + ret = read(numAttributes, &outStream); + if (ret != 4) { + return false; + } + + for (uint32_t i = 0; i < numAttributes; i++) { + std::string key; + ret = readUTF(key, &outStream, true); + if (ret <= 0) { + return false; + } + std::string value; + ret = readUTF(value, &outStream, true); + if (ret <= 0) { + return false; + } + this->_attributes[key] = value; + } + + ret = readUTF(this->_contentFullPath, &outStream); + if (ret <= 0) { + return false; + } + + ret = read(this->_size, &outStream); + if (ret != 8) { + return false; + } + + ret = read(this->_offset, &outStream); + if (ret != 8) { + return false; + } + + ret = readUTF(this->_sourceQueueIdentifier, &outStream); + if (ret <= 0) { + return false; + } + + if (this->_eventType == ProvenanceEventRecord::FORK + || this->_eventType == ProvenanceEventRecord::CLONE + || this->_eventType == ProvenanceEventRecord::JOIN) { + // read UUIDs + uint32_t number = 0; + ret = read(number, &outStream); + if (ret != 4) { + return false; + } + + for (uint32_t i = 0; i < number; i++) { + std::string parentUUID; + ret = readUTF(parentUUID, &outStream); + if (ret <= 0) { + return false; + } + this->addParentUuid(parentUUID); + } + number = 0; + ret = read(number, &outStream); + if (ret != 4) { + return false; + } + for (uint32_t i = 0; i < number; i++) { + std::string childUUID; + ret = readUTF(childUUID, &outStream); + if (ret <= 0) { + return false; + } + this->addChildUuid(childUUID); + } + } else if (this->_eventType == ProvenanceEventRecord::SEND + || this->_eventType == ProvenanceEventRecord::FETCH) { + ret = readUTF(this->_transitUri, &outStream); + if (ret <= 0) { + return false; + } + } else if (this->_eventType == ProvenanceEventRecord::RECEIVE) { + ret = readUTF(this->_transitUri, &outStream); + if (ret <= 0) { + return false; + } + ret = readUTF(this->_sourceSystemFlowFileIdentifier, &outStream); + if (ret <= 0) { + return false; + } + } + + return true; +} + +void ProvenanceReporter::commit() { + for (auto event : _events) { + if (!repo_->isFull()) { + event->Serialize(repo_); + } else { + logger_->log_debug("Provenance Repository is full"); + } + } +} + +void ProvenanceReporter::create(std::shared_ptr flow, + std::string detail) { + ProvenanceEventRecord *event = allocate(ProvenanceEventRecord::CREATE, flow); + + if (event) { + event->setDetails(detail); + add(event); + } +} + +void ProvenanceReporter::route(std::shared_ptr flow, + core::Relationship relation, std::string detail, + uint64_t processingDuration) { + ProvenanceEventRecord *event = allocate(ProvenanceEventRecord::ROUTE, flow); + + if (event) { + event->setDetails(detail); + event->setRelationship(relation.getName()); + event->setEventDuration(processingDuration); + add(event); + } +} + +void ProvenanceReporter::modifyAttributes(std::shared_ptr flow, + std::string detail) { + ProvenanceEventRecord *event = allocate( + ProvenanceEventRecord::ATTRIBUTES_MODIFIED, flow); + + if (event) { + event->setDetails(detail); + add(event); + } +} + +void ProvenanceReporter::modifyContent(std::shared_ptr flow, + std::string detail, + uint64_t processingDuration) { + ProvenanceEventRecord *event = allocate( + ProvenanceEventRecord::CONTENT_MODIFIED, flow); + + if (event) { + event->setDetails(detail); + event->setEventDuration(processingDuration); + add(event); + } +} + +void ProvenanceReporter::clone(std::shared_ptr parent, + std::shared_ptr child) { + ProvenanceEventRecord *event = allocate(ProvenanceEventRecord::CLONE, parent); + + if (event) { + event->addChildFlowFile(child); + event->addParentFlowFile(parent); + add(event); + } +} + +void ProvenanceReporter::join( + std::vector > parents, + std::shared_ptr child, std::string detail, + uint64_t processingDuration) { + ProvenanceEventRecord *event = allocate(ProvenanceEventRecord::JOIN, child); + + if (event) { + event->addChildFlowFile(child); + std::vector >::iterator it; + for (it = parents.begin(); it != parents.end(); it++) { + std::shared_ptr record = *it; + event->addParentFlowFile(record); + } + event->setDetails(detail); + event->setEventDuration(processingDuration); + add(event); + } +} + +void ProvenanceReporter::fork( + std::vector > child, + std::shared_ptr parent, std::string detail, + uint64_t processingDuration) { + ProvenanceEventRecord *event = allocate(ProvenanceEventRecord::FORK, parent); + + if (event) { + event->addParentFlowFile(parent); + std::vector >::iterator it; + for (it = child.begin(); it != child.end(); it++) { + std::shared_ptr record = *it; + event->addChildFlowFile(record); + } + event->setDetails(detail); + event->setEventDuration(processingDuration); + add(event); + } +} + +void ProvenanceReporter::expire(std::shared_ptr flow, + std::string detail) { + ProvenanceEventRecord *event = allocate(ProvenanceEventRecord::EXPIRE, flow); + + if (event) { + event->setDetails(detail); + add(event); + } +} + +void ProvenanceReporter::drop(std::shared_ptr flow, + std::string reason) { + ProvenanceEventRecord *event = allocate(ProvenanceEventRecord::DROP, flow); + + if (event) { + std::string dropReason = "Discard reason: " + reason; + event->setDetails(dropReason); + add(event); + } +} + +void ProvenanceReporter::send(std::shared_ptr flow, + std::string transitUri, std::string detail, + uint64_t processingDuration, bool force) { + ProvenanceEventRecord *event = allocate(ProvenanceEventRecord::SEND, flow); + + if (event) { + event->setTransitUri(transitUri); + event->setDetails(detail); + event->setEventDuration(processingDuration); + if (!force) { + add(event); + } else { + if (!repo_->isFull()) + event->Serialize(repo_); + delete event; + } + } +} + +void ProvenanceReporter::receive(std::shared_ptr flow, + std::string transitUri, + std::string sourceSystemFlowFileIdentifier, + std::string detail, + uint64_t processingDuration) { + ProvenanceEventRecord *event = allocate(ProvenanceEventRecord::RECEIVE, flow); + + if (event) { + event->setTransitUri(transitUri); + event->setDetails(detail); + event->setEventDuration(processingDuration); + event->setSourceSystemFlowFileIdentifier(sourceSystemFlowFileIdentifier); + add(event); + } +} + +void ProvenanceReporter::fetch(std::shared_ptr flow, + std::string transitUri, std::string detail, + uint64_t processingDuration) { + ProvenanceEventRecord *event = allocate(ProvenanceEventRecord::FETCH, flow); + + if (event) { + event->setTransitUri(transitUri); + event->setDetails(detail); + event->setEventDuration(processingDuration); + add(event); + } +} + +} /* namespace provenance */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/provenance/ProvenanceRepository.cpp b/libminifi/src/provenance/ProvenanceRepository.cpp new file mode 100644 index 0000000000..88455be9be --- /dev/null +++ b/libminifi/src/provenance/ProvenanceRepository.cpp @@ -0,0 +1,75 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "provenance/Provenance.h" +#include "provenance/ProvenanceRepository.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace provenance{ + + + +void ProvenanceRepository::run() { + // threshold for purge + uint64_t purgeThreshold = max_partition_bytes_ * 3 / 4; + while (running_) { + std::this_thread::sleep_for(std::chrono::milliseconds(purge_period_)); + uint64_t curTime = getTimeMillis(); + uint64_t size = repoSize(); + if (size >= purgeThreshold) { + std::vector purgeList; + leveldb::Iterator* it = db_->NewIterator(leveldb::ReadOptions()); + for (it->SeekToFirst(); it->Valid(); it->Next()) { + ProvenanceEventRecord eventRead; + std::string key = it->key().ToString(); + if (eventRead.DeSerialize((uint8_t *) it->value().data(), + (int) it->value().size())) { + if ((curTime - eventRead.getEventTime()) + > max_partition_millis_) + purgeList.push_back(key); + } else { + logger_->log_debug("NiFi Provenance retrieve event %s fail", + key.c_str()); + purgeList.push_back(key); + } + } + delete it; + std::vector::iterator itPurge; + for (itPurge = purgeList.begin(); itPurge != purgeList.end(); itPurge++) { + std::string eventId = *itPurge; + logger_->log_info("ProvenanceRepository Repo Purge %s", + eventId.c_str()); + Delete(eventId); + } + } + if (size > max_partition_bytes_) + repo_full_ = true; + else + repo_full_ = false; + } + return; +} +} /* namespace provenance */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + diff --git a/libminifi/test/Server.cpp b/libminifi/test/Server.cpp index 65245f6f68..9428ee068d 100644 --- a/libminifi/test/Server.cpp +++ b/libminifi/test/Server.cpp @@ -22,7 +22,7 @@ #define DEFAULT_REPORT_INTERVAL 1000 // 1 sec #define MAX_READ_TIMEOUT 30000 // 30 seconds -//! FlowControl Protocol Msg Type +// FlowControl Protocol Msg Type typedef enum { REGISTER_REQ, // Device Register Request from device to server which contain device serial number, current running flow YAML version REGISTER_RESP, // Device Register Respond from server to device, may contain new flow.yml from server ask device to apply and also device report interval @@ -31,7 +31,7 @@ typedef enum { MAX_FLOW_CONTROL_MSG_TYPE } FlowControlMsgType; -//! FlowControl Protocol Msg Type String +// FlowControl Protocol Msg Type String static const char *FlowControlMsgTypeStr[MAX_FLOW_CONTROL_MSG_TYPE] = { "REGISTER_REQ", @@ -40,7 +40,7 @@ static const char *FlowControlMsgTypeStr[MAX_FLOW_CONTROL_MSG_TYPE] = "REPORT_RESP" }; -//! Flow Control Msg Type to String +// Flow Control Msg Type to String inline const char *FlowControlMsgTypeToStr(FlowControlMsgType type) { if (type < MAX_FLOW_CONTROL_MSG_TYPE) @@ -49,7 +49,7 @@ inline const char *FlowControlMsgTypeToStr(FlowControlMsgType type) return NULL; } -//! FlowControll Protocol Msg ID (Some Messages are fix length, Some are variable length (TLV) +// FlowControll Protocol Msg ID (Some Messages are fix length, Some are variable length (TLV) typedef enum { //Fix length 8 bytes: client to server in register request, required field FLOW_SERIAL_NUMBER, @@ -70,7 +70,7 @@ typedef enum { MAX_FLOW_MSG_ID } FlowControlMsgID; -//! FlowControl Protocol Msg ID String +// FlowControl Protocol Msg ID String static const char *FlowControlMsgIDStr[MAX_FLOW_MSG_ID] = { "FLOW_SERIAL_NUMBER", @@ -86,7 +86,7 @@ static const char *FlowControlMsgIDStr[MAX_FLOW_MSG_ID] = #define TYPE_HDR_LEN 4 // Fix Hdr Type #define TLV_HDR_LEN 8 // Type 4 bytes and Len 4 bytes -//! FlowControl Protocol Msg Len +// FlowControl Protocol Msg Len inline int FlowControlMsgIDEncodingLen(FlowControlMsgID id, int payLoadLen) { if (id == FLOW_SERIAL_NUMBER) @@ -99,7 +99,7 @@ inline int FlowControlMsgIDEncodingLen(FlowControlMsgID id, int payLoadLen) return -1; } -//! Flow Control Msg Id to String +// Flow Control Msg Id to String inline const char *FlowControlMsgIdToStr(FlowControlMsgID id) { if (id < MAX_FLOW_MSG_ID) @@ -108,7 +108,7 @@ inline const char *FlowControlMsgIdToStr(FlowControlMsgID id) return NULL; } -//! Flow Control Respond status code +// Flow Control Respond status code typedef enum { RESP_SUCCESS, RESP_TRIGGER_REGISTER, // Server respond to client report to re trigger register @@ -118,7 +118,7 @@ typedef enum { MAX_RESP_CODE } FlowControlRespCode; -//! FlowControl Resp Code str +// FlowControl Resp Code str static const char *FlowControlRespCodeStr[MAX_RESP_CODE] = { "RESP_SUCCESS", @@ -128,7 +128,7 @@ static const char *FlowControlRespCodeStr[MAX_RESP_CODE] = "RESP_FAILURE" }; -//! Flow Control Resp Code to String +// Flow Control Resp Code to String inline const char *FlowControlRespCodeToStr(FlowControlRespCode code) { if (code < MAX_RESP_CODE) @@ -137,16 +137,16 @@ inline const char *FlowControlRespCodeToStr(FlowControlRespCode code) return NULL; } -//! Common FlowControlProtocol Header +// Common FlowControlProtocol Header typedef struct { - uint32_t msgType; //! Msg Type - uint32_t seqNumber; //! Seq Number to match Req with Resp - uint32_t status; //! Resp Code, see FlowControlRespCode - uint32_t payloadLen; //! Msg Payload length + uint32_t msgType; // Msg Type + uint32_t seqNumber; // Seq Number to match Req with Resp + uint32_t status; // Resp Code, see FlowControlRespCode + uint32_t payloadLen; // Msg Payload length } FlowControlProtocolHeader; -//! encode uint32_t +// encode uint32_t uint8_t *encode(uint8_t *buf, uint32_t value) { *buf++ = (value & 0xFF000000) >> 24; @@ -156,14 +156,14 @@ uint8_t *encode(uint8_t *buf, uint32_t value) return buf; } -//! encode uint32_t +// encode uint32_t uint8_t *decode(uint8_t *buf, uint32_t &value) { value = ((buf[0]<<24)|(buf[1]<<16)|(buf[2]<<8)|(buf[3])); return (buf + 4); } -//! encode byte array +// encode byte array uint8_t *encode(uint8_t *buf, uint8_t *bufArray, int size) { memcpy(buf, bufArray, size); @@ -171,7 +171,7 @@ uint8_t *encode(uint8_t *buf, uint8_t *bufArray, int size) return buf; } -//! encode std::string +// encode std::string uint8_t *encode(uint8_t *buf, std::string value) { // add the \0 for size diff --git a/libminifi/test/TestBase.h b/libminifi/test/TestBase.h index 97c32e630e..f73174ba7a 100644 --- a/libminifi/test/TestBase.h +++ b/libminifi/test/TestBase.h @@ -23,63 +23,52 @@ #include "ResourceClaim.h" #include "catch.hpp" #include -#include "Logger.h" +#include "core/logging/Logger.h" +#include "core/core.h" class LogTestController { -public: - LogTestController(const std::string level = "debug") { - Logger::getLogger()->setLogLevel(level); - } - - - void enableDebug() - { - Logger::getLogger()->setLogLevel("debug"); - } - - ~LogTestController() { - Logger::getLogger()->setLogLevel(LOG_LEVEL_E::info); - } + public: + LogTestController(const std::string level = "debug") { + logging::Logger::getLogger()->setLogLevel(level); + } + + void enableDebug() { + logging::Logger::getLogger()->setLogLevel("debug"); + } + + ~LogTestController() { + logging::Logger::getLogger()->setLogLevel(logging::LOG_LEVEL_E::info); + } }; -class TestController{ -public: - - +class TestController { + public: - TestController() : log("info") - { - ResourceClaim::default_directory_path = "./"; - } + TestController() + : log("info") { + minifi::ResourceClaim::default_directory_path = "./"; + } - ~TestController() - { - for(auto dir : directories) - { - rmdir(dir); - } - } + ~TestController() { + for (auto dir : directories) { + rmdir(dir); + } + } - void enableDebug() { - log.enableDebug(); - } + void enableDebug() { + log.enableDebug(); + } - char *createTempDirectory(char *format) - { - char *dir = mkdtemp(format); - return dir; - } - -protected: - LogTestController log; - std::vector directories; + char *createTempDirectory(char *format) { + char *dir = mkdtemp(format); + return dir; + } + protected: + LogTestController log; + std::vector directories; }; - - - - #endif /* LIBMINIFI_TEST_TESTBASE_H_ */ diff --git a/libminifi/test/unit/CRCTests.cpp b/libminifi/test/unit/CRCTests.cpp index 197b5be041..54f27bc334 100644 --- a/libminifi/test/unit/CRCTests.cpp +++ b/libminifi/test/unit/CRCTests.cpp @@ -22,60 +22,58 @@ #include "io/DataStream.h" #include "../TestBase.h" - TEST_CASE("Test CRC1", "[testcrc1]") { - BaseStream base; - CRCStream test(&base); - test.writeData((uint8_t*)"cow",3); - REQUIRE(2580823964 == test.getCRC()); - - + org::apache::nifi::minifi::io::BaseStream base; + org::apache::nifi::minifi::io::CRCStream< + org::apache::nifi::minifi::io::BaseStream> test(&base); + test.writeData((uint8_t*) "cow", 3); + REQUIRE(2580823964 == test.getCRC()); + + } TEST_CASE("Test CRC2", "[testcrc2]") { - BaseStream base; - CRCStream test(&base); - std::string fox = "the quick brown fox jumped over the brown fox"; - std::vector charvect(fox.begin(), fox.end()); - test.writeData(charvect,charvect.size()); - REQUIRE(1922388889 == test.getCRC()); - - -} + org::apache::nifi::minifi::io::BaseStream base; + org::apache::nifi::minifi::io::CRCStream< + org::apache::nifi::minifi::io::BaseStream> test(&base); + std::string fox = "the quick brown fox jumped over the brown fox"; + std::vector charvect(fox.begin(), fox.end()); + test.writeData(charvect, charvect.size()); + REQUIRE(1922388889 == test.getCRC()); +} TEST_CASE("Test CRC3", "[testcrc3]") { - BaseStream base; - CRCStream test(&base); - uint64_t number=7; - test.write(number); - REQUIRE(4215687882 == test.getCRC()); - - -} + org::apache::nifi::minifi::io::BaseStream base; + org::apache::nifi::minifi::io::CRCStream< + org::apache::nifi::minifi::io::BaseStream> test(&base); + uint64_t number = 7; + test.write(number); + REQUIRE(4215687882 == test.getCRC()); +} TEST_CASE("Test CRC4", "[testcrc4]") { - BaseStream base; - CRCStream test(&base); - uint32_t number=7; - test.write(number); - REQUIRE(3206564543 == test.getCRC()); - - + org::apache::nifi::minifi::io::BaseStream base; + org::apache::nifi::minifi::io::CRCStream< + org::apache::nifi::minifi::io::BaseStream> test(&base); + uint32_t number = 7; + test.write(number); + REQUIRE(3206564543 == test.getCRC()); + } TEST_CASE("Test CRC5", "[testcrc5]") { - BaseStream base; - CRCStream test(&base); - uint16_t number=7; - test.write(number); - REQUIRE(3753740124 == test.getCRC()); - - -} \ No newline at end of file + org::apache::nifi::minifi::io::BaseStream base; + org::apache::nifi::minifi::io::CRCStream< + org::apache::nifi::minifi::io::BaseStream> test(&base); + uint16_t number = 7; + test.write(number); + REQUIRE(3753740124 == test.getCRC()); + +} diff --git a/libminifi/test/unit/LoggerTests.cpp b/libminifi/test/unit/LoggerTests.cpp index 8359037ffc..f17cde1d8e 100644 --- a/libminifi/test/unit/LoggerTests.cpp +++ b/libminifi/test/unit/LoggerTests.cpp @@ -19,125 +19,140 @@ #include #include "../TestBase.h" -#include "../../include/LogAppenders.h" +#include "core/logging/LogAppenders.h" + +using namespace logging; bool contains(std::string stringA, std::string ending) { - return (ending.length() > 0 && stringA.find(ending) != std::string::npos); + return (ending.length() > 0 && stringA.find(ending) != std::string::npos); } TEST_CASE("Test log Levels", "[ttl1]") { - std::ostringstream oss; + std::ostringstream oss; - std::unique_ptr outputLogger = std::unique_ptr(new OutputStreamAppender(oss,0)); - std::shared_ptr logger = Logger::getLogger(); - logger->updateLogger(std::move(outputLogger)); - logger->setLogLevel("trace"); - logger->log_info("hello world"); + std::unique_ptr outputLogger = std::unique_ptr( + new org::apache::nifi::minifi::core::logging::OutputStreamAppender(oss, 0)); + std::shared_ptr logger = logging::Logger::getLogger(); + logger->updateLogger(std::move(outputLogger)); + logger->setLogLevel("trace"); + logger->log_info("hello world"); - REQUIRE( true == contains(oss.str(),"[minifi log -- OutputStreamAppender] [info] hello world")); + REQUIRE( + true == contains(oss.str(),"[minifi log -- org::apache::nifi::minifi::core::logging::OutputStreamAppender] [info] hello world")); } TEST_CASE("Test log Levels debug", "[ttl2]") { - std::ostringstream oss; + std::ostringstream oss; - std::unique_ptr outputLogger = std::unique_ptr(new OutputStreamAppender(oss,0)); - std::shared_ptr logger = Logger::getLogger(); - logger->updateLogger(std::move(outputLogger)); - logger->setLogLevel("trace"); - logger->log_debug("hello world"); + std::unique_ptr outputLogger = std::unique_ptr( + new org::apache::nifi::minifi::core::logging::OutputStreamAppender(oss, 0)); + std::shared_ptr logger = logging::Logger::getLogger(); + logger->updateLogger(std::move(outputLogger)); + logger->setLogLevel("trace"); + logger->log_debug("hello world"); - REQUIRE( true == contains(oss.str(),"[minifi log -- OutputStreamAppender] [debug] hello world")); + REQUIRE( + true == contains(oss.str(),"[minifi log -- org::apache::nifi::minifi::core::logging::OutputStreamAppender] [debug] hello world")); } TEST_CASE("Test log Levels trace", "[ttl3]") { - std::ostringstream oss; + std::ostringstream oss; - std::unique_ptr outputLogger = std::unique_ptr(new OutputStreamAppender(oss,0)); - std::shared_ptr logger = Logger::getLogger(); - logger->updateLogger(std::move(outputLogger)); - logger->setLogLevel("trace"); + std::unique_ptr outputLogger = std::unique_ptr( + new org::apache::nifi::minifi::core::logging::OutputStreamAppender(oss, 0)); + std::shared_ptr logger = logging::Logger::getLogger(); + logger->updateLogger(std::move(outputLogger)); + logger->setLogLevel("trace"); - logger->log_trace("hello world"); + logger->log_trace("hello world"); - REQUIRE( true == contains(oss.str(),"[minifi log -- OutputStreamAppender] [trace] hello world")); + REQUIRE( + true == contains(oss.str(),"[minifi log -- org::apache::nifi::minifi::core::logging::OutputStreamAppender] [trace] hello world")); } TEST_CASE("Test log Levels error", "[ttl4]") { - std::ostringstream oss; + std::ostringstream oss; - std::unique_ptr outputLogger = std::unique_ptr(new OutputStreamAppender(oss,0)); - std::shared_ptr logger = Logger::getLogger(); - logger->updateLogger(std::move(outputLogger)); - logger->setLogLevel("trace"); + std::unique_ptr outputLogger = std::unique_ptr( + new org::apache::nifi::minifi::core::logging::OutputStreamAppender(oss, 0)); + std::shared_ptr logger = logging::Logger::getLogger(); + logger->updateLogger(std::move(outputLogger)); + logger->setLogLevel("trace"); - logger->log_error("hello world"); + logger->log_error("hello world"); - REQUIRE( true == contains(oss.str(),"[minifi log -- OutputStreamAppender] [error] hello world")); + REQUIRE( + true == contains(oss.str(),"[minifi log -- org::apache::nifi::minifi::core::logging::OutputStreamAppender] [error] hello world")); } TEST_CASE("Test log Levels change", "[ttl5]") { - std::ostringstream oss; + std::ostringstream oss; - std::unique_ptr outputLogger = std::unique_ptr(new OutputStreamAppender(oss,0)); - std::shared_ptr logger = Logger::getLogger(); - logger->updateLogger(std::move(outputLogger)); - logger->setLogLevel("trace"); + std::unique_ptr outputLogger = std::unique_ptr( + new org::apache::nifi::minifi::core::logging::OutputStreamAppender(oss, 0)); + std::shared_ptr logger = logging::Logger::getLogger(); + logger->updateLogger(std::move(outputLogger)); + logger->setLogLevel("trace"); - logger->log_error("hello world"); + logger->log_error("hello world"); - REQUIRE( true == contains(oss.str(),"[minifi log -- OutputStreamAppender] [error] hello world")); - oss.str(""); - oss.clear(); - REQUIRE( 0 == oss.str().length() ); - logger->setLogLevel("off"); + REQUIRE( + true == contains(oss.str(),"[minifi log -- org::apache::nifi::minifi::core::logging::OutputStreamAppender] [error] hello world")); + oss.str(""); + oss.clear(); + REQUIRE(0 == oss.str().length()); + logger->setLogLevel("off"); - logger->log_error("hello world"); + logger->log_error("hello world"); - REQUIRE( 0 == oss.str().length() ); + REQUIRE(0 == oss.str().length()); } TEST_CASE("Test log LevelsConfigured", "[ttl6]") { - std::ostringstream oss; + std::ostringstream oss; - Configure *config = Configure::getConfigure(); + minifi::Configure *config = minifi::Configure::getConfigure(); - config->set(BaseLogger::nifi_log_appender,"outputstreamappender"); - config->set(OutputStreamAppender::nifi_log_output_stream_error_stderr,"true"); + config->set(BaseLogger::nifi_log_appender, "org::apache::nifi::minifi::core::logging::OutputStreamAppender"); + config->set(org::apache::nifi::minifi::core::logging::OutputStreamAppender::nifi_log_output_stream_error_stderr, + "true"); - std::shared_ptr logger = Logger::getLogger(); + std::shared_ptr logger = logging::Logger::getLogger(); - auto oldrdbuf = std::cerr.rdbuf(); - std::cerr.rdbuf(oss.rdbuf()); + auto oldrdbuf = std::cerr.rdbuf(); + std::cerr.rdbuf(oss.rdbuf()); - std::unique_ptr newLogger =LogInstance::getConfiguredLogger(config); + std::unique_ptr newLogger = LogInstance::getConfiguredLogger( + config); - logger->updateLogger(std::move(newLogger)); + logger->updateLogger(std::move(newLogger)); - logger->setLogLevel("trace"); + logger->setLogLevel("trace"); - // capture stderr - logger->log_error("hello world"); + // capture stderr + logger->log_error("hello world"); - REQUIRE( true == contains(oss.str(),"[minifi log -- OutputStreamAppender] [error] hello world")); + REQUIRE( + true == contains(oss.str(),"[minifi log -- org::apache::nifi::minifi::core::logging::OutputStreamAppender] [error] hello world")); - std::cerr.rdbuf(oldrdbuf); + std::cerr.rdbuf(oldrdbuf); - config->set(BaseLogger::nifi_log_appender,"nullappender"); + config->set(BaseLogger::nifi_log_appender, "nullappender"); - newLogger =LogInstance::getConfiguredLogger(config); + newLogger = LogInstance::getConfiguredLogger(config); - logger->updateLogger(std::move(newLogger)); + logger->updateLogger(std::move(newLogger)); - oss.str(""); - oss.clear(); - REQUIRE( 0 == oss.str().length() ); + oss.str(""); + oss.clear(); + REQUIRE(0 == oss.str().length()); - // should have nothing from the null appender - logger->log_info("hello world"); - logger->log_debug("hello world"); - logger->log_trace("hello world"); + // should have nothing from the null appender + logger->log_info("hello world"); + logger->log_debug("hello world"); + logger->log_trace("hello world"); // logger->log_error("hello world"); - REQUIRE( 0 == oss.str().length() ); + REQUIRE(0 == oss.str().length()); } diff --git a/libminifi/test/unit/ProcessorTests.cpp b/libminifi/test/unit/ProcessorTests.cpp index ae5f7e5d3e..e9b11b4001 100644 --- a/libminifi/test/unit/ProcessorTests.cpp +++ b/libminifi/test/unit/ProcessorTests.cpp @@ -20,21 +20,40 @@ #include "FlowController.h" #include "ProvenanceTestHelper.h" #include "../TestBase.h" +<<<<<<< HEAD #include #include "../../include/LogAppenders.h" #include "GetFile.h" +======= +#include "core/logging/LogAppenders.h" +#include "core/logging/BaseLogger.h" +#include "processors/GetFile.h" +#include "core/core.h" +#include "../../include/core/FlowFile.h" +#include "core/Processor.h" +#include "core/ProcessContext.h" +#include "core/ProcessSession.h" +#include "core/ProcessorNode.h" + +TEST_CASE("Test Creation of GetFile", "[getfileCreate]") { + org::apache::nifi::minifi::processors::GetFile processor("processorname"); + REQUIRE(processor.getName() == "processorname"); +} +>>>>>>> MINIFI-217: First commit. Updates namespaces and removes +TEST_CASE("Test Find file", "[getfileCreate2]") { -TEST_CASE("Test Creation of GetFile", "[getfileCreate]"){ - GetFile processor("processorname"); - REQUIRE( processor.getName() == "processorname"); -} + TestController testController; + testController.enableDebug(); -TEST_CASE("Test Find file", "[getfileCreate2]"){ + std::shared_ptr repo = std::make_shared< + ProvenanceTestRepository>(); - TestController testController; + std::shared_ptr processor = std::make_shared< + org::apache::nifi::minifi::processors::GetFile>("getfileCreate2"); +<<<<<<< HEAD Configure *config = Configure::getConfigure(); config->set(BaseLogger::nifi_log_appender,"rollingappender"); @@ -48,47 +67,101 @@ TEST_CASE("Test Find file", "[getfileCreate2]"){ FlowTestRepository flowRepo; TestFlowController controller(provenanceRepo, flowRepo); FlowControllerFactory::getFlowController( dynamic_cast(&controller)); +======= + char format[] = "/tmp/gt.XXXXXX"; + char *dir = testController.createTempDirectory(format); - GetFile processor("getfileCreate2"); + uuid_t processoruuid; + REQUIRE(true == processor->getUUID(processoruuid)); +>>>>>>> MINIFI-217: First commit. Updates namespaces and removes - char format[] ="/tmp/gt.XXXXXX"; - char *dir = testController.createTempDirectory(format); + std::shared_ptr connection = std::make_shared< + minifi::Connection>("getfileCreate2Connection"); + connection->setRelationship(core::Relationship("success", "description")); + // link the connections so that we can test results at the end for this + connection->setSource(processor); - uuid_t processoruuid; - REQUIRE( true == processor.getUUID(processoruuid) ); + connection->setSourceUUID(processoruuid); + connection->setDestinationUUID(processoruuid); - Connection connection("getfileCreate2Connection"); - connection.setRelationship(Relationship("success","description")); + processor->addConnection(connection); + REQUIRE(dir != NULL); - // link the connections so that we can test results at the end for this + core::ProcessorNode node(processor); + core::ProcessContext context(node, repo); + context.setProperty(org::apache::nifi::minifi::processors::GetFile::Directory, + dir); + core::ProcessSession session(&context); + +<<<<<<< HEAD connection.setSourceProcessor(&processor); connection.setDestinationProcessor(&processor); - - connection.setSourceProcessorUUID(processoruuid); - connection.setDestinationProcessorUUID(processoruuid); - - processor.addConnection(&connection); - REQUIRE( dir != NULL ); - - ProcessContext context(&processor); - context.setProperty(GetFile::Directory,dir); - ProcessSession session(&context); - +======= + REQUIRE(processor->getName() == "getfileCreate2"); + + std::shared_ptr record; + processor->setScheduledState(core::ScheduledState::RUNNING); + processor->onTrigger(&context, &session); +>>>>>>> MINIFI-217: First commit. Updates namespaces and removes + + provenance::ProvenanceReporter *reporter = session.getProvenanceReporter(); + std::set records = reporter->getEvents(); + record = session.get(); + REQUIRE(record == nullptr); + REQUIRE(records.size() == 0); + + std::fstream file; + std::stringstream ss; + ss << dir << "/" << "tstFile.ext"; + file.open(ss.str(), std::ios::out); + file << "tempFile"; + file.close(); + + processor->incrementActiveTasks(); + processor->setScheduledState(core::ScheduledState::RUNNING); + processor->onTrigger(&context, &session); + unlink(ss.str().c_str()); + rmdir(dir); + reporter = session.getProvenanceReporter(); + +<<<<<<< HEAD REQUIRE( processor.getName() == "getfileCreate2"); - - FlowFileRecord *record; - processor.setScheduledState(ScheduledState::RUNNING); - processor.onTrigger(&context,&session); - - ProvenanceReporter *reporter = session.getProvenanceReporter(); - std::set records = reporter->getEvents(); - - record = session.get(); - REQUIRE( record== 0 ); - REQUIRE( records.size() == 0 ); - +======= + records = reporter->getEvents(); + + for (provenance::ProvenanceEventRecord *provEventRecord : records) { + REQUIRE(provEventRecord->getComponentType() == processor->getName()); + } + session.commit(); + std::shared_ptr ffr = session.get(); +>>>>>>> MINIFI-217: First commit. Updates namespaces and removes + + ffr->getResourceClaim()->decreaseFlowFileRecordOwnedCount(); + REQUIRE(2 == repo->getRepoMap().size()); + + for (auto entry : repo->getRepoMap()) { + provenance::ProvenanceEventRecord newRecord; + newRecord.DeSerialize((uint8_t*) entry.second.data(), + entry.second.length()); + + bool found = false; + for (auto provRec : records) { + if (provRec->getEventId() == newRecord.getEventId()) { + REQUIRE(provRec->getEventId() == newRecord.getEventId()); + REQUIRE(provRec->getComponentId() == newRecord.getComponentId()); + REQUIRE(provRec->getComponentType() == newRecord.getComponentType()); + REQUIRE(provRec->getDetails() == newRecord.getDetails()); + REQUIRE(provRec->getEventDuration() == newRecord.getEventDuration()); + found = true; + break; + } + } + if (!found) + throw std::runtime_error("Did not find record"); + +<<<<<<< HEAD std::fstream file; std::stringstream ss; std::string fileName("tstFile.ext"); @@ -104,17 +177,26 @@ TEST_CASE("Test Find file", "[getfileCreate2]"){ processor.onTrigger(&context,&session); unlink(ss.str().c_str()); rmdir(dir); +======= + } - reporter = session.getProvenanceReporter(); +} +>>>>>>> MINIFI-217: First commit. Updates namespaces and removes - records = reporter->getEvents(); +TEST_CASE("LogAttributeTest", "[getfileCreate3]") { + std::ostringstream oss; + std::unique_ptr outputLogger = std::unique_ptr< + logging::BaseLogger>( + new org::apache::nifi::minifi::core::logging::OutputStreamAppender(oss, + 0)); + std::shared_ptr logger = logging::Logger::getLogger(); + logger->updateLogger(std::move(outputLogger)); - for(ProvenanceEventRecord *provEventRecord : records) - { - REQUIRE (provEventRecord->getComponentType() == processor.getName()); - } - session.commit(); + TestController testController; + + testController.enableDebug(); +<<<<<<< HEAD // verify flow file repo REQUIRE( 1 == flowRepo.getRepoMap().size() ); @@ -130,13 +212,27 @@ TEST_CASE("Test Find file", "[getfileCreate2]"){ } FlowFileRecord *ffr = session.get(); +======= + std::shared_ptr repo = std::make_shared< + ProvenanceTestRepository>(); + + std::shared_ptr processor = std::make_shared< + org::apache::nifi::minifi::processors::GetFile>("getfileCreate2"); + + std::shared_ptr logAttribute = std::make_shared< + org::apache::nifi::minifi::processors::LogAttribute>("logattribute"); +>>>>>>> MINIFI-217: First commit. Updates namespaces and removes - ffr->getResourceClaim()->decreaseFlowFileRecordOwnedCount(); + char format[] = "/tmp/gt.XXXXXX"; + char *dir = testController.createTempDirectory(format); - delete ffr; + uuid_t processoruuid; + REQUIRE(true == processor->getUUID(processoruuid)); - std::set expiredFlows; + uuid_t logattribute_uuid; + REQUIRE(true == logAttribute->getUUID(logattribute_uuid)); +<<<<<<< HEAD REQUIRE( 2 == provenanceRepo.getRepoMap().size() ); for(auto entry: provenanceRepo.getRepoMap()) @@ -161,16 +257,99 @@ TEST_CASE("Test Find file", "[getfileCreate2]"){ if (!found) throw std::runtime_error("Did not find record"); } - - - - - - - +======= + std::shared_ptr connection = std::make_shared< + minifi::Connection>("getfileCreate2Connection"); + connection->setRelationship(core::Relationship("success", "description")); + + std::shared_ptr connection2 = std::make_shared< + minifi::Connection>("logattribute"); + connection2->setRelationship(core::Relationship("success", "description")); + + // link the connections so that we can test results at the end for this + connection->setSource(processor); +>>>>>>> MINIFI-217: First commit. Updates namespaces and removes + + // link the connections so that we can test results at the end for this + connection->setDestination(logAttribute); + +<<<<<<< HEAD +======= + connection2->setSource(logAttribute); +>>>>>>> MINIFI-217: First commit. Updates namespaces and removes + + connection2->setSourceUUID(logattribute_uuid); + connection->setSourceUUID(processoruuid); + connection->setDestinationUUID(logattribute_uuid); + + processor->addConnection(connection); + logAttribute->addConnection(connection); + logAttribute->addConnection(connection2); + REQUIRE(dir != NULL); + + core::ProcessorNode node(processor); + core::ProcessorNode node2(logAttribute); + + core::ProcessContext context(node, repo); + core::ProcessContext context2(node2, repo); + context.setProperty(org::apache::nifi::minifi::processors::GetFile::Directory, + dir); + core::ProcessSession session(&context); + core::ProcessSession session2(&context2); + + REQUIRE(processor->getName() == "getfileCreate2"); + + std::shared_ptr record; + processor->setScheduledState(core::ScheduledState::RUNNING); + processor->onTrigger(&context, &session); + + logAttribute->incrementActiveTasks(); + logAttribute->setScheduledState(core::ScheduledState::RUNNING); + logAttribute->onTrigger(&context2, &session2); + + provenance::ProvenanceReporter *reporter = session.getProvenanceReporter(); + std::set records = reporter->getEvents(); + record = session.get(); + REQUIRE(record == nullptr); + REQUIRE(records.size() == 0); + + std::fstream file; + std::stringstream ss; + ss << dir << "/" << "tstFile.ext"; + file.open(ss.str(), std::ios::out); + file << "tempFile"; + file.close(); + + processor->incrementActiveTasks(); + processor->setScheduledState(core::ScheduledState::RUNNING); + processor->onTrigger(&context, &session); + unlink(ss.str().c_str()); + rmdir(dir); + reporter = session.getProvenanceReporter(); + + records = reporter->getEvents(); + session.commit(); + oss.str(""); + oss.clear(); + + logAttribute->incrementActiveTasks(); + logAttribute->setScheduledState(core::ScheduledState::RUNNING); + logAttribute->onTrigger(&context2, &session2); + + //session2.commit(); + records = reporter->getEvents(); + + std::string log_attribute_output = oss.str(); + REQUIRE( + log_attribute_output.find("key:absolute.path value:" + ss.str()) + != std::string::npos); + REQUIRE(log_attribute_output.find("Size:8 Offset:0") != std::string::npos); + REQUIRE( + log_attribute_output.find("key:path value:" + std::string(dir)) + != std::string::npos); + + outputLogger = std::unique_ptr( + new org::apache::nifi::minifi::core::logging::NullAppender()); + logger->updateLogger(std::move(outputLogger)); } - - - - diff --git a/libminifi/test/unit/PropertyTests.cpp b/libminifi/test/unit/PropertyTests.cpp index e9f1c19122..dee809fc97 100644 --- a/libminifi/test/unit/PropertyTests.cpp +++ b/libminifi/test/unit/PropertyTests.cpp @@ -16,22 +16,23 @@ * limitations under the License. */ +#include "../../include/core/Property.h" #include "utils/StringUtils.h" #include "../TestBase.h" -#include "Property.h" + TEST_CASE("Test Boolean Conversion", "[testboolConversion]") { bool b; - REQUIRE(true == StringUtils::StringToBool("true",b)); - REQUIRE(true == StringUtils::StringToBool("True",b)); - REQUIRE(true == StringUtils::StringToBool("TRue",b)); - REQUIRE(true == StringUtils::StringToBool("tRUE",b)); + REQUIRE(true == org::apache::nifi::minifi::utils::StringUtils::StringToBool("true",b)); + REQUIRE(true == org::apache::nifi::minifi::utils::StringUtils::StringToBool("True",b)); + REQUIRE(true == org::apache::nifi::minifi::utils::StringUtils::StringToBool("TRue",b)); + REQUIRE(true == org::apache::nifi::minifi::utils::StringUtils::StringToBool("tRUE",b)); - REQUIRE(false == StringUtils::StringToBool("FALSE",b)); - REQUIRE(false == StringUtils::StringToBool("FALLSEY",b)); - REQUIRE(false == StringUtils::StringToBool("FaLSE",b)); - REQUIRE(false == StringUtils::StringToBool("false",b)); + REQUIRE(false == org::apache::nifi::minifi::utils::StringUtils::StringToBool("FALSE",b)); + REQUIRE(false == org::apache::nifi::minifi::utils::StringUtils::StringToBool("FALLSEY",b)); + REQUIRE(false == org::apache::nifi::minifi::utils::StringUtils::StringToBool("FaLSE",b)); + REQUIRE(false == org::apache::nifi::minifi::utils::StringUtils::StringToBool("false",b)); } @@ -41,7 +42,7 @@ TEST_CASE("Test Trimmer Right", "[testTrims]") { REQUIRE(test.c_str()[test.length() - 1] == '\n'); REQUIRE(test.c_str()[test.length() - 2] == '\t'); - test = StringUtils::trimRight(test); + test = org::apache::nifi::minifi::utils::StringUtils::trimRight(test); REQUIRE(test.c_str()[test.length() - 1] == 'd'); REQUIRE(test.c_str()[test.length() - 2] == 'a'); @@ -51,7 +52,7 @@ TEST_CASE("Test Trimmer Right", "[testTrims]") { REQUIRE(test.c_str()[test.length() - 1] == '\t'); REQUIRE(test.c_str()[test.length() - 2] == '\v'); - test = StringUtils::trimRight(test); + test = org::apache::nifi::minifi::utils::StringUtils::trimRight(test); REQUIRE(test.c_str()[test.length() - 1] == 'd'); REQUIRE(test.c_str()[test.length() - 2] == 'a'); @@ -61,7 +62,7 @@ TEST_CASE("Test Trimmer Right", "[testTrims]") { REQUIRE(test.c_str()[test.length() - 1] == '\f'); REQUIRE(test.c_str()[test.length() - 2] == ' '); - test = StringUtils::trimRight(test); + test = org::apache::nifi::minifi::utils::StringUtils::trimRight(test); REQUIRE(test.c_str()[test.length() - 1] == 'd'); @@ -74,7 +75,7 @@ TEST_CASE("Test Trimmer Left", "[testTrims]") { REQUIRE(test.c_str()[0] == '\t'); REQUIRE(test.c_str()[1] == '\n'); - test = StringUtils::trimLeft(test); + test = org::apache::nifi::minifi::utils::StringUtils::trimLeft(test); REQUIRE(test.c_str()[0] == 'a'); REQUIRE(test.c_str()[1] == ' '); @@ -84,7 +85,7 @@ TEST_CASE("Test Trimmer Left", "[testTrims]") { REQUIRE(test.c_str()[0] == '\v'); REQUIRE(test.c_str()[1] == '\t'); - test = StringUtils::trimLeft(test); + test = org::apache::nifi::minifi::utils::StringUtils::trimLeft(test); REQUIRE(test.c_str()[0] == 'a'); REQUIRE(test.c_str()[1] == ' '); @@ -94,7 +95,7 @@ TEST_CASE("Test Trimmer Left", "[testTrims]") { REQUIRE(test.c_str()[0] == ' '); REQUIRE(test.c_str()[1] == '\f'); - test = StringUtils::trimLeft(test); + test = org::apache::nifi::minifi::utils::StringUtils::trimLeft(test); REQUIRE(test.c_str()[0] == 'a'); REQUIRE(test.c_str()[1] == ' '); diff --git a/libminifi/test/unit/ProvenanceTestHelper.h b/libminifi/test/unit/ProvenanceTestHelper.h index 1ee6a4c856..bc73641e34 100644 --- a/libminifi/test/unit/ProvenanceTestHelper.h +++ b/libminifi/test/unit/ProvenanceTestHelper.h @@ -18,8 +18,9 @@ #ifndef LIBMINIFI_TEST_UNIT_PROVENANCETESTHELPER_H_ #define LIBMINIFI_TEST_UNIT_PROVENANCETESTHELPER_H_ -#include "Provenance.h" +#include "provenance/Provenance.h" #include "FlowController.h" +<<<<<<< HEAD #include "FlowFileRepository.h" /** @@ -31,13 +32,13 @@ class FlowTestRepository : public FlowFileRepository FlowTestRepository() { } - //! initialize + // initialize bool initialize() { return true; } - //! Destructor + // Destructor virtual ~FlowTestRepository() { } @@ -47,13 +48,13 @@ class FlowTestRepository : public FlowFileRepository repositoryResults.insert(std::pair(key,std::string((const char*)buf,bufLen))); return true; } - //! Delete + // Delete bool Delete(std::string key) { repositoryResults.erase(key); return true; } - //! Get + // Get bool Get(std::string key, std::string &value) { auto result = repositoryResults.find(key); @@ -86,13 +87,13 @@ class ProvenanceTestRepository : public ProvenanceRepository ProvenanceTestRepository() { } - //! initialize + // initialize bool initialize() { return true; } - //! Destructor + // Destructor virtual ~ProvenanceTestRepository() { } @@ -102,13 +103,13 @@ class ProvenanceTestRepository : public ProvenanceRepository repositoryResults.insert(std::pair(key,std::string((const char*)buf,bufLen))); return true; } - //! Delete + // Delete bool Delete(std::string key) { repositoryResults.erase(key); return true; } - //! Get + // Get bool Get(std::string key, std::string &value) { auto result = repositoryResults.find(key); @@ -152,13 +153,13 @@ class TestFlowController : public FlowController bool start() { - _running.store(true); + running_.store(true); return true; } void stop(bool force) { - _running.store(false); + running_.store(false); } void waitUnload(const uint64_t timeToWaitMs) { @@ -188,6 +189,54 @@ class TestFlowController : public FlowController ProcessGroup *createRemoteProcessGroup(std::string name, uuid_t uuid){ return 0; } Connection *createConnection(std::string name, uuid_t uuid){ return 0; } +======= +#include "core/core.h" +/** + * Test repository + */ +class ProvenanceTestRepository : public provenance::ProvenanceRepository { + public: + ProvenanceTestRepository() { + } + // initialize + bool initialize() { + return true; + } + + // Destructor + virtual ~ProvenanceTestRepository() { + + } + + bool Put(std::string key, uint8_t *buf, int bufLen) { + repositoryResults.insert( + std::pair( + key, std::string((const char*) buf, bufLen))); + return true; + } + // Delete + bool Delete(std::string key) { + repositoryResults.erase(key); + return true; + } + // Get + bool Get(std::string key, std::string &value) { + auto result = repositoryResults.find(key); + if (result != repositoryResults.end()) { + value = result->second; + return true; + } else { + return false; + } + } + + const std::map &getRepoMap() const { + return repositoryResults; + } + + protected: + std::map repositoryResults; +>>>>>>> MINIFI-217: First commit. Updates namespaces and removes }; diff --git a/libminifi/test/unit/ProvenanceTests.cpp b/libminifi/test/unit/ProvenanceTests.cpp index d78de47957..015ee17053 100644 --- a/libminifi/test/unit/ProvenanceTests.cpp +++ b/libminifi/test/unit/ProvenanceTests.cpp @@ -16,81 +16,81 @@ * limitations under the License. */ - #ifndef PROVENANCE_TESTS #define PROVENANCE_TESTS #include "../TestBase.h" #include "ProvenanceTestHelper.h" -#include "Provenance.h" +#include "provenance/Provenance.h" #include "FlowFileRecord.h" +#include "core/core.h" +TEST_CASE("Test Provenance record create", "[Testprovenance::ProvenanceEventRecord]") { - -TEST_CASE("Test Provenance record create", "[TestProvenanceEventRecord]"){ - - ProvenanceEventRecord record1(ProvenanceEventRecord::ProvenanceEventType::CREATE,"blah","blahblah"); - REQUIRE( record1.getAttributes().size() == 0); - REQUIRE( record1.getAlternateIdentifierUri().length() == 0); + provenance::ProvenanceEventRecord record1( + provenance::ProvenanceEventRecord::ProvenanceEventType::CREATE, "blah", + "blahblah"); + REQUIRE(record1.getAttributes().size() == 0); + REQUIRE(record1.getAlternateIdentifierUri().length() == 0); } +TEST_CASE("Test Provenance record serialization", "[Testprovenance::ProvenanceEventRecordSerializeDeser]") { -TEST_CASE("Test Provenance record serialization", "[TestProvenanceEventRecordSerializeDeser]"){ + provenance::ProvenanceEventRecord record1( + provenance::ProvenanceEventRecord::ProvenanceEventType::CREATE, "componentid", + "componenttype"); - ProvenanceEventRecord record1(ProvenanceEventRecord::ProvenanceEventType::CREATE,"componentid","componenttype"); + std::string eventId = record1.getEventId(); - std::string eventId = record1.getEventId(); - - std::string smileyface = ":)" ; - record1.setDetails(smileyface); + std::string smileyface = ":)"; + record1.setDetails(smileyface); - ProvenanceTestRepository repo; - uint64_t sample = 65555; - ProvenanceRepository *testRepository = dynamic_cast(&repo); - record1.setEventDuration(sample); + ProvenanceTestRepository repo; + uint64_t sample = 65555; + std::shared_ptr testRepository =std::make_shared(); + record1.setEventDuration(sample); - record1.Serialize(testRepository); - ProvenanceEventRecord record2; - REQUIRE( record2.DeSerialize(testRepository,eventId) == true); - REQUIRE( record2.getEventId() == record1.getEventId()); - REQUIRE( record2.getComponentId() == record1.getComponentId()); - REQUIRE( record2.getComponentType() == record1.getComponentType()); - REQUIRE( record2.getDetails() == record1.getDetails()); - REQUIRE( record2.getDetails() == smileyface); - REQUIRE( record2.getEventDuration() == sample); + record1.Serialize(testRepository); + provenance::ProvenanceEventRecord record2; + REQUIRE(record2.DeSerialize(testRepository,eventId) == true); + REQUIRE(record2.getEventId() == record1.getEventId()); + REQUIRE(record2.getComponentId() == record1.getComponentId()); + REQUIRE(record2.getComponentType() == record1.getComponentType()); + REQUIRE(record2.getDetails() == record1.getDetails()); + REQUIRE(record2.getDetails() == smileyface); + REQUIRE(record2.getEventDuration() == sample); } - -TEST_CASE("Test Flowfile record added to provenance", "[TestFlowAndProv1]"){ - - ProvenanceEventRecord record1(ProvenanceEventRecord::ProvenanceEventType::CLONE,"componentid","componenttype"); - std::string eventId = record1.getEventId(); - std::map attributes; - attributes.insert(std::pair("potato","potatoe")); - attributes.insert(std::pair("tomato","tomatoe")); - FlowFileRecord ffr1(attributes); - - record1.addChildFlowFile(&ffr1); - - ProvenanceTestRepository repo; - uint64_t sample = 65555; - ProvenanceRepository *testRepository = dynamic_cast(&repo); - record1.setEventDuration(sample); - - record1.Serialize(testRepository); - ProvenanceEventRecord record2; - REQUIRE( record2.DeSerialize(testRepository,eventId) == true); - REQUIRE( record1.getChildrenUuids().size() == 1); - REQUIRE( record2.getChildrenUuids().size() == 1); - std::string childId = record2.getChildrenUuids().at(0); - REQUIRE( childId == ffr1.getUUIDStr()); - record2.removeChildUuid(childId); - REQUIRE( record2.getChildrenUuids().size() == 0); - +TEST_CASE("Test Flowfile record added to provenance", "[TestFlowAndProv1]") { + + provenance::ProvenanceEventRecord record1( + provenance::ProvenanceEventRecord::ProvenanceEventType::CLONE, "componentid", + "componenttype"); + std::string eventId = record1.getEventId(); + std::map attributes; + attributes.insert(std::pair("potato", "potatoe")); + attributes.insert(std::pair("tomato", "tomatoe")); + std::shared_ptr ffr1 = std::make_shared< + minifi::FlowFileRecord>(attributes); + + record1.addChildFlowFile(ffr1); + + ProvenanceTestRepository repo; + uint64_t sample = 65555; + std::shared_ptr testRepository =std::make_shared(); + record1.setEventDuration(sample); + + record1.Serialize(testRepository); + provenance::ProvenanceEventRecord record2; + REQUIRE(record2.DeSerialize(testRepository,eventId) == true); + REQUIRE(record1.getChildrenUuids().size() == 1); + REQUIRE(record2.getChildrenUuids().size() == 1); + std::string childId = record2.getChildrenUuids().at(0); + REQUIRE(childId == ffr1->getUUIDStr()); + record2.removeChildUuid(childId); + REQUIRE(record2.getChildrenUuids().size() == 0); } - - #endif diff --git a/libminifi/test/unit/SerializationTests.cpp b/libminifi/test/unit/SerializationTests.cpp index 96540bdbf9..4d1b89b81f 100644 --- a/libminifi/test/unit/SerializationTests.cpp +++ b/libminifi/test/unit/SerializationTests.cpp @@ -28,13 +28,13 @@ #define FMT_DEFAULT fmt_lower - +using namespace org::apache::nifi::minifi::io; TEST_CASE("TestSetPortId", "[S2S1]"){ - Site2SitePeer peer(std::unique_ptr(new DataStream()),"fake_host",65433); + minifi::Site2SitePeer peer(std::unique_ptr(new DataStream()),"fake_host",65433); - Site2SiteClientProtocol protocol(&peer); + minifi::Site2SiteClientProtocol protocol(&peer); std::string uuid_str = "c56a4180-65aa-42ec-a945-5fd21dec0538"; @@ -54,9 +54,9 @@ TEST_CASE("TestSetPortId", "[S2S1]"){ TEST_CASE("TestSetPortIdUppercase", "[S2S2]"){ - Site2SitePeer peer(std::unique_ptr(new DataStream()),"fake_host",65433); + minifi::Site2SitePeer peer(std::unique_ptr(new DataStream()),"fake_host",65433); - Site2SiteClientProtocol protocol(&peer); + minifi::Site2SiteClientProtocol protocol(&peer); std::string uuid_str = "C56A4180-65AA-42EC-A945-5FD21DEC0538"; diff --git a/libminifi/test/unit/SocketTests.cpp b/libminifi/test/unit/SocketTests.cpp index 3fb7c9e6e4..984e9073b9 100644 --- a/libminifi/test/unit/SocketTests.cpp +++ b/libminifi/test/unit/SocketTests.cpp @@ -17,13 +17,15 @@ */ #include "../TestBase.h" -#include "../../include/io/ClientSocket.h" +#include "io/ClientSocket.h" + +using namespace org::apache::nifi::minifi::io; TEST_CASE("TestSocket", "[TestSocket1]") { - Socket socket("localhost",8183); - REQUIRE(-1 == socket.initialize() ); - REQUIRE("localhost" == socket.getHostname()); - socket.closeStream(); + Socket socket("localhost", 8183); + REQUIRE(-1 == socket.initialize()); + REQUIRE("localhost" == socket.getHostname()); + socket.closeStream(); } @@ -32,155 +34,155 @@ TEST_CASE("TestSocketWriteTest1", "[TestSocket2]") { Socket socket("localhost",8183); REQUIRE(-1 == socket.initialize() ); - socket.writeData(0,0); + socket.writeData(0, 0); - std::vector buffer; - buffer.push_back('a'); + std::vector buffer; + buffer.push_back('a'); - REQUIRE(-1 == socket.writeData(buffer,1)); + REQUIRE(-1 == socket.writeData(buffer, 1)); - socket.closeStream(); + socket.closeStream(); } TEST_CASE("TestSocketWriteTest2", "[TestSocket3]") { - std::vector buffer; - buffer.push_back('a'); + std::vector buffer; + buffer.push_back('a'); + + Socket server(Socket::getMyHostName(), 9183, 1); Socket server("localhost",9183,1); - REQUIRE(-1 != server.initialize() ); + Socket client(Socket::getMyHostName(), 9183); Socket client("localhost",9183); - REQUIRE(-1 != client.initialize() ); + REQUIRE(1 == client.writeData(buffer, 1)); - REQUIRE( 1 == client.writeData(buffer,1) ); + std::vector readBuffer; + readBuffer.resize(1); - std::vector readBuffer; - readBuffer.resize(1); + REQUIRE(1 == server.readData(readBuffer, 1)); - REQUIRE( 1== server.readData(readBuffer,1) ); + REQUIRE(readBuffer == buffer); - REQUIRE( readBuffer == buffer ); + server.closeStream(); - server.closeStream(); - - - client.closeStream(); + client.closeStream(); } TEST_CASE("TestGetHostName", "[TestSocket4]") { - REQUIRE( Socket::getMyHostName().length() > 0 ); - - + REQUIRE(Socket::getMyHostName().length() > 0); } TEST_CASE("TestWriteEndian64", "[TestSocket4]") { - std::vector buffer; - buffer.push_back('a'); + std::vector buffer; + buffer.push_back('a'); Socket server("localhost",9183,1); - REQUIRE(-1 != server.initialize() ); + REQUIRE(-1 != server.initialize()); +<<<<<<< HEAD Socket client("localhost",9183); +======= + Socket client(Socket::getMyHostName(), 9183); +>>>>>>> MINIFI-217: First commit. Updates namespaces and removes - REQUIRE(-1 != client.initialize() ); - - uint64_t negative_one = -1; - REQUIRE( 8 == client.write(negative_one) ); + REQUIRE(-1 != client.initialize()); + uint64_t negative_one = -1; + REQUIRE(8 == client.write(negative_one)); - uint64_t negative_two = 0; - REQUIRE( 8 == server.read(negative_two) ); + uint64_t negative_two = 0; + REQUIRE(8 == server.read(negative_two)); - REQUIRE( negative_two == negative_one ); + REQUIRE(negative_two == negative_one); + server.closeStream(); - server.closeStream(); - - - client.closeStream(); + client.closeStream(); } TEST_CASE("TestWriteEndian32", "[TestSocket5]") { - std::vector buffer; - buffer.push_back('a'); + std::vector buffer; + buffer.push_back('a'); +<<<<<<< HEAD Socket server("localhost",9183,1); +======= + Socket server(Socket::getMyHostName(), 9183, 1); +>>>>>>> MINIFI-217: First commit. Updates namespaces and removes - REQUIRE(-1 != server.initialize() ); + REQUIRE(-1 != server.initialize()); +<<<<<<< HEAD Socket client("localhost",9183); +======= + Socket client(Socket::getMyHostName(), 9183); +>>>>>>> MINIFI-217: First commit. Updates namespaces and removes - REQUIRE(-1 != client.initialize() ); + REQUIRE(-1 != client.initialize()); - { - uint32_t negative_one = -1; - REQUIRE( 4 == client.write(negative_one) ); + { + uint32_t negative_one = -1; + REQUIRE(4 == client.write(negative_one)); + uint32_t negative_two = 0; + REQUIRE(4 == server.read(negative_two)); - uint32_t negative_two = 0; - REQUIRE( 4 == server.read(negative_two) ); + REQUIRE(negative_two == negative_one); + } - REQUIRE( negative_two == negative_one ); - } - - { - uint16_t negative_one = -1; - REQUIRE( 2 == client.write(negative_one) ); + { + uint16_t negative_one = -1; + REQUIRE(2 == client.write(negative_one)); + uint16_t negative_two = 0; + REQUIRE(2 == server.read(negative_two)); - uint16_t negative_two = 0; - REQUIRE( 2 == server.read(negative_two) ); + REQUIRE(negative_two == negative_one); + } + server.closeStream(); - REQUIRE( negative_two == negative_one ); - } - server.closeStream(); - - - client.closeStream(); + client.closeStream(); } - TEST_CASE("TestSocketWriteTestAfterClose", "[TestSocket6]") { - std::vector buffer; - buffer.push_back('a'); + std::vector buffer; + buffer.push_back('a'); + + Socket server(Socket::getMyHostName(), 9183, 1); Socket server("localhost",9183,1); - REQUIRE(-1 != server.initialize() ); + Socket client(Socket::getMyHostName(), 9183); Socket client("localhost",9183); - REQUIRE(-1 != client.initialize() ); + REQUIRE(1 == client.writeData(buffer, 1)); - REQUIRE( 1 == client.writeData(buffer,1) ); + std::vector readBuffer; + readBuffer.resize(1); - std::vector readBuffer; - readBuffer.resize(1); + REQUIRE(1 == server.readData(readBuffer, 1)); - REQUIRE( 1== server.readData(readBuffer,1) ); + REQUIRE(readBuffer == buffer); - REQUIRE( readBuffer == buffer ); - - client.closeStream(); - - REQUIRE( -1 == client.writeData(buffer,1) ); - - server.closeStream(); + client.closeStream(); + REQUIRE(-1 == client.writeData(buffer, 1)); + server.closeStream(); } diff --git a/libminifi/test/unit/TimeUtilsTest.cpp b/libminifi/test/unit/TimeUtilsTest.cpp new file mode 100644 index 0000000000..9470338365 --- /dev/null +++ b/libminifi/test/unit/TimeUtilsTest.cpp @@ -0,0 +1,21 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + + diff --git a/main/MiNiFiMain.cpp b/main/MiNiFiMain.cpp index 3e5202ddd3..fac9bfe22e 100644 --- a/main/MiNiFiMain.cpp +++ b/main/MiNiFiMain.cpp @@ -28,11 +28,11 @@ #include #include -#include "../libminifi/include/BaseLogger.h" -#include "../libminifi/include/LogAppenders.h" +#include "core/logging/BaseLogger.h" +#include "core/logging/LogAppenders.h" #include "spdlog/spdlog.h" -#include "Logger.h" +#include "core/logging/Logger.h" #include "Configure.h" #include "FlowController.h" @@ -56,7 +56,7 @@ // Variables that allow us to avoid a timed wait. sem_t *running; //! Flow Controller -static FlowController *controller = NULL; +static std::unique_ptr controller = nullptr; /** * Removed the stop command from the signal handler so that we could trigger @@ -69,104 +69,110 @@ static FlowController *controller = NULL; */ void sigHandler(int signal) { - if (signal == SIGINT || signal == SIGTERM) { - // avoid stopping the controller here. - sem_post(running); - } + if (signal == SIGINT || signal == SIGTERM) { + // avoid stopping the controller here. + sem_post(running); + } } int main(int argc, char **argv) { - std::shared_ptr logger = Logger::getLogger(); - - logger->setLogLevel(info); - - uint16_t stop_wait_time = STOP_WAIT_TIME_MS; - - std::string graceful_shutdown_seconds = ""; - - running = sem_open("MiNiFiMain", O_CREAT, 0644, 0); - if (running == SEM_FAILED || running == 0) { - - logger->log_error("could not initialize semaphore"); - perror("initialization failure"); - } - // assumes POSIX compliant environment - std::string minifiHome; - if (const char* env_p = std::getenv(MINIFI_HOME_ENV_KEY)) { - minifiHome = env_p; - } else { - logger->log_info( - "MINIFI_HOME was not found, determining based on executable path."); - char *path = NULL; - char full_path[PATH_MAX]; - path = realpath(argv[0], full_path); - std::string minifiHomePath(path); - minifiHomePath = minifiHomePath.substr(0, - minifiHomePath.find_last_of("/\\")); //Remove /minifi from path - minifiHome = minifiHomePath.substr(0, - minifiHomePath.find_last_of("/\\")); //Remove /bin from path - } - - if (signal(SIGINT, sigHandler) == SIG_ERR - || signal(SIGTERM, sigHandler) == SIG_ERR - || signal(SIGPIPE, SIG_IGN) == SIG_ERR) { - logger->log_error("Can not install signal handler"); - return -1; - } - - Configure *configure = Configure::getConfigure(); - configure->setHome(minifiHome); - configure->loadConfigureFile(DEFAULT_NIFI_PROPERTIES_FILE); - - if (configure->get(Configure::nifi_graceful_shutdown_seconds, - graceful_shutdown_seconds)) { - try { - stop_wait_time = std::stoi(graceful_shutdown_seconds); - } catch (const std::out_of_range &e) { - logger->log_error("%s is out of range. %s", - Configure::nifi_graceful_shutdown_seconds, e.what()); - } catch (const std::invalid_argument &e) { - logger->log_error("%s contains an invalid argument set. %s", - Configure::nifi_graceful_shutdown_seconds, e.what()); - } - } else { - logger->log_debug("%s not set, defaulting to %d", - Configure::nifi_graceful_shutdown_seconds, STOP_WAIT_TIME_MS); - } - - // set the log configuration. - std::unique_ptr configured_logger = LogInstance::getConfiguredLogger( - configure); - - logger->updateLogger(std::move(configured_logger)); - - controller = FlowControllerFactory::getFlowController(); - - // Load flow from specified configuration file - controller->load(); - // Start Processing the flow - - controller->start(); - logger->log_info("MiNiFi started"); - - /** - * Sem wait provides us the ability to have a controlled - * yield without the need for a more complex construct and - * a spin lock - */ - if (sem_wait(running) != -1) - perror("sem_wait"); - - sem_unlink("MiNiFiMain"); - - /** - * Trigger unload -- wait stop_wait_time - */ - controller->waitUnload(stop_wait_time); - - delete controller; - - logger->log_info("MiNiFi exit"); - - return 0; + std::shared_ptr logger = logging::Logger::getLogger(); + + logger->setLogLevel(logging::info); + + uint16_t stop_wait_time = STOP_WAIT_TIME_MS; + + std::string graceful_shutdown_seconds = ""; + + running = sem_open("MiNiFiMain", O_CREAT, 0644, 0); + if (running == SEM_FAILED || running == 0) { + + logger->log_error("could not initialize semaphore"); + perror("initialization failure"); + } + // assumes POSIX compliant environment + std::string minifiHome; + if (const char* env_p = std::getenv(MINIFI_HOME_ENV_KEY)) { + minifiHome = env_p; + } else { + logger->log_info( + "MINIFI_HOME was not found, determining based on executable path."); + char *path = NULL; + char full_path[PATH_MAX]; + path = realpath(argv[0], full_path); + std::string minifiHomePath(path); + minifiHomePath = minifiHomePath.substr(0, + minifiHomePath.find_last_of("/\\")); //Remove /minifi from path + minifiHome = minifiHomePath.substr(0, minifiHomePath.find_last_of("/\\")); //Remove /bin from path + } + + if (signal(SIGINT, sigHandler) == SIG_ERR + || signal(SIGTERM, sigHandler) == SIG_ERR + || signal(SIGPIPE, SIG_IGN) == SIG_ERR) { + logger->log_error("Can not install signal handler"); + return -1; + } + + minifi::Configure *configure = minifi::Configure::getConfigure(); + configure->setHome(minifiHome); + configure->loadConfigureFile(DEFAULT_NIFI_PROPERTIES_FILE); + + if (configure->get(minifi::Configure::nifi_graceful_shutdown_seconds, + graceful_shutdown_seconds)) { + try { + stop_wait_time = std::stoi(graceful_shutdown_seconds); + } catch (const std::out_of_range &e) { + logger->log_error("%s is out of range. %s", + minifi::Configure::nifi_graceful_shutdown_seconds, + e.what()); + } catch (const std::invalid_argument &e) { + logger->log_error("%s contains an invalid argument set. %s", + minifi::Configure::nifi_graceful_shutdown_seconds, + e.what()); + } + } else { + logger->log_debug("%s not set, defaulting to %d", + minifi::Configure::nifi_graceful_shutdown_seconds, + STOP_WAIT_TIME_MS); + } + + // set the log configuration. + std::unique_ptr configured_logger = + logging::LogInstance::getConfiguredLogger(configure); + + logger->updateLogger(std::move(configured_logger)); + + // Create repos for flow record and provenance + std::shared_ptr repo = std::make_shared< + provenance::ProvenanceRepository>(); + repo->initialize(); + + controller = std::unique_ptr( + new minifi::FlowControllerImpl(repo)); + + // Load flow from specified configuration file + controller->load(); + // Start Processing the flow + + controller->start(); + logger->log_info("MiNiFi started"); + + /** + * Sem wait provides us the ability to have a controlled + * yield without the need for a more complex construct and + * a spin lock + */ + if (sem_wait(running) != -1) + perror("sem_wait"); + + sem_unlink("MiNiFiMain"); + + /** + * Trigger unload -- wait stop_wait_time + */ + controller->waitUnload(stop_wait_time); + + logger->log_info("MiNiFi exit"); + + return 0; } From 54ba1cf92ca05ae5a2867e77b684b0b968390380 Mon Sep 17 00:00:00 2001 From: Marc Parisi Date: Mon, 13 Mar 2017 10:47:05 -0400 Subject: [PATCH 2/4] MINIFI-217: Update based on PR and rebasing against master --- CMakeLists.txt | 1 + libminifi/CMakeLists.txt | 2 +- libminifi/include/Connection.h | 4 +- .../include/EventDrivenSchedulingAgent.h | 2 +- libminifi/include/FlowControlProtocol.h | 2 +- libminifi/include/FlowController.h | 26 +- libminifi/include/FlowFileRecord.h | 53 ++++- libminifi/include/RemoteProcessorGroupPort.h | 28 ++- libminifi/include/ResourceClaim.h | 2 +- libminifi/include/SchedulingAgent.h | 8 +- libminifi/include/Site2SiteClientProtocol.h | 18 +- libminifi/include/Site2SitePeer.h | 2 +- libminifi/include/ThreadedSchedulingAgent.h | 5 +- .../include/TimerDrivenSchedulingAgent.h | 3 +- libminifi/include/core/FlowFile.h | 17 ++ libminifi/include/core/ProcessGroup.h | 3 +- libminifi/include/core/ProcessSession.h | 80 ++++--- libminifi/include/core/Repository.h | 12 +- libminifi/include/core/core.h | 18 +- libminifi/include/core/logging/LogAppenders.h | 2 +- ...wFIleRepository.h => FlowFileRepository.h} | 60 ++++- libminifi/include/io/SocketFactory.h | 2 +- libminifi/include/io/TLSSocket.h | 2 +- libminifi/include/provenance/Provenance.h | 2 +- .../include/provenance/ProvenanceRepository.h | 6 +- libminifi/src/Configure.cpp | 2 +- libminifi/src/Connection.cpp | 5 +- libminifi/src/FlowController.cpp | 62 ++--- libminifi/src/FlowFileRecord.cpp | 224 +++++++++++++++++- libminifi/src/RemoteProcessorGroupPort.cpp | 80 ++++--- libminifi/src/Site2SiteClientProtocol.cpp | 4 +- libminifi/src/core/Core.cpp | 6 +- libminifi/src/core/ProcessGroup.cpp | 20 +- libminifi/src/core/ProcessSession.cpp | 145 +++++------- libminifi/src/core/Record.cpp | 12 +- libminifi/src/core/Repository.cpp | 10 +- .../core/repository/FlowFileRepository.cpp | 34 +-- libminifi/src/io/ClientSocket.cpp | 224 +++--------------- libminifi/src/io/TLSSocket.cpp | 2 +- libminifi/test/unit/LoggerTests.cpp | 81 +++++-- libminifi/test/unit/ProcessorTests.cpp | 129 ++-------- libminifi/test/unit/ProvenanceTestHelper.h | 218 +++++------------ libminifi/test/unit/ProvenanceTests.cpp | 12 +- libminifi/test/unit/SerializationTests.cpp | 8 +- libminifi/test/unit/SocketTests.cpp | 41 ++-- main/MiNiFiMain.cpp | 15 +- 46 files changed, 852 insertions(+), 842 deletions(-) rename libminifi/include/core/repository/{FlowFIleRepository.h => FlowFileRepository.h} (70%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 62c0360dae..d2084a35b8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -129,6 +129,7 @@ enable_testing(test) target_include_directories(tests PRIVATE BEFORE "include") target_include_directories(tests PRIVATE BEFORE "libminifi/include/") target_include_directories(tests PRIVATE BEFORE "libminifi/include/core") + target_include_directories(tests PRIVATE BEFORE "libminifi/include/core/repository") target_include_directories(tests PRIVATE BEFORE "libminifi/include/io") target_include_directories(tests PRIVATE BEFORE "libminifi/include/utils") target_include_directories(tests PRIVATE BEFORE "libminifi/include/processors") diff --git a/libminifi/CMakeLists.txt b/libminifi/CMakeLists.txt index 6a48483b86..b562e24293 100644 --- a/libminifi/CMakeLists.txt +++ b/libminifi/CMakeLists.txt @@ -60,7 +60,7 @@ include_directories(../thirdparty/yaml-cpp-yaml-cpp-0.5.3/include) include_directories(../thirdparty/civetweb-1.9.1/include) include_directories(include) -file(GLOB SOURCES "src/core/logging/*.cpp" "src/io/*.cpp" "src/core/*.cpp" "src/provenance/*.cpp" "src/processors/*.cpp" "src/*.cpp") +file(GLOB SOURCES "src/core/logging/*.cpp" "src/io/*.cpp" "src/core/*.cpp" "src/core/repository/*.cpp" "src/provenance/*.cpp" "src/processors/*.cpp" "src/*.cpp") file(GLOB SPD_SOURCES "../include/spdlog/*") # Workaround the limitations of having a diff --git a/libminifi/include/Connection.h b/libminifi/include/Connection.h index 1ea38b9a18..1c7b9a475f 100644 --- a/libminifi/include/Connection.h +++ b/libminifi/include/Connection.h @@ -27,12 +27,13 @@ #include #include #include - +#include "core/core.h" #include "core/Connectable.h" #include "core/logging/Logger.h" #include "core/Relationship.h" #include "core/Connectable.h" #include "core/FlowFile.h" +#include "core/Repository.h" namespace org { namespace apache { @@ -181,7 +182,6 @@ class Connection : public core::Connectable, // Queue for the Flow File std::queue> queue_; // flow repository - std::shared_ptr flow_repository_; // Logger std::shared_ptr logger_; // Prevent default copy constructor and assignment operation diff --git a/libminifi/include/EventDrivenSchedulingAgent.h b/libminifi/include/EventDrivenSchedulingAgent.h index ab59725b93..9d53c5c7ba 100644 --- a/libminifi/include/EventDrivenSchedulingAgent.h +++ b/libminifi/include/EventDrivenSchedulingAgent.h @@ -38,7 +38,7 @@ class EventDrivenSchedulingAgent : public ThreadedSchedulingAgent { /*! * Create a new processor */ - EventDrivenSchedulingAgent(std::shared_ptr repo) + EventDrivenSchedulingAgent(std::shared_ptr repo) : ThreadedSchedulingAgent(repo) { } // Destructor diff --git a/libminifi/include/FlowControlProtocol.h b/libminifi/include/FlowControlProtocol.h index 664f612f1b..c37c8f8231 100644 --- a/libminifi/include/FlowControlProtocol.h +++ b/libminifi/include/FlowControlProtocol.h @@ -34,7 +34,7 @@ #include #include "core/Property.h" -#include "Configure.h" +#include "properties/Configure.h" #include "core/logging/Logger.h" diff --git a/libminifi/include/FlowController.h b/libminifi/include/FlowController.h index 4144409739..ba7b82f0df 100644 --- a/libminifi/include/FlowController.h +++ b/libminifi/include/FlowController.h @@ -28,11 +28,18 @@ #include #include #include +<<<<<<< HEAD #include "Configure.h" +======= +#include "yaml-cpp/yaml.h" + +#include "properties/Configure.h" +>>>>>>> MINIFI-217: Update based on PR and rebasing against master #include "core/Relationship.h" #include "FlowFileRecord.h" #include "Connection.h" #include "core/Processor.h" +#include "core/logging/Logger.h" #include "core/ProcessContext.h" #include "core/ProcessSession.h" #include "core/ProcessGroup.h" @@ -107,7 +114,7 @@ class FlowController : public core::CoreComponent { return max_event_driven_threads_; } // Get the provenance repository - virtual std::shared_ptr getProvenanceRepository() { + virtual std::shared_ptr getProvenanceRepository() { return this->provenance_repo_; } @@ -182,7 +189,7 @@ class FlowController : public core::CoreComponent { // Whether it has already been initialized (load the flow XML already) std::atomic initialized_; // Provenance Repo - std::shared_ptr provenance_repo_; + std::shared_ptr provenance_repo_; // FlowFile Repo std::shared_ptr flow_file_repo_; @@ -201,7 +208,7 @@ class FlowController : public core::CoreComponent { FlowController(std::shared_ptr provenance_repo, std::shared_ptr flow_file_repo) - : root_(0), + : CoreComponent(core::getClassName()), root_(0), max_timer_driven_threads_(0), max_event_driven_threads_(0), running_(false), @@ -210,14 +217,13 @@ class FlowController : public core::CoreComponent { flow_file_repo_(flow_file_repo), protocol_(0), _timerScheduler(provenance_repo_), - _eventScheduler(provenance_repo_), - logger_(logging::Logger::getLogger()) { + _eventScheduler(provenance_repo_){ + if (provenance_repo == nullptr) + throw std::runtime_error("Provenance Repo should not be null"); + if (flow_file_repo == nullptr) + throw std::runtime_error("Flow Repo should not be null"); } - private: - - // Logger - std::shared_ptr logger_; }; @@ -263,7 +269,7 @@ class FlowControllerImpl : public FlowController { /*! * Create a new Flow Controller */ - FlowControllerImpl(std::shared_ptr repo, + FlowControllerImpl(std::shared_ptr repo,std::shared_ptr flow_file_repo, std::string name = DEFAULT_ROOT_GROUP_NAME); // Prevent default copy constructor and assignment operation diff --git a/libminifi/include/FlowFileRecord.h b/libminifi/include/FlowFileRecord.h index f479787d30..ca0856c126 100644 --- a/libminifi/include/FlowFileRecord.h +++ b/libminifi/include/FlowFileRecord.h @@ -31,6 +31,7 @@ #include #include +#include "io/Serializable.h" #include "core/FlowFile.h" #include "utils/TimeUtil.h" #include "core/logging/Logger.h" @@ -89,17 +90,29 @@ class OutputStreamCallback { virtual void process(std::ofstream *stream) = 0; }; -class FlowFileRecord : public core::FlowFile { +class FlowFileRecord : public core::FlowFile, public io::Serializable { public: // Constructor - /*! + /* * Create a new flow record */ explicit FlowFileRecord(std::shared_ptr flow_repository, std::map attributes, std::shared_ptr claim = nullptr); - explicit FlowFileRecord(std::shared_ptr flow_repository,std::shared_ptr event); + explicit FlowFileRecord(std::shared_ptr flow_repository, + std::shared_ptr &event); + + explicit FlowFileRecord(std::shared_ptr flow_repository, + std::shared_ptr &event, + const std::string &uuidConnection); + + explicit FlowFileRecord(std::shared_ptr flow_repository) + : FlowFile(), + flow_repository_(flow_repository), + snapshot_("") { + + } // Destructor virtual ~FlowFileRecord(); // addAttribute key is enum @@ -111,17 +124,45 @@ class FlowFileRecord : public core::FlowFile { // getAttribute key is enum bool getKeyedAttribute(FlowAttribute key, std::string &value); + //! Serialize and Persistent to the repository + bool Serialize(); + //! DeSerialize + bool DeSerialize(const uint8_t *buffer, const int bufferSize); + //! DeSerialize + bool DeSerialize(io::DataStream &stream) { + return DeSerialize(stream.getBuffer(), stream.getSize()); + } + //! DeSerialize + bool DeSerialize(std::string key); + void setSnapShot(bool snapshot) { snapshot_ = snapshot; } - FlowFileRecord &operator=(const FlowFileRecord &) = delete; + /** + * gets the UUID connection. + * @return uuidConnection + */ + const std::string getConnectionUuid() { + return uuid_connection_; + } + + const std::string getContentFullPath() + { + return content_full_fath_; + } + + + FlowFileRecord &operator=(const FlowFileRecord &); FlowFileRecord(const FlowFileRecord &parent) = delete; protected: - private: + // connection uuid + std::string uuid_connection_; + // Full path to the content + std::string content_full_fath_; // Local flow sequence ID static std::atomic local_flow_seq_number_; @@ -129,8 +170,6 @@ class FlowFileRecord : public core::FlowFile { // repository reference. std::shared_ptr flow_repository_; - // Logger - std::shared_ptr logger_; // Snapshot flow record for session rollback bool snapshot_; // Prevent default copy constructor and assignment operation diff --git a/libminifi/include/RemoteProcessorGroupPort.h b/libminifi/include/RemoteProcessorGroupPort.h index f28469bcc5..e9a4228908 100644 --- a/libminifi/include/RemoteProcessorGroupPort.h +++ b/libminifi/include/RemoteProcessorGroupPort.h @@ -20,12 +20,15 @@ #ifndef __REMOTE_PROCESSOR_GROUP_PORT_H__ #define __REMOTE_PROCESSOR_GROUP_PORT_H__ +#include #include +#include #include "FlowFileRecord.h" #include "core/Processor.h" #include "core/ProcessSession.h" #include "Site2SiteClientProtocol.h" + namespace org { namespace apache { namespace nifi { @@ -41,12 +44,9 @@ class RemoteProcessorGroupPort : RemoteProcessorGroupPort(std::string name, uuid_t uuid = NULL) : core::Processor(name, uuid), direction_(SEND), - transmitting_(false), - peer_() { + transmitting_(false){ logger_ = logging::Logger::getLogger(); - protocol_ = std::unique_ptr( - new Site2SiteClientProtocol(0)); - protocol_->setPortId(uuid); + uuid_copy(protocol_uuid_,uuid); } // Destructor virtual ~RemoteProcessorGroupPort() { @@ -74,7 +74,7 @@ class RemoteProcessorGroupPort : } // Set Timeout void setTimeOut(uint64_t timeout) { - protocol_->setTimeOut(timeout); + timeout_ = timeout; } // SetTransmitting void setTransmitting(bool val) { @@ -84,17 +84,23 @@ class RemoteProcessorGroupPort : protected: private: + + std::unique_ptr getNextProtocol(); + void returnProtocol(std::unique_ptr protocol); + + std::stack> available_protocols_; + std::mutex protocol_mutex_; // Logger std::shared_ptr logger_; - // Peer Connection - Site2SitePeer peer_; - // Peer Protocol - std::unique_ptr protocol_; // Transaction Direction TransferDirection direction_; // Transmitting bool transmitting_; - + // timeout + uint64_t timeout_; + + uuid_t protocol_uuid_; + }; } /* namespace minifi */ diff --git a/libminifi/include/ResourceClaim.h b/libminifi/include/ResourceClaim.h index 6178ee2a54..4c5438cb6d 100644 --- a/libminifi/include/ResourceClaim.h +++ b/libminifi/include/ResourceClaim.h @@ -27,7 +27,7 @@ #include #include #include -#include "Configure.h" +#include "properties/Configure.h" namespace org { namespace apache { diff --git a/libminifi/include/SchedulingAgent.h b/libminifi/include/SchedulingAgent.h index 67cab47e1f..04936402a2 100644 --- a/libminifi/include/SchedulingAgent.h +++ b/libminifi/include/SchedulingAgent.h @@ -28,12 +28,14 @@ #include #include #include "utils/TimeUtil.h" +#include "core/core.h" #include "core/logging/Logger.h" -#include "Configure.h" +#include "properties/Configure.h" #include "FlowFileRecord.h" #include "core/logging/Logger.h" #include "core/Processor.h" #include "core/ProcessContext.h" +#include "provenance/ProvenanceRepository.h" namespace org { @@ -49,7 +51,7 @@ class SchedulingAgent { /*! * Create a new processor */ - SchedulingAgent(std::shared_ptr repo) { + SchedulingAgent(std::shared_ptr repo) { configure_ = Configure::getConfigure(); logger_ = logging::Logger::getLogger(); running_ = false; @@ -102,7 +104,7 @@ class SchedulingAgent { // BoredYieldDuration int64_t _boredYieldDuration; - std::shared_ptr repo_; + std::shared_ptr repo_; private: // Prevent default copy constructor and assignment operation diff --git a/libminifi/include/Site2SiteClientProtocol.h b/libminifi/include/Site2SiteClientProtocol.h index 705909e5a4..6120e3e5b9 100644 --- a/libminifi/include/Site2SiteClientProtocol.h +++ b/libminifi/include/Site2SiteClientProtocol.h @@ -36,7 +36,7 @@ #include #include "core/Property.h" -#include "Configure.h" +#include "properties/Configure.h" #include "Site2SitePeer.h" #include "FlowFileRecord.h" #include "core/logging/Logger.h" @@ -396,10 +396,10 @@ class Site2SiteClientProtocol { /*! * Create a new control protocol */ - Site2SiteClientProtocol(Site2SitePeer *peer) { + Site2SiteClientProtocol(std::unique_ptr peer) { logger_ = logging::Logger::getLogger(); configure_ = Configure::getConfigure(); - peer_ = peer; + peer_ = std::move(peer); _batchSize = 0; _batchCount = 0; _batchDuration = 0; @@ -419,6 +419,7 @@ class Site2SiteClientProtocol { } // Destructor virtual ~Site2SiteClientProtocol() { + tearDown(); } public: @@ -442,8 +443,8 @@ class Site2SiteClientProtocol { } - void setPeer(Site2SitePeer *peer) { - peer_ = peer; + void setPeer(std::unique_ptr peer) { + peer_ = std::move(peer); } /** * Provides a reference to the time out @@ -507,10 +508,7 @@ class Site2SiteClientProtocol { } return NULL; } - // getPeer - Site2SitePeer *getPeer() { - return peer_; - } + // Creation of a new transaction, return the transaction ID if success, // Return NULL when any error occurs Transaction *createTransaction(std::string &transactionID, @@ -611,7 +609,7 @@ class Site2SiteClientProtocol { // Timeout in msec std::atomic _timeOut; // Peer Connection - Site2SitePeer *peer_; + std::unique_ptr peer_; // portId uuid_t _portId; // portIDStr diff --git a/libminifi/include/Site2SitePeer.h b/libminifi/include/Site2SitePeer.h index f14993bbfc..de3a42f84a 100644 --- a/libminifi/include/Site2SitePeer.h +++ b/libminifi/include/Site2SitePeer.h @@ -32,7 +32,7 @@ #include "core/Property.h" #include "core/logging/Logger.h" -#include "Configure.h" +#include "properties/Configure.h" #include "io/ClientSocket.h" #include "io/BaseStream.h" #include "utils/TimeUtil.h" diff --git a/libminifi/include/ThreadedSchedulingAgent.h b/libminifi/include/ThreadedSchedulingAgent.h index 8c1c3e8c18..4e39da347a 100644 --- a/libminifi/include/ThreadedSchedulingAgent.h +++ b/libminifi/include/ThreadedSchedulingAgent.h @@ -20,9 +20,10 @@ #ifndef __THREADED_SCHEDULING_AGENT_H__ #define __THREADED_SCHEDULING_AGENT_H__ -#include "Configure.h" +#include "properties/Configure.h" #include "core/logging/Logger.h" #include "core/Processor.h" +#include "core/Repository.h" #include "core/ProcessContext.h" #include "SchedulingAgent.h" @@ -41,7 +42,7 @@ class ThreadedSchedulingAgent : public SchedulingAgent { /*! * Create a new processor */ - ThreadedSchedulingAgent(std::shared_ptr repo) + ThreadedSchedulingAgent(std::shared_ptr repo) : SchedulingAgent(repo) { } // Destructor diff --git a/libminifi/include/TimerDrivenSchedulingAgent.h b/libminifi/include/TimerDrivenSchedulingAgent.h index b34f9238b4..7da2abd26a 100644 --- a/libminifi/include/TimerDrivenSchedulingAgent.h +++ b/libminifi/include/TimerDrivenSchedulingAgent.h @@ -23,6 +23,7 @@ #include "core/logging/Logger.h" #include "core/Processor.h" #include "core/ProcessContext.h" +#include "core/Repository.h" #include "ThreadedSchedulingAgent.h" namespace org { @@ -36,7 +37,7 @@ class TimerDrivenSchedulingAgent : public ThreadedSchedulingAgent { /*! * Create a new processor */ - TimerDrivenSchedulingAgent(std::shared_ptr repo) + TimerDrivenSchedulingAgent(std::shared_ptr repo) : ThreadedSchedulingAgent(repo) { } // Destructor diff --git a/libminifi/include/core/FlowFile.h b/libminifi/include/core/FlowFile.h index 229a4aa069..247ad26555 100644 --- a/libminifi/include/core/FlowFile.h +++ b/libminifi/include/core/FlowFile.h @@ -73,6 +73,12 @@ class FlowFile { * @return entry date uint64_t */ uint64_t getEntryDate(); + + /** + * Gets the event time. + * @return event time. + */ + uint64_t getEventTime(); /** * Get lineage start date * @return lineage start date uint64_t @@ -177,6 +183,12 @@ class FlowFile { std::string getUUIDStr() { return uuid_str_; } + + bool getUUID(uuid_t other) + { + uuid_copy(other,uuid_); + return true; + } // Check whether it is still being penalized bool isPenalized() { @@ -227,6 +239,8 @@ class FlowFile { bool marked_delete_; // Date at which the flow file entered the flow uint64_t entry_date_; + // event time + uint64_t event_time_; // Date at which the origin of this flow file entered the flow uint64_t lineage_start_date_; // Date at which the flow file was queued @@ -250,6 +264,9 @@ class FlowFile { // UUID string for all parents std::set lineage_Identifiers_; + // Logger + std::shared_ptr logger_; + // Connection queue that this flow file will be transfer or current in std::shared_ptr connection_; // Orginal connection queue that this flow file was dequeued from diff --git a/libminifi/include/core/ProcessGroup.h b/libminifi/include/core/ProcessGroup.h index 05a50a0f3c..75bb0bac1f 100644 --- a/libminifi/include/core/ProcessGroup.h +++ b/libminifi/include/core/ProcessGroup.h @@ -146,7 +146,8 @@ class ProcessGroup { void updatePropertyValue(std::string processorName, std::string propertyName, std::string propertyValue); - void getConnections(std::map &connectionMap); + void getConnections( + std::map> &connectionMap); protected: // A global unique identifier diff --git a/libminifi/include/core/ProcessSession.h b/libminifi/include/core/ProcessSession.h index d50c8ec4ec..b516817f4e 100644 --- a/libminifi/include/core/ProcessSession.h +++ b/libminifi/include/core/ProcessSession.h @@ -56,105 +56,109 @@ class ProcessSession { provenance_report_ = new provenance::ProvenanceReporter( repo, process_context_->getProcessorNode().getUUIDStr(), process_context_->getProcessorNode().getName()); - } } - // Destructor + +// Destructor virtual ~ProcessSession() { if (provenance_report_) delete provenance_report_; } - // Commit the session +// Commit the session void commit(); - // Roll Back the session +// Roll Back the session void rollback(); - // Get Provenance Report +// Get Provenance Report provenance::ProvenanceReporter *getProvenanceReporter() { return provenance_report_; } - // - // Get the FlowFile from the highest priority queue +// +// Get the FlowFile from the highest priority queue std::shared_ptr get(); - // Create a new UUID FlowFile with no content resource claim and without parent +// Create a new UUID FlowFile with no content resource claim and without parent std::shared_ptr create(); - // Create a new UUID FlowFile with no content resource claim and inherit all attributes from parent +// Create a new UUID FlowFile with no content resource claim and inherit all attributes from parent std::shared_ptr create(std::shared_ptr &parent); - // Clone a new UUID FlowFile from parent both for content resource claim and attributes - std::shared_ptr clone(std::shared_ptr &parent); - // Clone a new UUID FlowFile from parent for attributes and sub set of parent content resource claim +// Clone a new UUID FlowFile from parent both for content resource claim and attributes + std::shared_ptr clone( + std::shared_ptr &parent); +// Clone a new UUID FlowFile from parent for attributes and sub set of parent content resource claim std::shared_ptr clone(std::shared_ptr &parent, - long offset, long size); - // Duplicate a FlowFile with the same UUID and all attributes and content resource claim for the roll back of the session + long offset, long size); +// Duplicate a FlowFile with the same UUID and all attributes and content resource claim for the roll back of the session std::shared_ptr duplicate( std::shared_ptr &original); - // Transfer the FlowFile to the relationship - void transfer(std::shared_ptr &flow, Relationship relationship); +// Transfer the FlowFile to the relationship + void transfer(std::shared_ptr &flow, + Relationship relationship); void transfer(std::shared_ptr &&flow, Relationship relationship); - // Put Attribute +// Put Attribute void putAttribute(std::shared_ptr &flow, std::string key, std::string value); void putAttribute(std::shared_ptr &&flow, std::string key, std::string value); - // Remove Attribute +// Remove Attribute void removeAttribute(std::shared_ptr &flow, std::string key); void removeAttribute(std::shared_ptr &&flow, std::string key); - // Remove Flow File +// Remove Flow File void remove(std::shared_ptr &flow); void remove(std::shared_ptr &&flow); - // Execute the given read callback against the content - void read(std::shared_ptr &flow, InputStreamCallback *callback); +// Execute the given read callback against the content + void read(std::shared_ptr &flow, + InputStreamCallback *callback); void read(std::shared_ptr &&flow, InputStreamCallback *callback); - // Execute the given write callback against the content +// Execute the given write callback against the content void write(std::shared_ptr &flow, OutputStreamCallback *callback); void write(std::shared_ptr &&flow, OutputStreamCallback *callback); - // Execute the given write/append callback against the content +// Execute the given write/append callback against the content void append(std::shared_ptr &flow, OutputStreamCallback *callback); void append(std::shared_ptr &&flow, OutputStreamCallback *callback); - // Penalize the flow +// Penalize the flow void penalize(std::shared_ptr &flow); void penalize(std::shared_ptr &&flow); - // Import the existed file into the flow +// Import the existed file into the flow void import(std::string source, std::shared_ptr &flow, bool keepSource = true, uint64_t offset = 0); void import(std::string source, std::shared_ptr &&flow, bool keepSource = true, uint64_t offset = 0); - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer +// Prevent default copy constructor and assignment operation +// Only support pass by reference or pointer ProcessSession(const ProcessSession &parent) = delete; ProcessSession &operator=(const ProcessSession &parent) = delete; protected: - // FlowFiles being modified by current process session +// FlowFiles being modified by current process session std::map > _updatedFlowFiles; - // Copy of the original FlowFiles being modified by current process session as above +// Copy of the original FlowFiles being modified by current process session as above std::map > _originalFlowFiles; - // FlowFiles being added by current process session +// FlowFiles being added by current process session std::map > _addedFlowFiles; - // FlowFiles being deleted by current process session +// FlowFiles being deleted by current process session std::map > _deletedFlowFiles; - // FlowFiles being transfered to the relationship +// FlowFiles being transfered to the relationship std::map _transferRelationship; - // FlowFiles being cloned for multiple connections per relationship +// FlowFiles being cloned for multiple connections per relationship std::map > _clonedFlowFiles; private: - // Clone the flow file during transfer to multiple connections for a relationship +// Clone the flow file during transfer to multiple connections for a relationship std::shared_ptr cloneDuringTransfer( std::shared_ptr &parent); - // ProcessContext +// ProcessContext ProcessContext *process_context_; - // Logger +// Logger std::shared_ptr logger_; - // Provenance Report +// Provenance Report provenance::ProvenanceReporter *provenance_report_; -}; +} +; } /* namespace core */ } /* namespace minifi */ } /* namespace nifi */ diff --git a/libminifi/include/core/Repository.h b/libminifi/include/core/Repository.h index d0a1d56044..059f38b18c 100644 --- a/libminifi/include/core/Repository.h +++ b/libminifi/include/core/Repository.h @@ -32,13 +32,7 @@ #include #include -#include "leveldb/db.h" -#include "leveldb/options.h" -#include "leveldb/slice.h" -#include "leveldb/status.h" -#include "Configure.h" -#include "Connection.h" -#include "FlowFileRecord.h" +#include "properties/Configure.h" #include "core/logging/Logger.h" #include "core/Property.h" #include "ResourceClaim.h" @@ -87,7 +81,7 @@ class Repository : public CoreComponent { virtual bool Get(std::string key, std::string &value) = 0; // Run function for the thread - void run() = 0; + virtual void run() = 0; // Start the repository monitor thread virtual void start(); // Stop the repository monitor thread @@ -102,7 +96,7 @@ class Repository : public CoreComponent { } uint64_t incrementSize(const char *fpath, const struct stat *sb, int typeflag) { - return (repo_full_ += sb->st_size); + return (repo_size_ += sb->st_size); } // Prevent default copy constructor and assignment operation diff --git a/libminifi/include/core/core.h b/libminifi/include/core/core.h index b639b2d6ae..17f921e3db 100644 --- a/libminifi/include/core/core.h +++ b/libminifi/include/core/core.h @@ -17,6 +17,10 @@ */ #ifndef LIBMINIFI_INCLUDE_CORE_CORE_H_ #define LIBMINIFI_INCLUDE_CORE_CORE_H_ + +#include +#include +#include "core/logging/Logger.h" /** * namespace aliasing */ @@ -35,7 +39,9 @@ namespace core { template static inline std::string getClassName() { - std::string name = abi::__cxa_demangle(typeid(T).name(), 0, 0, 0); + char *b = abi::__cxa_demangle(typeid(T).name(), 0, 0, 0); + std::string name = b; + delete [] b; return name; } @@ -75,7 +81,7 @@ class CoreComponent { } // Get component name Name - std::string getName() const; + std::string getName(); /** * Set name. @@ -94,14 +100,14 @@ class CoreComponent { * @param uuid uuid struct to which we will copy the memory * @return success of request */ - bool getUUID(uuid_t uuid) const; + bool getUUID(uuid_t uuid); - unsigned const char *getUUID() const; + unsigned const char *getUUID(); /** * Return the UUID string * @param constant reference to the UUID str */ - const std::string & getUUIDStr() const { + const std::string & getUUIDStr() { return uuidStr_; } @@ -112,7 +118,7 @@ class CoreComponent { std::string uuidStr_; // logger shared ptr - std::shared_ptr logger_; + std::shared_ptr logger_; // Connectable's name std::string name_; diff --git a/libminifi/include/core/logging/LogAppenders.h b/libminifi/include/core/logging/LogAppenders.h index 36892e9880..7bdc3befb0 100644 --- a/libminifi/include/core/logging/LogAppenders.h +++ b/libminifi/include/core/logging/LogAppenders.h @@ -22,7 +22,7 @@ #include "spdlog/sinks/null_sink.h" #include "spdlog/sinks/ostream_sink.h" #include -#include "Configure.h" +#include "properties/Configure.h" namespace org { namespace apache { diff --git a/libminifi/include/core/repository/FlowFIleRepository.h b/libminifi/include/core/repository/FlowFileRepository.h similarity index 70% rename from libminifi/include/core/repository/FlowFIleRepository.h rename to libminifi/include/core/repository/FlowFileRepository.h index 231148a51b..6980e6fd59 100644 --- a/libminifi/include/core/repository/FlowFIleRepository.h +++ b/libminifi/include/core/repository/FlowFileRepository.h @@ -18,8 +18,13 @@ #ifndef LIBMINIFI_INCLUDE_CORE_REPOSITORY_FLOWFILEREPOSITORY_H_ #define LIBMINIFI_INCLUDE_CORE_REPOSITORY_FLOWFILEREPOSITORY_H_ +#include "leveldb/db.h" +#include "leveldb/options.h" +#include "leveldb/slice.h" +#include "leveldb/status.h" #include "core/Repository.h" #include "core/core.h" +#include "Connection.h" namespace org { namespace apache { @@ -39,18 +44,25 @@ namespace repository { * Flow File repository * Design: Extends Repository and implements the run function, using LevelDB as the primary substrate. */ -class FlowFileRepository : public Repository { +class FlowFileRepository : public core::Repository, public std::enable_shared_from_this { public: // Constructor + + FlowFileRepository(std::string directory, int64_t maxPartitionMillis, int64_t maxPartitionBytes, uint64_t purgePeriod) - : Repository(getClassName(), directory, + : Repository(core::getClassName(), directory, maxPartitionMillis, maxPartitionBytes, purgePeriod) { db_ = NULL; } + + FlowFileRepository() : FlowFileRepository(FLOWFILE_REPOSITORY_DIRECTORY, + MAX_FLOWFILE_REPOSITORY_ENTRY_LIFE_TIME, MAX_FLOWFILE_REPOSITORY_STORAGE_SIZE, FLOWFILE_REPOSITORY_PURGE_PERIOD) + { + } // Destructor virtual ~FlowFileRepository() { @@ -101,7 +113,49 @@ class FlowFileRepository : public Repository { } virtual void run(); - + + virtual bool Put(std::string key, uint8_t *buf, int bufLen) + { + + // persistent to the DB + leveldb::Slice value((const char *) buf, bufLen); + leveldb::Status status; + status = db_->Put(leveldb::WriteOptions(), key, value); + if (status.ok()) + return true; + else + return false; + } + /** + * + * Deletes the key + * @return status of the delete operation + */ + virtual bool Delete(std::string key) + { + leveldb::Status status; + status = db_->Delete(leveldb::WriteOptions(), key); + if (status.ok()) + return true; + else + return false; + } + /** + * Sets the value from the provided key + * @return status of the get operation. + */ + virtual bool Get(std::string key, std::string &value) + { + leveldb::Status status; + status = db_->Get(leveldb::ReadOptions(), key, &value); + if (status.ok()) + return true; + else + return false; + } + + void loadFlowFileToConnections(std::map> &connectionMap); + private: leveldb::DB* db_; }; diff --git a/libminifi/include/io/SocketFactory.h b/libminifi/include/io/SocketFactory.h index d2349a5607..d6221062c5 100644 --- a/libminifi/include/io/SocketFactory.h +++ b/libminifi/include/io/SocketFactory.h @@ -21,7 +21,7 @@ #include "ClientSocket.h" #include "TLSSocket.h" #include "ClientSocket.h" -#include "Configure.h" +#include "properties/Configure.h" #include "utils/StringUtils.h" namespace org { diff --git a/libminifi/include/io/TLSSocket.h b/libminifi/include/io/TLSSocket.h index 469fcf7477..28858cd83f 100644 --- a/libminifi/include/io/TLSSocket.h +++ b/libminifi/include/io/TLSSocket.h @@ -24,7 +24,7 @@ #include #include -#include "Configure.h" +#include "properties/Configure.h" namespace org { namespace apache { diff --git a/libminifi/include/provenance/Provenance.h b/libminifi/include/provenance/Provenance.h index bdbffac894..9e7bbff7e8 100644 --- a/libminifi/include/provenance/Provenance.h +++ b/libminifi/include/provenance/Provenance.h @@ -37,7 +37,7 @@ #include "core/Repository.h" #include "core/Property.h" -#include "Configure.h" +#include "properties/Configure.h" #include "Connection.h" #include "FlowFileRecord.h" #include "core/logging/Logger.h" diff --git a/libminifi/include/provenance/ProvenanceRepository.h b/libminifi/include/provenance/ProvenanceRepository.h index de5ce2680c..fa3c3c9dff 100644 --- a/libminifi/include/provenance/ProvenanceRepository.h +++ b/libminifi/include/provenance/ProvenanceRepository.h @@ -17,6 +17,10 @@ #ifndef LIBMINIFI_INCLUDE_PROVENANCE_PROVENANCEREPOSITORY_H_ #define LIBMINIFI_INCLUDE_PROVENANCE_PROVENANCEREPOSITORY_H_ +#include "leveldb/db.h" +#include "leveldb/options.h" +#include "leveldb/slice.h" +#include "leveldb/status.h" #include "core/Repository.h" #include "core/core.h" namespace org { @@ -141,7 +145,7 @@ class ProvenanceRepository : public core::Repository, } } // Run function for the thread - static void run(); + void run(); // Prevent default copy constructor and assignment operation // Only support pass by reference or pointer diff --git a/libminifi/src/Configure.cpp b/libminifi/src/Configure.cpp index 76bd03ba03..74d2cc16ea 100644 --- a/libminifi/src/Configure.cpp +++ b/libminifi/src/Configure.cpp @@ -17,7 +17,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "Configure.h" +#include "properties/Configure.h" #include "utils/StringUtils.h" #include "core/core.h" diff --git a/libminifi/src/Connection.cpp b/libminifi/src/Connection.cpp index bf216ace9c..6f64ff3ed6 100644 --- a/libminifi/src/Connection.cpp +++ b/libminifi/src/Connection.cpp @@ -95,9 +95,8 @@ void Connection::put(std::shared_ptr flow) { if (!flow->isStored()) { // Save to the flowfile repo - FlowFileEventRecord event; - event.fromFlowFile(flow, this->uuidStr_); - if (event.Serialize(flow_repository_)) { + FlowFileRecord event(flow_repository_,flow,this->uuidStr_); + if (event.Serialize()) { flow->setStoredToRepository(true); } } diff --git a/libminifi/src/FlowController.cpp b/libminifi/src/FlowController.cpp index 74e0b2bb5f..c3eec13534 100644 --- a/libminifi/src/FlowController.cpp +++ b/libminifi/src/FlowController.cpp @@ -34,6 +34,7 @@ #include "core/ProcessGroup.h" #include "utils/StringUtils.h" #include "core/core.h" +#include "core/repository/FlowFileRepository.h" namespace org { namespace apache { @@ -47,7 +48,7 @@ FlowControllerImpl::FlowControllerImpl( setUUID(uuid_); // Setup the default values - configuration_filename_ = DEFAULT_FLOW_YAML_FILENAME; + configuration_filename_ = DEFAULT_FLOW_YAML_FILE_NAME; max_event_driven_threads_ = DEFAULT_MAX_EVENT_DRIVEN_THREAD; max_timer_driven_threads_ = DEFAULT_MAX_TIMER_DRIVEN_THREAD; running_ = false; @@ -258,10 +259,9 @@ core::ProcessGroup *FlowControllerImpl::createRemoteProcessGroup( std::shared_ptr FlowControllerImpl::createConnection( std::string name, uuid_t uuid) { - return std::make_shared(name, uuid); + return std::make_shared(flow_file_repo_, name, uuid); } -#ifdef YAML_SUPPORT void FlowControllerImpl::parseRootProcessGroupYaml(YAML::Node rootFlowNode) { uuid_t uuid; core::ProcessGroup *group = NULL; @@ -386,10 +386,10 @@ void FlowControllerImpl::parseProcessorNodeYaml(YAML::Node processorsNode, parsePropertiesNodeYaml(&propertiesNode, processor); // Take care of scheduling - TimeUnit unit; - if (Property::StringToTime(procCfg.schedulingPeriod, + core::TimeUnit unit; + if (core::Property::StringToTime(procCfg.schedulingPeriod, schedulingPeriod, unit) - && Property::ConvertTimeUnitToNS(schedulingPeriod, unit, + && core::Property::ConvertTimeUnitToNS(schedulingPeriod, unit, schedulingPeriod)) { logger_->log_debug( "convert: parseProcessorNode: schedulingPeriod => [%d] ns", @@ -397,9 +397,9 @@ void FlowControllerImpl::parseProcessorNodeYaml(YAML::Node processorsNode, processor->setSchedulingPeriodNano(schedulingPeriod); } - if (Property::StringToTime(procCfg.penalizationPeriod, + if (core::Property::StringToTime(procCfg.penalizationPeriod, penalizationPeriod, unit) - && Property::ConvertTimeUnitToMS(penalizationPeriod, + && core::Property::ConvertTimeUnitToMS(penalizationPeriod, unit, penalizationPeriod)) { logger_->log_debug( "convert: parseProcessorNode: penalizationPeriod => [%d] ms", @@ -407,9 +407,9 @@ void FlowControllerImpl::parseProcessorNodeYaml(YAML::Node processorsNode, processor->setPenalizationPeriodMsec(penalizationPeriod); } - if (Property::StringToTime(procCfg.yieldPeriod, yieldPeriod, + if (core::Property::StringToTime(procCfg.yieldPeriod, yieldPeriod, unit) - && Property::ConvertTimeUnitToMS(yieldPeriod, unit, + && core::Property::ConvertTimeUnitToMS(yieldPeriod, unit, yieldPeriod)) { logger_->log_debug( "convert: parseProcessorNode: yieldPeriod => [%d] ms", @@ -436,7 +436,7 @@ void FlowControllerImpl::parseProcessorNodeYaml(YAML::Node processorsNode, } int64_t maxConcurrentTasks; - if (Property::StringToInt(procCfg.maxConcurrentTasks, + if (core::Property::StringToInt(procCfg.maxConcurrentTasks, maxConcurrentTasks)) { logger_->log_debug( "parseProcessorNode: maxConcurrentTasks => [%d]", @@ -444,7 +444,7 @@ void FlowControllerImpl::parseProcessorNodeYaml(YAML::Node processorsNode, processor->setMaxConcurrentTasks(maxConcurrentTasks); } - if (Property::StringToInt(procCfg.runDurationNanos, + if (core::Property::StringToInt(procCfg.runDurationNanos, runDurationNanos)) { logger_->log_debug( "parseProcessorNode: runDurationNanos => [%d]", @@ -524,10 +524,10 @@ void FlowControllerImpl::parseRemoteProcessGroupYaml(YAML::Node *rpgNode, group->setParent(parentGroup); parentGroup->addProcessGroup(group); - TimeUnit unit; + core::TimeUnit unit; - if (Property::StringToTime(yieldPeriod, yieldPeriodValue, unit) - && Property::ConvertTimeUnitToMS(yieldPeriodValue, unit, + if (core::Property::StringToTime(yieldPeriod, yieldPeriodValue, unit) + && core::Property::ConvertTimeUnitToMS(yieldPeriodValue, unit, yieldPeriodValue) && group) { logger_->log_debug( "parseRemoteProcessGroupYaml: yieldPeriod => [%d] ms", @@ -535,8 +535,8 @@ void FlowControllerImpl::parseRemoteProcessGroupYaml(YAML::Node *rpgNode, group->setYieldPeriodMsec(yieldPeriodValue); } - if (Property::StringToTime(timeout, timeoutValue, unit) - && Property::ConvertTimeUnitToMS(timeoutValue, unit, + if (core::Property::StringToTime(timeout, timeoutValue, unit) + && core::Property::ConvertTimeUnitToMS(timeoutValue, unit, timeoutValue) && group) { logger_->log_debug( "parseRemoteProcessGroupYaml: timeoutValue => [%d] ms", @@ -615,7 +615,7 @@ void FlowControllerImpl::parseConnectionYaml(YAML::Node *connectionsNode, uuid_t srcUUID; uuid_parse(connectionSrcProcId.c_str(), srcUUID); - Processor *srcProcessor = this->root_->findProcessor( + auto srcProcessor = this->root_->findProcessor( srcUUID); if (!srcProcessor) { @@ -629,7 +629,7 @@ void FlowControllerImpl::parseConnectionYaml(YAML::Node *connectionsNode, uuid_t destUUID; uuid_parse(destId.c_str(), destUUID); - Processor *destProcessor = this->root_->findProcessor(destUUID); + auto destProcessor = this->root_->findProcessor(destUUID); // If we could not find name, try by UUID if (!destProcessor) { uuid_t destUuid; @@ -729,19 +729,8 @@ void FlowControllerImpl::parsePropertiesNodeYaml( } } } -#endif /* ifdef YAML_SUPPORT */ void FlowControllerImpl::load() { -<<<<<<< HEAD - if (_running) { - stop(true); - } - if (!_initialized) { - logger_->log_info("Load Flow Controller from file %s", _configurationFileName.c_str()); - -#ifdef YAML_SUPPORT - YAML::Node flow = YAML::LoadFile(_configurationFileName); -======= if (running_) { stop(true); } @@ -750,7 +739,6 @@ void FlowControllerImpl::load() { configuration_filename_.c_str()); YAML::Node flow = YAML::LoadFile(configuration_filename_); ->>>>>>> MINIFI-217: First commit. Updates namespaces and removes YAML::Node flowControllerNode = flow["Flow Controller"]; YAML::Node processorsNode = flow[CONFIG_YAML_PROCESSORS_KEY]; @@ -763,11 +751,10 @@ void FlowControllerImpl::load() { parseRemoteProcessGroupYaml(&remoteProcessingGroupNode, this->root_); parseConnectionYaml(&connectionsNode, this->root_); - // Load Flow File from Repo - loadFlowRepo(); -#endif - + // Load Flow File from Repo + loadFlowRepo(); + initialized_=true; } } @@ -795,9 +782,10 @@ void FlowControllerImpl::loadFlowRepo() { if (this->flow_file_repo_) { - std::map connectionMap; + std::map> connectionMap; this->root_->getConnections(connectionMap); - this->flow_file_repo_->loadFlowFileToConnections(&connectionMap); + auto rep = std::static_pointer_cast(flow_file_repo_); + rep->loadFlowFileToConnections(connectionMap); } } diff --git a/libminifi/src/FlowFileRecord.cpp b/libminifi/src/FlowFileRecord.cpp index 8bc132f2e0..7383574dc0 100644 --- a/libminifi/src/FlowFileRecord.cpp +++ b/libminifi/src/FlowFileRecord.cpp @@ -69,8 +69,30 @@ FlowFileRecord::FlowFileRecord( FlowFileRecord::FlowFileRecord( std::shared_ptr flow_repository, - std::shared_ptr event) + std::shared_ptr &event, const std::string &uuidConnection) : FlowFile(), + snapshot_(""), + flow_repository_(flow_repository) { + entry_date_ = event->getEntryDate(); + lineage_start_date_ = event->getlineageStartDate(); + lineage_Identifiers_ = event->getlineageIdentifiers(); + uuid_str_ = event->getUUIDStr(); + attributes_ = event->getAttributes(); + size_ = event->getSize(); + offset_ = event->getOffset(); + event->getUUID(uuid_); + uuid_connection_ = uuidConnection; + if (event->getResourceClaim()) { + content_full_fath_ = event->getResourceClaim()->getContentFullPath(); + } +} + +FlowFileRecord::FlowFileRecord( + std::shared_ptr flow_repository, + std::shared_ptr &event) + : FlowFile(), + uuid_connection_(""), + snapshot_(""), flow_repository_(flow_repository) { } @@ -83,10 +105,11 @@ FlowFileRecord::~FlowFileRecord() { if (claim_) { // Decrease the flow file record owned count for the resource claim claim_->decreaseFlowFileRecordOwnedCount(); + std::string value; if (claim_->getFlowFileRecordOwnedCount() <= 0) { logger_->log_debug("Delete Resource Claim %s", claim_->getContentFullPath().c_str()); - if (!this->stored || !flow_repository_->Get(_uuidStr, value)) { + if (!this->stored || !flow_repository_->Get(uuid_str_, value)) { std::remove(claim_->getContentFullPath().c_str()); } } @@ -136,9 +159,206 @@ bool FlowFileRecord::getKeyedAttribute(FlowAttribute key, std::string &value) { FlowFileRecord &FlowFileRecord::operator=(const FlowFileRecord &other) { core::FlowFile::operator=(other); + uuid_connection_ = other.uuid_connection_; + content_full_fath_ = other.content_full_fath_; + snapshot_ = other.snapshot_; return *this; } +bool FlowFileRecord::DeSerialize(std::string key) { + std::string value; + bool ret; + + ret = flow_repository_->Get(key, value); + + if (!ret) { + logger_->log_error("NiFi FlowFile Store event %s can not found", + key.c_str()); + return false; + } else + logger_->log_debug("NiFi FlowFile Read event %s length %d", key.c_str(), + value.length()); + + io::DataStream stream((const uint8_t*) value.data(), value.length()); + + ret = DeSerialize(stream); + + if (ret) { + logger_->log_debug( + "NiFi FlowFile retrieve uuid %s size %d connection %s success", + uuid_str_.c_str(), stream.getSize(), uuid_connection_.c_str()); + } else { + logger_->log_debug( + "NiFi FlowFile retrieve uuid %s size %d connection %d fail", + uuid_str_.c_str(), stream.getSize(), uuid_connection_.c_str()); + } + + return ret; +} + +bool FlowFileRecord::Serialize() { + + io::DataStream outStream; + + int ret; + + ret = write(this->event_time_, &outStream); + if (ret != 8) { + + return false; + } + + ret = write(this->entry_date_, &outStream); + if (ret != 8) { + return false; + } + + ret = write(this->lineage_start_date_, &outStream); + if (ret != 8) { + + return false; + } + + ret = writeUTF(this->uuid_str_, &outStream); + if (ret <= 0) { + + return false; + } + + ret = writeUTF(this->uuid_connection_, &outStream); + if (ret <= 0) { + + return false; + } + // write flow attributes + uint32_t numAttributes = this->attributes_.size(); + ret = write(numAttributes, &outStream); + if (ret != 4) { + + return false; + } + + for (auto itAttribute : attributes_) { + ret = writeUTF(itAttribute.first, &outStream, true); + if (ret <= 0) { + + return false; + } + ret = writeUTF(itAttribute.second, &outStream, true); + if (ret <= 0) { + + return false; + } + } + + ret = writeUTF(this->content_full_fath_, &outStream); + if (ret <= 0) { + + return false; + } + + ret = write(this->size_, &outStream); + if (ret != 8) { + + return false; + } + + ret = write(this->offset_, &outStream); + if (ret != 8) { + + return false; + } + + // Persistent to the DB + + + if (flow_repository_->Put(uuid_str_, + const_cast(outStream.getBuffer()), + outStream.getSize())) { + logger_->log_debug("NiFi FlowFile Store event %s size %d success", + uuid_str_.c_str(), outStream.getSize()); + return true; + } else { + logger_->log_error("NiFi FlowFile Store event %s size %d fail", + uuid_str_.c_str(), outStream.getSize()); + return false; + } + + // cleanup + + return true; +} + +bool FlowFileRecord::DeSerialize(const uint8_t *buffer, const int bufferSize) { + + int ret; + + io::DataStream outStream(buffer, bufferSize); + + ret = read(this->event_time_, &outStream); + if (ret != 8) { + return false; + } + + ret = read(this->entry_date_, &outStream); + if (ret != 8) { + return false; + } + + ret = read(this->lineage_start_date_, &outStream); + if (ret != 8) { + return false; + } + + ret = readUTF(this->uuid_str_, &outStream); + if (ret <= 0) { + return false; + } + + ret = readUTF(this->uuid_connection_, &outStream); + if (ret <= 0) { + return false; + } + + // read flow attributes + uint32_t numAttributes = 0; + ret = read(numAttributes, &outStream); + if (ret != 4) { + return false; + } + + for (uint32_t i = 0; i < numAttributes; i++) { + std::string key; + ret = readUTF(key, &outStream, true); + if (ret <= 0) { + return false; + } + std::string value; + ret = readUTF(value, &outStream, true); + if (ret <= 0) { + return false; + } + this->attributes_[key] = value; + } + + ret = readUTF(this->content_full_fath_, &outStream); + if (ret <= 0) { + return false; + } + + ret = read(this->size_, &outStream); + if (ret != 8) { + return false; + } + + ret = read(this->offset_, &outStream); + if (ret != 8) { + return false; + } + + return true; +} + } /* namespace minifi */ } /* namespace nifi */ } /* namespace apache */ diff --git a/libminifi/src/RemoteProcessorGroupPort.cpp b/libminifi/src/RemoteProcessorGroupPort.cpp index a74aa28a54..adbcad98fc 100644 --- a/libminifi/src/RemoteProcessorGroupPort.cpp +++ b/libminifi/src/RemoteProcessorGroupPort.cpp @@ -43,12 +43,29 @@ namespace minifi { const std::string RemoteProcessorGroupPort::ProcessorName( "RemoteProcessorGroupPort"); -core::Property RemoteProcessorGroupPort::hostName( - "Host Name", "Remote Host Name.", "localhost"); -core::Property RemoteProcessorGroupPort::port( - "Port", "Remote Port", "9999"); +core::Property RemoteProcessorGroupPort::hostName("Host Name", + "Remote Host Name.", + "localhost"); +core::Property RemoteProcessorGroupPort::port("Port", "Remote Port", "9999"); core::Relationship RemoteProcessorGroupPort::relation; + +std::unique_ptr RemoteProcessorGroupPort::getNextProtocol() { + std::lock_guard protocol_lock_(protocol_mutex_); + if (available_protocols_.empty()) + return nullptr; + + std::unique_ptr return_pointer = std::move(available_protocols_.top()); + available_protocols_.pop(); + return std::move(return_pointer); +} + +void RemoteProcessorGroupPort::returnProtocol( + std::unique_ptr return_protocol) { + std::lock_guard protocol_lock_(protocol_mutex_); + available_protocols_.push(std::move(return_protocol)); +} + void RemoteProcessorGroupPort::initialize() { // Set the supported properties @@ -63,52 +80,44 @@ void RemoteProcessorGroupPort::initialize() { } -void RemoteProcessorGroupPort::onTrigger( - core::ProcessContext *context, - core::ProcessSession *session) { +void RemoteProcessorGroupPort::onTrigger(core::ProcessContext *context, + core::ProcessSession *session) { std::string value; if (!transmitting_) return; - std::string host = peer_.getHostName(); - uint16_t sport = peer_.getPort(); - int64_t lvalue; + std::unique_ptr protocol_ = getNextProtocol(); - if (context->getProperty(hostName.getName(), value)) { - host = value; - } - if (context->getProperty(port.getName(), value) - && core::Property::StringToInt(value, - lvalue)) { - sport = (uint16_t) lvalue; - } + // Peer Connection + if (protocol_ == nullptr) { - if (host != peer_.getHostName() || sport != peer_.getPort()) + protocol_ = std::unique_ptr( + new Site2SiteClientProtocol(0)); + protocol_->setPortId(protocol_uuid_); + protocol_->setTimeOut(timeout_); - { + std::string host = ""; + uint16_t sport = 0; + int64_t lvalue; + if (context->getProperty(hostName.getName(), value)) { + host = value; + } + if (context->getProperty(port.getName(), value) + && core::Property::StringToInt(value, lvalue)) { + sport = (uint16_t) lvalue; + } std::unique_ptr str = std::unique_ptr( org::apache::nifi::minifi::io::SocketFactory::getInstance() ->createSocket(host, sport)); - peer_ = std::move(Site2SitePeer(std::move(str), host, sport)); - protocol_->setPeer(&peer_); - - } - bool needReset = false; + std::unique_ptr peer_ = std::unique_ptr( + new Site2SitePeer(std::move(str), host, sport)); - if (host != peer_.getHostName()) { - peer_.setHostName(host); - needReset = true; + protocol_->setPeer(std::move(peer_)); } - if (sport != peer_.getPort()) { - peer_.setPort(sport); - needReset = true; - } - if (needReset) - protocol_->tearDown(); if (!protocol_->bootstrap()) { // bootstrap the client protocol if needeed @@ -125,10 +134,11 @@ void RemoteProcessorGroupPort::onTrigger( else protocol_->transferFlowFiles(context, session); + returnProtocol(std::move(protocol_)); + return; } - } /* namespace minifi */ } /* namespace nifi */ } /* namespace apache */ diff --git a/libminifi/src/Site2SiteClientProtocol.cpp b/libminifi/src/Site2SiteClientProtocol.cpp index 3c93866cd2..e0265bb698 100644 --- a/libminifi/src/Site2SiteClientProtocol.cpp +++ b/libminifi/src/Site2SiteClientProtocol.cpp @@ -518,7 +518,7 @@ Transaction* Site2SiteClientProtocol::createTransaction( return NULL; } - org::apache::nifi::minifi::io::CRCStream crcstream(peer_); + org::apache::nifi::minifi::io::CRCStream crcstream(peer_.get()); switch (code) { case MORE_DATA: dataAvailable = true; @@ -553,7 +553,7 @@ Transaction* Site2SiteClientProtocol::createTransaction( // tearDown(); return NULL; } else { - org::apache::nifi::minifi::io::CRCStream crcstream(peer_); + org::apache::nifi::minifi::io::CRCStream crcstream(peer_.get()); transaction = new Transaction(direction, crcstream); _transactionMap[transaction->getUUIDStr()] = transaction; transactionID = transaction->getUUIDStr(); diff --git a/libminifi/src/core/Core.cpp b/libminifi/src/core/Core.cpp index 6582783476..39969f6405 100644 --- a/libminifi/src/core/Core.cpp +++ b/libminifi/src/core/Core.cpp @@ -21,7 +21,7 @@ void CoreComponent::setUUID(uuid_t uuid) { uuidStr_ = uuidStr; } // Get UUID -bool CoreComponent::getUUID(uuid_t uuid) const { +bool CoreComponent::getUUID(uuid_t uuid) { if (uuid) { uuid_copy(uuid, uuid_); return true; @@ -31,7 +31,7 @@ bool CoreComponent::getUUID(uuid_t uuid) const { } // Get UUID -unsigned const char *CoreComponent::getUUID() const { +unsigned const char *CoreComponent::getUUID() { return uuid_; } @@ -41,7 +41,7 @@ void CoreComponent::setName(const std::string name) { } // Get Process Name -std::string CoreComponent::getName() const { +std::string CoreComponent::getName() { return name_; } } diff --git a/libminifi/src/core/ProcessGroup.cpp b/libminifi/src/core/ProcessGroup.cpp index ef5ffd97a9..baa3ebd706 100644 --- a/libminifi/src/core/ProcessGroup.cpp +++ b/libminifi/src/core/ProcessGroup.cpp @@ -248,18 +248,16 @@ void ProcessGroup::updatePropertyValue(std::string processorName, return; } -void ProcessGroup::getConnections(std::map &connectionMap) - { - for (auto connection : connections_) - { - connectionMap[connection->getUUIDStr()] = connection; - } - - for (auto processGroup: child_process_groups_) { - processGroup->getConnections(connectionMap); - } - } +void ProcessGroup::getConnections( + std::map> &connectionMap) { + for (auto connection : connections_) { + connectionMap[connection->getUUIDStr()] = connection; + } + for (auto processGroup : child_process_groups_) { + processGroup->getConnections(connectionMap); + } +} void ProcessGroup::addConnection(std::shared_ptr connection) { std::lock_guard lock(mutex_); diff --git a/libminifi/src/core/ProcessSession.cpp b/libminifi/src/core/ProcessSession.cpp index a948e8a52c..e6fa7c4051 100644 --- a/libminifi/src/core/ProcessSession.cpp +++ b/libminifi/src/core/ProcessSession.cpp @@ -37,8 +37,8 @@ namespace core { std::shared_ptr ProcessSession::create() { std::map empty; - std::shared_ptr record = - std::make_shared(empty); + std::shared_ptr record = std::make_shared(process_context_->getProvenanceRepository(), + empty); _addedFlowFiles[record->getUUIDStr()] = record; logger_->log_debug("Create FlowFile with UUID %s", @@ -50,11 +50,10 @@ std::shared_ptr ProcessSession::create() { return record; } -std::shared_ptr ProcessSession::create( - std::shared_ptr &parent) { +std::shared_ptr ProcessSession::create(std::shared_ptr &parent) { std::map empty; - std::shared_ptr record = - std::make_shared(empty); + std::shared_ptr record = std::make_shared(process_context_->getProvenanceRepository(), + empty); if (record) { _addedFlowFiles[record->getUUIDStr()] = record; @@ -85,8 +84,7 @@ std::shared_ptr ProcessSession::create( std::shared_ptr ProcessSession::clone( std::shared_ptr &parent) { - std::shared_ptr record = - this->create(parent); + std::shared_ptr record = this->create(parent); if (record) { // Copy Resource Claim std::shared_ptr parent_claim = parent->getResourceClaim(); @@ -105,8 +103,8 @@ std::shared_ptr ProcessSession::clone( std::shared_ptr ProcessSession::cloneDuringTransfer( std::shared_ptr &parent) { std::map empty; - std::shared_ptr record = - std::make_shared(empty); + std::shared_ptr record = std::make_shared(process_context_->getProvenanceRepository(), + empty); if (record) { this->_clonedFlowFiles[record->getUUIDStr()] = record; @@ -145,10 +143,8 @@ std::shared_ptr ProcessSession::cloneDuringTransfer( } std::shared_ptr ProcessSession::clone( - std::shared_ptr &parent, - long offset, long size) { - std::shared_ptr record = - this->create(parent); + std::shared_ptr &parent, long offset, long size) { + std::shared_ptr record = this->create(parent); if (record) { if (parent->getResourceClaim()) { @@ -157,8 +153,7 @@ std::shared_ptr ProcessSession::clone( logger_->log_error("clone offset %d and size %d exceed parent size %d", offset, size, parent->getSize()); // Remove the Add FlowFile for the session - std::map >::iterator it = + std::map >::iterator it = this->_addedFlowFiles.find(record->getUUIDStr()); if (it != this->_addedFlowFiles.end()) this->_addedFlowFiles.erase(record->getUUIDStr()); @@ -179,8 +174,7 @@ std::shared_ptr ProcessSession::clone( return record; } -void ProcessSession::remove( - std::shared_ptr &flow) { +void ProcessSession::remove(std::shared_ptr &flow) { flow->setDeleted(true); _deletedFlowFiles[flow->getUUIDStr()] = flow; std::string reason = process_context_->getProcessorNode().getName() @@ -188,8 +182,7 @@ void ProcessSession::remove( provenance_report_->drop(flow, reason); } -void ProcessSession::remove( - std::shared_ptr &&flow) { +void ProcessSession::remove(std::shared_ptr &&flow) { flow->setDeleted(true); _deletedFlowFiles[flow->getUUIDStr()] = flow; std::string reason = process_context_->getProcessorNode().getName() @@ -197,9 +190,8 @@ void ProcessSession::remove( provenance_report_->drop(flow, reason); } -void ProcessSession::putAttribute( - std::shared_ptr &flow, - std::string key, std::string value) { +void ProcessSession::putAttribute(std::shared_ptr &flow, + std::string key, std::string value) { flow->setAttribute(key, value); std::string details = process_context_->getProcessorNode().getName() + " modify flow record " + flow->getUUIDStr() + " attribute " + key + ":" @@ -207,18 +199,16 @@ void ProcessSession::putAttribute( provenance_report_->modifyAttributes(flow, details); } -void ProcessSession::removeAttribute( - std::shared_ptr &flow, - std::string key) { +void ProcessSession::removeAttribute(std::shared_ptr &flow, + std::string key) { flow->removeAttribute(key); std::string details = process_context_->getProcessorNode().getName() + " remove flow record " + flow->getUUIDStr() + " attribute " + key; provenance_report_->modifyAttributes(flow, details); } -void ProcessSession::putAttribute( - std::shared_ptr &&flow, - std::string key, std::string value) { +void ProcessSession::putAttribute(std::shared_ptr &&flow, + std::string key, std::string value) { flow->setAttribute(key, value); std::string details = process_context_->getProcessorNode().getName() + " modify flow record " + flow->getUUIDStr() + " attribute " + key + ":" @@ -226,46 +216,40 @@ void ProcessSession::putAttribute( provenance_report_->modifyAttributes(flow, details); } -void ProcessSession::removeAttribute( - std::shared_ptr &&flow, - std::string key) { +void ProcessSession::removeAttribute(std::shared_ptr &&flow, + std::string key) { flow->removeAttribute(key); std::string details = process_context_->getProcessorNode().getName() + " remove flow record " + flow->getUUIDStr() + " attribute " + key; provenance_report_->modifyAttributes(flow, details); } -void ProcessSession::penalize( - std::shared_ptr &flow) { +void ProcessSession::penalize(std::shared_ptr &flow) { flow->setPenaltyExpiration( getTimeMillis() + process_context_->getProcessorNode().getPenalizationPeriodMsec()); } -void ProcessSession::penalize( - std::shared_ptr &&flow) { +void ProcessSession::penalize(std::shared_ptr &&flow) { flow->setPenaltyExpiration( getTimeMillis() + process_context_->getProcessorNode().getPenalizationPeriodMsec()); } -void ProcessSession::transfer( - std::shared_ptr &flow, - Relationship relationship) { +void ProcessSession::transfer(std::shared_ptr &flow, + Relationship relationship) { _transferRelationship[flow->getUUIDStr()] = relationship; } -void ProcessSession::transfer( - std::shared_ptr &&flow, - Relationship relationship) { +void ProcessSession::transfer(std::shared_ptr &&flow, + Relationship relationship) { _transferRelationship[flow->getUUIDStr()] = relationship; } -void ProcessSession::write( - std::shared_ptr &flow, - OutputStreamCallback *callback) { +void ProcessSession::write(std::shared_ptr &flow, + OutputStreamCallback *callback) { std::shared_ptr claim = std::make_shared( - DEFAULT_CONTENT_DIRECTORY); + DEFAULT_CONTENT_DIRECTORY); try { std::ofstream fs; @@ -318,9 +302,8 @@ void ProcessSession::write( } } -void ProcessSession::write( - std::shared_ptr &&flow, - OutputStreamCallback *callback) { +void ProcessSession::write(std::shared_ptr &&flow, + OutputStreamCallback *callback) { std::shared_ptr claim = std::make_shared(); try { std::ofstream fs; @@ -373,9 +356,8 @@ void ProcessSession::write( } } -void ProcessSession::append( - std::shared_ptr &&flow, - OutputStreamCallback *callback) { +void ProcessSession::append(std::shared_ptr &&flow, + OutputStreamCallback *callback) { std::shared_ptr claim = nullptr; if (flow->getResourceClaim() == nullptr) { @@ -421,9 +403,8 @@ void ProcessSession::append( } } -void ProcessSession::append( - std::shared_ptr &flow, - OutputStreamCallback *callback) { +void ProcessSession::append(std::shared_ptr &flow, + OutputStreamCallback *callback) { std::shared_ptr claim = nullptr; if (flow->getResourceClaim() == nullptr) { @@ -469,9 +450,8 @@ void ProcessSession::append( } } -void ProcessSession::read( - std::shared_ptr &flow, - InputStreamCallback *callback) { +void ProcessSession::read(std::shared_ptr &flow, + InputStreamCallback *callback) { try { std::shared_ptr claim = nullptr; @@ -510,9 +490,8 @@ void ProcessSession::read( } } -void ProcessSession::read( - std::shared_ptr &&flow, - InputStreamCallback *callback) { +void ProcessSession::read(std::shared_ptr &&flow, + InputStreamCallback *callback) { try { std::shared_ptr claim = nullptr; @@ -551,10 +530,9 @@ void ProcessSession::read( } } -void ProcessSession::import( - std::string source, - std::shared_ptr &flow, - bool keepSource, uint64_t offset) { +void ProcessSession::import(std::string source, + std::shared_ptr &flow, + bool keepSource, uint64_t offset) { std::shared_ptr claim = std::make_shared(); char *buf = NULL; int size = 4096; @@ -633,14 +611,11 @@ void ProcessSession::import( } } -void ProcessSession::import( - std::string source, - std::shared_ptr &&flow, - bool keepSource, uint64_t offset) { +void ProcessSession::import(std::string source, + std::shared_ptr &&flow, + bool keepSource, uint64_t offset) { std::shared_ptr claim = std::make_shared(); - - char *buf = NULL; int size = 4096; buf = new char[size]; @@ -723,8 +698,7 @@ void ProcessSession::commit() { try { // First we clone the flow record based on the transfered relationship for updated flow record for (auto && it : _updatedFlowFiles) { - std::shared_ptr record = it - .second; + std::shared_ptr record = it.second; if (record->isDeleted()) continue; std::map::iterator itRelationship = this @@ -777,8 +751,7 @@ void ProcessSession::commit() { // Do the samething for added flow file for (const auto it : _addedFlowFiles) { - std::shared_ptr record = it - .second; + std::shared_ptr record = it.second; if (record->isDeleted()) continue; std::map::iterator itRelationship = this @@ -829,12 +802,10 @@ void ProcessSession::commit() { } } - std::shared_ptr connection = nullptr; // Complete process the added and update flow files for the session, send the flow file to its queue for (const auto &it : _updatedFlowFiles) { - std::shared_ptr record = it - .second; + std::shared_ptr record = it.second; if (record->isDeleted()) { continue; } @@ -845,8 +816,7 @@ void ProcessSession::commit() { connection->put(record); } for (const auto &it : _addedFlowFiles) { - std::shared_ptr record = it - .second; + std::shared_ptr record = it.second; if (record->isDeleted()) { continue; } @@ -857,8 +827,7 @@ void ProcessSession::commit() { } // Process the clone flow files for (const auto &it : _clonedFlowFiles) { - std::shared_ptr record = it - .second; + std::shared_ptr record = it.second; if (record->isDeleted()) { continue; } @@ -892,8 +861,7 @@ void ProcessSession::rollback() { std::shared_ptr connection = nullptr; // Requeue the snapshot of the flowfile back for (const auto &it : _originalFlowFiles) { - std::shared_ptr record = it - .second; + std::shared_ptr record = it.second; connection = std::static_pointer_cast( record->getOriginalConnection()); if ((connection) != nullptr) { @@ -933,12 +901,11 @@ std::shared_ptr ProcessSession::get() { do { std::set > expired; - std::shared_ptr ret = - current->poll(expired); + std::shared_ptr ret = current->poll(expired); if (expired.size() > 0) { // Remove expired flow record - for (std::set >::iterator it = - expired.begin(); it != expired.end(); ++it) { + for (std::set >::iterator it = expired + .begin(); it != expired.end(); ++it) { std::shared_ptr record = *it; std::string details = process_context_->getProcessorNode().getName() + " expire flow record " + record->getUUIDStr(); @@ -951,7 +918,7 @@ std::shared_ptr ProcessSession::get() { _updatedFlowFiles[ret->getUUIDStr()] = ret; std::map empty; std::shared_ptr snapshot = - std::make_shared(empty); + std::make_shared(process_context_->getProvenanceRepository(),empty); logger_->log_debug("Create Snapshot FlowFile with UUID %s", snapshot->getUUIDStr().c_str()); snapshot = ret; diff --git a/libminifi/src/core/Record.cpp b/libminifi/src/core/Record.cpp index a911b2f61e..dbf01027eb 100644 --- a/libminifi/src/core/Record.cpp +++ b/libminifi/src/core/Record.cpp @@ -16,6 +16,7 @@ */ #include "core/FlowFile.h" +#include "core/logging/Logger.h" namespace org { namespace apache { @@ -44,6 +45,8 @@ FlowFile::FlowFile() uuid_unparse_lower(uuid_, uuidStr); uuid_str_ = uuidStr; + + logger_ = logging::Logger::getLogger(); } @@ -106,6 +109,9 @@ void FlowFile::setResourceClaim(std::shared_ptr &claim) { uint64_t FlowFile::getEntryDate() { return entry_date_; } +uint64_t FlowFile::getEventTime() { + return event_time_; +} // ! Get Lineage Start Date uint64_t FlowFile::getlineageStartDate() { return lineage_start_date_; @@ -182,8 +188,7 @@ void FlowFile::setOriginalConnection( * Sets the connection with a shared pointer. * @param connection shared connection. */ -void FlowFile::setConnection( - std::shared_ptr &connection) { +void FlowFile::setConnection(std::shared_ptr &connection) { connection_ = connection; } @@ -191,8 +196,7 @@ void FlowFile::setConnection( * Sets the connection with a shared pointer. * @param connection shared connection. */ -void FlowFile::setConnection( - std::shared_ptr &&connection) { +void FlowFile::setConnection(std::shared_ptr &&connection) { connection_ = connection; } diff --git a/libminifi/src/core/Repository.cpp b/libminifi/src/core/Repository.cpp index 967ecc9baf..9a27785210 100644 --- a/libminifi/src/core/Repository.cpp +++ b/libminifi/src/core/Repository.cpp @@ -20,12 +20,12 @@ #include #include "io/DataStream.h" #include "io/Serializable.h" -#include "Relationship.h" -#include "Logger.h" +#include "core/Relationship.h" +#include "core/logging/Logger.h" #include "FlowController.h" #include "core/Repository.h" #include "provenance/Provenance.h" -#include "FlowFileRepository.h" +#include "core/repository/FlowFileRepository.h" namespace org { namespace apache { @@ -41,7 +41,7 @@ void Repository::start() { thread_ = std::thread(&Repository::threadExecutor, this); thread_.detach(); running_ = true; - logger_->log_info("%s Repository Monitor Thread Start", name_); + logger_->log_info("%s Repository Monitor Thread Start", name_.c_str()); } void Repository::stop() { @@ -50,7 +50,7 @@ void Repository::stop() { running_ = false; if (thread_.joinable()) thread_.join(); - logger_->log_info("%s Repository Monitor Thread Stop", name_); + logger_->log_info("%s Repository Monitor Thread Stop", name_.c_str()); } // repoSize diff --git a/libminifi/src/core/repository/FlowFileRepository.cpp b/libminifi/src/core/repository/FlowFileRepository.cpp index f27af10bf3..c495a67e21 100644 --- a/libminifi/src/core/repository/FlowFileRepository.cpp +++ b/libminifi/src/core/repository/FlowFileRepository.cpp @@ -1,4 +1,5 @@ #include "core/repository/FlowFileRepository.h" +#include "FlowFileRecord.h" namespace org { namespace apache { @@ -19,21 +20,21 @@ void FlowFileRepository::run() { leveldb::Iterator* it = db_->NewIterator(leveldb::ReadOptions()); for (it->SeekToFirst(); it->Valid(); it->Next()) { - FlowFileEventRecord eventRead; + std::shared_ptr eventRead = std::make_shared(shared_from_this()); std::string key = it->key().ToString(); - if (eventRead.DeSerialize((uint8_t *) it->value().data(), + if (eventRead->DeSerialize((uint8_t *) it->value().data(), (int) it->value().size())) { - if ((curTime - eventRead.getEventTime()) > max_partition_millis_) + if ((curTime - eventRead->getEventTime()) > max_partition_millis_) purgeList.push_back(key); } else { - logger_->log_debug("NiFi %s retrieve event %s fail", name_, + logger_->log_debug("NiFi %s retrieve event %s fail", name_.c_str(), key.c_str()); purgeList.push_back(key); } } delete it; for (auto eventId : purgeList) { - logger_->log_info("Repository Repo %s Purge %s", name_, + logger_->log_info("Repository Repo %s Purge %s", name_.c_str(), eventId.c_str()); Delete(eventId); } @@ -46,7 +47,7 @@ void FlowFileRepository::run() { return; } -void FlowFileRepository::loadFlowFileToConnections(std::map &connectionMap) +void FlowFileRepository::loadFlowFileToConnections(std::map> &connectionMap) { std::vector purgeList; @@ -55,25 +56,26 @@ void FlowFileRepository::loadFlowFileToConnections(std::mapSeekToFirst(); it->Valid(); it->Next()) { - FlowFileEventRecord eventRead; + std::shared_ptr eventRead = std::make_shared(shared_from_this()); std::string key = it->key().ToString(); - if (eventRead.DeSerialize((uint8_t *) it->value().data(), + if (eventRead->DeSerialize((uint8_t *) it->value().data(), (int) it->value().size())) { - auto search = connectionMap->find(eventRead.getConnectionUuid()); - if (search != connectionMap->end()) + auto search = connectionMap.find(eventRead->getConnectionUuid()); + if (search != connectionMap.end()) { // we find the connection for the persistent flowfile, create the flowfile and enqueue that - FlowFileRecord *record = new FlowFileRecord(&eventRead); + std::shared_ptr flow_file_ref = std::static_pointer_cast(eventRead); + std::shared_ptr record = std::make_shared(shared_from_this(),flow_file_ref); // set store to repo to true so that we do need to persistent again in enqueue record->setStoredToRepository(true); search->second->put(record); } else { - if (eventRead.getContentFullPath().length() > 0) + if (eventRead->getContentFullPath().length() > 0) { - std::remove(eventRead.getContentFullPath().c_str()); + std::remove(eventRead->getContentFullPath().c_str()); } purgeList.push_back(key); } @@ -91,13 +93,13 @@ void FlowFileRepository::loadFlowFileToConnections(std::maplog_info("Repository Repo %s Purge %s", - RepositoryTypeStr[_type], + name_.c_str(), eventId.c_str()); Delete(eventId); } - + + return; - +} +} } /* namespace repository */ } /* namespace core */ diff --git a/libminifi/src/io/ClientSocket.cpp b/libminifi/src/io/ClientSocket.cpp index e726f0ff86..ad6b04dbce 100644 --- a/libminifi/src/io/ClientSocket.cpp +++ b/libminifi/src/io/ClientSocket.cpp @@ -88,69 +88,7 @@ void Socket::closeStream() { } } -<<<<<<< HEAD -int8_t Socket::createConnection(const addrinfo *p,in_addr_t &addr) { - if ((socket_file_descriptor_ = socket(p->ai_family, p->ai_socktype, - p->ai_protocol)) == -1) { - logger_->log_error("error while connecting to server socket"); - return -1; - } - - setSocketOptions(socket_file_descriptor_); - - if (listeners_ > 0) { - - struct sockaddr_in *sa_loc = (struct sockaddr_in*) p->ai_addr; - sa_loc->sin_family = AF_INET; - sa_loc->sin_port = htons(port_); - sa_loc->sin_addr.s_addr = htonl(INADDR_ANY); - - if (bind(socket_file_descriptor_, p->ai_addr, p->ai_addrlen) == -1) { - logger_->log_error("Could not bind to socket", strerror(errno)); - return -1; - } - } - { - if (listeners_ <=0 ) - { - struct sockaddr_in *sa_loc = (struct sockaddr_in*) p->ai_addr; - sa_loc->sin_family = AF_INET; - //sa_loc->sin_port = htons(port); - sa_loc->sin_port = htons(port_); - // use any address if you are connecting to the local machine for testing - // otherwise we must use the requested hostname - if (IsNullOrEmpty(requested_hostname_) || requested_hostname_=="localhost") - { - sa_loc->sin_addr.s_addr = htonl(INADDR_ANY); - } - else - { - sa_loc->sin_addr.s_addr = addr; - } - if (connect(socket_file_descriptor_, p->ai_addr, p->ai_addrlen) == -1) { - close(socket_file_descriptor_); - socket_file_descriptor_ = -1; - logger_->log_warn("Could not connect to socket, error:%s", strerror(errno)); - return -1; - - } - } - } - - // listen - if (listeners_ > 0) { - if (listen(socket_file_descriptor_, listeners_) == -1) { - logger_->log_warn("attempted connection, saw %s", strerror(errno)); - return -1; - } - - } - // add the listener to the total set - FD_SET(socket_file_descriptor_, &total_list_); - socket_max_ = socket_file_descriptor_; - return 0; -======= -int8_t Socket::createConnection(const addrinfo *p) { +int8_t Socket::createConnection(const addrinfo *p, in_addr_t &addr) { if ((socket_file_descriptor_ = socket(p->ai_family, p->ai_socktype, p->ai_protocol)) == -1) { logger_->log_error("error while connecting to server socket"); @@ -177,7 +115,14 @@ int8_t Socket::createConnection(const addrinfo *p) { sa_loc->sin_family = AF_INET; //sa_loc->sin_port = htons(port); sa_loc->sin_port = htons(port_); - sa_loc->sin_addr.s_addr = htonl(INADDR_ANY); // inet_addr(requested_hostname.c_str()); + // use any address if you are connecting to the local machine for testing + // otherwise we must use the requested hostname + if (IsNullOrEmpty(requested_hostname_) + || requested_hostname_ == "localhost") { + sa_loc->sin_addr.s_addr = htonl(INADDR_ANY); + } else { + sa_loc->sin_addr.s_addr = addr; + } if (connect(socket_file_descriptor_, p->ai_addr, p->ai_addrlen) == -1) { close(socket_file_descriptor_); socket_file_descriptor_ = -1; @@ -201,24 +146,12 @@ int8_t Socket::createConnection(const addrinfo *p) { FD_SET(socket_file_descriptor_, &total_list_); socket_max_ = socket_file_descriptor_; return 0; ->>>>>>> MINIFI-217: First commit. Updates namespaces and removes } short Socket::initialize() { struct sockaddr_in servAddr; -<<<<<<< HEAD - addrinfo hints = { sizeof(addrinfo) }; - memset(&hints, 0, sizeof hints); // make sure the struct is empty - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - hints.ai_flags = AI_CANONNAME; - if (listeners_ > 0) - hints.ai_flags |= AI_PASSIVE; - - hints.ai_protocol = 0; /* any protocol */ -======= addrinfo hints = { sizeof(addrinfo) }; memset(&hints, 0, sizeof hints); // make sure the struct is empty hints.ai_family = AF_UNSPEC; @@ -226,7 +159,6 @@ short Socket::initialize() { hints.ai_flags = AI_CANONNAME; if (listeners_ > 0) hints.ai_flags |= AI_PASSIVE; ->>>>>>> MINIFI-217: First commit. Updates namespaces and removes hints.ai_protocol = 0; /* any protocol */ @@ -239,44 +171,26 @@ short Socket::initialize() { return -1; } -<<<<<<< HEAD - in_addr_t addr; - struct hostent *h; - #ifdef __MACH__ - h = gethostbyname(requested_hostname_.c_str()); - #else - const char *host; - uint16_t port; - - host = requested_hostname_.c_str(); - port = port_; - char buf[1024]; - struct hostent he; - int hh_errno; - gethostbyname_r(host, &he, buf, sizeof(buf), &h, &hh_errno); - #endif - - memcpy((char *) &addr, h->h_addr_list[0], h->h_length); - - - auto p = addr_info_; - for (; p != NULL; p = p->ai_next) { - if (IsNullOrEmpty(canonical_hostname_)) { - if (!IsNullOrEmpty(p) && !IsNullOrEmpty(p->ai_canonname)) - canonical_hostname_ = p->ai_canonname; - } - - - //we've successfully connected - if (port_ > 0 && createConnection(p,addr) >= 0) - { - return 0; - break; - } - } -======= socket_file_descriptor_ = -1; + in_addr_t addr; + struct hostent *h; +#ifdef __MACH__ + h = gethostbyname(requested_hostname_.c_str()); +#else + const char *host; + uint16_t port; + + host = requested_hostname_.c_str(); + port = port_; + char buf[1024]; + struct hostent he; + int hh_errno; + gethostbyname_r(host, &he, buf, sizeof(buf), &h, &hh_errno); +#endif + + memcpy((char *) &addr, h->h_addr_list[0], h->h_length); + auto p = addr_info_; for (; p != NULL; p = p->ai_next) { if (IsNullOrEmpty(canonical_hostname_)) { @@ -285,72 +199,17 @@ short Socket::initialize() { } //we've successfully connected - if (port_ > 0 && createConnection(p) >= 0) { + if (port_ > 0 && createConnection(p, addr) >= 0) { return 0; break; } } ->>>>>>> MINIFI-217: First commit. Updates namespaces and removes return -1; } short Socket::select_descriptor(const uint16_t msec) { -<<<<<<< HEAD - struct timeval tv; - int retval; - - read_fds_ = total_list_; - - tv.tv_sec = msec / 1000; - tv.tv_usec = (msec % 1000) * 1000; - - std::lock_guard guard(selection_mutex_); - - if (msec > 0) - retval = select(socket_max_ + 1, &read_fds_, NULL, NULL, &tv); - else - retval = select(socket_max_ + 1, &read_fds_, NULL, NULL, NULL); - - if (retval < 0) { - logger_->log_error("Saw error during selection, error:%i %s",retval,strerror(errno)); - return retval; - } - - for (int i = 0; i <= socket_max_; i++) { - if (FD_ISSET(i, &read_fds_)) { - - if (i == socket_file_descriptor_) { - if (listeners_ > 0) - { - struct sockaddr_storage remoteaddr; // client address - socklen_t addrlen = sizeof remoteaddr; - int newfd = accept(socket_file_descriptor_, - (struct sockaddr *) &remoteaddr, &addrlen); - FD_SET(newfd, &total_list_); // add to master set - if (newfd > socket_max_) { // keep track of the max - socket_max_ = newfd; - } - return newfd; - - - - } - else{ - return socket_file_descriptor_; - } - // we have a new connection - } else { - // data to be received on i - return i; - } - } - - } - - return -1; -======= struct timeval tv; int retval; @@ -400,7 +259,6 @@ short Socket::select_descriptor(const uint16_t msec) { } return -1; ->>>>>>> MINIFI-217: First commit. Updates namespaces and removes } short Socket::setSocketOptions(const int sock) { @@ -473,29 +331,11 @@ int Socket::writeData(uint8_t *value, int size) { } bytes += ret; -<<<<<<< HEAD - - ret = send(socket_file_descriptor_, value + bytes, size - bytes, 0); - //check for errors - if (ret <= 0) { - close(socket_file_descriptor_); - logger_->log_error("Could not send to %d, error: %s",socket_file_descriptor_, strerror(errno)); - return ret; - } - bytes += ret; - - } - - if (ret) - logger_->log_trace("Send data size %d over socket %d", size, - socket_file_descriptor_); -======= } if (ret) - logger_->log_debug("Send data size %d over socket %d", size, + logger_->log_trace("Send data size %d over socket %d", size, socket_file_descriptor_); ->>>>>>> MINIFI-217: First commit. Updates namespaces and removes return bytes; @@ -511,13 +351,7 @@ inline std::vector Socket::readBuffer(const T& t) { int Socket::write(uint64_t base_value, bool is_little_endian) { -<<<<<<< HEAD -int Socket::write(uint64_t base_value, bool is_little_endian){ - - return Serializable::write(base_value,this,is_little_endian); -======= return Serializable::write(base_value, this, is_little_endian); ->>>>>>> MINIFI-217: First commit. Updates namespaces and removes } int Socket::write(uint32_t base_value, bool is_little_endian) { diff --git a/libminifi/src/io/TLSSocket.cpp b/libminifi/src/io/TLSSocket.cpp index e7fc02f0c8..1a28eeadac 100644 --- a/libminifi/src/io/TLSSocket.cpp +++ b/libminifi/src/io/TLSSocket.cpp @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "Configure.h" +#include "properties/Configure.h" #include "io/TLSSocket.h" #include "utils/StringUtils.h" diff --git a/libminifi/test/unit/LoggerTests.cpp b/libminifi/test/unit/LoggerTests.cpp index f17cde1d8e..efa30bfd7f 100644 --- a/libminifi/test/unit/LoggerTests.cpp +++ b/libminifi/test/unit/LoggerTests.cpp @@ -31,35 +31,54 @@ TEST_CASE("Test log Levels", "[ttl1]") { std::ostringstream oss; std::unique_ptr outputLogger = std::unique_ptr( - new org::apache::nifi::minifi::core::logging::OutputStreamAppender(oss, 0)); + new org::apache::nifi::minifi::core::logging::OutputStreamAppender(oss, + 0)); std::shared_ptr logger = logging::Logger::getLogger(); logger->updateLogger(std::move(outputLogger)); logger->setLogLevel("trace"); logger->log_info("hello world"); REQUIRE( - true == contains(oss.str(),"[minifi log -- org::apache::nifi::minifi::core::logging::OutputStreamAppender] [info] hello world")); + true + == contains( + oss.str(), + "[minifi log -- org::apache::nifi::minifi::core::logging::OutputStreamAppender] [info] hello world")); + + std::unique_ptr nullAppender = std::unique_ptr( + new org::apache::nifi::minifi::core::logging::NullAppender()); + + logger->updateLogger(std::move(nullAppender)); } TEST_CASE("Test log Levels debug", "[ttl2]") { std::ostringstream oss; std::unique_ptr outputLogger = std::unique_ptr( - new org::apache::nifi::minifi::core::logging::OutputStreamAppender(oss, 0)); + new org::apache::nifi::minifi::core::logging::OutputStreamAppender(oss, + 0)); std::shared_ptr logger = logging::Logger::getLogger(); logger->updateLogger(std::move(outputLogger)); logger->setLogLevel("trace"); logger->log_debug("hello world"); REQUIRE( - true == contains(oss.str(),"[minifi log -- org::apache::nifi::minifi::core::logging::OutputStreamAppender] [debug] hello world")); + true + == contains( + oss.str(), + "[minifi log -- org::apache::nifi::minifi::core::logging::OutputStreamAppender] [debug] hello world")); + + std::unique_ptr nullAppender = std::unique_ptr( + new org::apache::nifi::minifi::core::logging::NullAppender()); + + logger->updateLogger(std::move(nullAppender)); } TEST_CASE("Test log Levels trace", "[ttl3]") { std::ostringstream oss; std::unique_ptr outputLogger = std::unique_ptr( - new org::apache::nifi::minifi::core::logging::OutputStreamAppender(oss, 0)); + new org::apache::nifi::minifi::core::logging::OutputStreamAppender(oss, + 0)); std::shared_ptr logger = logging::Logger::getLogger(); logger->updateLogger(std::move(outputLogger)); logger->setLogLevel("trace"); @@ -67,14 +86,23 @@ TEST_CASE("Test log Levels trace", "[ttl3]") { logger->log_trace("hello world"); REQUIRE( - true == contains(oss.str(),"[minifi log -- org::apache::nifi::minifi::core::logging::OutputStreamAppender] [trace] hello world")); + true + == contains( + oss.str(), + "[minifi log -- org::apache::nifi::minifi::core::logging::OutputStreamAppender] [trace] hello world")); + + std::unique_ptr nullAppender = std::unique_ptr( + new org::apache::nifi::minifi::core::logging::NullAppender()); + + logger->updateLogger(std::move(nullAppender)); } TEST_CASE("Test log Levels error", "[ttl4]") { std::ostringstream oss; std::unique_ptr outputLogger = std::unique_ptr( - new org::apache::nifi::minifi::core::logging::OutputStreamAppender(oss, 0)); + new org::apache::nifi::minifi::core::logging::OutputStreamAppender(oss, + 0)); std::shared_ptr logger = logging::Logger::getLogger(); logger->updateLogger(std::move(outputLogger)); logger->setLogLevel("trace"); @@ -82,14 +110,23 @@ TEST_CASE("Test log Levels error", "[ttl4]") { logger->log_error("hello world"); REQUIRE( - true == contains(oss.str(),"[minifi log -- org::apache::nifi::minifi::core::logging::OutputStreamAppender] [error] hello world")); + true + == contains( + oss.str(), + "[minifi log -- org::apache::nifi::minifi::core::logging::OutputStreamAppender] [error] hello world")); + + std::unique_ptr nullAppender = std::unique_ptr( + new org::apache::nifi::minifi::core::logging::NullAppender()); + + logger->updateLogger(std::move(nullAppender)); } TEST_CASE("Test log Levels change", "[ttl5]") { std::ostringstream oss; std::unique_ptr outputLogger = std::unique_ptr( - new org::apache::nifi::minifi::core::logging::OutputStreamAppender(oss, 0)); + new org::apache::nifi::minifi::core::logging::OutputStreamAppender(oss, + 0)); std::shared_ptr logger = logging::Logger::getLogger(); logger->updateLogger(std::move(outputLogger)); logger->setLogLevel("trace"); @@ -97,7 +134,10 @@ TEST_CASE("Test log Levels change", "[ttl5]") { logger->log_error("hello world"); REQUIRE( - true == contains(oss.str(),"[minifi log -- org::apache::nifi::minifi::core::logging::OutputStreamAppender] [error] hello world")); + true + == contains( + oss.str(), + "[minifi log -- org::apache::nifi::minifi::core::logging::OutputStreamAppender] [error] hello world")); oss.str(""); oss.clear(); REQUIRE(0 == oss.str().length()); @@ -106,6 +146,12 @@ TEST_CASE("Test log Levels change", "[ttl5]") { logger->log_error("hello world"); REQUIRE(0 == oss.str().length()); + + std::unique_ptr nullAppender = std::unique_ptr( + new org::apache::nifi::minifi::core::logging::NullAppender()); + + logger->updateLogger(std::move(nullAppender)); + } TEST_CASE("Test log LevelsConfigured", "[ttl6]") { @@ -113,9 +159,11 @@ TEST_CASE("Test log LevelsConfigured", "[ttl6]") { minifi::Configure *config = minifi::Configure::getConfigure(); - config->set(BaseLogger::nifi_log_appender, "org::apache::nifi::minifi::core::logging::OutputStreamAppender"); - config->set(org::apache::nifi::minifi::core::logging::OutputStreamAppender::nifi_log_output_stream_error_stderr, - "true"); + config->set(BaseLogger::nifi_log_appender, + "OutputStreamAppender"); + config->set( + org::apache::nifi::minifi::core::logging::OutputStreamAppender::nifi_log_output_stream_error_stderr, + "true"); std::shared_ptr logger = logging::Logger::getLogger(); @@ -133,7 +181,10 @@ TEST_CASE("Test log LevelsConfigured", "[ttl6]") { logger->log_error("hello world"); REQUIRE( - true == contains(oss.str(),"[minifi log -- org::apache::nifi::minifi::core::logging::OutputStreamAppender] [error] hello world")); + true + == contains( + oss.str(), + "[minifi log -- org::apache::nifi::minifi::core::logging::OutputStreamAppender] [error] hello world")); std::cerr.rdbuf(oldrdbuf); @@ -151,8 +202,8 @@ TEST_CASE("Test log LevelsConfigured", "[ttl6]") { logger->log_info("hello world"); logger->log_debug("hello world"); logger->log_trace("hello world"); -// logger->log_error("hello world"); REQUIRE(0 == oss.str().length()); + } diff --git a/libminifi/test/unit/ProcessorTests.cpp b/libminifi/test/unit/ProcessorTests.cpp index e9b11b4001..4f08d5df54 100644 --- a/libminifi/test/unit/ProcessorTests.cpp +++ b/libminifi/test/unit/ProcessorTests.cpp @@ -20,11 +20,6 @@ #include "FlowController.h" #include "ProvenanceTestHelper.h" #include "../TestBase.h" -<<<<<<< HEAD -#include -#include "../../include/LogAppenders.h" -#include "GetFile.h" -======= #include "core/logging/LogAppenders.h" #include "core/logging/BaseLogger.h" #include "processors/GetFile.h" @@ -39,7 +34,6 @@ TEST_CASE("Test Creation of GetFile", "[getfileCreate]") { org::apache::nifi::minifi::processors::GetFile processor("processorname"); REQUIRE(processor.getName() == "processorname"); } ->>>>>>> MINIFI-217: First commit. Updates namespaces and removes TEST_CASE("Test Find file", "[getfileCreate2]") { @@ -47,40 +41,32 @@ TEST_CASE("Test Find file", "[getfileCreate2]") { testController.enableDebug(); - std::shared_ptr repo = std::make_shared< - ProvenanceTestRepository>(); + std::shared_ptr processor = std::make_shared< org::apache::nifi::minifi::processors::GetFile>("getfileCreate2"); -<<<<<<< HEAD - Configure *config = Configure::getConfigure(); - - config->set(BaseLogger::nifi_log_appender,"rollingappender"); - config->set(OutputStreamAppender::nifi_log_output_stream_error_stderr,"true"); - std::shared_ptr logger = Logger::getLogger(); - std::unique_ptr newLogger =LogInstance::getConfiguredLogger(config); - logger->updateLogger(std::move(newLogger)); - logger->setLogLevel("debug"); - - ProvenanceTestRepository provenanceRepo; - FlowTestRepository flowRepo; - TestFlowController controller(provenanceRepo, flowRepo); - FlowControllerFactory::getFlowController( dynamic_cast(&controller)); -======= + std::shared_ptr test_repo = std::make_shared(); + + std::shared_ptr repo = std::static_pointer_cast(test_repo); + std::shared_ptr controller = std::make_shared(test_repo, test_repo); + + + char format[] = "/tmp/gt.XXXXXX"; char *dir = testController.createTempDirectory(format); uuid_t processoruuid; REQUIRE(true == processor->getUUID(processoruuid)); ->>>>>>> MINIFI-217: First commit. Updates namespaces and removes + std::shared_ptr connection = std::make_shared< - minifi::Connection>("getfileCreate2Connection"); + minifi::Connection>(test_repo,"getfileCreate2Connection"); connection->setRelationship(core::Relationship("success", "description")); // link the connections so that we can test results at the end for this connection->setSource(processor); + connection->setDestination(processor); connection->setSourceUUID(processoruuid); connection->setDestinationUUID(processoruuid); @@ -90,21 +76,17 @@ TEST_CASE("Test Find file", "[getfileCreate2]") { core::ProcessorNode node(processor); - core::ProcessContext context(node, repo); + core::ProcessContext context(node, test_repo); context.setProperty(org::apache::nifi::minifi::processors::GetFile::Directory, dir); core::ProcessSession session(&context); -<<<<<<< HEAD - connection.setSourceProcessor(&processor); - connection.setDestinationProcessor(&processor); -======= + REQUIRE(processor->getName() == "getfileCreate2"); std::shared_ptr record; processor->setScheduledState(core::ScheduledState::RUNNING); processor->onTrigger(&context, &session); ->>>>>>> MINIFI-217: First commit. Updates namespaces and removes provenance::ProvenanceReporter *reporter = session.getProvenanceReporter(); std::set records = reporter->getEvents(); @@ -126,9 +108,8 @@ TEST_CASE("Test Find file", "[getfileCreate2]") { rmdir(dir); reporter = session.getProvenanceReporter(); -<<<<<<< HEAD - REQUIRE( processor.getName() == "getfileCreate2"); -======= + REQUIRE( processor->getName() == "getfileCreate2"); + records = reporter->getEvents(); for (provenance::ProvenanceEventRecord *provEventRecord : records) { @@ -136,7 +117,6 @@ TEST_CASE("Test Find file", "[getfileCreate2]") { } session.commit(); std::shared_ptr ffr = session.get(); ->>>>>>> MINIFI-217: First commit. Updates namespaces and removes ffr->getResourceClaim()->decreaseFlowFileRecordOwnedCount(); REQUIRE(2 == repo->getRepoMap().size()); @@ -161,27 +141,10 @@ TEST_CASE("Test Find file", "[getfileCreate2]") { if (!found) throw std::runtime_error("Did not find record"); -<<<<<<< HEAD - std::fstream file; - std::stringstream ss; - std::string fileName("tstFile.ext"); - ss << dir << "/" << fileName; - file.open(ss.str(),std::ios::out); - file << "tempFile"; - int64_t fileSize = file.tellp(); - file.close(); - - processor.incrementActiveTasks(); - processor.setScheduledState(ScheduledState::RUNNING); - - processor.onTrigger(&context,&session); - unlink(ss.str().c_str()); - rmdir(dir); -======= + } } ->>>>>>> MINIFI-217: First commit. Updates namespaces and removes TEST_CASE("LogAttributeTest", "[getfileCreate3]") { std::ostringstream oss; @@ -196,32 +159,15 @@ TEST_CASE("LogAttributeTest", "[getfileCreate3]") { testController.enableDebug(); -<<<<<<< HEAD - // verify flow file repo - REQUIRE( 1 == flowRepo.getRepoMap().size() ); - - for(auto entry: flowRepo.getRepoMap()) - { - FlowFileEventRecord newRecord; - newRecord.DeSerialize((uint8_t*)entry.second.data(),entry.second.length()); - REQUIRE (fileSize == newRecord.getFileSize()); - REQUIRE (0 == newRecord.getFileOffset()); - std::map attrs = newRecord.getAttributes(); - std::string key = FlowAttributeKey(FILENAME); - REQUIRE (attrs[key] == fileName); - } - - FlowFileRecord *ffr = session.get(); -======= - std::shared_ptr repo = std::make_shared< - ProvenanceTestRepository>(); + + std::shared_ptr repo = std::make_shared< + TestRepository>(); std::shared_ptr processor = std::make_shared< org::apache::nifi::minifi::processors::GetFile>("getfileCreate2"); std::shared_ptr logAttribute = std::make_shared< org::apache::nifi::minifi::processors::LogAttribute>("logattribute"); ->>>>>>> MINIFI-217: First commit. Updates namespaces and removes char format[] = "/tmp/gt.XXXXXX"; char *dir = testController.createTempDirectory(format); @@ -232,51 +178,24 @@ TEST_CASE("LogAttributeTest", "[getfileCreate3]") { uuid_t logattribute_uuid; REQUIRE(true == logAttribute->getUUID(logattribute_uuid)); -<<<<<<< HEAD - REQUIRE( 2 == provenanceRepo.getRepoMap().size() ); - - for(auto entry: provenanceRepo.getRepoMap()) - { - ProvenanceEventRecord newRecord; - newRecord.DeSerialize((uint8_t*)entry.second.data(),entry.second.length()); - - bool found = false; - for ( auto provRec : records) - { - if (provRec->getEventId() == newRecord.getEventId() ) - { - REQUIRE( provRec->getEventId() == newRecord.getEventId()); - REQUIRE( provRec->getComponentId() == newRecord.getComponentId()); - REQUIRE( provRec->getComponentType() == newRecord.getComponentType()); - REQUIRE( provRec->getDetails() == newRecord.getDetails()); - REQUIRE( provRec->getEventDuration() == newRecord.getEventDuration()); - found = true; - break; - } - } - if (!found) - throw std::runtime_error("Did not find record"); - } -======= + std::shared_ptr connection = std::make_shared< - minifi::Connection>("getfileCreate2Connection"); + minifi::Connection>(repo,"getfileCreate2Connection"); connection->setRelationship(core::Relationship("success", "description")); std::shared_ptr connection2 = std::make_shared< - minifi::Connection>("logattribute"); + minifi::Connection>(repo,"logattribute"); connection2->setRelationship(core::Relationship("success", "description")); // link the connections so that we can test results at the end for this connection->setSource(processor); ->>>>>>> MINIFI-217: First commit. Updates namespaces and removes + // link the connections so that we can test results at the end for this connection->setDestination(logAttribute); -<<<<<<< HEAD -======= connection2->setSource(logAttribute); ->>>>>>> MINIFI-217: First commit. Updates namespaces and removes + connection2->setSourceUUID(logattribute_uuid); connection->setSourceUUID(processoruuid); diff --git a/libminifi/test/unit/ProvenanceTestHelper.h b/libminifi/test/unit/ProvenanceTestHelper.h index bc73641e34..1de133f65c 100644 --- a/libminifi/test/unit/ProvenanceTestHelper.h +++ b/libminifi/test/unit/ProvenanceTestHelper.h @@ -20,128 +20,67 @@ #include "provenance/Provenance.h" #include "FlowController.h" -<<<<<<< HEAD -#include "FlowFileRepository.h" - +#include "core/Repository.h" +#include "core/core.h" /** * Test repository */ -class FlowTestRepository : public FlowFileRepository -{ -public: - FlowTestRepository() -{ -} - // initialize - bool initialize() - { - return true; - } - - // Destructor - virtual ~FlowTestRepository() { - - } - - bool Put(std::string key, uint8_t *buf, int bufLen) - { - repositoryResults.insert(std::pair(key,std::string((const char*)buf,bufLen))); - return true; - } - // Delete - bool Delete(std::string key) - { - repositoryResults.erase(key); - return true; - } - // Get - bool Get(std::string key, std::string &value) - { - auto result = repositoryResults.find(key); - if (result != repositoryResults.end()) - { - value = result->second; - return true; - } - else - { - return false; - } - } - - const std::map &getRepoMap() const - { - return repositoryResults; - } - -protected: - std::map repositoryResults; -}; +class TestRepository : public core::Repository{ + public: + TestRepository() : Repository("repo_name", "./dir", + 1000, 100, 0) { + } + // initialize + bool initialize() { + return true; + } -/** - * Test repository - */ -class ProvenanceTestRepository : public ProvenanceRepository -{ -public: - ProvenanceTestRepository() -{ -} - // initialize - bool initialize() - { - return true; - } - - // Destructor - virtual ~ProvenanceTestRepository() { - - } - - bool Put(std::string key, uint8_t *buf, int bufLen) - { - repositoryResults.insert(std::pair(key,std::string((const char*)buf,bufLen))); - return true; - } - // Delete - bool Delete(std::string key) - { - repositoryResults.erase(key); - return true; - } - // Get - bool Get(std::string key, std::string &value) - { - auto result = repositoryResults.find(key); - if (result != repositoryResults.end()) - { - value = result->second; - return true; - } - else - { - return false; - } - } - - const std::map &getRepoMap() const - { - return repositoryResults; - } - -protected: - std::map repositoryResults; + // Destructor + virtual ~TestRepository() { + + } + + bool Put(std::string key, uint8_t *buf, int bufLen) { + repositoryResults.insert( + std::pair( + key, std::string((const char*) buf, bufLen))); + return true; + } + // Delete + bool Delete(std::string key) { + repositoryResults.erase(key); + return true; + } + // Get + bool Get(std::string key, std::string &value) { + auto result = repositoryResults.find(key); + if (result != repositoryResults.end()) { + value = result->second; + return true; + } else { + return false; + } + } + + const std::map &getRepoMap() const { + return repositoryResults; + } + + void run(){ + // do nothing + } + protected: + std::map repositoryResults; }; -class TestFlowController : public FlowController + +class TestFlowController : public minifi::FlowController { public: - TestFlowController(ProvenanceTestRepository &provenanceRepo, FlowTestRepository &flowRepo) : ::FlowController() + TestFlowController(std::shared_ptr repo,std::shared_ptr flow_file_repo) : minifi::FlowController(repo,flow_file_repo) { - _provenanceRepo = dynamic_cast(&provenanceRepo); - _flowfileRepo = dynamic_cast(&flowRepo); } ~TestFlowController() { @@ -182,62 +121,13 @@ class TestFlowController : public FlowController } - Processor *createProcessor(std::string name, uuid_t uuid){ return 0;} - - ProcessGroup *createRootProcessGroup(std::string name, uuid_t uuid){ return 0;} - - ProcessGroup *createRemoteProcessGroup(std::string name, uuid_t uuid){ return 0; } - - Connection *createConnection(std::string name, uuid_t uuid){ return 0; } -======= -#include "core/core.h" -/** - * Test repository - */ -class ProvenanceTestRepository : public provenance::ProvenanceRepository { - public: - ProvenanceTestRepository() { - } - // initialize - bool initialize() { - return true; - } - - // Destructor - virtual ~ProvenanceTestRepository() { - - } + std::shared_ptr createProcessor(std::string name, uuid_t uuid){ return 0;} - bool Put(std::string key, uint8_t *buf, int bufLen) { - repositoryResults.insert( - std::pair( - key, std::string((const char*) buf, bufLen))); - return true; - } - // Delete - bool Delete(std::string key) { - repositoryResults.erase(key); - return true; - } - // Get - bool Get(std::string key, std::string &value) { - auto result = repositoryResults.find(key); - if (result != repositoryResults.end()) { - value = result->second; - return true; - } else { - return false; - } - } + core::ProcessGroup *createRootProcessGroup(std::string name, uuid_t uuid){ return 0;} - const std::map &getRepoMap() const { - return repositoryResults; - } + core::ProcessGroup *createRemoteProcessGroup(std::string name, uuid_t uuid){ return 0; } - protected: - std::map repositoryResults; ->>>>>>> MINIFI-217: First commit. Updates namespaces and removes + std::shared_ptr createConnection(std::string name, uuid_t uuid){ return 0; } }; - #endif /* LIBMINIFI_TEST_UNIT_PROVENANCETESTHELPER_H_ */ diff --git a/libminifi/test/unit/ProvenanceTests.cpp b/libminifi/test/unit/ProvenanceTests.cpp index 015ee17053..624601c04e 100644 --- a/libminifi/test/unit/ProvenanceTests.cpp +++ b/libminifi/test/unit/ProvenanceTests.cpp @@ -24,6 +24,7 @@ #include "provenance/Provenance.h" #include "FlowFileRecord.h" #include "core/core.h" +#include "core/repository/FlowFileRepository.h" TEST_CASE("Test Provenance record create", "[Testprovenance::ProvenanceEventRecord]") { @@ -46,9 +47,8 @@ TEST_CASE("Test Provenance record serialization", "[Testprovenance::ProvenanceEv std::string smileyface = ":)"; record1.setDetails(smileyface); - ProvenanceTestRepository repo; uint64_t sample = 65555; - std::shared_ptr testRepository =std::make_shared(); + std::shared_ptr testRepository =std::make_shared(); record1.setEventDuration(sample); record1.Serialize(testRepository); @@ -71,14 +71,14 @@ TEST_CASE("Test Flowfile record added to provenance", "[TestFlowAndProv1]") { std::map attributes; attributes.insert(std::pair("potato", "potatoe")); attributes.insert(std::pair("tomato", "tomatoe")); + std::shared_ptr frepo = std::make_shared("./content_repository",0,0,0); std::shared_ptr ffr1 = std::make_shared< - minifi::FlowFileRecord>(attributes); + minifi::FlowFileRecord>(frepo,attributes); record1.addChildFlowFile(ffr1); - ProvenanceTestRepository repo; - uint64_t sample = 65555; - std::shared_ptr testRepository =std::make_shared(); + uint64_t sample = 65555; + std::shared_ptr testRepository =std::make_shared(); record1.setEventDuration(sample); record1.Serialize(testRepository); diff --git a/libminifi/test/unit/SerializationTests.cpp b/libminifi/test/unit/SerializationTests.cpp index 4d1b89b81f..c6ada3387b 100644 --- a/libminifi/test/unit/SerializationTests.cpp +++ b/libminifi/test/unit/SerializationTests.cpp @@ -32,9 +32,9 @@ using namespace org::apache::nifi::minifi::io; TEST_CASE("TestSetPortId", "[S2S1]"){ - minifi::Site2SitePeer peer(std::unique_ptr(new DataStream()),"fake_host",65433); + std::unique_ptr peer = std::unique_ptr( new minifi::Site2SitePeer(std::unique_ptr(new DataStream()),"fake_host",65433)); - minifi::Site2SiteClientProtocol protocol(&peer); + minifi::Site2SiteClientProtocol protocol(std::move(peer)); std::string uuid_str = "c56a4180-65aa-42ec-a945-5fd21dec0538"; @@ -54,9 +54,9 @@ TEST_CASE("TestSetPortId", "[S2S1]"){ TEST_CASE("TestSetPortIdUppercase", "[S2S2]"){ - minifi::Site2SitePeer peer(std::unique_ptr(new DataStream()),"fake_host",65433); + std::unique_ptr peer = std::unique_ptr( new minifi::Site2SitePeer(std::unique_ptr(new DataStream()),"fake_host",65433)); - minifi::Site2SiteClientProtocol protocol(&peer); + minifi::Site2SiteClientProtocol protocol(std::move(peer)); std::string uuid_str = "C56A4180-65AA-42EC-A945-5FD21DEC0538"; diff --git a/libminifi/test/unit/SocketTests.cpp b/libminifi/test/unit/SocketTests.cpp index 984e9073b9..e735f152b5 100644 --- a/libminifi/test/unit/SocketTests.cpp +++ b/libminifi/test/unit/SocketTests.cpp @@ -31,8 +31,8 @@ TEST_CASE("TestSocket", "[TestSocket1]") { TEST_CASE("TestSocketWriteTest1", "[TestSocket2]") { - Socket socket("localhost",8183); - REQUIRE(-1 == socket.initialize() ); + Socket socket("localhost", 8183); + REQUIRE(-1 == socket.initialize()); socket.writeData(0, 0); @@ -50,13 +50,13 @@ TEST_CASE("TestSocketWriteTest2", "[TestSocket3]") { std::vector buffer; buffer.push_back('a'); - Socket server(Socket::getMyHostName(), 9183, 1); + Socket server("localhost", 9183, 1); - Socket server("localhost",9183,1); + REQUIRE(-1 != server.initialize()); - Socket client(Socket::getMyHostName(), 9183); + Socket client("localhost", 9183); - Socket client("localhost",9183); + REQUIRE(-1 != client.initialize()); REQUIRE(1 == client.writeData(buffer, 1)); @@ -84,15 +84,11 @@ TEST_CASE("TestWriteEndian64", "[TestSocket4]") { std::vector buffer; buffer.push_back('a'); - Socket server("localhost",9183,1); + Socket server("localhost", 9183, 1); REQUIRE(-1 != server.initialize()); -<<<<<<< HEAD - Socket client("localhost",9183); -======= - Socket client(Socket::getMyHostName(), 9183); ->>>>>>> MINIFI-217: First commit. Updates namespaces and removes + Socket client("localhost", 9183); REQUIRE(-1 != client.initialize()); @@ -115,19 +111,11 @@ TEST_CASE("TestWriteEndian32", "[TestSocket5]") { std::vector buffer; buffer.push_back('a'); -<<<<<<< HEAD - Socket server("localhost",9183,1); -======= - Socket server(Socket::getMyHostName(), 9183, 1); ->>>>>>> MINIFI-217: First commit. Updates namespaces and removes + Socket server("localhost", 9183, 1); REQUIRE(-1 != server.initialize()); -<<<<<<< HEAD - Socket client("localhost",9183); -======= - Socket client(Socket::getMyHostName(), 9183); ->>>>>>> MINIFI-217: First commit. Updates namespaces and removes + Socket client("localhost", 9183); REQUIRE(-1 != client.initialize()); @@ -161,13 +149,13 @@ TEST_CASE("TestSocketWriteTestAfterClose", "[TestSocket6]") { std::vector buffer; buffer.push_back('a'); - Socket server(Socket::getMyHostName(), 9183, 1); + Socket server("localhost", 9183, 1); - Socket server("localhost",9183,1); + REQUIRE(-1 != server.initialize()); - Socket client(Socket::getMyHostName(), 9183); + Socket client("localhost", 9183); - Socket client("localhost",9183); + REQUIRE(-1 != client.initialize()); REQUIRE(1 == client.writeData(buffer, 1)); @@ -185,4 +173,3 @@ TEST_CASE("TestSocketWriteTestAfterClose", "[TestSocket6]") { server.closeStream(); } - diff --git a/main/MiNiFiMain.cpp b/main/MiNiFiMain.cpp index fac9bfe22e..e320133a59 100644 --- a/main/MiNiFiMain.cpp +++ b/main/MiNiFiMain.cpp @@ -28,12 +28,15 @@ #include #include +#include "core/core.h" + #include "core/logging/BaseLogger.h" #include "core/logging/LogAppenders.h" #include "spdlog/spdlog.h" +#include "core/repository/FlowFileRepository.h" #include "core/logging/Logger.h" -#include "Configure.h" +#include "properties/Configure.h" #include "FlowController.h" //! Main thread sleep interval 1 second @@ -143,12 +146,16 @@ int main(int argc, char **argv) { logger->updateLogger(std::move(configured_logger)); // Create repos for flow record and provenance - std::shared_ptr repo = std::make_shared< + std::shared_ptr prov_repo = std::make_shared< provenance::ProvenanceRepository>(); - repo->initialize(); + prov_repo->initialize(); + + std::shared_ptr flow_repo = std::make_shared< + core::repository::FlowFileRepository>(); + flow_repo->initialize(); controller = std::unique_ptr( - new minifi::FlowControllerImpl(repo)); + new minifi::FlowControllerImpl(prov_repo,flow_repo)); // Load flow from specified configuration file controller->load(); From fa31a6b19f87a71e4a423df0df8b99ab4e03d51a Mon Sep 17 00:00:00 2001 From: Marc Parisi Date: Mon, 20 Mar 2017 15:12:51 -0400 Subject: [PATCH 3/4] MINIFI-226: First Step is to Separate YamlConfiguration from FlowController. Next step will be to Create Controller services, which will be easier with the seperated FlowConfiguration --- libminifi/CMakeLists.txt | 2 +- libminifi/include/FlowController.h | 186 +---- libminifi/include/core/FlowConfiguration.h | 103 +++ libminifi/include/core/ProcessorConfig.h | 53 ++ .../include/core/yaml/YamlConfiguration.h | 97 +++ libminifi/include/io/SocketFactory.h | 128 ---- libminifi/include/io/Sockets.h | 28 + libminifi/include/io/StreamFactoryh | 135 ++++ libminifi/include/io/{ => tls}/TLSSocket.h | 2 +- libminifi/include/io/validation.h | 2 + libminifi/src/Configure.cpp | 2 - libminifi/src/FlowController.cpp | 671 +++--------------- libminifi/src/RemoteProcessorGroupPort.cpp | 5 +- libminifi/src/core/FlowConfiguration.cpp | 110 +++ libminifi/src/core/yaml/YamlConfiguration.cpp | 479 +++++++++++++ .../{SocketFactory.cpp => StreamFactory.cpp} | 7 +- libminifi/src/io/{ => tls}/TLSSocket.cpp | 2 +- libminifi/test/unit/ProvenanceTestHelper.h | 96 +-- main/CMakeLists.txt | 2 +- main/MiNiFiMain.cpp | 20 +- 20 files changed, 1193 insertions(+), 937 deletions(-) create mode 100644 libminifi/include/core/FlowConfiguration.h create mode 100644 libminifi/include/core/ProcessorConfig.h create mode 100644 libminifi/include/core/yaml/YamlConfiguration.h delete mode 100644 libminifi/include/io/SocketFactory.h create mode 100644 libminifi/include/io/Sockets.h create mode 100644 libminifi/include/io/StreamFactoryh rename libminifi/include/io/{ => tls}/TLSSocket.h (99%) create mode 100644 libminifi/src/core/FlowConfiguration.cpp create mode 100644 libminifi/src/core/yaml/YamlConfiguration.cpp rename libminifi/src/io/{SocketFactory.cpp => StreamFactory.cpp} (87%) rename libminifi/src/io/{ => tls}/TLSSocket.cpp (99%) diff --git a/libminifi/CMakeLists.txt b/libminifi/CMakeLists.txt index b562e24293..4c71cc158c 100644 --- a/libminifi/CMakeLists.txt +++ b/libminifi/CMakeLists.txt @@ -60,7 +60,7 @@ include_directories(../thirdparty/yaml-cpp-yaml-cpp-0.5.3/include) include_directories(../thirdparty/civetweb-1.9.1/include) include_directories(include) -file(GLOB SOURCES "src/core/logging/*.cpp" "src/io/*.cpp" "src/core/*.cpp" "src/core/repository/*.cpp" "src/provenance/*.cpp" "src/processors/*.cpp" "src/*.cpp") +file(GLOB SOURCES "src/core/logging/*.cpp" "src/io/*.cpp" "src/io/tls/*.cpp" "src/core/*.cpp" "src/core/repository/*.cpp" "src/core/yaml/*.cpp" "src/provenance/*.cpp" "src/processors/*.cpp" "src/*.cpp") file(GLOB SPD_SOURCES "../include/spdlog/*") # Workaround the limitations of having a diff --git a/libminifi/include/FlowController.h b/libminifi/include/FlowController.h index ba7b82f0df..bec580ddf1 100644 --- a/libminifi/include/FlowController.h +++ b/libminifi/include/FlowController.h @@ -28,13 +28,7 @@ #include #include #include -<<<<<<< HEAD -#include "Configure.h" -======= -#include "yaml-cpp/yaml.h" - #include "properties/Configure.h" ->>>>>>> MINIFI-217: Update based on PR and rebasing against master #include "core/Relationship.h" #include "FlowFileRecord.h" #include "Connection.h" @@ -43,21 +37,11 @@ #include "core/ProcessContext.h" #include "core/ProcessSession.h" #include "core/ProcessGroup.h" -#include "processors/GenerateFlowFile.h" -#include "processors/LogAttribute.h" -#include "processors/RealTimeDataCollector.h" +#include "core/FlowConfiguration.h" #include "TimerDrivenSchedulingAgent.h" #include "EventDrivenSchedulingAgent.h" #include "FlowControlProtocol.h" -#include "RemoteProcessorGroupPort.h" -#include "provenance/Provenance.h" -#include "processors/GetFile.h" -#include "processors/PutFile.h" -#include "processors/TailFile.h" -#include "processors/ListenSyslog.h" -#include "processors/ListenHTTP.h" -#include "processors/ExecuteProcess.h" -#include "processors/AppendHostInfo.h" + #include "core/Property.h" namespace org { @@ -67,22 +51,6 @@ namespace minifi { // Default NiFi Root Group Name #define DEFAULT_ROOT_GROUP_NAME "" -#define DEFAULT_FLOW_YAML_FILE_NAME "conf/flow.yml" -#define CONFIG_YAML_PROCESSORS_KEY "Processors" - -struct ProcessorConfig { - std::string id; - std::string name; - std::string javaClass; - std::string maxConcurrentTasks; - std::string schedulingStrategy; - std::string schedulingPeriod; - std::string penalizationPeriod; - std::string yieldPeriod; - std::string runDurationNanos; - std::vector autoTerminatedRelationships; - std::vector properties; -}; /** * Flow Controller class. Generally used by FlowController factory @@ -93,9 +61,16 @@ class FlowController : public core::CoreComponent { static const int DEFAULT_MAX_TIMER_DRIVEN_THREAD = 10; static const int DEFAULT_MAX_EVENT_DRIVEN_THREAD = 5; + /** + * Flow controller constructor + */ + FlowController(std::shared_ptr provenance_repo, + std::shared_ptr flow_file_repo, + std::unique_ptr flow_configuration, + const std::string name = DEFAULT_ROOT_GROUP_NAME,bool headless_mode=false); + // Destructor - virtual ~FlowController() { - } + virtual ~FlowController(); // Set MAX TimerDrivenThreads virtual void setMaxTimerDrivenThreads(int number) { @@ -124,7 +99,7 @@ class FlowController : public core::CoreComponent { } // Load flow xml from disk, after that, create the root process group and its children, initialize the flows - virtual void load() = 0; + virtual void load(); // Whether the Flow Controller is start running virtual bool isRunning() { @@ -135,15 +110,15 @@ class FlowController : public core::CoreComponent { return initialized_.load(); } // Start to run the Flow Controller which internally start the root process group and all its children - virtual bool start() = 0; + virtual bool start(); // Unload the current flow YAML, clean the root process group and all its children - virtual void stop(bool force) = 0; + virtual void stop(bool force); // Asynchronous function trigger unloading and wait for a period of time - virtual void waitUnload(const uint64_t timeToWaitMs) = 0; + virtual void waitUnload(const uint64_t timeToWaitMs); // Unload the current flow xml, clean the root process group and all its children - virtual void unload() = 0; + virtual void unload(); // Load new xml - virtual void reload(std::string yamlFile) = 0; + virtual void reload(std::string yamlFile); // update property value void updatePropertyValue(std::string processorName, std::string propertyName, std::string propertyValue) { @@ -151,18 +126,6 @@ class FlowController : public core::CoreComponent { root_->updatePropertyValue(processorName, propertyName, propertyValue); } - // Create Processor (Node/Input/Output Port) based on the name - virtual std::shared_ptr createProcessor(std::string name, - uuid_t uuid) = 0; - // Create Root Processor Group - virtual core::ProcessGroup *createRootProcessGroup(std::string name, - uuid_t uuid) = 0; - // Create Remote Processor Group - virtual core::ProcessGroup *createRemoteProcessGroup(std::string name, - uuid_t uuid) = 0; - // Create Connection - virtual std::shared_ptr createConnection(std::string name, - uuid_t uuid) = 0; // set 8 bytes SerialNumber virtual void setSerialNumber(uint8_t *number) { protocol_->setSerialNumber(number); @@ -170,17 +133,30 @@ class FlowController : public core::CoreComponent { protected: + // function to load the flow file repo. + void loadFlowRepo(); + + /** + * Initializes flow controller paths. + */ + virtual void initializePaths(const std::string &adjustedFilename); + + // flow controller mutex + std::recursive_mutex mutex_; + + // configuration object + Configure *configure_; + // Configuration File Name std::string configuration_file_name_; // NiFi property File Name std::string properties_file_name_; // Root Process Group - core::ProcessGroup *root_; + std::unique_ptr root_; // MAX Timer Driven Threads int max_timer_driven_threads_; // MAX Event Driven Threads int max_event_driven_threads_; - // Config // FlowFile Repo // Whether it is running std::atomic running_; @@ -206,102 +182,8 @@ class FlowController : public core::CoreComponent { // FlowControl Protocol FlowControlProtocol *protocol_; - FlowController(std::shared_ptr provenance_repo, - std::shared_ptr flow_file_repo) - : CoreComponent(core::getClassName()), root_(0), - max_timer_driven_threads_(0), - max_event_driven_threads_(0), - running_(false), - initialized_(false), - provenance_repo_(provenance_repo), - flow_file_repo_(flow_file_repo), - protocol_(0), - _timerScheduler(provenance_repo_), - _eventScheduler(provenance_repo_){ - if (provenance_repo == nullptr) - throw std::runtime_error("Provenance Repo should not be null"); - if (flow_file_repo == nullptr) - throw std::runtime_error("Flow Repo should not be null"); - } - - -}; - -class FlowControllerImpl : public FlowController { - public: - - // Destructor - virtual ~FlowControllerImpl(); - - // Life Cycle related function - // Load flow xml from disk, after that, create the root process group and its children, initialize the flows - void load(); - // Start to run the Flow Controller which internally start the root process group and all its children - bool start(); - // Stop to run the Flow Controller which internally stop the root process group and all its children - void stop(bool force); - // Asynchronous function trigger unloading and wait for a period of time - void waitUnload(const uint64_t timeToWaitMs); - // Unload the current flow xml, clean the root process group and all its children - void unload(); - // Load Flow File from persistent Flow Repo - void loadFlowRepo(); - // Load new xml - void reload(std::string yamlFile); - // update property value - void updatePropertyValue(std::string processorName, std::string propertyName, - std::string propertyValue) { - if (root_) - root_->updatePropertyValue(processorName, propertyName, propertyValue); - } - - // Create Processor (Node/Input/Output Port) based on the name - std::shared_ptr createProcessor(std::string name, - uuid_t uuid); - // Create Root Processor Group - core::ProcessGroup *createRootProcessGroup(std::string name, uuid_t uuid); - // Create Remote Processor Group - core::ProcessGroup *createRemoteProcessGroup(std::string name, uuid_t uuid); - // Create Connection - std::shared_ptr createConnection(std::string name, uuid_t uuid); - - // Constructor - /*! - * Create a new Flow Controller - */ - FlowControllerImpl(std::shared_ptr repo,std::shared_ptr flow_file_repo, - std::string name = DEFAULT_ROOT_GROUP_NAME); - - // Prevent default copy constructor and assignment operation - // Only support pass by reference or pointer - FlowControllerImpl(const FlowController &parent) = delete; - FlowControllerImpl &operator=(const FlowController &parent) = delete; - - private: - - std::mutex mutex_; - - // configuration object - Configure *configure_; - // Process Processor Node YAML - void parseProcessorNodeYaml(YAML::Node processorNode, - core::ProcessGroup *parent); - // Process Port YAML - void parsePortYaml(YAML::Node *portNode, core::ProcessGroup *parent, - TransferDirection direction); - // Process Root Processor Group YAML - void parseRootProcessGroupYaml(YAML::Node rootNode); - // Process Property YAML - void parseProcessorPropertyYaml(YAML::Node *doc, YAML::Node *node, - std::shared_ptr processor); - // Process connection YAML - void parseConnectionYaml(YAML::Node *node, core::ProcessGroup *parent); - // Process Remote Process Group YAML - void parseRemoteProcessGroupYaml(YAML::Node *node, - core::ProcessGroup *parent); - // Parse Properties Node YAML for a processor - void parsePropertiesNodeYaml(YAML::Node *propertiesNode, - std::shared_ptr processor); + // flow configuration object. + std::unique_ptr flow_configuration_; }; diff --git a/libminifi/include/core/FlowConfiguration.h b/libminifi/include/core/FlowConfiguration.h new file mode 100644 index 0000000000..fd1eaddc78 --- /dev/null +++ b/libminifi/include/core/FlowConfiguration.h @@ -0,0 +1,103 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBMINIFI_INCLUDE_CORE_FLOWCONFIGURATION_H_ +#define LIBMINIFI_INCLUDE_CORE_FLOWCONFIGURATION_H_ + +#include "core/core.h" +#include "Connection.h" +#include "RemoteProcessorGroupPort.h" +#include "provenance/Provenance.h" +#include "processors/GetFile.h" +#include "processors/PutFile.h" +#include "processors/TailFile.h" +#include "processors/ListenSyslog.h" +#include "processors/GenerateFlowFile.h" +#include "processors/RealTimeDataCollector.h" +#include "processors/ListenHTTP.h" +#include "processors/LogAttribute.h" +#include "processors/ExecuteProcess.h" +#include "processors/AppendHostInfo.h" +#include "core/Processor.h" +#include "core/logging/Logger.h" +#include "core/ProcessContext.h" +#include "core/ProcessSession.h" +#include "core/ProcessGroup.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +/** + * Purpose: Flow configuration defines the mechanism + * by which we will configure our flow controller + */ +class FlowConfiguration : public CoreComponent { + public: + FlowConfiguration(std::shared_ptr repo, + std::shared_ptr flow_file_repo, + const std::string path) + : CoreComponent(core::getClassName()), + flow_file_repo_(flow_file_repo), + config_path_(path) { + + } + + virtual ~FlowConfiguration(); + + // Create Processor (Node/Input/Output Port) based on the name + std::shared_ptr createProcessor(std::string name, + uuid_t uuid); + // Create Root Processor Group + std::unique_ptr createRootProcessGroup(std::string name, + uuid_t uuid); + // Create Remote Processor Group + std::unique_ptr createRemoteProcessGroup(std::string name, + uuid_t uuid); + // Create Connection + std::shared_ptr createConnection(std::string name, + uuid_t uuid); + + const std::string &getConfigurationPath() { + return config_path_; + } + + virtual std::unique_ptr getRoot() { + return getRoot(config_path_); + } + + virtual std::unique_ptr getRoot( + const std::string &from_config) =0; + + protected: + // configuration path + std::string config_path_; + // flow file repo + std::shared_ptr flow_file_repo_; + +}; + +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif /* LIBMINIFI_INCLUDE_CORE_FLOWCONFIGURATION_H_ */ + diff --git a/libminifi/include/core/ProcessorConfig.h b/libminifi/include/core/ProcessorConfig.h new file mode 100644 index 0000000000..6b4a00a724 --- /dev/null +++ b/libminifi/include/core/ProcessorConfig.h @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBMINIFI_INCLUDE_CORE_PROCESSORCONFIG_H_ +#define LIBMINIFI_INCLUDE_CORE_PROCESSORCONFIG_H_ + +#include "core/core.h" +#include "core/Property.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + + +struct ProcessorConfig { + std::string id; + std::string name; + std::string javaClass; + std::string maxConcurrentTasks; + std::string schedulingStrategy; + std::string schedulingPeriod; + std::string penalizationPeriod; + std::string yieldPeriod; + std::string runDurationNanos; + std::vector autoTerminatedRelationships; + std::vector properties; +}; + + + +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + + +#endif /* LIBMINIFI_INCLUDE_CORE_PROCESSORCONFIG_H_ */ diff --git a/libminifi/include/core/yaml/YamlConfiguration.h b/libminifi/include/core/yaml/YamlConfiguration.h new file mode 100644 index 0000000000..39d8d176b0 --- /dev/null +++ b/libminifi/include/core/yaml/YamlConfiguration.h @@ -0,0 +1,97 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBMINIFI_INCLUDE_CORE_YAMLCONFIGURATION_H_ +#define LIBMINIFI_INCLUDE_CORE_YAMLCONFIGURATION_H_ + +#include "core/ProcessorConfig.h" +#include "yaml-cpp/yaml.h" +#include "../FlowConfiguration.h" +#include + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { +#include "Site2SiteClientProtocol.h" + +#define DEFAULT_FLOW_YAML_FILE_NAME "conf/flow.yml" +#define CONFIG_YAML_PROCESSORS_KEY "Processors" + +class YamlConfiguration : public FlowConfiguration { + + public: + YamlConfiguration(std::shared_ptr repo, + std::shared_ptr flow_file_repo, + const std::string path = DEFAULT_FLOW_YAML_FILE_NAME) + : FlowConfiguration(repo, flow_file_repo, path) { + + } + + virtual ~YamlConfiguration() { + + } + + std::unique_ptr getRoot(const std::string &from_config) { + + YAML::Node flow = YAML::LoadFile(from_config); + + YAML::Node flowControllerNode = flow["Flow Controller"]; + YAML::Node processorsNode = flow[CONFIG_YAML_PROCESSORS_KEY]; + YAML::Node connectionsNode = flow["Connections"]; + YAML::Node remoteProcessingGroupNode = flow["Remote Processing Groups"]; + + // Create the root process group + core::ProcessGroup * root = parseRootProcessGroupYaml(flowControllerNode); + parseProcessorNodeYaml(processorsNode, root); + parseRemoteProcessGroupYaml(&remoteProcessingGroupNode, root); + parseConnectionYaml(&connectionsNode, root); + + return std::unique_ptr(root); + + } + protected: + // Process Processor Node YAML + void parseProcessorNodeYaml(YAML::Node processorNode, + core::ProcessGroup * parent); + // Process Port YAML + void parsePortYaml(YAML::Node *portNode, core::ProcessGroup *parent, + TransferDirection direction); + // Process Root Processor Group YAML + core::ProcessGroup *parseRootProcessGroupYaml(YAML::Node rootNode); + // Process Property YAML + void parseProcessorPropertyYaml(YAML::Node *doc, YAML::Node *node, + std::shared_ptr processor); + // Process connection YAML + void parseConnectionYaml(YAML::Node *node, core::ProcessGroup * parent); + // Process Remote Process Group YAML + void parseRemoteProcessGroupYaml(YAML::Node *node, + core::ProcessGroup * parent); + // Parse Properties Node YAML for a processor + void parsePropertiesNodeYaml(YAML::Node *propertiesNode, + std::shared_ptr processor); +}; + + +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif /* LIBMINIFI_INCLUDE_CORE_YAMLCONFIGURATION_H_ */ diff --git a/libminifi/include/io/SocketFactory.h b/libminifi/include/io/SocketFactory.h deleted file mode 100644 index d6221062c5..0000000000 --- a/libminifi/include/io/SocketFactory.h +++ /dev/null @@ -1,128 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef SOCKET_FACTORY_H -#define SOCKET_FACTORY_H - -#include "ClientSocket.h" -#include "TLSSocket.h" -#include "ClientSocket.h" -#include "properties/Configure.h" -#include "utils/StringUtils.h" - -namespace org { -namespace apache { -namespace nifi { -namespace minifi { -namespace io { - -/** - Purpose: Due to the current design this is the only mechanism by which we can - inject different socket types - - **/ -class SocketFactory { - public: - - /** - * Build an instance, creating a memory fence, which - * allows us to avoid locking. This is tantamount to double checked locking. - * @returns new SocketFactory; - */ - static SocketFactory *getInstance() { - SocketFactory* atomic_context = context_instance_.load( - std::memory_order_relaxed); - std::atomic_thread_fence(std::memory_order_acquire); - if (atomic_context == nullptr) { - std::lock_guard lock(context_mutex_); - atomic_context = context_instance_.load(std::memory_order_relaxed); - if (atomic_context == nullptr) { - atomic_context = new SocketFactory(); - std::atomic_thread_fence(std::memory_order_release); - context_instance_.store(atomic_context, std::memory_order_relaxed); - } - } - return atomic_context; - } - -<<<<<<< HEAD - /** - * Creates a socket and returns a unique ptr - * - */ - std::unique_ptr createSocket(const std::string &host, const uint16_t port) { - Socket *socket = 0; - if (is_secure_) { -#ifdef OPENSSL_SUPPORT - socket = new TLSSocket(host, port); -#else - socket = 0; -#endif - } else { - socket = new Socket(host, port); - } - return std::unique_ptr(socket); - } -protected: - SocketFactory() : - configure_(Configure::getConfigure()) { - std::string secureStr; - is_secure_ = false; - if (configure_->get(Configure::nifi_remote_input_secure, secureStr)) { - StringUtils::StringToBool(secureStr, is_secure_); - } - } -======= - /** - * Creates a socket and returns a unique ptr - * - */ - std::unique_ptr createSocket(const std::string &host, - const uint16_t port) { - Socket *socket = 0; - if (is_secure_) { - socket = new TLSSocket(host, port); - } else { - socket = new Socket(host, port); - } - return std::unique_ptr(socket); - } - protected: - SocketFactory() - : configure_(Configure::getConfigure()) { - std::string secureStr; - is_secure_ = false; - if (configure_->get(Configure::nifi_remote_input_secure, secureStr)) { - org::apache::nifi::minifi::utils::StringUtils::StringToBool(secureStr, is_secure_); - } - } ->>>>>>> MINIFI-217: First commit. Updates namespaces and removes - - bool is_secure_; - static std::atomic context_instance_; - static std::mutex context_mutex_; - - Configure *configure_; -}; - -} /* namespace io */ -} /* namespace minifi */ -} /* namespace nifi */ -} /* namespace apache */ -} /* namespace org */ - -#endif diff --git a/libminifi/include/io/Sockets.h b/libminifi/include/io/Sockets.h new file mode 100644 index 0000000000..fc548e7d26 --- /dev/null +++ b/libminifi/include/io/Sockets.h @@ -0,0 +1,28 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBMINIFI_INCLUDE_IO_SOCKET_H_ +#define LIBMINIFI_INCLUDE_IO_SOCKET_H_ + +#include "ClientSocket.h" + +#include "tls/TLSSocket.h" + + + + +#endif /* LIBMINIFI_INCLUDE_IO_TLS_SECURESOCKET_H_ */ diff --git a/libminifi/include/io/StreamFactoryh b/libminifi/include/io/StreamFactoryh new file mode 100644 index 0000000000..cdaf83fd06 --- /dev/null +++ b/libminifi/include/io/StreamFactoryh @@ -0,0 +1,135 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SOCKET_FACTORY_H +#define SOCKET_FACTORY_H + +#include "properties/Configure.h" +#include "Sockets.h" +#include "utils/StringUtils.h" +#include "validation.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace io { + +/** + * Purpose: Socket Creator is a class that will determine if the provided socket type + * exists per the compilation parameters + */ +template +class SocketCreator { + + template + using TypeCheck = typename std::enable_if< cond, U >::type; + +public: + template + TypeCheck *create(const std::string &host, const uint16_t port) { + return new T(host, port); + } + template + TypeCheck *create(const std::string &host, const uint16_t port) { + return new Socket(host, port); + } + +}; + +/** + Purpose: Due to the current design this is the only mechanism by which we can + inject different socket types + + **/ +class StreamFactory { +public: + + /** + * Build an instance, creating a memory fence, which + * allows us to avoid locking. This is tantamount to double checked locking. + * @returns new StreamFactory; + */ + static StreamFactory *getInstance() { + StreamFactory* atomic_context = context_instance_.load( + std::memory_order_relaxed); + std::atomic_thread_fence(std::memory_order_acquire); + if (atomic_context == nullptr) { + std::lock_guard < std::mutex > lock(context_mutex_); + atomic_context = context_instance_.load(std::memory_order_relaxed); + if (atomic_context == nullptr) { + atomic_context = new StreamFactory(); + std::atomic_thread_fence(std::memory_order_release); + context_instance_.store(atomic_context, + std::memory_order_relaxed); + } + } + return atomic_context; + } + + /** + * Creates a socket and returns a unique ptr + * + */ + std::unique_ptr createSocket(const std::string &host, + const uint16_t port) { + Socket *socket = 0; + + if (is_secure_) { + socket = createSocket(host, port); + } else { + socket = createSocket(host, port); + } + return std::unique_ptr < Socket > (socket); + } + +protected: + + /** + * Creates a socket and returns a unique ptr + * + */ + template + Socket *createSocket(const std::string &host, const uint16_t port) { + SocketCreator creator; + return creator.create(host, port); + } + + StreamFactory() : + configure_(Configure::getConfigure()) { + std::string secureStr; + is_secure_ = false; + if (configure_->get(Configure::nifi_remote_input_secure, secureStr)) { + org::apache::nifi::minifi::utils::StringUtils::StringToBool( + secureStr, is_secure_); + } + } + + bool is_secure_; + static std::atomic context_instance_; + static std::mutex context_mutex_; + + Configure *configure_; +}; + +} /* namespace io */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif diff --git a/libminifi/include/io/TLSSocket.h b/libminifi/include/io/tls/TLSSocket.h similarity index 99% rename from libminifi/include/io/TLSSocket.h rename to libminifi/include/io/tls/TLSSocket.h index 28858cd83f..9a55fbcade 100644 --- a/libminifi/include/io/TLSSocket.h +++ b/libminifi/include/io/tls/TLSSocket.h @@ -20,7 +20,7 @@ #ifdef OPENSSL_SUPPORT #include -#include "ClientSocket.h" +#include "../ClientSocket.h" #include #include diff --git a/libminifi/include/io/validation.h b/libminifi/include/io/validation.h index 9db554472c..c66c41287d 100644 --- a/libminifi/include/io/validation.h +++ b/libminifi/include/io/validation.h @@ -18,6 +18,7 @@ #ifndef VALIDATION_H #define VALIDATION_H +#include #include #include @@ -41,6 +42,7 @@ class size_function_functor_checker { }; }; + /** * Determines if the variable is null or ::size() == 0 */ diff --git a/libminifi/src/Configure.cpp b/libminifi/src/Configure.cpp index 74d2cc16ea..e14a9c1a61 100644 --- a/libminifi/src/Configure.cpp +++ b/libminifi/src/Configure.cpp @@ -1,6 +1,4 @@ /** - * @file Configure.cpp - * Configure class implementation * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with diff --git a/libminifi/src/FlowController.cpp b/libminifi/src/FlowController.cpp index c3eec13534..ff78d3c587 100644 --- a/libminifi/src/FlowController.cpp +++ b/libminifi/src/FlowController.cpp @@ -41,46 +41,76 @@ namespace apache { namespace nifi { namespace minifi { -FlowControllerImpl::FlowControllerImpl( - std::shared_ptr repo , std::shared_ptr flow_file_repo, std::string name) - : FlowController(repo,flow_file_repo) { +#define DEFAULT_CONFIG_NAME "conf/flow.yml" + +FlowController::FlowController( + std::shared_ptr provenance_repo, + std::shared_ptr flow_file_repo, + std::unique_ptr flow_configuration, + const std::string name, bool headless_mode) + : CoreComponent(core::getClassName()), + root_(nullptr), + max_timer_driven_threads_(0), + max_event_driven_threads_(0), + running_(false), + initialized_(false), + provenance_repo_(provenance_repo), + flow_file_repo_(flow_file_repo), + protocol_(0), + _timerScheduler(provenance_repo_), + _eventScheduler(provenance_repo_), + flow_configuration_(std::move(flow_configuration)) { + if (provenance_repo == nullptr) + throw std::runtime_error("Provenance Repo should not be null"); + if (flow_file_repo == nullptr) + throw std::runtime_error("Flow Repo should not be null"); + uuid_generate(uuid_); setUUID(uuid_); // Setup the default values - configuration_filename_ = DEFAULT_FLOW_YAML_FILE_NAME; + if (flow_configuration_ != nullptr) { + configuration_filename_ = flow_configuration_->getConfigurationPath(); + } max_event_driven_threads_ = DEFAULT_MAX_EVENT_DRIVEN_THREAD; max_timer_driven_threads_ = DEFAULT_MAX_TIMER_DRIVEN_THREAD; running_ = false; initialized_ = false; root_ = NULL; - logger_ = logging::Logger::getLogger(); + protocol_ = new FlowControlProtocol(this); // NiFi config properties configure_ = Configure::getConfigure(); - std::string rawConfigFileString; - configure_->get(Configure::nifi_flow_configuration_file, rawConfigFileString); - - if (!rawConfigFileString.empty()) { - configuration_filename_ = rawConfigFileString; - } + if (!headless_mode) { + std::string rawConfigFileString; + configure_->get(Configure::nifi_flow_configuration_file, + rawConfigFileString); - char *path = NULL; - char full_path[PATH_MAX]; + if (!rawConfigFileString.empty()) { + configuration_filename_ = rawConfigFileString; + } - std::string adjustedFilename; - if (!configuration_filename_.empty()) { - // perform a naive determination if this is a relative path - if (configuration_filename_.c_str()[0] != '/') { - adjustedFilename = adjustedFilename + configure_->getHome() + "/" - + configuration_filename_; - } else { - adjustedFilename = configuration_filename_; + std::string adjustedFilename; + if (!configuration_filename_.empty()) { + // perform a naive determination if this is a relative path + if (configuration_filename_.c_str()[0] != '/') { + adjustedFilename = adjustedFilename + configure_->getHome() + "/" + + configuration_filename_; + } else { + adjustedFilename = configuration_filename_; + } } + + initializePaths(adjustedFilename); } +} + +void FlowController::initializePaths(const std::string &adjustedFilename) { + char *path = NULL; + char full_path[PATH_MAX]; path = realpath(adjustedFilename.c_str(), full_path); if (path == NULL) { @@ -115,10 +145,9 @@ FlowControllerImpl::FlowControllerImpl( exit(1); } - } -FlowControllerImpl::~FlowControllerImpl() { +FlowController::~FlowController() { stop(true); unload(); if (NULL != protocol_) @@ -126,8 +155,8 @@ FlowControllerImpl::~FlowControllerImpl() { } -void FlowControllerImpl::stop(bool force) { - +void FlowController::stop(bool force) { + std::lock_guard flow_lock(mutex_); if (running_) { // immediately indicate that we are not running running_ = false; @@ -155,7 +184,7 @@ void FlowControllerImpl::stop(bool force) { * @param timeToWaitMs Maximum time to wait before manually * marking running as false. */ -void FlowControllerImpl::waitUnload(const uint64_t timeToWaitMs) { +void FlowController::waitUnload(const uint64_t timeToWaitMs) { if (running_) { // use the current time and increment with the provided argument. std::chrono::system_clock::time_point wait_time = @@ -173,15 +202,14 @@ void FlowControllerImpl::waitUnload(const uint64_t timeToWaitMs) { } } -void FlowControllerImpl::unload() { +void FlowController::unload() { + std::lock_guard flow_lock(mutex_); if (running_) { stop(true); } if (initialized_) { logger_->log_info("Unload Flow Controller"); - if (root_) - delete root_; - root_ = NULL; + root_ = nullptr; initialized_ = false; name_ = ""; } @@ -189,548 +217,8 @@ void FlowControllerImpl::unload() { return; } -std::shared_ptr FlowControllerImpl::createProcessor( - std::string name, uuid_t uuid) { - std::shared_ptr processor = nullptr; - if (name - == org::apache::nifi::minifi::processors::GenerateFlowFile::ProcessorName) { - processor = std::make_shared< - org::apache::nifi::minifi::processors::GenerateFlowFile>(name, uuid); - } else if (name - == org::apache::nifi::minifi::processors::LogAttribute::ProcessorName) { - processor = std::make_shared< - org::apache::nifi::minifi::processors::LogAttribute>(name, uuid); - } else if (name - == org::apache::nifi::minifi::processors::RealTimeDataCollector::ProcessorName) { - processor = std::make_shared< - org::apache::nifi::minifi::processors::RealTimeDataCollector>(name, - uuid); - } else if (name - == org::apache::nifi::minifi::processors::GetFile::ProcessorName) { - processor = - std::make_shared(name, - uuid); - } else if (name - == org::apache::nifi::minifi::processors::PutFile::ProcessorName) { - processor = - std::make_shared(name, - uuid); - } else if (name - == org::apache::nifi::minifi::processors::TailFile::ProcessorName) { - processor = - std::make_shared(name, - uuid); - } else if (name - == org::apache::nifi::minifi::processors::ListenSyslog::ProcessorName) { - processor = std::make_shared< - org::apache::nifi::minifi::processors::ListenSyslog>(name, uuid); - } else if (name - == org::apache::nifi::minifi::processors::ListenHTTP::ProcessorName) { - processor = std::make_shared< - org::apache::nifi::minifi::processors::ListenHTTP>(name, uuid); - } else if (name - == org::apache::nifi::minifi::processors::ExecuteProcess::ProcessorName) { - processor = std::make_shared< - org::apache::nifi::minifi::processors::ExecuteProcess>(name, uuid); - } else if (name - == org::apache::nifi::minifi::processors::AppendHostInfo::ProcessorName) { - processor = std::make_shared< - org::apache::nifi::minifi::processors::AppendHostInfo>(name, uuid); - } else { - logger_->log_error("No Processor defined for %s", name.c_str()); - return nullptr; - } - - // initialize the processor - processor->initialize(); - - return processor; -} - -core::ProcessGroup *FlowControllerImpl::createRootProcessGroup(std::string name, - uuid_t uuid) { - return new core::ProcessGroup(core::ROOT_PROCESS_GROUP, name, uuid); -} - -core::ProcessGroup *FlowControllerImpl::createRemoteProcessGroup( - std::string name, uuid_t uuid) { - return new core::ProcessGroup(core::REMOTE_PROCESS_GROUP, name, uuid); -} - -std::shared_ptr FlowControllerImpl::createConnection( - std::string name, uuid_t uuid) { - return std::make_shared(flow_file_repo_, name, uuid); -} - -void FlowControllerImpl::parseRootProcessGroupYaml(YAML::Node rootFlowNode) { - uuid_t uuid; - core::ProcessGroup *group = NULL; - std::string flowName = rootFlowNode["name"].as(); - std::string id = rootFlowNode["id"].as(); - - uuid_parse(id.c_str(), uuid); - - logger_->log_debug("parseRootProcessGroup: id => [%s]", id.c_str()); - logger_->log_debug("parseRootProcessGroup: name => [%s]", flowName.c_str()); - group = this->createRootProcessGroup(flowName, uuid); - this->root_ = group; - this->name_ = flowName; -} - -void FlowControllerImpl::parseProcessorNodeYaml(YAML::Node processorsNode, - core::ProcessGroup *parentGroup) { - int64_t schedulingPeriod = -1; - int64_t penalizationPeriod = -1; - int64_t yieldPeriod = -1; - int64_t runDurationNanos = -1; - uuid_t uuid; - std::shared_ptr processor = nullptr; - - if (!parentGroup) { - logger_->log_error("parseProcessNodeYaml: no parent group exists"); - return; - } - - if (processorsNode) { - - if (processorsNode.IsSequence()) { - // Evaluate sequence of processors - int numProcessors = processorsNode.size(); - - for (YAML::const_iterator iter = processorsNode.begin(); - iter != processorsNode.end(); ++iter) { - ProcessorConfig procCfg; - YAML::Node procNode = iter->as(); - - procCfg.name = procNode["name"].as(); - procCfg.id = procNode["id"].as(); - logger_->log_debug("parseProcessorNode: name => [%s] id => [%s]", - procCfg.name.c_str(), procCfg.id.c_str()); - procCfg.javaClass = procNode["class"].as(); - logger_->log_debug("parseProcessorNode: class => [%s]", - procCfg.javaClass.c_str()); - - uuid_parse(procCfg.id.c_str(), uuid); - - // Determine the processor name only from the Java class - int lastOfIdx = procCfg.javaClass.find_last_of("."); - if (lastOfIdx != std::string::npos) { - lastOfIdx++; // if a value is found, increment to move beyond the . - int nameLength = procCfg.javaClass.length() - lastOfIdx; - std::string processorName = procCfg.javaClass.substr( - lastOfIdx, nameLength); - processor = this->createProcessor(processorName, uuid); - } - - if (!processor) { - logger_->log_error( - "Could not create a processor %s with name %s", - procCfg.name.c_str(), procCfg.id.c_str()); - throw std::invalid_argument( - "Could not create processor " + procCfg.name); - } - processor->setName(procCfg.name); - - procCfg.maxConcurrentTasks = - procNode["max concurrent tasks"].as(); - logger_->log_debug( - "parseProcessorNode: max concurrent tasks => [%s]", - procCfg.maxConcurrentTasks.c_str()); - procCfg.schedulingStrategy = procNode["scheduling strategy"].as< - std::string>(); - logger_->log_debug( - "parseProcessorNode: scheduling strategy => [%s]", - procCfg.schedulingStrategy.c_str()); - procCfg.schedulingPeriod = procNode["scheduling period"].as< - std::string>(); - logger_->log_debug( - "parseProcessorNode: scheduling period => [%s]", - procCfg.schedulingPeriod.c_str()); - procCfg.penalizationPeriod = procNode["penalization period"].as< - std::string>(); - logger_->log_debug( - "parseProcessorNode: penalization period => [%s]", - procCfg.penalizationPeriod.c_str()); - procCfg.yieldPeriod = - procNode["yield period"].as(); - logger_->log_debug("parseProcessorNode: yield period => [%s]", - procCfg.yieldPeriod.c_str()); - procCfg.yieldPeriod = procNode["run duration nanos"].as< - std::string>(); - logger_->log_debug( - "parseProcessorNode: run duration nanos => [%s]", - procCfg.runDurationNanos.c_str()); - - // handle auto-terminated relationships - YAML::Node autoTerminatedSequence = - procNode["auto-terminated relationships list"]; - std::vector rawAutoTerminatedRelationshipValues; - if (autoTerminatedSequence.IsSequence() - && !autoTerminatedSequence.IsNull() - && autoTerminatedSequence.size() > 0) { - for (YAML::const_iterator relIter = - autoTerminatedSequence.begin(); - relIter != autoTerminatedSequence.end(); - ++relIter) { - std::string autoTerminatedRel = - relIter->as(); - rawAutoTerminatedRelationshipValues.push_back( - autoTerminatedRel); - } - } - procCfg.autoTerminatedRelationships = - rawAutoTerminatedRelationshipValues; - - // handle processor properties - YAML::Node propertiesNode = procNode["Properties"]; - parsePropertiesNodeYaml(&propertiesNode, processor); - - // Take care of scheduling - core::TimeUnit unit; - if (core::Property::StringToTime(procCfg.schedulingPeriod, - schedulingPeriod, unit) - && core::Property::ConvertTimeUnitToNS(schedulingPeriod, unit, - schedulingPeriod)) { - logger_->log_debug( - "convert: parseProcessorNode: schedulingPeriod => [%d] ns", - schedulingPeriod); - processor->setSchedulingPeriodNano(schedulingPeriod); - } - - if (core::Property::StringToTime(procCfg.penalizationPeriod, - penalizationPeriod, unit) - && core::Property::ConvertTimeUnitToMS(penalizationPeriod, - unit, penalizationPeriod)) { - logger_->log_debug( - "convert: parseProcessorNode: penalizationPeriod => [%d] ms", - penalizationPeriod); - processor->setPenalizationPeriodMsec(penalizationPeriod); - } - - if (core::Property::StringToTime(procCfg.yieldPeriod, yieldPeriod, - unit) - && core::Property::ConvertTimeUnitToMS(yieldPeriod, unit, - yieldPeriod)) { - logger_->log_debug( - "convert: parseProcessorNode: yieldPeriod => [%d] ms", - yieldPeriod); - processor->setYieldPeriodMsec(yieldPeriod); - } - - // Default to running - processor->setScheduledState(core::RUNNING); - - if (procCfg.schedulingStrategy == "TIMER_DRIVEN") { - processor->setSchedulingStrategy(core::TIMER_DRIVEN); - logger_->log_debug("setting scheduling strategy as %s", - procCfg.schedulingStrategy.c_str()); - } else if (procCfg.schedulingStrategy == "EVENT_DRIVEN") { - processor->setSchedulingStrategy(core::EVENT_DRIVEN); - logger_->log_debug("setting scheduling strategy as %s", - procCfg.schedulingStrategy.c_str()); - } else { - processor->setSchedulingStrategy(core::CRON_DRIVEN); - logger_->log_debug("setting scheduling strategy as %s", - procCfg.schedulingStrategy.c_str()); - - } - - int64_t maxConcurrentTasks; - if (core::Property::StringToInt(procCfg.maxConcurrentTasks, - maxConcurrentTasks)) { - logger_->log_debug( - "parseProcessorNode: maxConcurrentTasks => [%d]", - maxConcurrentTasks); - processor->setMaxConcurrentTasks(maxConcurrentTasks); - } - - if (core::Property::StringToInt(procCfg.runDurationNanos, - runDurationNanos)) { - logger_->log_debug( - "parseProcessorNode: runDurationNanos => [%d]", - runDurationNanos); - processor->setRunDurationNano(runDurationNanos); - } - - std::set autoTerminatedRelationships; - for (auto &&relString : procCfg.autoTerminatedRelationships) { - core::Relationship relationship(relString, ""); - logger_->log_debug( - "parseProcessorNode: autoTerminatedRelationship => [%s]", - relString.c_str()); - autoTerminatedRelationships.insert(relationship); - } - - processor->setAutoTerminatedRelationships( - autoTerminatedRelationships); - - parentGroup->addProcessor(processor); - } - } - } else { - throw new std::invalid_argument( - "Cannot instantiate a MiNiFi instance without a defined Processors configuration node."); - } -} - -void FlowControllerImpl::parseRemoteProcessGroupYaml(YAML::Node *rpgNode, - core::ProcessGroup *parentGroup) { - uuid_t uuid; - - if (!parentGroup) { - logger_->log_error( - "parseRemoteProcessGroupYaml: no parent group exists"); - return; - } - - if (rpgNode) { - if (rpgNode->IsSequence()) { - for (YAML::const_iterator iter = rpgNode->begin(); - iter != rpgNode->end(); ++iter) { - YAML::Node rpgNode = iter->as(); - - auto name = rpgNode["name"].as(); - auto id = rpgNode["id"].as(); - - logger_->log_debug("parseRemoteProcessGroupYaml: name => [%s], id => [%s]", - name.c_str(), id.c_str()); - - std::string url = rpgNode["url"].as(); - logger_->log_debug("parseRemoteProcessGroupYaml: url => [%s]", - url.c_str()); - - std::string timeout = rpgNode["timeout"].as(); - logger_->log_debug( - "parseRemoteProcessGroupYaml: timeout => [%s]", - timeout.c_str()); - - std::string yieldPeriod = - rpgNode["yield period"].as(); - logger_->log_debug( - "parseRemoteProcessGroupYaml: yield period => [%s]", - yieldPeriod.c_str()); - - YAML::Node inputPorts = rpgNode["Input Ports"].as(); - YAML::Node outputPorts = - rpgNode["Output Ports"].as(); - core::ProcessGroup *group = NULL; - - uuid_parse(id.c_str(), uuid); - - int64_t timeoutValue = -1; - int64_t yieldPeriodValue = -1; - - group = this->createRemoteProcessGroup(name.c_str(), uuid); - group->setParent(parentGroup); - parentGroup->addProcessGroup(group); - - core::TimeUnit unit; - - if (core::Property::StringToTime(yieldPeriod, yieldPeriodValue, unit) - && core::Property::ConvertTimeUnitToMS(yieldPeriodValue, unit, - yieldPeriodValue) && group) { - logger_->log_debug( - "parseRemoteProcessGroupYaml: yieldPeriod => [%d] ms", - yieldPeriodValue); - group->setYieldPeriodMsec(yieldPeriodValue); - } - - if (core::Property::StringToTime(timeout, timeoutValue, unit) - && core::Property::ConvertTimeUnitToMS(timeoutValue, unit, - timeoutValue) && group) { - logger_->log_debug( - "parseRemoteProcessGroupYaml: timeoutValue => [%d] ms", - timeoutValue); - group->setTimeOut(timeoutValue); - } - - group->setTransmitting(true); - group->setURL(url); - - if (inputPorts && inputPorts.IsSequence()) { - for (YAML::const_iterator portIter = inputPorts.begin(); - portIter != inputPorts.end(); ++portIter) { - logger_->log_debug("Got a current port, iterating..."); - - YAML::Node currPort = portIter->as(); - - this->parsePortYaml(&currPort, group, SEND); - } // for node - } - if (outputPorts && outputPorts.IsSequence()) { - for (YAML::const_iterator portIter = outputPorts.begin(); - portIter != outputPorts.end(); ++portIter) { - logger_->log_debug("Got a current port, iterating..."); - - YAML::Node currPort = portIter->as(); - - this->parsePortYaml(&currPort, group, RECEIVE); - } // for node - } - - } - } - } -} - -void FlowControllerImpl::parseConnectionYaml(YAML::Node *connectionsNode, - core::ProcessGroup *parent) { - uuid_t uuid; - std::shared_ptr connection = nullptr; - - if (!parent) { - logger_->log_error("parseProcessNode: no parent group was provided"); - return; - } - - if (connectionsNode) { - - if (connectionsNode->IsSequence()) { - for (YAML::const_iterator iter = connectionsNode->begin(); - iter != connectionsNode->end(); ++iter) { - - YAML::Node connectionNode = iter->as(); - - std::string name = connectionNode["name"].as(); - std::string id = connectionNode["id"].as(); - std::string destId = connectionNode["destination id"].as< - std::string>(); - - uuid_parse(id.c_str(), uuid); - - logger_->log_debug( - "Created connection with UUID %s and name %s", id.c_str(), - name.c_str()); - connection = this->createConnection(name, uuid); - auto rawRelationship = - connectionNode["source relationship name"].as< - std::string>(); - core::Relationship relationship(rawRelationship, ""); - logger_->log_debug("parseConnection: relationship => [%s]", - rawRelationship.c_str()); - if (connection) - connection->setRelationship(relationship); - std::string connectionSrcProcId = - connectionNode["source id"].as(); - uuid_t srcUUID; - uuid_parse(connectionSrcProcId.c_str(), srcUUID); - - auto srcProcessor = this->root_->findProcessor( - srcUUID); - - if (!srcProcessor) { - logger_->log_error( - "Could not locate a source with id %s to create a connection", - connectionSrcProcId.c_str()); - throw std::invalid_argument( - "Could not locate a source with id %s to create a connection " - + connectionSrcProcId); - } - - uuid_t destUUID; - uuid_parse(destId.c_str(), destUUID); - auto destProcessor = this->root_->findProcessor(destUUID); - // If we could not find name, try by UUID - if (!destProcessor) { - uuid_t destUuid; - uuid_parse(destId.c_str(), destUuid); - destProcessor = this->root_->findProcessor(destUuid); - } - if (destProcessor) { - std::string destUuid = destProcessor->getUUIDStr(); - } - - uuid_t srcUuid; - uuid_t destUuid; - srcProcessor->getUUID(srcUuid); - connection->setSourceUUID(srcUuid); - destProcessor->getUUID(destUuid); - connection->setDestinationUUID(destUuid); - - if (connection) { - parent->addConnection(connection); - } - } - } - - if (connection) - parent->addConnection(connection); - - return; - } -} - -void FlowControllerImpl::parsePortYaml(YAML::Node *portNode, - core::ProcessGroup *parent, - TransferDirection direction) { - uuid_t uuid; - std::shared_ptr processor = NULL; - RemoteProcessorGroupPort *port = NULL; - - if (!parent) { - logger_->log_error("parseProcessNode: no parent group existed"); - return; - } - - YAML::Node inputPortsObj = portNode->as(); - - // generate the random UIID - uuid_generate(uuid); - - auto portId = inputPortsObj["id"].as(); - auto nameStr = inputPortsObj["name"].as(); - uuid_parse(portId.c_str(), uuid); - - port = new RemoteProcessorGroupPort(nameStr.c_str(), uuid); - - processor = (std::shared_ptr) port; - port->setDirection(direction); - port->setTimeOut(parent->getTimeOut()); - port->setTransmitting(true); - processor->setYieldPeriodMsec(parent->getYieldPeriodMsec()); - processor->initialize(); - - // handle port properties - YAML::Node nodeVal = portNode->as(); - YAML::Node propertiesNode = nodeVal["Properties"]; - - parsePropertiesNodeYaml(&propertiesNode, processor); - - // add processor to parent - parent->addProcessor(processor); - processor->setScheduledState(core::RUNNING); - auto rawMaxConcurrentTasks = inputPortsObj["max concurrent tasks"] - .as(); - int64_t maxConcurrentTasks; - if (core::Property::StringToInt(rawMaxConcurrentTasks, maxConcurrentTasks)) { - processor->setMaxConcurrentTasks(maxConcurrentTasks); - } - logger_->log_debug("parseProcessorNode: maxConcurrentTasks => [%d]", - maxConcurrentTasks); - processor->setMaxConcurrentTasks(maxConcurrentTasks); - -} - -void FlowControllerImpl::parsePropertiesNodeYaml( - YAML::Node *propertiesNode, std::shared_ptr processor) { - // Treat generically as a YAML node so we can perform inspection on entries to ensure they are populated - for (YAML::const_iterator propsIter = propertiesNode->begin(); - propsIter != propertiesNode->end(); ++propsIter) { - std::string propertyName = propsIter->first.as(); - YAML::Node propertyValueNode = propsIter->second; - if (!propertyValueNode.IsNull() && propertyValueNode.IsDefined()) { - std::string rawValueString = propertyValueNode.as(); - if (!processor->setProperty(propertyName, rawValueString)) { - logger_->log_warn( - "Received property %s with value %s but is not one of the properties for %s", - propertyName.c_str(), rawValueString.c_str(), - processor->getName().c_str()); - } - } - } -} - -void FlowControllerImpl::load() { +void FlowController::load() { + std::lock_guard flow_lock(mutex_); if (running_) { stop(true); } @@ -738,27 +226,29 @@ void FlowControllerImpl::load() { logger_->log_info("Load Flow Controller from file %s", configuration_filename_.c_str()); - YAML::Node flow = YAML::LoadFile(configuration_filename_); + this->root_ = flow_configuration_->getRoot(configuration_filename_); + /*YAML::Node flow = YAML::LoadFile(configuration_filename_); - YAML::Node flowControllerNode = flow["Flow Controller"]; - YAML::Node processorsNode = flow[CONFIG_YAML_PROCESSORS_KEY]; - YAML::Node connectionsNode = flow["Connections"]; - YAML::Node remoteProcessingGroupNode = flow["Remote Processing Groups"]; + YAML::Node flowControllerNode = flow["Flow Controller"]; + YAML::Node processorsNode = flow[CONFIG_YAML_PROCESSORS_KEY]; + YAML::Node connectionsNode = flow["Connections"]; + YAML::Node remoteProcessingGroupNode = flow["Remote Processing Groups"]; - // Create the root process group - parseRootProcessGroupYaml(flowControllerNode); - parseProcessorNodeYaml(processorsNode, this->root_); - parseRemoteProcessGroupYaml(&remoteProcessingGroupNode, this->root_); - parseConnectionYaml(&connectionsNode, this->root_); + // Create the root process group + parseRootProcessGroupYaml(flowControllerNode); + parseProcessorNodeYaml(processorsNode, this->root_); + parseRemoteProcessGroupYaml(&remoteProcessingGroupNode, this->root_); + parseConnectionYaml(&connectionsNode, this->root_);*/ // Load Flow File from Repo loadFlowRepo(); - initialized_=true; + initialized_ = true; } } -void FlowControllerImpl::reload(std::string yamlFile) { +void FlowController::reload(std::string yamlFile) { + std::lock_guard flow_lock(mutex_); logger_->log_info("Starting to reload Flow Controller with yaml %s", yamlFile.c_str()); stop(true); @@ -778,19 +268,18 @@ void FlowControllerImpl::reload(std::string yamlFile) { } } -void FlowControllerImpl::loadFlowRepo() -{ - if (this->flow_file_repo_) - { +void FlowController::loadFlowRepo() { + if (this->flow_file_repo_) { std::map> connectionMap; this->root_->getConnections(connectionMap); - auto rep = std::static_pointer_cast(flow_file_repo_); + auto rep = std::static_pointer_cast( + flow_file_repo_); rep->loadFlowFileToConnections(connectionMap); } } - -bool FlowControllerImpl::start() { +bool FlowController::start() { + std::lock_guard flow_lock(mutex_); if (!initialized_) { logger_->log_error( "Can not start Flow Controller because it has not been initialized"); diff --git a/libminifi/src/RemoteProcessorGroupPort.cpp b/libminifi/src/RemoteProcessorGroupPort.cpp index adbcad98fc..8e05ea6d03 100644 --- a/libminifi/src/RemoteProcessorGroupPort.cpp +++ b/libminifi/src/RemoteProcessorGroupPort.cpp @@ -29,9 +29,8 @@ #include "RemoteProcessorGroupPort.h" +#include "../include/io/StreamFactoryh" #include "io/ClientSocket.h" -#include "io/SocketFactory.h" - #include "utils/TimeUtil.h" #include "core/ProcessContext.h" #include "core/ProcessSession.h" @@ -110,7 +109,7 @@ void RemoteProcessorGroupPort::onTrigger(core::ProcessContext *context, } std::unique_ptr str = std::unique_ptr( - org::apache::nifi::minifi::io::SocketFactory::getInstance() + org::apache::nifi::minifi::io::StreamFactory::getInstance() ->createSocket(host, sport)); std::unique_ptr peer_ = std::unique_ptr( diff --git a/libminifi/src/core/FlowConfiguration.cpp b/libminifi/src/core/FlowConfiguration.cpp new file mode 100644 index 0000000000..c6472cc773 --- /dev/null +++ b/libminifi/src/core/FlowConfiguration.cpp @@ -0,0 +1,110 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "core/FlowConfiguration.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +FlowConfiguration::~FlowConfiguration() { + +} + +std::shared_ptr FlowConfiguration::createProcessor( + std::string name, uuid_t uuid) { + std::shared_ptr processor = nullptr; + if (name + == org::apache::nifi::minifi::processors::GenerateFlowFile::ProcessorName) { + processor = std::make_shared< + org::apache::nifi::minifi::processors::GenerateFlowFile>(name, uuid); + } else if (name + == org::apache::nifi::minifi::processors::LogAttribute::ProcessorName) { + processor = std::make_shared< + org::apache::nifi::minifi::processors::LogAttribute>(name, uuid); + } else if (name + == org::apache::nifi::minifi::processors::RealTimeDataCollector::ProcessorName) { + processor = std::make_shared< + org::apache::nifi::minifi::processors::RealTimeDataCollector>(name, + uuid); + } else if (name + == org::apache::nifi::minifi::processors::GetFile::ProcessorName) { + processor = + std::make_shared(name, + uuid); + } else if (name + == org::apache::nifi::minifi::processors::PutFile::ProcessorName) { + processor = + std::make_shared(name, + uuid); + } else if (name + == org::apache::nifi::minifi::processors::TailFile::ProcessorName) { + processor = + std::make_shared(name, + uuid); + } else if (name + == org::apache::nifi::minifi::processors::ListenSyslog::ProcessorName) { + processor = std::make_shared< + org::apache::nifi::minifi::processors::ListenSyslog>(name, uuid); + } else if (name + == org::apache::nifi::minifi::processors::ListenHTTP::ProcessorName) { + processor = std::make_shared< + org::apache::nifi::minifi::processors::ListenHTTP>(name, uuid); + } else if (name + == org::apache::nifi::minifi::processors::ExecuteProcess::ProcessorName) { + processor = std::make_shared< + org::apache::nifi::minifi::processors::ExecuteProcess>(name, uuid); + } else if (name + == org::apache::nifi::minifi::processors::AppendHostInfo::ProcessorName) { + processor = std::make_shared< + org::apache::nifi::minifi::processors::AppendHostInfo>(name, uuid); + } else { + logger_->log_error("No Processor defined for %s", name.c_str()); + return nullptr; + } + + // initialize the processor + processor->initialize(); + + return processor; +} + +std::unique_ptr FlowConfiguration::createRootProcessGroup( + std::string name, uuid_t uuid) { + return std::unique_ptr( + new core::ProcessGroup(core::ROOT_PROCESS_GROUP, name, uuid)); +} + +std::unique_ptr FlowConfiguration::createRemoteProcessGroup( + std::string name, uuid_t uuid) { + return std::unique_ptr( + new core::ProcessGroup(core::REMOTE_PROCESS_GROUP, name, uuid)); +} + +std::shared_ptr FlowConfiguration::createConnection( + std::string name, uuid_t uuid) { + return std::make_shared(flow_file_repo_, name, uuid); +} + +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/core/yaml/YamlConfiguration.cpp b/libminifi/src/core/yaml/YamlConfiguration.cpp new file mode 100644 index 0000000000..26265fa8ae --- /dev/null +++ b/libminifi/src/core/yaml/YamlConfiguration.cpp @@ -0,0 +1,479 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "core/yaml/YamlConfiguration.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +core::ProcessGroup *YamlConfiguration::parseRootProcessGroupYaml( + YAML::Node rootFlowNode) { + uuid_t uuid; + + std::string flowName = rootFlowNode["name"].as(); + std::string id = rootFlowNode["id"].as(); + + uuid_parse(id.c_str(), uuid); + + logger_->log_debug("parseRootProcessGroup: id => [%s]", id.c_str()); + logger_->log_debug("parseRootProcessGroup: name => [%s]", flowName.c_str()); + std::unique_ptr group = + FlowConfiguration::createRootProcessGroup(flowName, uuid); + + this->name_ = flowName; + + return group.release(); +} + +void YamlConfiguration::parseProcessorNodeYaml( + YAML::Node processorsNode, core::ProcessGroup * parentGroup) { + int64_t schedulingPeriod = -1; + int64_t penalizationPeriod = -1; + int64_t yieldPeriod = -1; + int64_t runDurationNanos = -1; + uuid_t uuid; + std::shared_ptr processor = nullptr; + + if (!parentGroup) { + logger_->log_error("parseProcessNodeYaml: no parent group exists"); + return; + } + + if (processorsNode) { + + if (processorsNode.IsSequence()) { + // Evaluate sequence of processors + int numProcessors = processorsNode.size(); + + for (YAML::const_iterator iter = processorsNode.begin(); + iter != processorsNode.end(); ++iter) { + core::ProcessorConfig procCfg; + YAML::Node procNode = iter->as(); + + procCfg.name = procNode["name"].as(); + procCfg.id = procNode["id"].as(); + logger_->log_debug("parseProcessorNode: name => [%s] id => [%s]", + procCfg.name.c_str(), procCfg.id.c_str()); + procCfg.javaClass = procNode["class"].as(); + logger_->log_debug("parseProcessorNode: class => [%s]", + procCfg.javaClass.c_str()); + + uuid_parse(procCfg.id.c_str(), uuid); + + // Determine the processor name only from the Java class + int lastOfIdx = procCfg.javaClass.find_last_of("."); + if (lastOfIdx != std::string::npos) { + lastOfIdx++; // if a value is found, increment to move beyond the . + int nameLength = procCfg.javaClass.length() - lastOfIdx; + std::string processorName = procCfg.javaClass.substr(lastOfIdx, + nameLength); + processor = this->createProcessor(processorName, uuid); + } + + if (!processor) { + logger_->log_error("Could not create a processor %s with name %s", + procCfg.name.c_str(), procCfg.id.c_str()); + throw std::invalid_argument( + "Could not create processor " + procCfg.name); + } + processor->setName(procCfg.name); + + procCfg.maxConcurrentTasks = procNode["max concurrent tasks"] + .as(); + logger_->log_debug("parseProcessorNode: max concurrent tasks => [%s]", + procCfg.maxConcurrentTasks.c_str()); + procCfg.schedulingStrategy = procNode["scheduling strategy"] + .as(); + logger_->log_debug("parseProcessorNode: scheduling strategy => [%s]", + procCfg.schedulingStrategy.c_str()); + procCfg.schedulingPeriod = + procNode["scheduling period"].as(); + logger_->log_debug("parseProcessorNode: scheduling period => [%s]", + procCfg.schedulingPeriod.c_str()); + procCfg.penalizationPeriod = procNode["penalization period"] + .as(); + logger_->log_debug("parseProcessorNode: penalization period => [%s]", + procCfg.penalizationPeriod.c_str()); + procCfg.yieldPeriod = procNode["yield period"].as(); + logger_->log_debug("parseProcessorNode: yield period => [%s]", + procCfg.yieldPeriod.c_str()); + procCfg.yieldPeriod = procNode["run duration nanos"].as(); + logger_->log_debug("parseProcessorNode: run duration nanos => [%s]", + procCfg.runDurationNanos.c_str()); + + // handle auto-terminated relationships + YAML::Node autoTerminatedSequence = + procNode["auto-terminated relationships list"]; + std::vector rawAutoTerminatedRelationshipValues; + if (autoTerminatedSequence.IsSequence() + && !autoTerminatedSequence.IsNull() + && autoTerminatedSequence.size() > 0) { + for (YAML::const_iterator relIter = autoTerminatedSequence.begin(); + relIter != autoTerminatedSequence.end(); ++relIter) { + std::string autoTerminatedRel = relIter->as(); + rawAutoTerminatedRelationshipValues.push_back(autoTerminatedRel); + } + } + procCfg.autoTerminatedRelationships = + rawAutoTerminatedRelationshipValues; + + // handle processor properties + YAML::Node propertiesNode = procNode["Properties"]; + parsePropertiesNodeYaml(&propertiesNode, processor); + + // Take care of scheduling + core::TimeUnit unit; + if (core::Property::StringToTime(procCfg.schedulingPeriod, + schedulingPeriod, unit) + && core::Property::ConvertTimeUnitToNS(schedulingPeriod, unit, + schedulingPeriod)) { + logger_->log_debug( + "convert: parseProcessorNode: schedulingPeriod => [%d] ns", + schedulingPeriod); + processor->setSchedulingPeriodNano(schedulingPeriod); + } + + if (core::Property::StringToTime(procCfg.penalizationPeriod, + penalizationPeriod, unit) + && core::Property::ConvertTimeUnitToMS(penalizationPeriod, unit, + penalizationPeriod)) { + logger_->log_debug( + "convert: parseProcessorNode: penalizationPeriod => [%d] ms", + penalizationPeriod); + processor->setPenalizationPeriodMsec(penalizationPeriod); + } + + if (core::Property::StringToTime(procCfg.yieldPeriod, yieldPeriod, unit) + && core::Property::ConvertTimeUnitToMS(yieldPeriod, unit, + yieldPeriod)) { + logger_->log_debug( + "convert: parseProcessorNode: yieldPeriod => [%d] ms", + yieldPeriod); + processor->setYieldPeriodMsec(yieldPeriod); + } + + // Default to running + processor->setScheduledState(core::RUNNING); + + if (procCfg.schedulingStrategy == "TIMER_DRIVEN") { + processor->setSchedulingStrategy(core::TIMER_DRIVEN); + logger_->log_debug("setting scheduling strategy as %s", + procCfg.schedulingStrategy.c_str()); + } else if (procCfg.schedulingStrategy == "EVENT_DRIVEN") { + processor->setSchedulingStrategy(core::EVENT_DRIVEN); + logger_->log_debug("setting scheduling strategy as %s", + procCfg.schedulingStrategy.c_str()); + } else { + processor->setSchedulingStrategy(core::CRON_DRIVEN); + logger_->log_debug("setting scheduling strategy as %s", + procCfg.schedulingStrategy.c_str()); + + } + + int64_t maxConcurrentTasks; + if (core::Property::StringToInt(procCfg.maxConcurrentTasks, + maxConcurrentTasks)) { + logger_->log_debug("parseProcessorNode: maxConcurrentTasks => [%d]", + maxConcurrentTasks); + processor->setMaxConcurrentTasks(maxConcurrentTasks); + } + + if (core::Property::StringToInt(procCfg.runDurationNanos, + runDurationNanos)) { + logger_->log_debug("parseProcessorNode: runDurationNanos => [%d]", + runDurationNanos); + processor->setRunDurationNano(runDurationNanos); + } + + std::set autoTerminatedRelationships; + for (auto &&relString : procCfg.autoTerminatedRelationships) { + core::Relationship relationship(relString, ""); + logger_->log_debug( + "parseProcessorNode: autoTerminatedRelationship => [%s]", + relString.c_str()); + autoTerminatedRelationships.insert(relationship); + } + + processor->setAutoTerminatedRelationships(autoTerminatedRelationships); + + parentGroup->addProcessor(processor); + } + } + } else { + throw new std::invalid_argument( + "Cannot instantiate a MiNiFi instance without a defined Processors configuration node."); + } +} + +void YamlConfiguration::parseRemoteProcessGroupYaml( + YAML::Node *rpgNode, core::ProcessGroup * parentGroup) { + uuid_t uuid; + + if (!parentGroup) { + logger_->log_error("parseRemoteProcessGroupYaml: no parent group exists"); + return; + } + + if (rpgNode) { + if (rpgNode->IsSequence()) { + for (YAML::const_iterator iter = rpgNode->begin(); iter != rpgNode->end(); + ++iter) { + YAML::Node rpgNode = iter->as(); + + auto name = rpgNode["name"].as(); + auto id = rpgNode["id"].as(); + + logger_->log_debug( + "parseRemoteProcessGroupYaml: name => [%s], id => [%s]", + name.c_str(), id.c_str()); + + std::string url = rpgNode["url"].as(); + logger_->log_debug("parseRemoteProcessGroupYaml: url => [%s]", + url.c_str()); + + std::string timeout = rpgNode["timeout"].as(); + logger_->log_debug("parseRemoteProcessGroupYaml: timeout => [%s]", + timeout.c_str()); + + std::string yieldPeriod = rpgNode["yield period"].as(); + logger_->log_debug("parseRemoteProcessGroupYaml: yield period => [%s]", + yieldPeriod.c_str()); + + YAML::Node inputPorts = rpgNode["Input Ports"].as(); + YAML::Node outputPorts = rpgNode["Output Ports"].as(); + core::ProcessGroup *group = NULL; + + uuid_parse(id.c_str(), uuid); + + int64_t timeoutValue = -1; + int64_t yieldPeriodValue = -1; + + group = this->createRemoteProcessGroup(name.c_str(), uuid).release(); + group->setParent(parentGroup); + parentGroup->addProcessGroup(group); + + core::TimeUnit unit; + + if (core::Property::StringToTime(yieldPeriod, yieldPeriodValue, unit) + && core::Property::ConvertTimeUnitToMS(yieldPeriodValue, unit, + yieldPeriodValue) && group) { + logger_->log_debug( + "parseRemoteProcessGroupYaml: yieldPeriod => [%d] ms", + yieldPeriodValue); + group->setYieldPeriodMsec(yieldPeriodValue); + } + + if (core::Property::StringToTime(timeout, timeoutValue, unit) + && core::Property::ConvertTimeUnitToMS(timeoutValue, unit, + timeoutValue) && group) { + logger_->log_debug( + "parseRemoteProcessGroupYaml: timeoutValue => [%d] ms", + timeoutValue); + group->setTimeOut(timeoutValue); + } + + group->setTransmitting(true); + group->setURL(url); + + if (inputPorts && inputPorts.IsSequence()) { + for (YAML::const_iterator portIter = inputPorts.begin(); + portIter != inputPorts.end(); ++portIter) { + logger_->log_debug("Got a current port, iterating..."); + + YAML::Node currPort = portIter->as(); + + this->parsePortYaml(&currPort, group, SEND); + } // for node + } + if (outputPorts && outputPorts.IsSequence()) { + for (YAML::const_iterator portIter = outputPorts.begin(); + portIter != outputPorts.end(); ++portIter) { + logger_->log_debug("Got a current port, iterating..."); + + YAML::Node currPort = portIter->as(); + + this->parsePortYaml(&currPort, group, RECEIVE); + } // for node + } + + } + } + } +} + +void YamlConfiguration::parseConnectionYaml(YAML::Node *connectionsNode, + core::ProcessGroup *parent) { + uuid_t uuid; + std::shared_ptr connection = nullptr; + + if (!parent) { + logger_->log_error("parseProcessNode: no parent group was provided"); + return; + } + + if (connectionsNode) { + + if (connectionsNode->IsSequence()) { + for (YAML::const_iterator iter = connectionsNode->begin(); + iter != connectionsNode->end(); ++iter) { + + YAML::Node connectionNode = iter->as(); + + std::string name = connectionNode["name"].as(); + std::string id = connectionNode["id"].as(); + std::string destId = connectionNode["destination id"].as(); + + uuid_parse(id.c_str(), uuid); + + logger_->log_debug("Created connection with UUID %s and name %s", + id.c_str(), name.c_str()); + connection = this->createConnection(name, uuid); + auto rawRelationship = connectionNode["source relationship name"] + .as(); + core::Relationship relationship(rawRelationship, ""); + logger_->log_debug("parseConnection: relationship => [%s]", + rawRelationship.c_str()); + if (connection) + connection->setRelationship(relationship); + std::string connectionSrcProcId = connectionNode["source id"] + .as(); + uuid_t srcUUID; + uuid_parse(connectionSrcProcId.c_str(), srcUUID); + + auto srcProcessor = parent->findProcessor(srcUUID); + + if (!srcProcessor) { + logger_->log_error( + "Could not locate a source with id %s to create a connection", + connectionSrcProcId.c_str()); + throw std::invalid_argument( + "Could not locate a source with id %s to create a connection " + + connectionSrcProcId); + } + + uuid_t destUUID; + uuid_parse(destId.c_str(), destUUID); + auto destProcessor = parent->findProcessor(destUUID); + // If we could not find name, try by UUID + if (!destProcessor) { + uuid_t destUuid; + uuid_parse(destId.c_str(), destUuid); + destProcessor = parent->findProcessor(destUuid); + } + if (destProcessor) { + std::string destUuid = destProcessor->getUUIDStr(); + } + + uuid_t srcUuid; + uuid_t destUuid; + srcProcessor->getUUID(srcUuid); + connection->setSourceUUID(srcUuid); + destProcessor->getUUID(destUuid); + connection->setDestinationUUID(destUuid); + + if (connection) { + parent->addConnection(connection); + } + } + } + + if (connection) + parent->addConnection(connection); + + return; + } +} + +void YamlConfiguration::parsePortYaml(YAML::Node *portNode, + core::ProcessGroup *parent, + TransferDirection direction) { + uuid_t uuid; + std::shared_ptr processor = NULL; + minifi::RemoteProcessorGroupPort *port = NULL; + + if (!parent) { + logger_->log_error("parseProcessNode: no parent group existed"); + return; + } + + YAML::Node inputPortsObj = portNode->as(); + + // generate the random UIID + uuid_generate(uuid); + + auto portId = inputPortsObj["id"].as(); + auto nameStr = inputPortsObj["name"].as(); + uuid_parse(portId.c_str(), uuid); + + port = new minifi::RemoteProcessorGroupPort(nameStr.c_str(), uuid); + + processor = (std::shared_ptr) port; + port->setDirection(direction); + port->setTimeOut(parent->getTimeOut()); + port->setTransmitting(true); + processor->setYieldPeriodMsec(parent->getYieldPeriodMsec()); + processor->initialize(); + + // handle port properties + YAML::Node nodeVal = portNode->as(); + YAML::Node propertiesNode = nodeVal["Properties"]; + + parsePropertiesNodeYaml(&propertiesNode, processor); + + // add processor to parent + parent->addProcessor(processor); + processor->setScheduledState(core::RUNNING); + auto rawMaxConcurrentTasks = inputPortsObj["max concurrent tasks"] + .as(); + int64_t maxConcurrentTasks; + if (core::Property::StringToInt(rawMaxConcurrentTasks, maxConcurrentTasks)) { + processor->setMaxConcurrentTasks(maxConcurrentTasks); + } + logger_->log_debug("parseProcessorNode: maxConcurrentTasks => [%d]", + maxConcurrentTasks); + processor->setMaxConcurrentTasks(maxConcurrentTasks); + +} + +void YamlConfiguration::parsePropertiesNodeYaml( + YAML::Node *propertiesNode, std::shared_ptr processor) { + // Treat generically as a YAML node so we can perform inspection on entries to ensure they are populated + for (YAML::const_iterator propsIter = propertiesNode->begin(); + propsIter != propertiesNode->end(); ++propsIter) { + std::string propertyName = propsIter->first.as(); + YAML::Node propertyValueNode = propsIter->second; + if (!propertyValueNode.IsNull() && propertyValueNode.IsDefined()) { + std::string rawValueString = propertyValueNode.as(); + if (!processor->setProperty(propertyName, rawValueString)) { + logger_->log_warn( + "Received property %s with value %s but is not one of the properties for %s", + propertyName.c_str(), rawValueString.c_str(), + processor->getName().c_str()); + } + } + } +} + +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/io/SocketFactory.cpp b/libminifi/src/io/StreamFactory.cpp similarity index 87% rename from libminifi/src/io/SocketFactory.cpp rename to libminifi/src/io/StreamFactory.cpp index 34833d833f..a8fa6702df 100644 --- a/libminifi/src/io/SocketFactory.cpp +++ b/libminifi/src/io/StreamFactory.cpp @@ -15,10 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "io/SocketFactory.h" - #include #include +#include "../../include/io/StreamFactoryh" namespace org { namespace apache { @@ -26,8 +25,8 @@ namespace nifi { namespace minifi { namespace io { -std::atomic SocketFactory::context_instance_; -std::mutex SocketFactory::context_mutex_; +std::atomic StreamFactory::context_instance_; +std::mutex StreamFactory::context_mutex_; } /* namespace io */ diff --git a/libminifi/src/io/TLSSocket.cpp b/libminifi/src/io/tls/TLSSocket.cpp similarity index 99% rename from libminifi/src/io/TLSSocket.cpp rename to libminifi/src/io/tls/TLSSocket.cpp index 1a28eeadac..b2df394d36 100644 --- a/libminifi/src/io/TLSSocket.cpp +++ b/libminifi/src/io/tls/TLSSocket.cpp @@ -16,7 +16,7 @@ * limitations under the License. */ #include "properties/Configure.h" -#include "io/TLSSocket.h" +#include "io/tls/TLSSocket.h" #include "utils/StringUtils.h" #include "core/Property.h" diff --git a/libminifi/test/unit/ProvenanceTestHelper.h b/libminifi/test/unit/ProvenanceTestHelper.h index 1de133f65c..cb8f520888 100644 --- a/libminifi/test/unit/ProvenanceTestHelper.h +++ b/libminifi/test/unit/ProvenanceTestHelper.h @@ -25,10 +25,10 @@ /** * Test repository */ -class TestRepository : public core::Repository{ +class TestRepository : public core::Repository { public: - TestRepository() : Repository("repo_name", "./dir", - 1000, 100, 0) { + TestRepository() + : Repository("repo_name", "./dir", 1000, 100, 0) { } // initialize bool initialize() { @@ -66,68 +66,72 @@ class TestRepository : public core::Repository{ return repositoryResults; } - void run(){ + void run() { // do nothing } protected: std::map repositoryResults; }; +class TestFlowController : public minifi::FlowController { + public: + TestFlowController(std::shared_ptr repo, + std::shared_ptr flow_file_repo) + : minifi::FlowController(repo, flow_file_repo, nullptr, "",true) { + } + ~TestFlowController() { -class TestFlowController : public minifi::FlowController -{ - -public: - TestFlowController(std::shared_ptr repo,std::shared_ptr flow_file_repo) : minifi::FlowController(repo,flow_file_repo) - { - } - ~TestFlowController() - { - - } - void load(){ - - } + } + void load() { - bool start() - { - running_.store(true); - return true; - } + } - void stop(bool force) - { - running_.store(false); - } - void waitUnload(const uint64_t timeToWaitMs) - { - stop(true); - } + bool start() { + running_.store(true); + return true; + } - void unload() - { - stop(true); - } + void stop(bool force) { + running_.store(false); + } + void waitUnload(const uint64_t timeToWaitMs) { + stop(true); + } - void reload(std::string file) - { + void unload() { + stop(true); + } - } + void reload(std::string file) { - bool isRunning() - { - return true; - } + } + bool isRunning() { + return true; + } - std::shared_ptr createProcessor(std::string name, uuid_t uuid){ return 0;} + std::shared_ptr createProcessor(std::string name, + uuid_t uuid) { + return 0; + } - core::ProcessGroup *createRootProcessGroup(std::string name, uuid_t uuid){ return 0;} + core::ProcessGroup *createRootProcessGroup(std::string name, uuid_t uuid) { + return 0; + } - core::ProcessGroup *createRemoteProcessGroup(std::string name, uuid_t uuid){ return 0; } + core::ProcessGroup *createRemoteProcessGroup(std::string name, uuid_t uuid) { + return 0; + } - std::shared_ptr createConnection(std::string name, uuid_t uuid){ return 0; } + std::shared_ptr createConnection(std::string name, + uuid_t uuid) { + return 0; + } + protected: + void initializePaths(const std::string &adjustedFilename) { + std::cout << "what" << std::endl; + } }; #endif /* LIBMINIFI_TEST_UNIT_PROVENANCETESTHELPER_H_ */ diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt index 126411bdc0..f7bd6e31b7 100644 --- a/main/CMakeLists.txt +++ b/main/CMakeLists.txt @@ -23,7 +23,7 @@ IF(POLICY CMP0048) CMAKE_POLICY(SET CMP0048 OLD) ENDIF(POLICY CMP0048) -include_directories(../include ../libminifi/include ../thirdparty/yaml-cpp-yaml-cpp-0.5.3/include ../thirdparty/civetweb-1.9.1/include ../thirdparty/leveldb-1.18/include ../thirdparty/) +include_directories(../include ../libminifi/include ../libminifi/include/core/yaml ../libminifi/include/core ../thirdparty/yaml-cpp-yaml-cpp-0.5.3/include ../thirdparty/civetweb-1.9.1/include ../thirdparty/leveldb-1.18/include ../thirdparty/) find_package(Boost REQUIRED) include_directories(${Boost_INCLUDE_DIRS}) diff --git a/main/MiNiFiMain.cpp b/main/MiNiFiMain.cpp index e320133a59..07481f8158 100644 --- a/main/MiNiFiMain.cpp +++ b/main/MiNiFiMain.cpp @@ -33,7 +33,8 @@ #include "core/logging/BaseLogger.h" #include "core/logging/LogAppenders.h" #include "spdlog/spdlog.h" - +#include "core/FlowConfiguration.h" +#include "core/yaml/YamlConfiguration.h" #include "core/repository/FlowFileRepository.h" #include "core/logging/Logger.h" #include "properties/Configure.h" @@ -146,16 +147,21 @@ int main(int argc, char **argv) { logger->updateLogger(std::move(configured_logger)); // Create repos for flow record and provenance - std::shared_ptr prov_repo = std::make_shared< - provenance::ProvenanceRepository>(); + std::shared_ptr prov_repo = + std::make_shared(); prov_repo->initialize(); - - std::shared_ptr flow_repo = std::make_shared< - core::repository::FlowFileRepository>(); + + std::shared_ptr flow_repo = + std::make_shared(); flow_repo->initialize(); + std::unique_ptr flow_configuration = std::unique_ptr< + core::FlowConfiguration>( + new core::YamlConfiguration(prov_repo, flow_repo)); + controller = std::unique_ptr( - new minifi::FlowControllerImpl(prov_repo,flow_repo)); + new minifi::FlowController(prov_repo, flow_repo, + std::move(flow_configuration))); // Load flow from specified configuration file controller->load(); From 566f0e7ac91641ee33c2c33ae00bde41bc903692 Mon Sep 17 00:00:00 2001 From: Marc Parisi Date: Fri, 24 Mar 2017 20:42:53 -0400 Subject: [PATCH 4/4] MINIFI-217: Rebase against master --- libminifi/include/FlowController.h | 2 +- libminifi/include/core/ConfigurationFactory.h | 65 +++++++ libminifi/include/core/FlowConfiguration.h | 17 +- libminifi/include/core/Repository.h | 20 ++- libminifi/include/core/RepositoryFactory.h | 44 +++++ libminifi/include/core/core.h | 30 ++++ .../core/repository/FlowFileRepository.h | 3 +- .../include/core/yaml/YamlConfiguration.h | 8 +- libminifi/include/io/Sockets.h | 5 +- .../io/{StreamFactoryh => StreamFactory.h} | 5 +- libminifi/include/io/tls/TLSSocket.h | 82 +-------- libminifi/include/properties/Configure.h | 167 +++++++++--------- libminifi/include/provenance/Provenance.h | 4 - .../include/provenance/ProvenanceRepository.h | 1 + libminifi/src/Configure.cpp | 3 + libminifi/src/FlowController.cpp | 21 +-- libminifi/src/RemoteProcessorGroupPort.cpp | 2 +- libminifi/src/core/ConfigurationFactory.cpp | 81 +++++++++ libminifi/src/core/Property.cpp | 5 +- libminifi/src/core/RepositoryFactory.cpp | 69 ++++++++ libminifi/src/core/yaml/YamlConfiguration.cpp | 17 +- libminifi/src/io/StreamFactory.cpp | 3 +- libminifi/test/nodefs/NoLevelDB.cpp | 34 ++++ libminifi/test/nodefs/NoYamlConfiguration.cpp | 38 ++++ main/MiNiFiMain.cpp | 32 ++-- 25 files changed, 537 insertions(+), 221 deletions(-) create mode 100644 libminifi/include/core/ConfigurationFactory.h create mode 100644 libminifi/include/core/RepositoryFactory.h rename libminifi/include/io/{StreamFactoryh => StreamFactory.h} (98%) create mode 100644 libminifi/src/core/ConfigurationFactory.cpp create mode 100644 libminifi/src/core/RepositoryFactory.cpp create mode 100644 libminifi/test/nodefs/NoLevelDB.cpp create mode 100644 libminifi/test/nodefs/NoYamlConfiguration.cpp diff --git a/libminifi/include/FlowController.h b/libminifi/include/FlowController.h index bec580ddf1..0475623296 100644 --- a/libminifi/include/FlowController.h +++ b/libminifi/include/FlowController.h @@ -122,7 +122,7 @@ class FlowController : public core::CoreComponent { // update property value void updatePropertyValue(std::string processorName, std::string propertyName, std::string propertyValue) { - if (root_) + if (root_ != nullptr) root_->updatePropertyValue(processorName, propertyName, propertyValue); } diff --git a/libminifi/include/core/ConfigurationFactory.h b/libminifi/include/core/ConfigurationFactory.h new file mode 100644 index 0000000000..19ed5f4abc --- /dev/null +++ b/libminifi/include/core/ConfigurationFactory.h @@ -0,0 +1,65 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBMINIFI_INCLUDE_CORE_CONFIGURATIONFACTORY_H_ +#define LIBMINIFI_INCLUDE_CORE_CONFIGURATIONFACTORY_H_ + +#include "FlowConfiguration.h" +#include + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + + + + +template +typename std::enable_if::value, T*>::type instantiate( + std::shared_ptr repo, + std::shared_ptr flow_file_repo,const std::string path ) { + throw std::runtime_error("Cannot instantiate class"); +} + +template +typename std::enable_if::value, T*>::type instantiate( + std::shared_ptr repo, + std::shared_ptr flow_file_repo,const std::string path ) { + return new T(repo,flow_file_repo,path); +} + + +/** + * Configuration factory is used to create a new FlowConfiguration + * object. + */ + std::unique_ptr createFlowConfiguration( + std::shared_ptr repo, + std::shared_ptr flow_file_repo, + const std::string configuration_class_name, const std::string path = "", + bool fail_safe = false); + +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif /* LIBMINIFI_INCLUDE_CORE_CONFIGURATIONFACTORY_H_ */ diff --git a/libminifi/include/core/FlowConfiguration.h b/libminifi/include/core/FlowConfiguration.h index fd1eaddc78..c7eedd21b7 100644 --- a/libminifi/include/core/FlowConfiguration.h +++ b/libminifi/include/core/FlowConfiguration.h @@ -50,6 +50,10 @@ namespace core { */ class FlowConfiguration : public CoreComponent { public: + /** + * Constructor that will be used for configuring + * the flow controller. + */ FlowConfiguration(std::shared_ptr repo, std::shared_ptr flow_file_repo, const std::string path) @@ -74,6 +78,10 @@ class FlowConfiguration : public CoreComponent { std::shared_ptr createConnection(std::string name, uuid_t uuid); + /** + * Returns the configuration path string + * @return config_path_ + */ const std::string &getConfigurationPath() { return config_path_; } @@ -82,8 +90,15 @@ class FlowConfiguration : public CoreComponent { return getRoot(config_path_); } + /** + * Base implementation that returns a null root pointer. + * @return Extensions should return a non-null pointer in order to + * properly configure flow controller. + */ virtual std::unique_ptr getRoot( - const std::string &from_config) =0; + const std::string &from_config) { + return nullptr; + } protected: // configuration path diff --git a/libminifi/include/core/Repository.h b/libminifi/include/core/Repository.h index 059f38b18c..a668df5fe1 100644 --- a/libminifi/include/core/Repository.h +++ b/libminifi/include/core/Repository.h @@ -72,16 +72,26 @@ class Repository : public CoreComponent { } // initialize - virtual bool initialize() = 0; + virtual bool initialize(){ + return true; + } // Put - virtual bool Put(std::string key, uint8_t *buf, int bufLen) = 0; + virtual bool Put(std::string key, uint8_t *buf, int bufLen){ + return true; + } // Delete - virtual bool Delete(std::string key) = 0; + virtual bool Delete(std::string key){ + return true; + } - virtual bool Get(std::string key, std::string &value) = 0; + virtual bool Get(std::string key, std::string &value) { + return true; + } // Run function for the thread - virtual void run() = 0; + virtual void run(){ + // no op + } // Start the repository monitor thread virtual void start(); // Stop the repository monitor thread diff --git a/libminifi/include/core/RepositoryFactory.h b/libminifi/include/core/RepositoryFactory.h new file mode 100644 index 0000000000..03ed5242cf --- /dev/null +++ b/libminifi/include/core/RepositoryFactory.h @@ -0,0 +1,44 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBMINIFI_INCLUDE_CORE_REPOSITORYFACTORY_H_ +#define LIBMINIFI_INCLUDE_CORE_REPOSITORYFACTORY_H_ + + +#include "core/Repository.h" +#include "core.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { + +namespace core { + + std::shared_ptr createRepository( + const std::string configuration_class_name, bool fail_safe = false); + + + +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif /* LIBMINIFI_INCLUDE_CORE_REPOSITORYFACTORY_H_ */ diff --git a/libminifi/include/core/core.h b/libminifi/include/core/core.h index 17f921e3db..9f86100582 100644 --- a/libminifi/include/core/core.h +++ b/libminifi/include/core/core.h @@ -45,6 +45,36 @@ static inline std::string getClassName() { return name; } +template +struct class_operations { + + template + static std::true_type canDestruct(decltype(std::declval().~Q()) *) { + return std::true_type(); + } + + + template + static std::false_type canDestruct(...) { + return std::false_type(); + } + + typedef decltype(canDestruct(0)) type; + + static const bool value = type::value; /* Which is it? */ +}; + + +template +typename std::enable_if::value, T*>::type instantiate() { + throw std::runtime_error("Cannot instantiate class"); +} + +template +typename std::enable_if::value, T*>::type instantiate() { + return new T(); +} + /** * Base component within MiNiFi * Purpose: Many objects store a name and UUID, therefore diff --git a/libminifi/include/core/repository/FlowFileRepository.h b/libminifi/include/core/repository/FlowFileRepository.h index 6980e6fd59..31e655a27d 100644 --- a/libminifi/include/core/repository/FlowFileRepository.h +++ b/libminifi/include/core/repository/FlowFileRepository.h @@ -65,14 +65,13 @@ class FlowFileRepository : public core::Repository, public std::enable_shared_fr } // Destructor - virtual ~FlowFileRepository() { + ~FlowFileRepository() { if (db_) delete db_; } // initialize virtual bool initialize() { - std::string value; if (configure_->get(Configure::nifi_flowfile_repository_directory_default, diff --git a/libminifi/include/core/yaml/YamlConfiguration.h b/libminifi/include/core/yaml/YamlConfiguration.h index 39d8d176b0..0ca9190a25 100644 --- a/libminifi/include/core/yaml/YamlConfiguration.h +++ b/libminifi/include/core/yaml/YamlConfiguration.h @@ -21,14 +21,15 @@ #include "core/ProcessorConfig.h" #include "yaml-cpp/yaml.h" #include "../FlowConfiguration.h" +#include "Site2SiteClientProtocol.h" #include +#include "io/validation.h" namespace org { namespace apache { namespace nifi { namespace minifi { namespace core { -#include "Site2SiteClientProtocol.h" #define DEFAULT_FLOW_YAML_FILE_NAME "conf/flow.yml" #define CONFIG_YAML_PROCESSORS_KEY "Processors" @@ -40,7 +41,9 @@ class YamlConfiguration : public FlowConfiguration { std::shared_ptr flow_file_repo, const std::string path = DEFAULT_FLOW_YAML_FILE_NAME) : FlowConfiguration(repo, flow_file_repo, path) { - + if (IsNullOrEmpty(config_path_)) { + config_path_ = DEFAULT_FLOW_YAML_FILE_NAME; + } } virtual ~YamlConfiguration() { @@ -87,7 +90,6 @@ class YamlConfiguration : public FlowConfiguration { std::shared_ptr processor); }; - } /* namespace core */ } /* namespace minifi */ } /* namespace nifi */ diff --git a/libminifi/include/io/Sockets.h b/libminifi/include/io/Sockets.h index fc548e7d26..2c0b163487 100644 --- a/libminifi/include/io/Sockets.h +++ b/libminifi/include/io/Sockets.h @@ -20,9 +20,8 @@ #include "ClientSocket.h" +#ifdef OPENSSL_SUPPORT #include "tls/TLSSocket.h" - - - +#endif #endif /* LIBMINIFI_INCLUDE_IO_TLS_SECURESOCKET_H_ */ diff --git a/libminifi/include/io/StreamFactoryh b/libminifi/include/io/StreamFactory.h similarity index 98% rename from libminifi/include/io/StreamFactoryh rename to libminifi/include/io/StreamFactory.h index cdaf83fd06..faa10b5605 100644 --- a/libminifi/include/io/StreamFactoryh +++ b/libminifi/include/io/StreamFactory.h @@ -1,5 +1,4 @@ /** - * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -23,6 +22,10 @@ #include "utils/StringUtils.h" #include "validation.h" +#ifdef OPENSSL_SUPPORT +#include "tls/TLSSocket.h" +#endif + namespace org { namespace apache { namespace nifi { diff --git a/libminifi/include/io/tls/TLSSocket.h b/libminifi/include/io/tls/TLSSocket.h index 9a55fbcade..f86f8bcd59 100644 --- a/libminifi/include/io/tls/TLSSocket.h +++ b/libminifi/include/io/tls/TLSSocket.h @@ -18,7 +18,6 @@ #ifndef LIBMINIFI_INCLUDE_IO_TLSSOCKET_H_ #define LIBMINIFI_INCLUDE_IO_TLSSOCKET_H_ -#ifdef OPENSSL_SUPPORT #include #include "../ClientSocket.h" #include @@ -43,84 +42,6 @@ namespace io { class TLSContext { -<<<<<<< HEAD -public: - - /** - * Build an instance, creating a memory fence, which - * allows us to avoid locking. This is tantamount to double checked locking. - * @returns new TLSContext; - */ - static TLSContext *getInstance() { - TLSContext* atomic_context = context_instance.load( - std::memory_order_relaxed); - std::atomic_thread_fence(std::memory_order_acquire); - if (atomic_context == nullptr) { - std::lock_guard lock(context_mutex); - atomic_context = context_instance.load(std::memory_order_relaxed); - if (atomic_context == nullptr) { - atomic_context = new TLSContext(); - atomic_context->initialize(); - std::atomic_thread_fence(std::memory_order_release); - context_instance.store(atomic_context, - std::memory_order_relaxed); - } - } - return atomic_context; - } - - virtual ~TLSContext() { - if (0 != ctx) - SSL_CTX_free(ctx); - } - - SSL_CTX *getContext() { - return ctx; - } - - short getError() { - return error_value; - } - - short initialize(); - -private: - - static int pemPassWordCb(char *buf, int size, int rwflag, void *userdata) { - std::string passphrase; - - if (Configure::getConfigure()->get( - Configure::nifi_security_client_pass_phrase, passphrase)) { - - std::ifstream file(passphrase.c_str(), std::ifstream::in); - if (!file.good()) { - memset(buf, 0x00, size); - return 0; - } - - std::string password; - password.assign((std::istreambuf_iterator(file)), - std::istreambuf_iterator()); - file.close(); - memset(buf,0x00,size); - memcpy(buf, password.c_str(), password.length()-1); - - return password.length()-1; - } - return 0; - } - - TLSContext(); - - std::shared_ptr logger_; - Configure *configuration; - SSL_CTX *ctx; - - short error_value; - static std::atomic context_instance; - - static std::mutex context_mutex; -======= public: /** @@ -196,7 +117,7 @@ class TLSContext { static std::atomic context_instance; static std::mutex context_mutex; ->>>>>>> MINIFI-217: First commit. Updates namespaces and removes + }; class TLSSocket : public Socket { @@ -267,7 +188,6 @@ class TLSSocket : public Socket { SSL* ssl; }; -#endif } /* namespace io */ } /* namespace minifi */ diff --git a/libminifi/include/properties/Configure.h b/libminifi/include/properties/Configure.h index 8f598539f8..c0d9bd4e99 100644 --- a/libminifi/include/properties/Configure.h +++ b/libminifi/include/properties/Configure.h @@ -30,105 +30,98 @@ #include "core/core.h" #include "core/logging/Logger.h" - namespace org { namespace apache { namespace nifi { namespace minifi { class Configure { -public: - // Get the singleton logger instance - static Configure * getConfigure() - { - if (!configure_) - { - configure_ = new Configure(); - } - return configure_; - } - // nifi.flow.configuration.file - static const char *nifi_flow_configuration_file; - static const char *nifi_administrative_yield_duration; - static const char *nifi_bored_yield_duration; - static const char *nifi_graceful_shutdown_seconds; - static const char *nifi_log_level; - static const char *nifi_server_name; - static const char *nifi_server_port; - static const char *nifi_server_report_interval; - static const char *nifi_provenance_repository_max_storage_time; - static const char *nifi_provenance_repository_max_storage_size; - static const char *nifi_provenance_repository_directory_default; - static const char *nifi_provenance_repository_enable; - static const char *nifi_flowfile_repository_max_storage_time; - static const char *nifi_flowfile_repository_max_storage_size; - static const char *nifi_flowfile_repository_directory_default; - static const char *nifi_flowfile_repository_enable; - static const char *nifi_remote_input_secure; - static const char *nifi_security_need_ClientAuth; - static const char *nifi_security_client_certificate; - static const char *nifi_security_client_private_key; - static const char *nifi_security_client_pass_phrase; - static const char *nifi_security_client_ca_certificate; - - // Clear the load config - void clear() - { - std::lock_guard lock(mutex_); - properties_.clear(); - } - // Set the config value - void set(std::string key, std::string value) - { - std::lock_guard lock(mutex_); - properties_[key] = value; - } - // Check whether the config value existed - bool has(std::string key) - { - std::lock_guard lock(mutex_); - return (properties_.find(key) != properties_.end()); - } - // Get the config value - bool get(std::string key, std::string &value); - // Parse one line in configure file like key=value - void parseConfigureFileLine(char *buf); - // Load Configure File - void loadConfigureFile(const char *fileName); - // Set the determined MINIFI_HOME - void setHome(std::string minifiHome) - { - minifi_home_ = minifiHome; + public: + // Get the singleton logger instance + static Configure * getConfigure() { + if (!configure_) { + configure_ = new Configure(); } + return configure_; + } + // nifi.flow.configuration.file + static const char *nifi_flow_configuration_file; + static const char *nifi_administrative_yield_duration; + static const char *nifi_bored_yield_duration; + static const char *nifi_graceful_shutdown_seconds; + static const char *nifi_log_level; + static const char *nifi_server_name; + static const char *nifi_configuration_class_name; + static const char *nifi_flow_repository_class_name; + static const char *nifi_provenance_repository_class_name; + static const char *nifi_server_port; + static const char *nifi_server_report_interval; + static const char *nifi_provenance_repository_max_storage_time; + static const char *nifi_provenance_repository_max_storage_size; + static const char *nifi_provenance_repository_directory_default; + static const char *nifi_provenance_repository_enable; + static const char *nifi_flowfile_repository_max_storage_time; + static const char *nifi_flowfile_repository_max_storage_size; + static const char *nifi_flowfile_repository_directory_default; + static const char *nifi_flowfile_repository_enable; + static const char *nifi_remote_input_secure; + static const char *nifi_security_need_ClientAuth; + static const char *nifi_security_client_certificate; + static const char *nifi_security_client_private_key; + static const char *nifi_security_client_pass_phrase; + static const char *nifi_security_client_ca_certificate; - // Get the determined MINIFI_HOME - std::string getHome() - { - return minifi_home_; - } - // Parse Command Line - void parseCommandLine(int argc, char **argv); + // Clear the load config + void clear() { + std::lock_guard lock(mutex_); + properties_.clear(); + } + // Set the config value + void set(std::string key, std::string value) { + std::lock_guard lock(mutex_); + properties_[key] = value; + } + // Check whether the config value existed + bool has(std::string key) { + std::lock_guard lock(mutex_); + return (properties_.find(key) != properties_.end()); + } + // Get the config value + bool get(std::string key, std::string &value); + // Parse one line in configure file like key=value + void parseConfigureFileLine(char *buf); + // Load Configure File + void loadConfigureFile(const char *fileName); + // Set the determined MINIFI_HOME + void setHome(std::string minifiHome) { + minifi_home_ = minifiHome; + } + + // Get the determined MINIFI_HOME + std::string getHome() { + return minifi_home_; + } + // Parse Command Line + void parseCommandLine(int argc, char **argv); -private: - // Mutex for protection - std::mutex mutex_; - // Logger - std::shared_ptr logger_; - // Home location for this executable - std::string minifi_home_; + private: + // Mutex for protection + std::mutex mutex_; + // Logger + std::shared_ptr logger_; + // Home location for this executable + std::string minifi_home_; - Configure() - { - logger_ = logging::Logger::getLogger(); - } - virtual ~Configure() - { + Configure() { + logger_ = logging::Logger::getLogger(); + } + virtual ~Configure() { - } - static Configure *configure_; + } + static Configure *configure_; -protected: - std::map properties_; + protected: + std::map properties_; }; } /* namespace minifi */ diff --git a/libminifi/include/provenance/Provenance.h b/libminifi/include/provenance/Provenance.h index 9e7bbff7e8..3d5d19ea24 100644 --- a/libminifi/include/provenance/Provenance.h +++ b/libminifi/include/provenance/Provenance.h @@ -30,10 +30,6 @@ #include #include -#include "leveldb/db.h" -#include "leveldb/options.h" -#include "leveldb/slice.h" -#include "leveldb/status.h" #include "core/Repository.h" #include "core/Property.h" diff --git a/libminifi/include/provenance/ProvenanceRepository.h b/libminifi/include/provenance/ProvenanceRepository.h index fa3c3c9dff..0f8ee5d3fb 100644 --- a/libminifi/include/provenance/ProvenanceRepository.h +++ b/libminifi/include/provenance/ProvenanceRepository.h @@ -23,6 +23,7 @@ #include "leveldb/status.h" #include "core/Repository.h" #include "core/core.h" +#include "provenance/Provenance.h" namespace org { namespace apache { namespace nifi { diff --git a/libminifi/src/Configure.cpp b/libminifi/src/Configure.cpp index e14a9c1a61..96ed7c79dc 100644 --- a/libminifi/src/Configure.cpp +++ b/libminifi/src/Configure.cpp @@ -31,6 +31,9 @@ const char *Configure::nifi_bored_yield_duration = "nifi.bored.yield.duration"; const char *Configure::nifi_graceful_shutdown_seconds = "nifi.flowcontroller.graceful.shutdown.period"; const char *Configure::nifi_log_level = "nifi.log.level"; const char *Configure::nifi_server_name = "nifi.server.name"; +const char *Configure::nifi_configuration_class_name = "nifi.flow.configuration.class.name"; +const char *Configure::nifi_flow_repository_class_name = "nifi.flow.repository.class.name"; +const char *Configure::nifi_provenance_repository_class_name = "nifi.provenance.repository.class.name"; const char *Configure::nifi_server_port = "nifi.server.port"; const char *Configure::nifi_server_report_interval= "nifi.server.report.interval"; const char *Configure::nifi_provenance_repository_max_storage_size = "nifi.provenance.repository.max.storage.size"; diff --git a/libminifi/src/FlowController.cpp b/libminifi/src/FlowController.cpp index ff78d3c587..e472a9ad69 100644 --- a/libminifi/src/FlowController.cpp +++ b/libminifi/src/FlowController.cpp @@ -227,18 +227,6 @@ void FlowController::load() { configuration_filename_.c_str()); this->root_ = flow_configuration_->getRoot(configuration_filename_); - /*YAML::Node flow = YAML::LoadFile(configuration_filename_); - - YAML::Node flowControllerNode = flow["Flow Controller"]; - YAML::Node processorsNode = flow[CONFIG_YAML_PROCESSORS_KEY]; - YAML::Node connectionsNode = flow["Connections"]; - YAML::Node remoteProcessingGroupNode = flow["Remote Processing Groups"]; - - // Create the root process group - parseRootProcessGroupYaml(flowControllerNode); - parseProcessorNodeYaml(processorsNode, this->root_); - parseRemoteProcessGroupYaml(&remoteProcessingGroupNode, this->root_); - parseConnectionYaml(&connectionsNode, this->root_);*/ // Load Flow File from Repo loadFlowRepo(); @@ -257,7 +245,7 @@ void FlowController::reload(std::string yamlFile) { this->configuration_filename_ = yamlFile; load(); start(); - if (!this->root_) { + if (this->root_ != nullptr) { this->configuration_filename_ = oldYamlFile; logger_->log_info("Rollback Flow Controller to YAML %s", oldYamlFile.c_str()); @@ -271,7 +259,9 @@ void FlowController::reload(std::string yamlFile) { void FlowController::loadFlowRepo() { if (this->flow_file_repo_) { std::map> connectionMap; - this->root_->getConnections(connectionMap); + if (this->root_ != nullptr) { + this->root_->getConnections(connectionMap); + } auto rep = std::static_pointer_cast( flow_file_repo_); rep->loadFlowFileToConnections(connectionMap); @@ -290,9 +280,10 @@ bool FlowController::start() { logger_->log_info("Starting Flow Controller"); this->_timerScheduler.start(); this->_eventScheduler.start(); - if (this->root_) + if (this->root_ != nullptr) { this->root_->startProcessing(&this->_timerScheduler, &this->_eventScheduler); + } running_ = true; this->protocol_->start(); this->provenance_repo_->start(); diff --git a/libminifi/src/RemoteProcessorGroupPort.cpp b/libminifi/src/RemoteProcessorGroupPort.cpp index 8e05ea6d03..9790256463 100644 --- a/libminifi/src/RemoteProcessorGroupPort.cpp +++ b/libminifi/src/RemoteProcessorGroupPort.cpp @@ -29,7 +29,7 @@ #include "RemoteProcessorGroupPort.h" -#include "../include/io/StreamFactoryh" +#include "../include/io/StreamFactory.h" #include "io/ClientSocket.h" #include "utils/TimeUtil.h" #include "core/ProcessContext.h" diff --git a/libminifi/src/core/ConfigurationFactory.cpp b/libminifi/src/core/ConfigurationFactory.cpp new file mode 100644 index 0000000000..52bde692f3 --- /dev/null +++ b/libminifi/src/core/ConfigurationFactory.cpp @@ -0,0 +1,81 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "core/ConfigurationFactory.h" +#include "core/FlowConfiguration.h" +#include +#ifdef YAML_SUPPORT +#include "core/yaml/YamlConfiguration.h" +#endif + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { +#ifndef YAML_SUPPORT + class YamlConfiguration; +#endif + + std::unique_ptr createFlowConfiguration( + std::shared_ptr repo, + std::shared_ptr flow_file_repo, + const std::string configuration_class_name, const std::string path, + bool fail_safe) { + + std::string class_name_lc = configuration_class_name; + std::transform(class_name_lc.begin(), class_name_lc.end(), + class_name_lc.begin(), ::tolower); + try { + + if (class_name_lc == "flowconfiguration") { + // load the base configuration. + return std::unique_ptr( + new core::FlowConfiguration(repo, flow_file_repo, path)); + + } else if (class_name_lc == "yamlconfiguration") { + // only load if the class is defined. + return std::unique_ptr(instantiate(repo, flow_file_repo, path)); + + + } else { + if (fail_safe) { + return std::unique_ptr( + new core::FlowConfiguration(repo, flow_file_repo, path)); + } else { + throw std::runtime_error( + "Support for the provided configuration class could not be found"); + } + } + } catch (const std::runtime_error &r) { + if (fail_safe) { + return std::unique_ptr( + new core::FlowConfiguration(repo, flow_file_repo, path)); + } + } + + throw std::runtime_error( + "Support for the provided configuration class could not be found"); + } + +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + diff --git a/libminifi/src/core/Property.cpp b/libminifi/src/core/Property.cpp index 13028a4b89..287b7ecb64 100644 --- a/libminifi/src/core/Property.cpp +++ b/libminifi/src/core/Property.cpp @@ -20,8 +20,7 @@ namespace org { namespace apache { -namespace nifi { -namespace minifi { +namespace nifi {namespace minifi { namespace core { // Get Name for the property @@ -51,7 +50,7 @@ const Property &Property::operator=(const Property &other) { return *this; } -} /* namespace components */ +} /* namespace core */ } /* namespace minifi */ } /* namespace nifi */ } /* namespace apache */ diff --git a/libminifi/src/core/RepositoryFactory.cpp b/libminifi/src/core/RepositoryFactory.cpp new file mode 100644 index 0000000000..ef0b9efc23 --- /dev/null +++ b/libminifi/src/core/RepositoryFactory.cpp @@ -0,0 +1,69 @@ +#include "core/Repository.h" +#include "core/Repository.h" + +#ifdef LEVELDB_SUPPORT +#include "core/repository/FlowFileRepository.h" +#include "provenance/ProvenanceRepository.h" +#endif + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +#ifndef LEVELDB_SUPPORT + namespace provenance{ + class ProvenanceRepository; + } +#endif +namespace core { + +#ifndef LEVELDB_SUPPORT + class FlowFileRepository; +#endif + + + std::shared_ptr createRepository( + const std::string configuration_class_name, bool fail_safe = false) { + + std::string class_name_lc = configuration_class_name; + std::transform(class_name_lc.begin(), class_name_lc.end(), + class_name_lc.begin(), ::tolower); + try { + std::shared_ptr return_obj = nullptr; + if (class_name_lc == "flowfilerepository") { + + return_obj = std::shared_ptr((core::Repository*)instantiate()); + } else if (class_name_lc == "provenancerepository") { + + + return_obj = std::shared_ptr((core::Repository*)instantiate()); + + } + + if (return_obj){ + return return_obj; + } + if (fail_safe) { + return std::make_shared("fail_safe", "fail_safe", 1, + 1, 1); + } else { + throw std::runtime_error( + "Support for the provided configuration class could not be found"); + } + } catch (const std::runtime_error &r) { + if (fail_safe) { + return std::make_shared("fail_safe", "fail_safe", 1, + 1, 1); + } + } + + throw std::runtime_error( + "Support for the provided configuration class could not be found"); + } + + +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/core/yaml/YamlConfiguration.cpp b/libminifi/src/core/yaml/YamlConfiguration.cpp index 26265fa8ae..8e593631b6 100644 --- a/libminifi/src/core/yaml/YamlConfiguration.cpp +++ b/libminifi/src/core/yaml/YamlConfiguration.cpp @@ -29,9 +29,20 @@ core::ProcessGroup *YamlConfiguration::parseRootProcessGroupYaml( uuid_t uuid; std::string flowName = rootFlowNode["name"].as(); - std::string id = rootFlowNode["id"].as(); - - uuid_parse(id.c_str(), uuid); + std::string id ; + + try { + rootFlowNode["id"].as(); + + uuid_parse(id.c_str(), uuid); + }catch(...) + { + logger_->log_warn("Generating random ID for root node"); + uuid_generate(uuid); + char uuid_str[37]; + uuid_unparse(uuid,uuid_str); + id = uuid_str; + } logger_->log_debug("parseRootProcessGroup: id => [%s]", id.c_str()); logger_->log_debug("parseRootProcessGroup: name => [%s]", flowName.c_str()); diff --git a/libminifi/src/io/StreamFactory.cpp b/libminifi/src/io/StreamFactory.cpp index a8fa6702df..e3aa2900ea 100644 --- a/libminifi/src/io/StreamFactory.cpp +++ b/libminifi/src/io/StreamFactory.cpp @@ -15,9 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include "../../include/io/StreamFactory.h" + #include #include -#include "../../include/io/StreamFactoryh" namespace org { namespace apache { diff --git a/libminifi/test/nodefs/NoLevelDB.cpp b/libminifi/test/nodefs/NoLevelDB.cpp new file mode 100644 index 0000000000..00c9212fa8 --- /dev/null +++ b/libminifi/test/nodefs/NoLevelDB.cpp @@ -0,0 +1,34 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../TestBase.h" + +#include "core/core.h" +#include "core/RepositoryFactory.h" + +TEST_CASE("NoLevelDBTest1", "[NoLevelDBTest]") { + std::shared_ptr prov_repo = core::createRepository( + "provenancerepository", true); + REQUIRE(nullptr != prov_repo); +} + +TEST_CASE("NoLevelDBTest2", "[NoLevelDBTest]") { + std::shared_ptr prov_repo = core::createRepository( + "flowfilerepository", true); + REQUIRE(nullptr != prov_repo); +} diff --git a/libminifi/test/nodefs/NoYamlConfiguration.cpp b/libminifi/test/nodefs/NoYamlConfiguration.cpp new file mode 100644 index 0000000000..9a9b10ee68 --- /dev/null +++ b/libminifi/test/nodefs/NoYamlConfiguration.cpp @@ -0,0 +1,38 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include "core/core.h" +#include "core/RepositoryFactory.h" + + +#include "core/ConfigurationFactory.h" + +TEST_CASE("NoYamlSupport1", "[NoYamlSupport1]") { + std::shared_ptr prov_repo = core::createRepository( + "provenancerepository", true); +REQUIRE(nullptr != prov_repo); +std::unique_ptr flow_configuration = std::move( + core::createFlowConfiguration(prov_repo, prov_repo, + "yamlconfiguration")); + + + REQUIRE(nullptr != flow_configuration); + +} + diff --git a/main/MiNiFiMain.cpp b/main/MiNiFiMain.cpp index 07481f8158..6bfd9c9720 100644 --- a/main/MiNiFiMain.cpp +++ b/main/MiNiFiMain.cpp @@ -34,8 +34,8 @@ #include "core/logging/LogAppenders.h" #include "spdlog/spdlog.h" #include "core/FlowConfiguration.h" -#include "core/yaml/YamlConfiguration.h" -#include "core/repository/FlowFileRepository.h" +#include "core/ConfigurationFactory.h" +#include "core/RepositoryFactory.h" #include "core/logging/Logger.h" #include "properties/Configure.h" #include "FlowController.h" @@ -87,6 +87,9 @@ int main(int argc, char **argv) { uint16_t stop_wait_time = STOP_WAIT_TIME_MS; std::string graceful_shutdown_seconds = ""; + std::string prov_repo_class = "provenancerepository"; + std::string flow_repo_class = "flowfilerepository"; + std::string nifi_configuration_class_name = "yamlconfiguration"; running = sem_open("MiNiFiMain", O_CREAT, 0644, 0); if (running == SEM_FAILED || running == 0) { @@ -146,22 +149,31 @@ int main(int argc, char **argv) { logger->updateLogger(std::move(configured_logger)); + configure->get(minifi::Configure::nifi_provenance_repository_class_name, + prov_repo_class); // Create repos for flow record and provenance - std::shared_ptr prov_repo = - std::make_shared(); + std::shared_ptr prov_repo = core::createRepository( + prov_repo_class, true); prov_repo->initialize(); - std::shared_ptr flow_repo = - std::make_shared(); + configure->get(minifi::Configure::nifi_flow_repository_class_name, + flow_repo_class); + + std::shared_ptr flow_repo = core::createRepository( + flow_repo_class, true); + flow_repo->initialize(); - std::unique_ptr flow_configuration = std::unique_ptr< - core::FlowConfiguration>( - new core::YamlConfiguration(prov_repo, flow_repo)); + configure->get(minifi::Configure::nifi_configuration_class_name, + nifi_configuration_class_name); + + std::unique_ptr flow_configuration = std::move( + core::createFlowConfiguration(prov_repo, flow_repo, + nifi_configuration_class_name)); controller = std::unique_ptr( new minifi::FlowController(prov_repo, flow_repo, - std::move(flow_configuration))); + std::move(flow_configuration))); // Load flow from specified configuration file controller->load();