From dfc3a76c4557153e7f104a9b4e24e6d19e412f0e Mon Sep 17 00:00:00 2001 From: Bin Qiu Date: Mon, 15 May 2017 08:59:58 -0700 Subject: [PATCH 01/11] Configuration Listener --- .../include/core/HttpConfigurationListener.h | 125 ++++++++++++++ .../src/core/HttpConfigurationListener.cpp | 159 ++++++++++++++++++ 2 files changed, 284 insertions(+) create mode 100644 libminifi/include/core/HttpConfigurationListener.h create mode 100644 libminifi/src/core/HttpConfigurationListener.cpp diff --git a/libminifi/include/core/HttpConfigurationListener.h b/libminifi/include/core/HttpConfigurationListener.h new file mode 100644 index 0000000000..f8c094bf93 --- /dev/null +++ b/libminifi/include/core/HttpConfigurationListener.h @@ -0,0 +1,125 @@ +/** + * HttpConfigurationListener class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __HTTP_CONFIGURATION_LISTENER__ +#define __HTTP_CONFIGURATION_LISTENER__ + +#include + +#include +#include "FlowController.h" +#include "core/Processor.h" +#include "core/Core.h" +#include "core/Property.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +/** + * HTTP Response object + */ +struct HTTPRequestResponse { + std::vector data; + + /** + * Receive HTTP Response. + */ + static size_t recieve_write(char * data, size_t size, size_t nmemb, + void * p) { + return static_cast(p)->write_content(data, size, + nmemb); + } + + size_t write_content(char* ptr, size_t size, size_t nmemb) { + data.insert(data.end(), ptr, ptr + size * nmemb); + return size * nmemb; + } + +}; + +// HttpConfigurationListener Class +class HttpConfigurationListener : public core::Processor { + public: + + // Constructor + /*! + * Create a new processor + */ + HttpConfigurationListener(std::string name, uuid_t uuid = NULL) + : Processor(name, uuid), + connect_timeout_(20000), + read_timeout_(20000), + use_etag_(false) { + curl_global_init(CURL_GLOBAL_DEFAULT); + } + // Destructor + virtual ~HttpConfigurationListener(); + // Processor Name + static const char *ProcessorName; + + static const char* STATUS_CODE; + static const char* STATUS_MESSAGE; + static const char* RESPONSE_BODY; + static const char* REQUEST_URL; + static const char* TRANSACTION_ID; + static const char* REMOTE_DN; + static const char* EXCEPTION_CLASS; + static const char* EXCEPTION_MESSAGE; + + void onTrigger(core::ProcessContext *context, core::ProcessSession *session); + void initialize(); + void onSchedule(core::ProcessContext *context, + core::ProcessSessionFactory *sessionFactory); + + protected: + + /** + * Generate a transaction ID + * @return transaction ID string. + */ + std::string generateId(); + /** + * Set the request method on the curl struct. + * @param curl pointer to this instance. + * @param string request method + */ + void set_request_method(CURL *curl, const std::string &); + + CURLcode res; + // http method + std::string method_; + // url + std::string url_; + // connection timeout + int64_t connect_timeout_; + // read timeout. + int64_t read_timeout_; + bool use_etag_; + std::string last_etag_; +}; + +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif diff --git a/libminifi/src/core/HttpConfigurationListener.cpp b/libminifi/src/core/HttpConfigurationListener.cpp new file mode 100644 index 0000000000..964f035354 --- /dev/null +++ b/libminifi/src/core/HttpConfigurationListener.cpp @@ -0,0 +1,159 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "core/HttpConfigurationListener.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/logging/Logger.h" +#include "core/ProcessContext.h" +#include "core/Relationship.h" +#include "io/DataStream.h" +#include "io/StreamFactory.h" +#include "utils/StringUtils.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace core { + +const char *HttpConfigurationListener::ProcessorName = "HttpConfigurationListener"; + +const char* HttpConfigurationListener::STATUS_CODE = "invokehttp.status.code"; +const char* HttpConfigurationListener::STATUS_MESSAGE = "invokehttp.status.message"; +const char* HttpConfigurationListener::RESPONSE_BODY = "invokehttp.response.body"; +const char* HttpConfigurationListener::REQUEST_URL = "invokehttp.request.url"; +const char* HttpConfigurationListener::TRANSACTION_ID = "invokehttp.tx.id"; +const char* HttpConfigurationListener::REMOTE_DN = "invokehttp.remote.dn"; +const char* HttpConfigurationListener::EXCEPTION_CLASS = "invokehttp.java.exception.class"; +const char* HttpConfigurationListener::EXCEPTION_MESSAGE = "invokehttp.java.exception.message"; + +void HttpConfigurationListener::set_request_method(CURL *curl, const std::string &method) { + std::string my_method = method; + std::transform(my_method.begin(), my_method.end(), my_method.begin(), ::toupper); + if (my_method == "POST") { + curl_easy_setopt(curl, CURLOPT_POST, 1); + } else if (my_method == "PUT") { + curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); + } else if (my_method == "GET") { + } else { + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, my_method.c_str()); + } +} + +void HttpConfigurationListener::initialize() { + logger_->log_info("Initializing HttpConfigurationListener"); + +} + +void HttpConfigurationListener::onSchedule(core::ProcessContext *context, + core::ProcessSessionFactory *sessionFactory) { + +} + +HttpConfigurationListener::~HttpConfigurationListener() { + curl_global_cleanup(); +} + +std::string HttpConfigurationListener::generateId() { + uuid_t txId; + uuid_generate(txId); + char uuidStr[37]; + uuid_unparse_lower(txId, uuidStr); + return uuidStr; +} + +void HttpConfigurationListener::onTrigger(core::ProcessContext *context, + core::ProcessSession *session) { + std::shared_ptr flowFile = std::static_pointer_cast< + FlowFileRecord>(session->get()); + + logger_->log_info("onTrigger HttpConfigurationListener with %s", method_.c_str()); + + // create a transaction id + std::string tx_id = generateId(); + + CURL *http_session = curl_easy_init(); + // set the HTTP request method from libCURL + set_request_method(http_session, method_); + curl_easy_setopt(http_session, CURLOPT_URL, url_.c_str()); + + if (connect_timeout_ > 0) { + curl_easy_setopt(http_session, CURLOPT_TIMEOUT, connect_timeout_); + } + + if (read_timeout_ > 0) { + curl_easy_setopt(http_session, CURLOPT_TIMEOUT, read_timeout_); + } + HTTPRequestResponse content; + curl_easy_setopt(http_session, CURLOPT_WRITEFUNCTION, + &HTTPRequestResponse::recieve_write); + + curl_easy_setopt(http_session, CURLOPT_WRITEDATA, + static_cast(&content)); + + logger_->log_info("HttpConfigurationListener -- curl performed"); + res = curl_easy_perform(http_session); + + if (res == CURLE_OK) { + logger_->log_info("HttpConfigurationListener -- curl successful"); + + std::string response_body(content.data.begin(), content.data.end()); + int64_t http_code = 0; + curl_easy_getinfo(http_session, CURLINFO_RESPONSE_CODE, &http_code); + char *content_type; + /* ask for the content-type */ + curl_easy_getinfo(http_session, CURLINFO_CONTENT_TYPE, &content_type); + + bool isSuccess = ((int32_t) (http_code / 100)) == 2 + && res != CURLE_ABORTED_BY_CALLBACK; + bool output_body_to_content = isSuccess; + bool body_empty = IsNullOrEmpty(content.data); + + logger_->log_info("isSuccess: %d", isSuccess); + + if (output_body_to_content) { + } else { + logger_->log_info("Cannot output body to content"); + } + } else { + logger_->log_error("HttpConfigurationListener -- curl_easy_perform() failed %s\n", + curl_easy_strerror(res)); + } + curl_easy_cleanup(http_session); +} + +} /* namespace core */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ From a63e6df28612c18cc4a0418a97b98e00edad2e1d Mon Sep 17 00:00:00 2001 From: Bin Qiu Date: Sat, 20 May 2017 13:44:40 -0700 Subject: [PATCH 02/11] More configuration listener --- libminifi/include/ConfigurationListener.h | 112 ++++++++++++ libminifi/include/FlowController.h | 55 +++++- .../{core => }/HttpConfigurationListener.h | 78 +++------ libminifi/include/core/FlowConfiguration.h | 9 +- libminifi/include/core/ProcessGroup.h | 9 +- .../core/repository/FlowFileRepository.h | 2 +- .../include/core/yaml/YamlConfiguration.h | 16 ++ libminifi/include/properties/Configure.h | 4 + libminifi/src/ConfigurationListener.cpp | 84 +++++++++ libminifi/src/Configure.cpp | 23 ++- libminifi/src/FlowController.cpp | 64 ++++++- libminifi/src/HttpConfigurationListener.cpp | 103 ++++++++++++ libminifi/src/core/FlowConfiguration.cpp | 6 +- .../src/core/HttpConfigurationListener.cpp | 159 ------------------ libminifi/src/core/ProcessGroup.cpp | 4 +- libminifi/src/core/yaml/YamlConfiguration.cpp | 14 +- 16 files changed, 509 insertions(+), 233 deletions(-) create mode 100644 libminifi/include/ConfigurationListener.h rename libminifi/include/{core => }/HttpConfigurationListener.h (51%) create mode 100644 libminifi/src/ConfigurationListener.cpp create mode 100644 libminifi/src/HttpConfigurationListener.cpp delete mode 100644 libminifi/src/core/HttpConfigurationListener.cpp diff --git a/libminifi/include/ConfigurationListener.h b/libminifi/include/ConfigurationListener.h new file mode 100644 index 0000000000..0eed4a9a28 --- /dev/null +++ b/libminifi/include/ConfigurationListener.h @@ -0,0 +1,112 @@ +/** + * ConfigurationListener class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __CONFIGURATION_LISTENER__ +#define __CONFIGURATION_LISTENER__ + +#include +#include +#include +#include +#include +#include +#include + +#include "yaml-cpp/yaml.h" +#include "core/Core.h" +#include "core/Property.h" +#include "properties/Configure.h" +#include "core/logging/Logger.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +// Forwarder declaration +class FlowController; +// ConfigurationListener Class +class ConfigurationListener { +public: + + // Constructor + /*! + * Create a new processor + */ + ConfigurationListener(FlowController *controller, std::shared_ptr configure, std::string type): + connect_timeout_(20000), read_timeout_(20000), type_(type), configure_(configure), controller_(controller) { + logger_ = logging::Logger::getLogger(); + } + // Destructor + virtual ~ConfigurationListener() { + stop(); + } + + + // Start the thread + void start(); + // Stop the thread + void stop(); + // whether the thread is enable + bool isRunning() { + return running_; + } + // pull the new configuration from the remote host + virtual bool pullConfiguration(std::string &configuration) { + return false; + } + + +protected: + + // Run function for the thread + void run(); + + // Run function for the thread + void threadExecutor() { + run(); + } + + // Mutex for protection + std::mutex mutex_; + // thread + std::thread thread_; + // whether the thread is running + bool running_; + + // url + std::string url_; + // connection timeout + int64_t connect_timeout_; + // read timeout. + int64_t read_timeout_; + // pull interval + int64_t pull_interval_; + // type (http/rest) + std::string type_; + + std::shared_ptr configure_; + std::shared_ptr logger_; + FlowController *controller_; +}; + +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif diff --git a/libminifi/include/FlowController.h b/libminifi/include/FlowController.h index dc0d610f21..8450182aaa 100644 --- a/libminifi/include/FlowController.h +++ b/libminifi/include/FlowController.h @@ -21,6 +21,7 @@ #define __FLOW_CONTROLLER_H__ #include +#include #include #include #include @@ -43,6 +44,8 @@ #include "TimerDrivenSchedulingAgent.h" #include "EventDrivenSchedulingAgent.h" #include "FlowControlProtocol.h" +#include "ConfigurationListener.h" +#include "HttpConfigurationListener.h" #include "core/Property.h" @@ -127,9 +130,50 @@ class FlowController : public core::controller::ControllerServiceProvider, publi root_->updatePropertyValue(processorName, propertyName, propertyValue); } - // set 8 bytes SerialNumber - virtual void setSerialNumber(uint8_t *number) { - protocol_->setSerialNumber(number); + // set 6 bytes SerialNumber + void setSerialNumber(uint8_t *number) { + memcpy(serial_number_, number, 6); + } + + //get serial number + uint8_t *getSerialNumber() { + return serial_number_; + } + + // get serial number as string + std::string getSerialNumberStr() { + char str[18]; + char *strPtr = &str[0]; + + for (int i = 0; i < 6; i++) { + strPtr += sprintf(strPtr, "%02x", serial_number_[i]); + if (i != 5) + strPtr += sprintf(strPtr, ":"); + } + std::string value(str); + return value; + } + + // validate and apply passing yaml configuration payload + // first it will validate the payload with the current root node config for flowController + // like FlowController id/name is the same and new version is greater than the current version + // after that, it will apply the configuration + bool applyConfiguration(std::string &configurePayload); + + // get name + std::string getName() { + if (root_ != nullptr) + return root_->getName(); + else + return ""; + } + + // get version + int getVersion() { + if (root_ != nullptr) + return root_->getVersion(); + else + return 0; } /** @@ -292,6 +336,11 @@ class FlowController : public core::controller::ControllerServiceProvider, publi private: std::shared_ptr logger_; + // http configuration listener object. + std::unique_ptr http_configuration_listener_; + + // Serial Number + uint8_t serial_number_[6]; }; } /* namespace minifi */ diff --git a/libminifi/include/core/HttpConfigurationListener.h b/libminifi/include/HttpConfigurationListener.h similarity index 51% rename from libminifi/include/core/HttpConfigurationListener.h rename to libminifi/include/HttpConfigurationListener.h index f8c094bf93..f7cfabcd38 100644 --- a/libminifi/include/core/HttpConfigurationListener.h +++ b/libminifi/include/HttpConfigurationListener.h @@ -19,20 +19,15 @@ #ifndef __HTTP_CONFIGURATION_LISTENER__ #define __HTTP_CONFIGURATION_LISTENER__ -#include - #include -#include "FlowController.h" -#include "core/Processor.h" #include "core/Core.h" #include "core/Property.h" +#include "ConfigurationListener.h" namespace org { namespace apache { namespace nifi { namespace minifi { -namespace core { - /** * HTTP Response object */ @@ -43,9 +38,9 @@ struct HTTPRequestResponse { * Receive HTTP Response. */ static size_t recieve_write(char * data, size_t size, size_t nmemb, - void * p) { + void * p) { return static_cast(p)->write_content(data, size, - nmemb); + nmemb); } size_t write_content(char* ptr, size_t size, size_t nmemb) { @@ -56,67 +51,38 @@ struct HTTPRequestResponse { }; // HttpConfigurationListener Class -class HttpConfigurationListener : public core::Processor { - public: +class HttpConfigurationListener: public ConfigurationListener { +public: // Constructor /*! * Create a new processor */ - HttpConfigurationListener(std::string name, uuid_t uuid = NULL) - : Processor(name, uuid), - connect_timeout_(20000), - read_timeout_(20000), - use_etag_(false) { - curl_global_init(CURL_GLOBAL_DEFAULT); + HttpConfigurationListener(FlowController *controller, + std::shared_ptr configure) : + minifi::ConfigurationListener(controller, configure, "http") { + std::string value; + + if (configure->get(Configure::nifi_configuration_listener_http_url, value)) { + url_ = value; + logger_->log_info("Http configuration listener URL %s", url_.c_str()); + } else { + url_ = ""; + } + curl_global_init (CURL_GLOBAL_DEFAULT); } - // Destructor - virtual ~HttpConfigurationListener(); - // Processor Name - static const char *ProcessorName; - static const char* STATUS_CODE; - static const char* STATUS_MESSAGE; - static const char* RESPONSE_BODY; - static const char* REQUEST_URL; - static const char* TRANSACTION_ID; - static const char* REMOTE_DN; - static const char* EXCEPTION_CLASS; - static const char* EXCEPTION_MESSAGE; + bool pullConfiguration(std::string &configuration); - void onTrigger(core::ProcessContext *context, core::ProcessSession *session); - void initialize(); - void onSchedule(core::ProcessContext *context, - core::ProcessSessionFactory *sessionFactory); + // Destructor + virtual ~HttpConfigurationListener() { - protected: + } - /** - * Generate a transaction ID - * @return transaction ID string. - */ - std::string generateId(); - /** - * Set the request method on the curl struct. - * @param curl pointer to this instance. - * @param string request method - */ - void set_request_method(CURL *curl, const std::string &); +protected: - CURLcode res; - // http method - std::string method_; - // url - std::string url_; - // connection timeout - int64_t connect_timeout_; - // read timeout. - int64_t read_timeout_; - bool use_etag_; - std::string last_etag_; }; -} /* namespace core */ } /* namespace minifi */ } /* namespace nifi */ } /* namespace apache */ diff --git a/libminifi/include/core/FlowConfiguration.h b/libminifi/include/core/FlowConfiguration.h index edcb2b66f5..6e2b700ab5 100644 --- a/libminifi/include/core/FlowConfiguration.h +++ b/libminifi/include/core/FlowConfiguration.h @@ -75,8 +75,8 @@ class FlowConfiguration : public CoreComponent { // Create Processor (Node/Input/Output Port) based on the name std::shared_ptr createProcessor(std::string name, uuid_t uuid); // Create Root Processor Group - std::unique_ptr createRootProcessGroup(std::string name, uuid_t uuid); - + std::unique_ptr createRootProcessGroup(std::string name, + uuid_t uuid, int version); std::shared_ptr createControllerService(const std::string &class_name, const std::string &name, uuid_t uuid); // Create Remote Processor Group @@ -98,6 +98,11 @@ class FlowConfiguration : public CoreComponent { return getRoot(config_path_); } + virtual std::unique_ptr getRootFromPayload( + std::string &yamlConfigPayload) { + return nullptr; + } + /** * Base implementation that returns a null root pointer. * @return Extensions should return a non-null pointer in order to diff --git a/libminifi/include/core/ProcessGroup.h b/libminifi/include/core/ProcessGroup.h index f54f5b4788..497888653a 100644 --- a/libminifi/include/core/ProcessGroup.h +++ b/libminifi/include/core/ProcessGroup.h @@ -55,7 +55,8 @@ class ProcessGroup { /*! * Create a new process group */ - ProcessGroup(ProcessGroupType type, std::string name, uuid_t uuid = NULL, ProcessGroup *parent = NULL); + ProcessGroup(ProcessGroupType type, std::string name, uuid_t uuid = NULL, int version = 0, + ProcessGroup *parent = NULL); // Destructor virtual ~ProcessGroup(); // Set Processor Name @@ -109,6 +110,10 @@ class ProcessGroup { } else return false; } + // getVersion + int getVersion() { + return version_; + } // Start Processing void startProcessing(TimerDrivenSchedulingAgent *timeScheduler, EventDrivenSchedulingAgent *eventScheduler); // Stop Processing @@ -165,6 +170,8 @@ class ProcessGroup { uuid_t uuid_; // Processor Group Name std::string name_; + // version + int version_; // Process Group Type ProcessGroupType type_; // Processors (ProcessNode) inside this process group which include Input/Output Port, Remote Process Group input/Output port diff --git a/libminifi/include/core/repository/FlowFileRepository.h b/libminifi/include/core/repository/FlowFileRepository.h index b7c43b2a3c..2e19286d84 100644 --- a/libminifi/include/core/repository/FlowFileRepository.h +++ b/libminifi/include/core/repository/FlowFileRepository.h @@ -37,7 +37,7 @@ namespace repository { #define FLOWFILE_REPOSITORY_DIRECTORY "./flowfile_repository" #define MAX_FLOWFILE_REPOSITORY_STORAGE_SIZE (10*1024*1024) // 10M #define MAX_FLOWFILE_REPOSITORY_ENTRY_LIFE_TIME (600000) // 10 minute -#define FLOWFILE_REPOSITORY_PURGE_PERIOD (2500) // 2500 msec +#define FLOWFILE_REPOSITORY_PURGE_PERIOD (2000) // 2000 msec /** * Flow File repository diff --git a/libminifi/include/core/yaml/YamlConfiguration.h b/libminifi/include/core/yaml/YamlConfiguration.h index 3bfaefd5d9..61bf27105a 100644 --- a/libminifi/include/core/yaml/YamlConfiguration.h +++ b/libminifi/include/core/yaml/YamlConfiguration.h @@ -91,6 +91,22 @@ class YamlConfiguration : public FlowConfiguration { return getRoot(&rootYamlNode); } + /** + * Returns a shared pointer to a ProcessGroup object containing the + * flow configuration. The yamlConfigPayload argument must be + * a payload for the raw YAML configuration. + * + * @param yamlConfigPayload an input payload for the raw YAML configuration + * to be parsed and loaded into the flow + * configuration tree + * @return the root ProcessGroup node of the flow + * configuration tree + */ + std::unique_ptr getRootFromPayload(std::string &yamlConfigPayload) { + YAML::Node rootYamlNode = YAML::Load(yamlConfigPayload); + return getRoot(&rootYamlNode); + } + protected: /** diff --git a/libminifi/include/properties/Configure.h b/libminifi/include/properties/Configure.h index 58f6679d69..52f9d638ed 100644 --- a/libminifi/include/properties/Configure.h +++ b/libminifi/include/properties/Configure.h @@ -58,6 +58,10 @@ class Configure : public Properties { static const char *nifi_security_client_private_key; static const char *nifi_security_client_pass_phrase; static const char *nifi_security_client_ca_certificate; + static const char *nifi_configuration_listener_pull_interval; + static const char *nifi_configuration_listener_http_url; + static const char *nifi_configuration_listener_rest_url; + static const char *nifi_configuration_listener_type; // http or rest }; } /* namespace minifi */ diff --git a/libminifi/src/ConfigurationListener.cpp b/libminifi/src/ConfigurationListener.cpp new file mode 100644 index 0000000000..795e1ae71e --- /dev/null +++ b/libminifi/src/ConfigurationListener.cpp @@ -0,0 +1,84 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "ConfigurationListener.h" +#include "FlowController.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { + +void ConfigurationListener::start() { + if (running_) + return; + + std::string value; + // grab the value for configuration + if (configure_->get(Configure::nifi_configuration_listener_pull_interval, value)) { + core::TimeUnit unit; + if (core::Property::StringToTime(value, pull_interval_, unit) + && core::Property::ConvertTimeUnitToMS(pull_interval_, unit, + pull_interval_)) { + logger_->log_info("Configuration Listener pull interval: [%d] ms", + pull_interval_); + } + } else { + pull_interval_ = 60*1000; + logger_->log_info("Configuration Listener pull interval: [%d] ms", + pull_interval_); + } + + thread_ = std::thread(&ConfigurationListener::threadExecutor, this); + thread_.detach(); + running_ = true; + logger_->log_info("%s ConfigurationListener Thread Start", type_.c_str()); +} + +void ConfigurationListener::stop() { + if (!running_) + return; + running_ = false; + if (thread_.joinable()) + thread_.join(); + logger_->log_info("%s ConfigurationListener Thread Stop", type_.c_str()); +} + +void ConfigurationListener::run() { + + int64_t interval = 0; + while (running_) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + interval += 100; + if (interval >= pull_interval_) { + std::string payload; + bool ret = false; + ret = pullConfiguration(payload); + if (ret) { + ret = this->controller_->applyConfiguration(payload); + } + interval = 0; + } + } + return; + +} + +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/Configure.cpp b/libminifi/src/Configure.cpp index d8e049c721..5002e295cc 100644 --- a/libminifi/src/Configure.cpp +++ b/libminifi/src/Configure.cpp @@ -43,11 +43,24 @@ const char *Configure::nifi_flowfile_repository_max_storage_size = "nifi.flowfil const char *Configure::nifi_flowfile_repository_max_storage_time = "nifi.flowfile.repository.max.storage.time"; const char *Configure::nifi_flowfile_repository_directory_default = "nifi.flowfile.repository.directory.default"; const char *Configure::nifi_remote_input_secure = "nifi.remote.input.secure"; -const char *Configure::nifi_security_need_ClientAuth = "nifi.security.need.ClientAuth"; -const char *Configure::nifi_security_client_certificate = "nifi.security.client.certificate"; -const char *Configure::nifi_security_client_private_key = "nifi.security.client.private.key"; -const char *Configure::nifi_security_client_pass_phrase = "nifi.security.client.pass.phrase"; -const char *Configure::nifi_security_client_ca_certificate = "nifi.security.client.ca.certificate"; +const char *Configure::nifi_security_need_ClientAuth = + "nifi.security.need.ClientAuth"; +const char *Configure::nifi_security_client_certificate = + "nifi.security.client.certificate"; +const char *Configure::nifi_security_client_private_key = + "nifi.security.client.private.key"; +const char *Configure::nifi_security_client_pass_phrase = + "nifi.security.client.pass.phrase"; +const char *Configure::nifi_security_client_ca_certificate = + "nifi.security.client.ca.certificate"; +const char *Configure::nifi_configuration_listener_pull_interval = + "nifi.configuration.listener.pull.interval"; +const char *Configure::nifi_configuration_listener_http_url = + "nifi.configuration.listener.http.url"; +const char *Configure::nifi_configuration_listener_rest_url = + "nifi.configuration.listener.rest.url"; +const char *Configure::nifi_configuration_listener_type = + "nifi.configuration.listener.type"; } /* namespace minifi */ } /* namespace nifi */ diff --git a/libminifi/src/FlowController.cpp b/libminifi/src/FlowController.cpp index 62cf21c609..3ebd8c2cb6 100644 --- a/libminifi/src/FlowController.cpp +++ b/libminifi/src/FlowController.cpp @@ -33,6 +33,7 @@ #include #include #include +#include "yaml-cpp/yaml.h" #include "core/ProcessContext.h" #include "core/ProcessGroup.h" #include "utils/StringUtils.h" @@ -84,10 +85,23 @@ FlowController::FlowController(std::shared_ptr provenance_repo max_timer_driven_threads_ = DEFAULT_MAX_TIMER_DRIVEN_THREAD; running_ = false; initialized_ = false; +<<<<<<< HEAD root_ = nullptr; +======= + root_ = NULL; + memset(serial_number_, 0, 6); +>>>>>>> More configuration listener protocol_ = new FlowControlProtocol(this, configure); + std::string listenerType; + // grab the value for configuration + if (configure->get(Configure::nifi_configuration_listener_type, listenerType)) { + if (listenerType == "http") { + this->http_configuration_listener_ = std::unique_ptr(new minifi::HttpConfigurationListener(this, configure)); + } + } + if (!headless_mode) { std::string rawConfigFileString; configure->get(Configure::nifi_flow_configuration_file, rawConfigFileString); @@ -152,6 +166,54 @@ FlowController::~FlowController() { provenance_repo_ = nullptr; } +bool FlowController::applyConfiguration(std::string &configurePayload) { + + std::unique_ptr newRoot; + try { + newRoot = std::move(flow_configuration_->getRootFromPayload(configurePayload)); + } + catch (const YAML::Exception& e) { + logger_->log_error("Invalid configuration payload"); + return false; + } + + if (newRoot == nullptr) + return false; + + if (this->root_ != nullptr) { + // check for version/name/uuid for the flow controller + if (newRoot->getName() != root_->getName()) { + logger_->log_error("Configuration flow controller name mismatching"); + return false; + } + + uuid_t oldUUID; + uuid_t newUUID; + + if (newRoot->getUUID(newUUID) && root_->getUUID(oldUUID) && + uuid_compare(newUUID, oldUUID) != 0) { + logger_->log_error("Configuration flow controller UUID mismatching"); + return false; + } + + if (newRoot->getVersion() <= root_->getVersion()) { + logger_->log_error("Configuration flow controller version is not latest"); + return false; + } + } + + logger_->log_info("Starting to reload Flow Controller with flow control name %s, version %d", + newRoot->getName().c_str(), newRoot->getVersion()); + + std::lock_guard flow_lock(mutex_); + stop(true); + waitUnload(30000); + this->root_ = std::move(newRoot); + loadFlowRepo(); + initialized_ = true; + return start(); +} + void FlowController::stop(bool force) { std::lock_guard flow_lock(mutex_); if (running_) { @@ -164,7 +226,7 @@ void FlowController::stop(bool force) { this->flow_file_repo_->stop(); this->provenance_repo_->stop(); // Wait for sometime for thread stop - std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + std::this_thread::sleep_for(std::chrono::milliseconds(3000)); if (this->root_) this->root_->stopProcessing(this->timer_scheduler_.get(), this->event_scheduler_.get()); } diff --git a/libminifi/src/HttpConfigurationListener.cpp b/libminifi/src/HttpConfigurationListener.cpp new file mode 100644 index 0000000000..521baacb6f --- /dev/null +++ b/libminifi/src/HttpConfigurationListener.cpp @@ -0,0 +1,103 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "HttpConfigurationListener.h" +#include "FlowController.h" +#include +#include +#include +#include +#include +#include + +#include "core/logging/Logger.h" +#include "core/ProcessContext.h" +#include "core/Relationship.h" +#include "io/DataStream.h" +#include "io/StreamFactory.h" +#include "utils/StringUtils.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { + + +bool HttpConfigurationListener::pullConfiguration(std::string &configuration) { + + if (url_.empty()) + return false; + + bool ret = false; + + std::string fullUrl = url_ + "?" + "flowName=" + controller_->getName() + "&flowVersion=" + + std::to_string(controller_->getVersion()) + "&serialNumber=" + controller_->getSerialNumberStr(); + + CURL *http_session = curl_easy_init(); + + curl_easy_setopt(http_session, CURLOPT_URL, fullUrl.c_str()); + + if (connect_timeout_ > 0) { + curl_easy_setopt(http_session, CURLOPT_TIMEOUT, connect_timeout_); + } + + if (read_timeout_ > 0) { + curl_easy_setopt(http_session, CURLOPT_TIMEOUT, read_timeout_); + } + + HTTPRequestResponse content; + curl_easy_setopt(http_session, CURLOPT_WRITEFUNCTION, + &HTTPRequestResponse::recieve_write); + + curl_easy_setopt(http_session, CURLOPT_WRITEDATA, + static_cast(&content)); + + CURLcode res = curl_easy_perform(http_session); + + if (res == CURLE_OK) { + logger_->log_debug("HttpConfigurationListener -- curl successful to %s", fullUrl.c_str()); + + std::string response_body(content.data.begin(), content.data.end()); + int64_t http_code = 0; + curl_easy_getinfo(http_session, CURLINFO_RESPONSE_CODE, &http_code); + char *content_type; + /* ask for the content-type */ + curl_easy_getinfo(http_session, CURLINFO_CONTENT_TYPE, &content_type); + + bool isSuccess = ((int32_t) (http_code / 100)) == 2 + && res != CURLE_ABORTED_BY_CALLBACK; + bool body_empty = IsNullOrEmpty(content.data); + + if (isSuccess && !body_empty) { + ret = controller_->applyConfiguration(response_body); + } else { + logger_->log_error("Cannot output body to content"); + } + } else { + logger_->log_error("HttpConfigurationListener -- curl_easy_perform() failed %s\n", + curl_easy_strerror(res)); + } + curl_easy_cleanup(http_session); + + return ret; +} + +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ diff --git a/libminifi/src/core/FlowConfiguration.cpp b/libminifi/src/core/FlowConfiguration.cpp index 6635701dc9..cc6e0e5657 100644 --- a/libminifi/src/core/FlowConfiguration.cpp +++ b/libminifi/src/core/FlowConfiguration.cpp @@ -53,8 +53,10 @@ std::shared_ptr FlowConfiguration::createProvenanceReportTask() return processor; } -std::unique_ptr FlowConfiguration::createRootProcessGroup(std::string name, uuid_t uuid) { - return std::unique_ptr(new core::ProcessGroup(core::ROOT_PROCESS_GROUP, name, uuid)); +std::unique_ptr FlowConfiguration::createRootProcessGroup( + std::string name, uuid_t uuid, int version) { + return std::unique_ptr( + new core::ProcessGroup(core::ROOT_PROCESS_GROUP, name, uuid, version)); } std::unique_ptr FlowConfiguration::createRemoteProcessGroup(std::string name, uuid_t uuid) { diff --git a/libminifi/src/core/HttpConfigurationListener.cpp b/libminifi/src/core/HttpConfigurationListener.cpp deleted file mode 100644 index 964f035354..0000000000 --- a/libminifi/src/core/HttpConfigurationListener.cpp +++ /dev/null @@ -1,159 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "core/HttpConfigurationListener.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "core/logging/Logger.h" -#include "core/ProcessContext.h" -#include "core/Relationship.h" -#include "io/DataStream.h" -#include "io/StreamFactory.h" -#include "utils/StringUtils.h" - -namespace org { -namespace apache { -namespace nifi { -namespace minifi { -namespace core { - -const char *HttpConfigurationListener::ProcessorName = "HttpConfigurationListener"; - -const char* HttpConfigurationListener::STATUS_CODE = "invokehttp.status.code"; -const char* HttpConfigurationListener::STATUS_MESSAGE = "invokehttp.status.message"; -const char* HttpConfigurationListener::RESPONSE_BODY = "invokehttp.response.body"; -const char* HttpConfigurationListener::REQUEST_URL = "invokehttp.request.url"; -const char* HttpConfigurationListener::TRANSACTION_ID = "invokehttp.tx.id"; -const char* HttpConfigurationListener::REMOTE_DN = "invokehttp.remote.dn"; -const char* HttpConfigurationListener::EXCEPTION_CLASS = "invokehttp.java.exception.class"; -const char* HttpConfigurationListener::EXCEPTION_MESSAGE = "invokehttp.java.exception.message"; - -void HttpConfigurationListener::set_request_method(CURL *curl, const std::string &method) { - std::string my_method = method; - std::transform(my_method.begin(), my_method.end(), my_method.begin(), ::toupper); - if (my_method == "POST") { - curl_easy_setopt(curl, CURLOPT_POST, 1); - } else if (my_method == "PUT") { - curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); - } else if (my_method == "GET") { - } else { - curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, my_method.c_str()); - } -} - -void HttpConfigurationListener::initialize() { - logger_->log_info("Initializing HttpConfigurationListener"); - -} - -void HttpConfigurationListener::onSchedule(core::ProcessContext *context, - core::ProcessSessionFactory *sessionFactory) { - -} - -HttpConfigurationListener::~HttpConfigurationListener() { - curl_global_cleanup(); -} - -std::string HttpConfigurationListener::generateId() { - uuid_t txId; - uuid_generate(txId); - char uuidStr[37]; - uuid_unparse_lower(txId, uuidStr); - return uuidStr; -} - -void HttpConfigurationListener::onTrigger(core::ProcessContext *context, - core::ProcessSession *session) { - std::shared_ptr flowFile = std::static_pointer_cast< - FlowFileRecord>(session->get()); - - logger_->log_info("onTrigger HttpConfigurationListener with %s", method_.c_str()); - - // create a transaction id - std::string tx_id = generateId(); - - CURL *http_session = curl_easy_init(); - // set the HTTP request method from libCURL - set_request_method(http_session, method_); - curl_easy_setopt(http_session, CURLOPT_URL, url_.c_str()); - - if (connect_timeout_ > 0) { - curl_easy_setopt(http_session, CURLOPT_TIMEOUT, connect_timeout_); - } - - if (read_timeout_ > 0) { - curl_easy_setopt(http_session, CURLOPT_TIMEOUT, read_timeout_); - } - HTTPRequestResponse content; - curl_easy_setopt(http_session, CURLOPT_WRITEFUNCTION, - &HTTPRequestResponse::recieve_write); - - curl_easy_setopt(http_session, CURLOPT_WRITEDATA, - static_cast(&content)); - - logger_->log_info("HttpConfigurationListener -- curl performed"); - res = curl_easy_perform(http_session); - - if (res == CURLE_OK) { - logger_->log_info("HttpConfigurationListener -- curl successful"); - - std::string response_body(content.data.begin(), content.data.end()); - int64_t http_code = 0; - curl_easy_getinfo(http_session, CURLINFO_RESPONSE_CODE, &http_code); - char *content_type; - /* ask for the content-type */ - curl_easy_getinfo(http_session, CURLINFO_CONTENT_TYPE, &content_type); - - bool isSuccess = ((int32_t) (http_code / 100)) == 2 - && res != CURLE_ABORTED_BY_CALLBACK; - bool output_body_to_content = isSuccess; - bool body_empty = IsNullOrEmpty(content.data); - - logger_->log_info("isSuccess: %d", isSuccess); - - if (output_body_to_content) { - } else { - logger_->log_info("Cannot output body to content"); - } - } else { - logger_->log_error("HttpConfigurationListener -- curl_easy_perform() failed %s\n", - curl_easy_strerror(res)); - } - curl_easy_cleanup(http_session); -} - -} /* namespace core */ -} /* namespace minifi */ -} /* namespace nifi */ -} /* namespace apache */ -} /* namespace org */ diff --git a/libminifi/src/core/ProcessGroup.cpp b/libminifi/src/core/ProcessGroup.cpp index 9e6778ce7d..7ac139b53e 100644 --- a/libminifi/src/core/ProcessGroup.cpp +++ b/libminifi/src/core/ProcessGroup.cpp @@ -37,10 +37,12 @@ namespace nifi { namespace minifi { namespace core { -ProcessGroup::ProcessGroup(ProcessGroupType type, std::string name, uuid_t uuid, ProcessGroup *parent) +ProcessGroup::ProcessGroup(ProcessGroupType type, std::string name, uuid_t uuid, int version, + ProcessGroup *parent) : logger_(logging::LoggerFactory::getLogger()), name_(name), type_(type), + version_(version), parent_process_group_(parent) { if (!uuid) // Generate the global UUID for the flow record diff --git a/libminifi/src/core/yaml/YamlConfiguration.cpp b/libminifi/src/core/yaml/YamlConfiguration.cpp index a11db2b5ee..44aec12581 100644 --- a/libminifi/src/core/yaml/YamlConfiguration.cpp +++ b/libminifi/src/core/yaml/YamlConfiguration.cpp @@ -31,14 +31,24 @@ namespace core { core::ProcessGroup *YamlConfiguration::parseRootProcessGroupYaml(YAML::Node rootFlowNode) { uuid_t uuid; + int64_t version = 0; checkRequiredField(&rootFlowNode, "name", CONFIG_YAML_REMOTE_PROCESS_GROUP_KEY); std::string flowName = rootFlowNode["name"].as(); std::string id = getOrGenerateId(&rootFlowNode); uuid_parse(id.c_str(), uuid); - logger_->log_debug("parseRootProcessGroup: id => [%s], name => [%s]", id, flowName); - std::unique_ptr group = FlowConfiguration::createRootProcessGroup(flowName, uuid); + if (rootFlowNode["version"]) { + std::string value = rootFlowNode["version"].as(); + if (core::Property::StringToInt(value, version)) { + logger_->log_debug("parseRootProcessorGroup: version => [%d]", version); + } + } + + logger_->log_debug( + "parseRootProcessGroup: id => [%s], name => [%s]", id, flowName); + std::unique_ptr group = + FlowConfiguration::createRootProcessGroup(flowName, uuid, version); this->name_ = flowName; From 2a5a05fe373b5753658c0dbc5411ff2319867855 Mon Sep 17 00:00:00 2001 From: Bin Qiu Date: Sat, 20 May 2017 14:50:01 -0700 Subject: [PATCH 03/11] More configuration Listener --- libminifi/test/minifi-app.log.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 libminifi/test/minifi-app.log.txt diff --git a/libminifi/test/minifi-app.log.txt b/libminifi/test/minifi-app.log.txt new file mode 100644 index 0000000000..e69de29bb2 From c1c249a847e89545c249143f3917f0ca4366fba2 Mon Sep 17 00:00:00 2001 From: Bin Qiu Date: Sun, 21 May 2017 08:57:47 -0700 Subject: [PATCH 04/11] more listener --- libminifi/src/ConfigurationListener.cpp | 10 ++++------ libminifi/src/HttpConfigurationListener.cpp | 3 ++- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/libminifi/src/ConfigurationListener.cpp b/libminifi/src/ConfigurationListener.cpp index 795e1ae71e..8f67a161b4 100644 --- a/libminifi/src/ConfigurationListener.cpp +++ b/libminifi/src/ConfigurationListener.cpp @@ -27,6 +27,7 @@ void ConfigurationListener::start() { if (running_) return; + pull_interval_ = 60*1000; std::string value; // grab the value for configuration if (configure_->get(Configure::nifi_configuration_listener_pull_interval, value)) { @@ -34,15 +35,12 @@ void ConfigurationListener::start() { if (core::Property::StringToTime(value, pull_interval_, unit) && core::Property::ConvertTimeUnitToMS(pull_interval_, unit, pull_interval_)) { - logger_->log_info("Configuration Listener pull interval: [%d] ms", - pull_interval_); } - } else { - pull_interval_ = 60*1000; - logger_->log_info("Configuration Listener pull interval: [%d] ms", - pull_interval_); } + logger_->log_info("Configuration Listener pull interval: [%d] ms", + pull_interval_); + thread_ = std::thread(&ConfigurationListener::threadExecutor, this); thread_.detach(); running_ = true; diff --git a/libminifi/src/HttpConfigurationListener.cpp b/libminifi/src/HttpConfigurationListener.cpp index 521baacb6f..358a7e2993 100644 --- a/libminifi/src/HttpConfigurationListener.cpp +++ b/libminifi/src/HttpConfigurationListener.cpp @@ -84,7 +84,8 @@ bool HttpConfigurationListener::pullConfiguration(std::string &configuration) { bool body_empty = IsNullOrEmpty(content.data); if (isSuccess && !body_empty) { - ret = controller_->applyConfiguration(response_body); + configuration = std::move(response_body); + ret = true; } else { logger_->log_error("Cannot output body to content"); } From 36dbb58b6f7ff7b815916e0e5317c391287a8417 Mon Sep 17 00:00:00 2001 From: Bin Qiu Date: Thu, 25 May 2017 22:37:49 -0700 Subject: [PATCH 05/11] More client auth for config listener --- README.md | 18 +++++ libminifi/include/ConfigurationListener.h | 19 +++-- libminifi/include/HttpConfigurationListener.h | 15 +++- libminifi/include/properties/Configure.h | 1 + libminifi/src/ConfigurationListener.cpp | 62 +++++++++++++-- libminifi/src/Configure.cpp | 11 +++ libminifi/src/FlowController.cpp | 23 ------ libminifi/src/HttpConfigurationListener.cpp | 79 +++++++++++++++++-- 8 files changed, 183 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index acaf8dda4d..0fb25f3b92 100644 --- a/README.md +++ b/README.md @@ -323,6 +323,24 @@ Additionally, users can utilize the MiNiFi Toolkit Converter (version 0.0.1 - sc host: localhost port uuid: 471deef6-2a6e-4a7d-912a-81cc17e3a204 batch size: 100 + +### Http Configuration Listener + + Http Configuration Listener will pull flow file configuration from the remote command control server, + validate and apply the new flow configuration + + in minifi.properties + + nifi.configuration.listener.type=http + nifi.configuration.listener.http.url=https://localhost:8080 + nifi.configuration.listener.pull.interval=1 sec + nifi.configuration.listener.client.ca.certificate=./conf/nifi-cert.pem + + if you want to enable client certificate + nifi.configuration.listener.need.ClientAuth=true + nifi.configuration.listener.client.certificate=./conf/client.pem + nifi.configuration.listener.client.private.key=./conf/client.key + nifi.configuration.listener.client.pass.phrase=./conf/password ### Controller Services If you need to reference a controller service in your config.yml file, use the following template. In the example, below, ControllerServiceClass is the name of the class defining the controller Service. ControllerService1 diff --git a/libminifi/include/ConfigurationListener.h b/libminifi/include/ConfigurationListener.h index 0eed4a9a28..e381fafb30 100644 --- a/libminifi/include/ConfigurationListener.h +++ b/libminifi/include/ConfigurationListener.h @@ -47,30 +47,31 @@ class ConfigurationListener { /*! * Create a new processor */ - ConfigurationListener(FlowController *controller, std::shared_ptr configure, std::string type): - connect_timeout_(20000), read_timeout_(20000), type_(type), configure_(configure), controller_(controller) { + ConfigurationListener(FlowController *controller, + std::shared_ptr configure, std::string type) : + connect_timeout_(20000), read_timeout_(20000), type_(type), configure_( + configure), controller_(controller), need_client_certificate_(false) { logger_ = logging::Logger::getLogger(); + running_ = false; } // Destructor virtual ~ConfigurationListener() { stop(); } - // Start the thread void start(); // Stop the thread void stop(); // whether the thread is enable bool isRunning() { - return running_; + return running_; } // pull the new configuration from the remote host virtual bool pullConfiguration(std::string &configuration) { return false; } - protected: // Run function for the thread @@ -98,10 +99,18 @@ class ConfigurationListener { int64_t pull_interval_; // type (http/rest) std::string type_; + // last applied configuration + std::string lastAppliedConfiguration; std::shared_ptr configure_; std::shared_ptr logger_; FlowController *controller_; + + bool need_client_certificate_; + std::string certificate_; + std::string private_key_; + std::string passphrase_; + std::string ca_certificate_; }; } /* namespace minifi */ diff --git a/libminifi/include/HttpConfigurationListener.h b/libminifi/include/HttpConfigurationListener.h index f7cfabcd38..3d3a52afa1 100644 --- a/libminifi/include/HttpConfigurationListener.h +++ b/libminifi/include/HttpConfigurationListener.h @@ -69,14 +69,25 @@ class HttpConfigurationListener: public ConfigurationListener { } else { url_ = ""; } - curl_global_init (CURL_GLOBAL_DEFAULT); + + curl_global_init(CURL_GLOBAL_DEFAULT); + this->start(); } bool pullConfiguration(std::string &configuration); + /** + * Configures a secure connection + */ + void configureSecureConnection(CURL *http_session); + + static CURLcode configureSSLContext(CURL *curl, void *ctx, void *param); + static int pemPassWordCb(char *buf, int size, int rwflag, void *param); + // Destructor virtual ~HttpConfigurationListener() { - + this->stop(); + curl_global_cleanup(); } protected: diff --git a/libminifi/include/properties/Configure.h b/libminifi/include/properties/Configure.h index 52f9d638ed..422f51992e 100644 --- a/libminifi/include/properties/Configure.h +++ b/libminifi/include/properties/Configure.h @@ -54,6 +54,7 @@ class Configure : public Properties { static const char *nifi_flowfile_repository_enable; static const char *nifi_remote_input_secure; static const char *nifi_security_need_ClientAuth; + // site2site security config static const char *nifi_security_client_certificate; static const char *nifi_security_client_private_key; static const char *nifi_security_client_pass_phrase; diff --git a/libminifi/src/ConfigurationListener.cpp b/libminifi/src/ConfigurationListener.cpp index 8f67a161b4..b024a74069 100644 --- a/libminifi/src/ConfigurationListener.cpp +++ b/libminifi/src/ConfigurationListener.cpp @@ -17,6 +17,11 @@ */ #include "ConfigurationListener.h" #include "FlowController.h" +#include +#include +#include +#include +#include namespace org { namespace apache { @@ -27,19 +32,59 @@ void ConfigurationListener::start() { if (running_) return; - pull_interval_ = 60*1000; + pull_interval_ = 60 * 1000; std::string value; // grab the value for configuration - if (configure_->get(Configure::nifi_configuration_listener_pull_interval, value)) { + if (configure_->get(Configure::nifi_configuration_listener_pull_interval, + value)) { core::TimeUnit unit; if (core::Property::StringToTime(value, pull_interval_, unit) && core::Property::ConvertTimeUnitToMS(pull_interval_, unit, pull_interval_)) { + logger_->log_info("Configuration Listener pull interval: [%d] ms", + pull_interval_); } } - logger_->log_info("Configuration Listener pull interval: [%d] ms", - pull_interval_); + std::string clientAuthStr; + if (configure_->get(Configure::nifi_configuration_listener_need_ClientAuth, clientAuthStr)) { + org::apache::nifi::minifi::utils::StringUtils::StringToBool(clientAuthStr, this->need_client_certificate_); + } + + if (configure_->get( + Configure::nifi_configuration_listener_client_ca_certificate, + this->ca_certificate_)) { + logger_->log_info("Configuration Listener CA certificates: [%s]", + this->ca_certificate_.c_str()); + } + + if (this->need_client_certificate_) { + std::string passphrase_file; + + if (!(configure_->get( + Configure::nifi_configuration_listener_client_certificate, this->certificate_) + && configure_->get(Configure::nifi_configuration_listener_private_key, + this->private_key_))) { + logger_->log_error( + "Certificate and Private Key PEM file not configured for configuration listener, error: %s.", + std::strerror(errno)); + } + + if (configure_->get( + Configure::nifi_configuration_listener_client_pass_phrase, + passphrase_file)) { + // load the passphase from file + std::ifstream file(passphrase_file.c_str(), std::ifstream::in); + if (file.good()) { + this->passphrase_.assign((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + file.close(); + } + } + + logger_->log_info("Configuration Listener certificate: [%s], private key: [%s], passphrase file: [%s]", + this->certificate_.c_str(), this->private_key_.c_str(), passphrase_file.c_str()); + } thread_ = std::thread(&ConfigurationListener::threadExecutor, this); thread_.detach(); @@ -57,7 +102,6 @@ void ConfigurationListener::stop() { } void ConfigurationListener::run() { - int64_t interval = 0; while (running_) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); @@ -67,13 +111,17 @@ void ConfigurationListener::run() { bool ret = false; ret = pullConfiguration(payload); if (ret) { + if (payload.empty() || payload == lastAppliedConfiguration) { + interval = 0; + continue; + } ret = this->controller_->applyConfiguration(payload); + if (ret) + this->lastAppliedConfiguration = payload; } interval = 0; } } - return; - } } /* namespace minifi */ diff --git a/libminifi/src/Configure.cpp b/libminifi/src/Configure.cpp index 5002e295cc..e1bc2259f4 100644 --- a/libminifi/src/Configure.cpp +++ b/libminifi/src/Configure.cpp @@ -61,6 +61,17 @@ const char *Configure::nifi_configuration_listener_rest_url = "nifi.configuration.listener.rest.url"; const char *Configure::nifi_configuration_listener_type = "nifi.configuration.listener.type"; +const char *Configure::nifi_configuration_listener_need_ClientAuth = + "nifi.configuration.listener.need.ClientAuth"; +const char *Configure::nifi_configuration_listener_client_certificate = + "nifi.configuration.listener.client.certificate"; +const char *Configure::nifi_configuration_listener_private_key = + "nifi.configuration.listener.client.private.key"; +const char *Configure::nifi_configuration_listener_client_pass_phrase = + "nifi.configuration.listener.client.pass.phrase"; +const char *Configure::nifi_configuration_listener_client_ca_certificate = + "nifi.configuration.listener.client.ca.certificate"; + } /* namespace minifi */ } /* namespace nifi */ diff --git a/libminifi/src/FlowController.cpp b/libminifi/src/FlowController.cpp index 3ebd8c2cb6..acd364ec16 100644 --- a/libminifi/src/FlowController.cpp +++ b/libminifi/src/FlowController.cpp @@ -167,7 +167,6 @@ FlowController::~FlowController() { } bool FlowController::applyConfiguration(std::string &configurePayload) { - std::unique_ptr newRoot; try { newRoot = std::move(flow_configuration_->getRootFromPayload(configurePayload)); @@ -180,28 +179,6 @@ bool FlowController::applyConfiguration(std::string &configurePayload) { if (newRoot == nullptr) return false; - if (this->root_ != nullptr) { - // check for version/name/uuid for the flow controller - if (newRoot->getName() != root_->getName()) { - logger_->log_error("Configuration flow controller name mismatching"); - return false; - } - - uuid_t oldUUID; - uuid_t newUUID; - - if (newRoot->getUUID(newUUID) && root_->getUUID(oldUUID) && - uuid_compare(newUUID, oldUUID) != 0) { - logger_->log_error("Configuration flow controller UUID mismatching"); - return false; - } - - if (newRoot->getVersion() <= root_->getVersion()) { - logger_->log_error("Configuration flow controller version is not latest"); - return false; - } - } - logger_->log_info("Starting to reload Flow Controller with flow control name %s, version %d", newRoot->getName().c_str(), newRoot->getVersion()); diff --git a/libminifi/src/HttpConfigurationListener.cpp b/libminifi/src/HttpConfigurationListener.cpp index 358a7e2993..7dad6ed358 100644 --- a/libminifi/src/HttpConfigurationListener.cpp +++ b/libminifi/src/HttpConfigurationListener.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include "core/logging/Logger.h" #include "core/ProcessContext.h" @@ -37,16 +38,71 @@ namespace apache { namespace nifi { namespace minifi { +int HttpConfigurationListener::pemPassWordCb(char *buf, int size, int rwflag, + void *param) { + minifi::HttpConfigurationListener *listener = + static_cast(param); -bool HttpConfigurationListener::pullConfiguration(std::string &configuration) { + if (listener->passphrase_.length() > 0) { + memset(buf, 0x00, size); + memcpy(buf, listener->passphrase_.c_str(), + listener->passphrase_.length() - 1); + return listener->passphrase_.length() - 1; + } + return 0; +} + +CURLcode HttpConfigurationListener::configureSSLContext(CURL *curl, void *ctx, + void *param) { + minifi::HttpConfigurationListener *listener = + static_cast(param); + SSL_CTX* sslCtx = static_cast(ctx); + + SSL_CTX_load_verify_locations(sslCtx, listener->ca_certificate_.c_str(), 0); + SSL_CTX_use_certificate_file(sslCtx, listener->certificate_.c_str(), + SSL_FILETYPE_PEM); + SSL_CTX_set_default_passwd_cb(sslCtx, + HttpConfigurationListener::pemPassWordCb); + SSL_CTX_set_default_passwd_cb_userdata(sslCtx, param); + SSL_CTX_use_PrivateKey_file(sslCtx, listener->private_key_.c_str(), + SSL_FILETYPE_PEM); + // verify private key + if (!SSL_CTX_check_private_key(sslCtx)) { + listener->logger_->log_error( + "Private key does not match the public certificate, error : %s", + std::strerror(errno)); + return CURLE_FAILED_INIT; + } + + listener->logger_->log_debug( + "HttpConfigurationListener load Client Certificates OK"); + return CURLE_OK; +} +void HttpConfigurationListener::configureSecureConnection(CURL *http_session) { + curl_easy_setopt(http_session, CURLOPT_VERBOSE, 1L); + curl_easy_setopt(http_session, CURLOPT_CAINFO, this->ca_certificate_.c_str()); + curl_easy_setopt(http_session, CURLOPT_SSLCERTTYPE, "PEM"); + curl_easy_setopt(http_session, CURLOPT_SSL_VERIFYPEER, 1L); + if (this->need_client_certificate_) { + CURLcode ret; + ret = curl_easy_setopt(http_session, CURLOPT_SSL_CTX_FUNCTION, + &HttpConfigurationListener::configureSSLContext); + if (ret != CURLE_OK) + logger_->log_error("CURLOPT_SSL_CTX_FUNCTION not supported %d", ret); + curl_easy_setopt(http_session, CURLOPT_SSL_CTX_DATA, + static_cast(this)); + curl_easy_setopt(http_session, CURLOPT_SSLKEYTYPE, "PEM"); + } +} + +bool HttpConfigurationListener::pullConfiguration(std::string &configuration) { if (url_.empty()) return false; bool ret = false; - std::string fullUrl = url_ + "?" + "flowName=" + controller_->getName() + "&flowVersion=" + - std::to_string(controller_->getVersion()) + "&serialNumber=" + controller_->getSerialNumberStr(); + std::string fullUrl = url_; CURL *http_session = curl_easy_init(); @@ -60,17 +116,22 @@ bool HttpConfigurationListener::pullConfiguration(std::string &configuration) { curl_easy_setopt(http_session, CURLOPT_TIMEOUT, read_timeout_); } + if (fullUrl.find("https") != std::string::npos) { + configureSecureConnection(http_session); + } + HTTPRequestResponse content; curl_easy_setopt(http_session, CURLOPT_WRITEFUNCTION, - &HTTPRequestResponse::recieve_write); + &HTTPRequestResponse::recieve_write); curl_easy_setopt(http_session, CURLOPT_WRITEDATA, - static_cast(&content)); + static_cast(&content)); CURLcode res = curl_easy_perform(http_session); if (res == CURLE_OK) { - logger_->log_debug("HttpConfigurationListener -- curl successful to %s", fullUrl.c_str()); + logger_->log_debug("HttpConfigurationListener -- curl successful to %s", + fullUrl.c_str()); std::string response_body(content.data.begin(), content.data.end()); int64_t http_code = 0; @@ -85,13 +146,15 @@ bool HttpConfigurationListener::pullConfiguration(std::string &configuration) { if (isSuccess && !body_empty) { configuration = std::move(response_body); + logger_->log_debug("config %s", configuration.c_str()); ret = true; } else { logger_->log_error("Cannot output body to content"); } } else { - logger_->log_error("HttpConfigurationListener -- curl_easy_perform() failed %s\n", - curl_easy_strerror(res)); + logger_->log_error( + "HttpConfigurationListener -- curl_easy_perform() failed %s\n", + curl_easy_strerror(res)); } curl_easy_cleanup(http_session); From f1139814593fa4b64aff65bddbde33385fdfcfd3 Mon Sep 17 00:00:00 2001 From: Bin Qiu Date: Thu, 25 May 2017 22:41:13 -0700 Subject: [PATCH 06/11] More client auth for config listener --- minifi-app.log.txt | 856 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 856 insertions(+) create mode 100644 minifi-app.log.txt diff --git a/minifi-app.log.txt b/minifi-app.log.txt new file mode 100644 index 0000000000..3b2a4d3af4 --- /dev/null +++ b/minifi-app.log.txt @@ -0,0 +1,856 @@ +[2017-05-25 22:29:59.723] [minifi log] [info] Using configuration file located at /Users/binqiu/today/nifi-minifi-cpp/conf/minifi.properties +[2017-05-25 22:29:59.724] [minifi log] [debug] nifi.flowcontroller.graceful.shutdown.period not set, defaulting to 30000 +[2017-05-25 22:29:59.724] [minifi log] [info] NiFi Provenance Repository Directory ./provenance_repository +[2017-05-25 22:29:59.724] [minifi log] [info] NiFi Provenance Max Partition Bytes 1048576 +[2017-05-25 22:29:59.724] [minifi log] [info] NiFi Provenance Max Storage Time: [60000] ms +[2017-05-25 22:29:59.746] [minifi log] [info] NiFi Provenance Repository database open ./provenance_repository success +[2017-05-25 22:29:59.746] [minifi log] [info] NiFi FlowFile Repository Directory ./flowfile_repository +[2017-05-25 22:29:59.746] [minifi log] [info] NiFi FlowFile Max Partition Bytes 10485760 +[2017-05-25 22:29:59.746] [minifi log] [info] NiFi FlowFile Max Storage Time: [600000] ms +[2017-05-25 22:29:59.754] [minifi log] [info] NiFi FlowFile Repository database open ./flowfile_repository success +[2017-05-25 22:29:59.754] [minifi log] [info] Http configuration listener URL https://bbende:443 +[2017-05-25 22:29:59.758] [minifi log] [info] Configuration Listener pull interval: [1000] ms +[2017-05-25 22:29:59.758] [minifi log] [info] Configuration Listener CA certificates: [./conf/nifi-cert.pem] +[2017-05-25 22:29:59.758] [minifi log] [info] Configuration Listener certificate: [./conf/client.pem], private key: [./conf/client.key], passphrase file: [./conf/password] +[2017-05-25 22:29:59.758] [minifi log] [info] http ConfigurationListener Thread Start +[2017-05-25 22:29:59.758] [minifi log] [info] FlowController NiFi Configuration file /Users/binqiu/today/nifi-minifi-cpp/conf/config.yml +[2017-05-25 22:29:59.758] [minifi log] [info] FlowController content directory /Users/binqiu/today/nifi-minifi-cpp/content_repository +[2017-05-25 22:29:59.758] [minifi log] [info] Loading FlowController +[2017-05-25 22:29:59.758] [minifi log] [info] Initializing timers +[2017-05-25 22:29:59.759] [minifi log] [info] Load Flow Controller from file /Users/binqiu/today/nifi-minifi-cpp/conf/config.yml +[2017-05-25 22:29:59.761] [minifi log] [debug] parseRootProcessGroup: id => [471deef6-2a6e-4a7d-912a-81cc17e3a205], name => [MiNiFi Flow] +[2017-05-25 22:29:59.761] [minifi log] [info] ProcessGroup MiNiFi Flow created +[2017-05-25 22:29:59.761] [minifi log] [debug] parseProcessorNode: name => [GetFile] id => [471deef6-2a6e-4a7d-912a-81cc17e3a206] +[2017-05-25 22:29:59.761] [minifi log] [debug] parseProcessorNode: class => [org.apache.nifi.processors.standard.GetFile] +[2017-05-25 22:29:59.761] [minifi log] [info] Processor GetFile created UUID 471deef6-2a6e-4a7d-912a-81cc17e3a206 +[2017-05-25 22:29:59.762] [minifi log] [info] Processor GetFile supported relationship name success +[2017-05-25 22:29:59.762] [minifi log] [debug] parseProcessorNode: scheduling strategy => [TIMER_DRIVEN] +[2017-05-25 22:29:59.762] [minifi log] [debug] parseProcessorNode: scheduling period => [1 sec] +[2017-05-25 22:29:59.762] [minifi log] [debug] parseProcessorNode: max concurrent tasks => [1] +[2017-05-25 22:29:59.762] [minifi log] [debug] parseProcessorNode: penalization period => [30 sec] +[2017-05-25 22:29:59.762] [minifi log] [debug] parseProcessorNode: yield period => [10 sec] +[2017-05-25 22:29:59.762] [minifi log] [debug] parseProcessorNode: run duration nanos => [] +[2017-05-25 22:29:59.762] [minifi log] [info] Component Input Directory property name Input Directory value /tmp/getfile +[2017-05-25 22:29:59.762] [minifi log] [info] Component Keep Source File property name Keep Source File value true +[2017-05-25 22:29:59.762] [minifi log] [debug] convert: parseProcessorNode: schedulingPeriod => [1000000000] ns +[2017-05-25 22:29:59.762] [minifi log] [debug] convert: parseProcessorNode: penalizationPeriod => [30000] ms +[2017-05-25 22:29:59.762] [minifi log] [debug] setting scheduling strategy as TIMER_DRIVEN +[2017-05-25 22:29:59.762] [minifi log] [debug] parseProcessorNode: maxConcurrentTasks => [1] +[2017-05-25 22:29:59.762] [minifi log] [info] Add processor GetFile into process group MiNiFi Flow +[2017-05-25 22:29:59.762] [minifi log] [debug] parseRemoteProcessGroupYaml: name => [NiFi Flow], id => [471deef6-2a6e-4a7d-912a-81cc17e3a208] +[2017-05-25 22:29:59.762] [minifi log] [debug] parseRemoteProcessGroupYaml: url => [https://localhost:8081/nifi] +[2017-05-25 22:29:59.762] [minifi log] [info] ProcessGroup NiFi Flow created +[2017-05-25 22:29:59.762] [minifi log] [info] Add child process group NiFi Flow into process group MiNiFi Flow +[2017-05-25 22:29:59.762] [minifi log] [debug] parseRemoteProcessGroupYaml: yield period => [10 sec] +[2017-05-25 22:29:59.762] [minifi log] [debug] parseRemoteProcessGroupYaml: yieldPeriod => [10000] ms +[2017-05-25 22:29:59.762] [minifi log] [debug] parseRemoteProcessGroupYaml: timeout => [30 secs] +[2017-05-25 22:29:59.762] [minifi log] [debug] parseRemoteProcessGroupYaml: timeoutValue => [30000] ms +[2017-05-25 22:29:59.762] [minifi log] [debug] Got a current port, iterating... +[2017-05-25 22:29:59.762] [minifi log] [info] Processor From Node A created UUID 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:29:59.762] [minifi log] [info] Processor From Node A supported relationship name undefined +[2017-05-25 22:29:59.762] [minifi log] [info] Component Port property name Port value 10001 +[2017-05-25 22:29:59.762] [minifi log] [info] Component Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:29:59.762] [minifi log] [info] Component Host Name property name Host Name value localhost +[2017-05-25 22:29:59.762] [minifi log] [info] Add processor From Node A into process group NiFi Flow +[2017-05-25 22:29:59.762] [minifi log] [debug] parseProcessorNode: maxConcurrentTasks => [1] +[2017-05-25 22:29:59.762] [minifi log] [info] Connection GenerateFlowFileS2S created +[2017-05-25 22:29:59.762] [minifi log] [debug] Created connection with UUID 471deef6-2a6e-4a7d-912a-81cc17e3a207 and name GenerateFlowFileS2S +[2017-05-25 22:29:59.762] [minifi log] [debug] parseConnection: relationship => [success] +[2017-05-25 22:29:59.762] [minifi log] [debug] Using 'source id' to match source with same id for connection 'GenerateFlowFileS2S': source id => [471deef6-2a6e-4a7d-912a-81cc17e3a206] +[2017-05-25 22:29:59.762] [minifi log] [debug] Using 'destination id' to match destination with same id for connection 'GenerateFlowFileS2S': destination id => [471deef6-2a6e-4a7d-912a-81cc17e3a204] +[2017-05-25 22:29:59.762] [minifi log] [info] Add connection GenerateFlowFileS2S into process group MiNiFi Flow +[2017-05-25 22:29:59.762] [minifi log] [info] find processor GetFile +[2017-05-25 22:29:59.762] [minifi log] [info] Add connection GenerateFlowFileS2S into Processor GetFile outgoing connection for relationship success +[2017-05-25 22:29:59.762] [minifi log] [info] find processor GetFile +[2017-05-25 22:29:59.762] [minifi log] [info] find processor child NiFi Flow +[2017-05-25 22:29:59.762] [minifi log] [info] find processor From Node A +[2017-05-25 22:29:59.762] [minifi log] [info] Add connection GenerateFlowFileS2S into Processor From Node A incoming connection +[2017-05-25 22:29:59.762] [minifi log] [info] Processor SiteToSiteProvenanceReportingTask created UUID 8c70d581-18f1-46ff-af18-a5957798f2f0 +[2017-05-25 22:29:59.762] [minifi log] [info] Add processor SiteToSiteProvenanceReportingTask into process group MiNiFi Flow +[2017-05-25 22:29:59.762] [minifi log] [debug] ProvenanceReportingTask schedulingPeriod 1000000000 ns +[2017-05-25 22:29:59.762] [minifi log] [debug] ProvenanceReportingTask scheduling strategy TIMER_DRIVEN +[2017-05-25 22:29:59.763] [minifi log] [debug] ProvenanceReportingTask host localhost +[2017-05-25 22:29:59.763] [minifi log] [debug] ProvenanceReportingTask port 10001 +[2017-05-25 22:29:59.763] [minifi log] [debug] ProvenanceReportingTask port uuid 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:29:59.763] [minifi log] [info] Loaded root processor Group +[2017-05-25 22:29:59.763] [minifi log] [info] Loaded controller service provider +[2017-05-25 22:29:59.763] [minifi log] [debug] Getting connection map +[2017-05-25 22:29:59.763] [minifi log] [debug] Number of connections from connectionMap 1 +[2017-05-25 22:29:59.763] [minifi log] [info] Loaded flow repository +[2017-05-25 22:29:59.763] [minifi log] [info] Starting Flow Controller +[2017-05-25 22:29:59.763] [minifi log] [info] Enabling 0 controller services +[2017-05-25 22:29:59.763] [minifi log] [debug] Starting GetFile +[2017-05-25 22:29:59.763] [minifi log] [debug] nifi_administrative_yield_duration: [30000] ms +[2017-05-25 22:29:59.763] [minifi log] [debug] nifi_bored_yield_duration: [10] ms +[2017-05-25 22:29:59.763] [minifi log] [info] Processor Input Directory property name Input Directory value /tmp/getfile +[2017-05-25 22:29:59.763] [minifi log] [info] Processor Batch Size property name Batch Size value 10 +[2017-05-25 22:29:59.763] [minifi log] [info] Processor Ignore Hidden Files property name Ignore Hidden Files value true +[2017-05-25 22:29:59.763] [minifi log] [info] Processor Keep Source File property name Keep Source File value true +[2017-05-25 22:29:59.763] [minifi log] [info] Processor Maximum File Age property name Maximum File Age value 0 sec +[2017-05-25 22:29:59.763] [minifi log] [debug] successfully applied _maxAge +[2017-05-25 22:29:59.763] [minifi log] [info] Processor Minimum File Age property name Minimum File Age value 0 sec +[2017-05-25 22:29:59.763] [minifi log] [debug] successfully applied _minAge +[2017-05-25 22:29:59.763] [minifi log] [info] Processor Maximum File Size property name Maximum File Size value 0 B +[2017-05-25 22:29:59.763] [minifi log] [info] Processor Minimum File Size property name Minimum File Size value 0 B +[2017-05-25 22:29:59.763] [minifi log] [info] Processor Polling Interval property name Polling Interval value 0 sec +[2017-05-25 22:29:59.763] [minifi log] [debug] successfully applied _pollInterval +[2017-05-25 22:29:59.763] [minifi log] [info] Processor Recurse Subdirectories property name Recurse Subdirectories value true +[2017-05-25 22:29:59.763] [minifi log] [info] Processor File Filter property name File Filter value [^\.].* +[2017-05-25 22:29:59.763] [minifi log] [info] Scheduled thread 0 running for process GetFile +[2017-05-25 22:29:59.763] [minifi log] [debug] Starting SiteToSiteProvenanceReportingTask +[2017-05-25 22:29:59.763] [minifi log] [debug] nifi_administrative_yield_duration: [30000] ms +[2017-05-25 22:29:59.763] [minifi log] [debug] nifi_bored_yield_duration: [10] ms +[2017-05-25 22:29:59.763] [minifi log] [info] Is listing empty 1 +[2017-05-25 22:29:59.763] [minifi log] [info] Performing file listing against /tmp/getfile +[2017-05-25 22:29:59.763] [minifi log] [info] Scheduled thread 0 running for process SiteToSiteProvenanceReportingTask +[2017-05-25 22:29:59.763] [minifi log] [debug] Starting From Node A +[2017-05-25 22:29:59.763] [minifi log] [info] Performing file listing against /tmp/getfile +[2017-05-25 22:29:59.763] [minifi log] [debug] nifi_administrative_yield_duration: [30000] ms +[2017-05-25 22:29:59.763] [minifi log] [debug] nifi_bored_yield_duration: [10] ms +[2017-05-25 22:29:59.764] [minifi log] [info] Processor Host Name property name Host Name value localhost +[2017-05-25 22:29:59.764] [minifi log] [info] Processor Port property name Port value 10001 +[2017-05-25 22:29:59.764] [minifi log] [info] Processor Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:29:59.764] [minifi log] [info] Scheduled thread 0 running for process From Node A +[2017-05-25 22:29:59.764] [minifi log] [info] org::apache::nifi::minifi::provenance::ProvenanceRepository Repository Monitor Thread Start +[2017-05-25 22:29:59.764] [minifi log] [info] org::apache::nifi::minifi::core::repository::FlowFileRepository Repository Monitor Thread Start +[2017-05-25 22:29:59.764] [minifi log] [info] Started Flow Controller +[2017-05-25 22:29:59.764] [minifi log] [info] MiNiFi started +[2017-05-25 22:29:59.764] [minifi log] [info] Is listing empty 0 +[2017-05-25 22:29:59.764] [minifi log] [info] GetFile process /tmp/getfile/minifi-app.log.txt +[2017-05-25 22:29:59.764] [minifi log] [debug] Create FlowFile with UUID 06e4a308-a065-460e-97a1-75ba8b46efaa +[2017-05-25 22:29:59.764] [minifi log] [debug] Resource Claim created ./content_repository/ce638c86-375d-4213-9be7-1b6e33f8cf0b +[2017-05-25 22:29:59.765] [minifi log] [debug] Import offset 0 length 36219 into content ./content_repository/ce638c86-375d-4213-9be7-1b6e33f8cf0b for FlowFile UUID 06e4a308-a065-460e-97a1-75ba8b46efaa +[2017-05-25 22:29:59.766] [minifi log] [info] Negotiate protocol version with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 current version 5 +[2017-05-25 22:29:59.766] [minifi log] [info] result of writing resource name is 22 +[2017-05-25 22:29:59.767] [minifi log] [info] status code is 20 +[2017-05-25 22:29:59.767] [minifi log] [info] Site2Site Protocol Negotiate protocol version OK +[2017-05-25 22:29:59.767] [minifi log] [info] Site2Site socket established +[2017-05-25 22:29:59.767] [minifi log] [info] Site2Site Protocol Perform hand shake with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:29:59.767] [minifi log] [info] Site2Site Protocol Send handshake properties GZIP false +[2017-05-25 22:29:59.767] [minifi log] [info] Site2Site Protocol Send handshake properties PORT_IDENTIFIER 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:29:59.767] [minifi log] [info] Site2Site Protocol Send handshake properties REQUEST_EXPIRATION_MILLIS 30000 +[2017-05-25 22:29:59.768] [minifi log] [debug] Enqueue flow file UUID 06e4a308-a065-460e-97a1-75ba8b46efaa to connection GenerateFlowFileS2S +[2017-05-25 22:29:59.768] [minifi log] [debug] NiFi FlowFile Store event 06e4a308-a065-460e-97a1-75ba8b46efaa size 337 success +[2017-05-25 22:29:59.768] [minifi log] [debug] Delete SnapShot FlowFile UUID 06e4a308-a065-460e-97a1-75ba8b46efaa +[2017-05-25 22:29:59.768] [minifi log] [debug] NiFi Provenance Store event 6ba5ea25-689f-4654-85c4-2875f28ca15d size 345 success +[2017-05-25 22:29:59.768] [minifi log] [debug] NiFi Provenance Store event 68fec583-55bc-43c9-98bd-06453b0179ef size 471 success +[2017-05-25 22:29:59.776] [minifi log] [info] Processor Host Name property name Host Name value localhost +[2017-05-25 22:29:59.776] [minifi log] [info] Processor Port property name Port value 10001 +[2017-05-25 22:29:59.776] [minifi log] [info] Processor Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:29:59.778] [minifi log] [info] Site2Site HandShake Completed +[2017-05-25 22:29:59.778] [minifi log] [info] Negotiate protocol version with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 current version 5 +[2017-05-25 22:29:59.778] [minifi log] [info] Site2Site Protocol Negotiate Codec with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:29:59.778] [minifi log] [info] result of writing resource name is 22 +[2017-05-25 22:29:59.778] [minifi log] [info] Negotiate Codec version with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 current version 1 +[2017-05-25 22:29:59.779] [minifi log] [info] status code is 20 +[2017-05-25 22:29:59.779] [minifi log] [info] Site2Site Protocol Negotiate protocol version OK +[2017-05-25 22:29:59.779] [minifi log] [info] Site2Site socket established +[2017-05-25 22:29:59.779] [minifi log] [info] Site2Site Protocol Perform hand shake with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:29:59.779] [minifi log] [info] Site2Site Protocol Send handshake properties GZIP false +[2017-05-25 22:29:59.779] [minifi log] [info] Site2Site Protocol Send handshake properties PORT_IDENTIFIER 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:29:59.779] [minifi log] [info] Site2Site Protocol Send handshake properties REQUEST_EXPIRATION_MILLIS 30000 +[2017-05-25 22:29:59.790] [minifi log] [info] Site2Site HandShake Completed +[2017-05-25 22:29:59.791] [minifi log] [info] Site2Site Codec Negotiate version OK +[2017-05-25 22:29:59.791] [minifi log] [info] Site2Site Protocol Negotiate Codec with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:29:59.791] [minifi log] [info] Site2Site Codec Completed and move to READY state for data transfer +[2017-05-25 22:29:59.791] [minifi log] [info] Site2Site Ready For data transaction +[2017-05-25 22:29:59.791] [minifi log] [info] Negotiate Codec version with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 current version 1 +[2017-05-25 22:29:59.791] [minifi log] [info] Site2Site create transaction ea140105-e7a4-464b-8784-ab8b270bfa5f +[2017-05-25 22:29:59.791] [minifi log] [info] Site2Site transaction ea140105-e7a4-464b-8784-ab8b270bfa5f send flow record 1, total length 1902 +[2017-05-25 22:29:59.791] [minifi log] [info] Site2Site transaction ea140105-e7a4-464b-8784-ab8b270bfa5f send bytes length 1902 +[2017-05-25 22:29:59.791] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction ea140105-e7a4-464b-8784-ab8b270bfa5f +[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site Codec Negotiate version OK +[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site Codec Completed and move to READY state for data transfer +[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site Ready For data transaction +[2017-05-25 22:29:59.803] [minifi log] [debug] Dequeue flow file UUID 06e4a308-a065-460e-97a1-75ba8b46efaa from connection GenerateFlowFileS2S +[2017-05-25 22:29:59.803] [minifi log] [debug] Create Snapshot FlowFile with UUID 45ba7411-95e9-402a-a1b1-5196c831dcdf +[2017-05-25 22:29:59.803] [minifi log] [debug] Delete FlowFile UUID 45ba7411-95e9-402a-a1b1-5196c831dcdf +[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site create transaction 08d59ed6-daf6-4250-bb91-be486de22108 +[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site transaction 08d59ed6-daf6-4250-bb91-be486de22108 send attribute key absolute.path value /tmp/getfile/minifi-app.log.txt +[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site transaction 08d59ed6-daf6-4250-bb91-be486de22108 send attribute key filename value minifi-app.log.txt +[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site transaction 08d59ed6-daf6-4250-bb91-be486de22108 send attribute key path value /tmp/getfile +[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site transaction 08d59ed6-daf6-4250-bb91-be486de22108 send attribute key uuid value 06e4a308-a065-460e-97a1-75ba8b46efaa +[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site transaction ea140105-e7a4-464b-8784-ab8b270bfa5f peer confirm transaction with CRC 239634877 +[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site transaction ea140105-e7a4-464b-8784-ab8b270bfa5f CRC matched +[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site transaction 08d59ed6-daf6-4250-bb91-be486de22108 send flow record 1, total length 36219 +[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site transaction 08d59ed6-daf6-4250-bb91-be486de22108 send flow record 06e4a308-a065-460e-97a1-75ba8b46efaa +[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 08d59ed6-daf6-4250-bb91-be486de22108 +[2017-05-25 22:29:59.815] [minifi log] [info] Site2Site transaction ea140105-e7a4-464b-8784-ab8b270bfa5f peer finished transaction +[2017-05-25 22:29:59.815] [minifi log] [info] Site2Site transaction ea140105-e7a4-464b-8784-ab8b270bfa5f successfully send flow record 1, content bytes 1902 +[2017-05-25 22:29:59.815] [minifi log] [info] Site2Site delete transaction ea140105-e7a4-464b-8784-ab8b270bfa5f +[2017-05-25 22:29:59.816] [minifi log] [info] Site2Site transaction 08d59ed6-daf6-4250-bb91-be486de22108 peer confirm transaction with CRC 2452005643 +[2017-05-25 22:29:59.816] [minifi log] [info] Site2Site transaction 08d59ed6-daf6-4250-bb91-be486de22108 CRC matched +[2017-05-25 22:29:59.827] [minifi log] [info] Site2Site transaction 08d59ed6-daf6-4250-bb91-be486de22108 peer finished transaction +[2017-05-25 22:29:59.827] [minifi log] [info] Site2Site transaction 08d59ed6-daf6-4250-bb91-be486de22108 successfully send flow record 1, content bytes 36219 +[2017-05-25 22:29:59.827] [minifi log] [info] Site2Site delete transaction 08d59ed6-daf6-4250-bb91-be486de22108 +[2017-05-25 22:29:59.827] [minifi log] [debug] Delete FlowFile UUID 06e4a308-a065-460e-97a1-75ba8b46efaa +[2017-05-25 22:29:59.827] [minifi log] [debug] Delete Resource Claim ./content_repository/ce638c86-375d-4213-9be7-1b6e33f8cf0b +[2017-05-25 22:29:59.828] [minifi log] [debug] NiFi Provenance Store event d6c555da-ff88-406e-930d-f216570e7464 size 504 success +[2017-05-25 22:29:59.828] [minifi log] [debug] NiFi Provenance Store event c85bc571-8882-43fa-99ab-3eb29e274b7a size 550 success +[2017-05-25 22:30:00.770] [minifi log] [info] Is listing empty 1 +[2017-05-25 22:30:00.770] [minifi log] [info] Performing file listing against /tmp/getfile +[2017-05-25 22:30:00.770] [minifi log] [info] Performing file listing against /tmp/getfile +[2017-05-25 22:30:00.770] [minifi log] [info] Is listing empty 0 +[2017-05-25 22:30:00.770] [minifi log] [info] GetFile process /tmp/getfile/minifi-app.log.txt +[2017-05-25 22:30:00.770] [minifi log] [debug] Create FlowFile with UUID 03ce4edc-c0a9-45f7-9793-cbcc12f3e8c6 +[2017-05-25 22:30:00.770] [minifi log] [debug] Resource Claim created ./content_repository/157415c6-ef77-4004-9f9a-b1df8c6ce54c +[2017-05-25 22:30:00.771] [minifi log] [debug] Import offset 0 length 36219 into content ./content_repository/157415c6-ef77-4004-9f9a-b1df8c6ce54c for FlowFile UUID 03ce4edc-c0a9-45f7-9793-cbcc12f3e8c6 +[2017-05-25 22:30:00.774] [minifi log] [debug] Enqueue flow file UUID 03ce4edc-c0a9-45f7-9793-cbcc12f3e8c6 to connection GenerateFlowFileS2S +[2017-05-25 22:30:00.774] [minifi log] [debug] NiFi FlowFile Store event 03ce4edc-c0a9-45f7-9793-cbcc12f3e8c6 size 337 success +[2017-05-25 22:30:00.774] [minifi log] [debug] Delete SnapShot FlowFile UUID 03ce4edc-c0a9-45f7-9793-cbcc12f3e8c6 +[2017-05-25 22:30:00.774] [minifi log] [debug] NiFi Provenance Store event 042b7639-c864-46c2-a8b3-33e076c343c7 size 471 success +[2017-05-25 22:30:00.774] [minifi log] [debug] NiFi Provenance Store event 0c8d2130-7c79-4ea8-97be-9f3c92b93e38 size 345 success +[2017-05-25 22:30:00.778] [minifi log] [info] Processor Host Name property name Host Name value localhost +[2017-05-25 22:30:00.778] [minifi log] [info] Processor Port property name Port value 10001 +[2017-05-25 22:30:00.778] [minifi log] [info] Processor Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:30:00.778] [minifi log] [debug] Dequeue flow file UUID 03ce4edc-c0a9-45f7-9793-cbcc12f3e8c6 from connection GenerateFlowFileS2S +[2017-05-25 22:30:00.778] [minifi log] [debug] Create Snapshot FlowFile with UUID 5f216db1-abeb-4edc-aab5-7db36d33f5c3 +[2017-05-25 22:30:00.778] [minifi log] [debug] Delete FlowFile UUID 5f216db1-abeb-4edc-aab5-7db36d33f5c3 +[2017-05-25 22:30:00.778] [minifi log] [info] Site2Site create transaction 2307ba45-6195-40ed-9a32-83ac59b16609 +[2017-05-25 22:30:00.778] [minifi log] [info] Site2Site transaction 2307ba45-6195-40ed-9a32-83ac59b16609 send attribute key absolute.path value /tmp/getfile/minifi-app.log.txt +[2017-05-25 22:30:00.778] [minifi log] [info] Site2Site transaction 2307ba45-6195-40ed-9a32-83ac59b16609 send attribute key filename value minifi-app.log.txt +[2017-05-25 22:30:00.778] [minifi log] [info] Site2Site transaction 2307ba45-6195-40ed-9a32-83ac59b16609 send attribute key path value /tmp/getfile +[2017-05-25 22:30:00.778] [minifi log] [info] Site2Site transaction 2307ba45-6195-40ed-9a32-83ac59b16609 send attribute key uuid value 03ce4edc-c0a9-45f7-9793-cbcc12f3e8c6 +[2017-05-25 22:30:00.778] [minifi log] [info] Site2Site transaction 2307ba45-6195-40ed-9a32-83ac59b16609 send flow record 1, total length 36219 +[2017-05-25 22:30:00.778] [minifi log] [info] Site2Site transaction 2307ba45-6195-40ed-9a32-83ac59b16609 send flow record 03ce4edc-c0a9-45f7-9793-cbcc12f3e8c6 +[2017-05-25 22:30:00.778] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 2307ba45-6195-40ed-9a32-83ac59b16609 +[2017-05-25 22:30:00.790] [minifi log] [info] Site2Site transaction 2307ba45-6195-40ed-9a32-83ac59b16609 peer confirm transaction with CRC 1532199305 +[2017-05-25 22:30:00.790] [minifi log] [info] Site2Site transaction 2307ba45-6195-40ed-9a32-83ac59b16609 CRC matched +[2017-05-25 22:30:00.802] [minifi log] [info] Site2Site transaction 2307ba45-6195-40ed-9a32-83ac59b16609 peer finished transaction +[2017-05-25 22:30:00.802] [minifi log] [info] Site2Site transaction 2307ba45-6195-40ed-9a32-83ac59b16609 successfully send flow record 1, content bytes 36219 +[2017-05-25 22:30:00.802] [minifi log] [info] Site2Site delete transaction 2307ba45-6195-40ed-9a32-83ac59b16609 +[2017-05-25 22:30:00.802] [minifi log] [debug] Delete FlowFile UUID 03ce4edc-c0a9-45f7-9793-cbcc12f3e8c6 +[2017-05-25 22:30:00.802] [minifi log] [debug] Delete Resource Claim ./content_repository/157415c6-ef77-4004-9f9a-b1df8c6ce54c +[2017-05-25 22:30:00.802] [minifi log] [debug] NiFi Provenance Store event a84e3e3c-f886-4ea3-9a28-b04388e08e88 size 550 success +[2017-05-25 22:30:00.803] [minifi log] [debug] NiFi Provenance Store event 6d959d3e-a18f-4964-9d3b-b8816909af10 size 504 success +[2017-05-25 22:30:00.819] [minifi log] [info] Site2Site create transaction 3758fa47-3a50-49f6-98b1-869ab08e0a6b +[2017-05-25 22:30:00.819] [minifi log] [info] Site2Site transaction 3758fa47-3a50-49f6-98b1-869ab08e0a6b send flow record 1, total length 5982 +[2017-05-25 22:30:00.819] [minifi log] [info] Site2Site transaction 3758fa47-3a50-49f6-98b1-869ab08e0a6b send bytes length 5982 +[2017-05-25 22:30:00.819] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 3758fa47-3a50-49f6-98b1-869ab08e0a6b +[2017-05-25 22:30:00.824] [minifi log] [info] Site2Site transaction 3758fa47-3a50-49f6-98b1-869ab08e0a6b peer confirm transaction with CRC 2217791887 +[2017-05-25 22:30:00.824] [minifi log] [info] Site2Site transaction 3758fa47-3a50-49f6-98b1-869ab08e0a6b CRC matched +[2017-05-25 22:30:00.833] [minifi log] [debug] HttpConfigurationListener load Client Certificates OK +[2017-05-25 22:30:00.837] [minifi log] [info] Site2Site transaction 3758fa47-3a50-49f6-98b1-869ab08e0a6b peer finished transaction +[2017-05-25 22:30:00.837] [minifi log] [info] Site2Site transaction 3758fa47-3a50-49f6-98b1-869ab08e0a6b successfully send flow record 1, content bytes 5982 +[2017-05-25 22:30:00.837] [minifi log] [info] Site2Site delete transaction 3758fa47-3a50-49f6-98b1-869ab08e0a6b +[2017-05-25 22:30:00.933] [minifi log] [debug] HttpConfigurationListener -- curl successful to https://bbende:443 +[2017-05-25 22:30:00.933] [minifi log] [debug] config # Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Flow Controller: + id: 471deef6-2a6e-4a7d-912a-81cc17e3a205 + name: MiNiFi Flow + version: 1 + +Processors: + - name: GetFile + id: 471deef6-2a6e-4a7d-912a-81cc17e3a206 + class: org.apache.nifi.processors.standard.Ge +[2017-05-25 22:30:00.934] [minifi log] [debug] parseRootProcessorGroup: version => [1] +[2017-05-25 22:30:00.934] [minifi log] [debug] parseRootProcessGroup: id => [471deef6-2a6e-4a7d-912a-81cc17e3a205], name => [MiNiFi Flow] +[2017-05-25 22:30:00.934] [minifi log] [info] ProcessGroup MiNiFi Flow created +[2017-05-25 22:30:00.934] [minifi log] [debug] parseProcessorNode: name => [GetFile] id => [471deef6-2a6e-4a7d-912a-81cc17e3a206] +[2017-05-25 22:30:00.934] [minifi log] [debug] parseProcessorNode: class => [org.apache.nifi.processors.standard.GetFile] +[2017-05-25 22:30:00.934] [minifi log] [info] Processor GetFile created UUID 471deef6-2a6e-4a7d-912a-81cc17e3a206 +[2017-05-25 22:30:00.935] [minifi log] [info] Processor GetFile supported relationship name success +[2017-05-25 22:30:00.935] [minifi log] [debug] parseProcessorNode: scheduling strategy => [TIMER_DRIVEN] +[2017-05-25 22:30:00.935] [minifi log] [debug] parseProcessorNode: scheduling period => [1 sec] +[2017-05-25 22:30:00.935] [minifi log] [debug] parseProcessorNode: max concurrent tasks => [1] +[2017-05-25 22:30:00.935] [minifi log] [debug] parseProcessorNode: penalization period => [30 sec] +[2017-05-25 22:30:00.935] [minifi log] [debug] parseProcessorNode: yield period => [10 sec] +[2017-05-25 22:30:00.935] [minifi log] [debug] parseProcessorNode: run duration nanos => [] +[2017-05-25 22:30:00.935] [minifi log] [info] Component Input Directory property name Input Directory value /tmp/getfile +[2017-05-25 22:30:00.935] [minifi log] [info] Component Keep Source File property name Keep Source File value true +[2017-05-25 22:30:00.935] [minifi log] [debug] convert: parseProcessorNode: schedulingPeriod => [1000000000] ns +[2017-05-25 22:30:00.935] [minifi log] [debug] convert: parseProcessorNode: penalizationPeriod => [30000] ms +[2017-05-25 22:30:00.935] [minifi log] [debug] setting scheduling strategy as TIMER_DRIVEN +[2017-05-25 22:30:00.935] [minifi log] [debug] parseProcessorNode: maxConcurrentTasks => [1] +[2017-05-25 22:30:00.935] [minifi log] [info] Add processor GetFile into process group MiNiFi Flow +[2017-05-25 22:30:00.935] [minifi log] [debug] parseRemoteProcessGroupYaml: name => [NiFi Flow], id => [471deef6-2a6e-4a7d-912a-81cc17e3a208] +[2017-05-25 22:30:00.935] [minifi log] [debug] parseRemoteProcessGroupYaml: url => [https://localhost:8081/nifi] +[2017-05-25 22:30:00.935] [minifi log] [info] ProcessGroup NiFi Flow created +[2017-05-25 22:30:00.935] [minifi log] [info] Add child process group NiFi Flow into process group MiNiFi Flow +[2017-05-25 22:30:00.935] [minifi log] [debug] parseRemoteProcessGroupYaml: yield period => [10 sec] +[2017-05-25 22:30:00.935] [minifi log] [debug] parseRemoteProcessGroupYaml: yieldPeriod => [10000] ms +[2017-05-25 22:30:00.935] [minifi log] [debug] parseRemoteProcessGroupYaml: timeout => [30 secs] +[2017-05-25 22:30:00.935] [minifi log] [debug] parseRemoteProcessGroupYaml: timeoutValue => [30000] ms +[2017-05-25 22:30:00.935] [minifi log] [debug] Got a current port, iterating... +[2017-05-25 22:30:00.935] [minifi log] [info] Processor From Node A created UUID 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:30:00.935] [minifi log] [info] Processor From Node A supported relationship name undefined +[2017-05-25 22:30:00.935] [minifi log] [info] Component Port property name Port value 10001 +[2017-05-25 22:30:00.935] [minifi log] [info] Component Host Name property name Host Name value localhost +[2017-05-25 22:30:00.935] [minifi log] [info] Component Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:30:00.935] [minifi log] [info] Add processor From Node A into process group NiFi Flow +[2017-05-25 22:30:00.935] [minifi log] [debug] parseProcessorNode: maxConcurrentTasks => [1] +[2017-05-25 22:30:00.935] [minifi log] [info] Connection GenerateFlowFileS2S created +[2017-05-25 22:30:00.935] [minifi log] [debug] Created connection with UUID 471deef6-2a6e-4a7d-912a-81cc17e3a207 and name GenerateFlowFileS2S +[2017-05-25 22:30:00.935] [minifi log] [debug] parseConnection: relationship => [success] +[2017-05-25 22:30:00.935] [minifi log] [debug] Using 'source id' to match source with same id for connection 'GenerateFlowFileS2S': source id => [471deef6-2a6e-4a7d-912a-81cc17e3a206] +[2017-05-25 22:30:00.935] [minifi log] [debug] Using 'destination id' to match destination with same id for connection 'GenerateFlowFileS2S': destination id => [471deef6-2a6e-4a7d-912a-81cc17e3a204] +[2017-05-25 22:30:00.935] [minifi log] [info] Add connection GenerateFlowFileS2S into process group MiNiFi Flow +[2017-05-25 22:30:00.935] [minifi log] [info] find processor GetFile +[2017-05-25 22:30:00.935] [minifi log] [info] Add connection GenerateFlowFileS2S into Processor GetFile outgoing connection for relationship success +[2017-05-25 22:30:00.935] [minifi log] [info] find processor GetFile +[2017-05-25 22:30:00.935] [minifi log] [info] find processor child NiFi Flow +[2017-05-25 22:30:00.935] [minifi log] [info] find processor From Node A +[2017-05-25 22:30:00.935] [minifi log] [info] Add connection GenerateFlowFileS2S into Processor From Node A incoming connection +[2017-05-25 22:30:00.935] [minifi log] [info] Processor SiteToSiteProvenanceReportingTask created UUID d4a7865a-c352-4fff-a918-4c837749a705 +[2017-05-25 22:30:00.935] [minifi log] [info] Add processor SiteToSiteProvenanceReportingTask into process group MiNiFi Flow +[2017-05-25 22:30:00.935] [minifi log] [debug] ProvenanceReportingTask schedulingPeriod 1000000000 ns +[2017-05-25 22:30:00.935] [minifi log] [debug] ProvenanceReportingTask scheduling strategy TIMER_DRIVEN +[2017-05-25 22:30:00.935] [minifi log] [debug] ProvenanceReportingTask host localhost +[2017-05-25 22:30:00.935] [minifi log] [debug] ProvenanceReportingTask port 10001 +[2017-05-25 22:30:00.935] [minifi log] [debug] ProvenanceReportingTask port uuid 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:30:00.935] [minifi log] [info] Starting to reload Flow Controller with flow control name MiNiFi Flow, version 1 +[2017-05-25 22:30:00.935] [minifi log] [info] Stop Flow Controller +[2017-05-25 22:30:00.935] [minifi log] [info] org::apache::nifi::minifi::core::repository::FlowFileRepository Repository Monitor Thread Stop +[2017-05-25 22:30:00.935] [minifi log] [info] org::apache::nifi::minifi::provenance::ProvenanceRepository Repository Monitor Thread Stop +[2017-05-25 22:30:03.939] [minifi log] [info] Shutting down threads for processor GetFile/471deef6-2a6e-4a7d-912a-81cc17e3a206 +[2017-05-25 22:30:03.939] [minifi log] [info] Scheduled thread 0 deleted for process GetFile +[2017-05-25 22:30:03.939] [minifi log] [info] Shutting down threads for processor SiteToSiteProvenanceReportingTask/8c70d581-18f1-46ff-af18-a5957798f2f0 +[2017-05-25 22:30:03.939] [minifi log] [info] Scheduled thread 0 deleted for process SiteToSiteProvenanceReportingTask +[2017-05-25 22:30:03.939] [minifi log] [info] Shutting down threads for processor From Node A/471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:30:03.939] [minifi log] [info] Scheduled thread 0 deleted for process From Node A +[2017-05-25 22:30:03.939] [minifi log] [debug] Getting connection map +[2017-05-25 22:30:03.939] [minifi log] [debug] Number of connections from connectionMap 1 +[2017-05-25 22:30:03.939] [minifi log] [info] Starting Flow Controller +[2017-05-25 22:30:03.939] [minifi log] [info] Enabling 0 controller services +[2017-05-25 22:30:03.939] [minifi log] [debug] Starting GetFile +[2017-05-25 22:30:03.939] [minifi log] [debug] nifi_administrative_yield_duration: [30000] ms +[2017-05-25 22:30:03.939] [minifi log] [debug] nifi_bored_yield_duration: [10] ms +[2017-05-25 22:30:03.939] [minifi log] [info] Processor Input Directory property name Input Directory value /tmp/getfile +[2017-05-25 22:30:03.939] [minifi log] [info] Processor Batch Size property name Batch Size value 10 +[2017-05-25 22:30:03.939] [minifi log] [info] Processor Ignore Hidden Files property name Ignore Hidden Files value true +[2017-05-25 22:30:03.939] [minifi log] [info] Processor Keep Source File property name Keep Source File value true +[2017-05-25 22:30:03.939] [minifi log] [info] Processor Maximum File Age property name Maximum File Age value 0 sec +[2017-05-25 22:30:03.939] [minifi log] [debug] successfully applied _maxAge +[2017-05-25 22:30:03.939] [minifi log] [info] Processor Minimum File Age property name Minimum File Age value 0 sec +[2017-05-25 22:30:03.939] [minifi log] [debug] successfully applied _minAge +[2017-05-25 22:30:03.939] [minifi log] [info] Processor Maximum File Size property name Maximum File Size value 0 B +[2017-05-25 22:30:03.939] [minifi log] [info] Processor Minimum File Size property name Minimum File Size value 0 B +[2017-05-25 22:30:03.939] [minifi log] [info] Processor Polling Interval property name Polling Interval value 0 sec +[2017-05-25 22:30:03.939] [minifi log] [debug] successfully applied _pollInterval +[2017-05-25 22:30:03.939] [minifi log] [info] Processor Recurse Subdirectories property name Recurse Subdirectories value true +[2017-05-25 22:30:03.939] [minifi log] [info] Processor File Filter property name File Filter value [^\.].* +[2017-05-25 22:30:03.939] [minifi log] [info] Is listing empty 1 +[2017-05-25 22:30:03.939] [minifi log] [info] Scheduled thread 0 running for process GetFile +[2017-05-25 22:30:03.939] [minifi log] [info] Performing file listing against /tmp/getfile +[2017-05-25 22:30:03.939] [minifi log] [debug] Starting SiteToSiteProvenanceReportingTask +[2017-05-25 22:30:03.939] [minifi log] [debug] nifi_administrative_yield_duration: [30000] ms +[2017-05-25 22:30:03.939] [minifi log] [debug] nifi_bored_yield_duration: [10] ms +[2017-05-25 22:30:03.939] [minifi log] [info] Performing file listing against /tmp/getfile +[2017-05-25 22:30:03.940] [minifi log] [info] Is listing empty 0 +[2017-05-25 22:30:03.940] [minifi log] [info] GetFile process /tmp/getfile/minifi-app.log.txt +[2017-05-25 22:30:03.940] [minifi log] [info] Scheduled thread 0 running for process SiteToSiteProvenanceReportingTask +[2017-05-25 22:30:03.940] [minifi log] [debug] Starting From Node A +[2017-05-25 22:30:03.940] [minifi log] [debug] nifi_administrative_yield_duration: [30000] ms +[2017-05-25 22:30:03.940] [minifi log] [debug] nifi_bored_yield_duration: [10] ms +[2017-05-25 22:30:03.940] [minifi log] [debug] Create FlowFile with UUID 01bac7f1-ac06-447d-879b-c3065ad5d689 +[2017-05-25 22:30:03.940] [minifi log] [info] Processor Host Name property name Host Name value localhost +[2017-05-25 22:30:03.940] [minifi log] [debug] Resource Claim created ./content_repository/ea1adede-b3e3-4870-ae2a-b3ca33b3314e +[2017-05-25 22:30:03.940] [minifi log] [info] Processor Port property name Port value 10001 +[2017-05-25 22:30:03.940] [minifi log] [info] Processor Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:30:03.940] [minifi log] [info] Scheduled thread 0 running for process From Node A +[2017-05-25 22:30:03.940] [minifi log] [info] org::apache::nifi::minifi::provenance::ProvenanceRepository Repository Monitor Thread Start +[2017-05-25 22:30:03.940] [minifi log] [info] org::apache::nifi::minifi::core::repository::FlowFileRepository Repository Monitor Thread Start +[2017-05-25 22:30:03.940] [minifi log] [info] Started Flow Controller +[2017-05-25 22:30:03.940] [minifi log] [debug] Import offset 0 length 36219 into content ./content_repository/ea1adede-b3e3-4870-ae2a-b3ca33b3314e for FlowFile UUID 01bac7f1-ac06-447d-879b-c3065ad5d689 +[2017-05-25 22:30:03.942] [minifi log] [info] Negotiate protocol version with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 current version 5 +[2017-05-25 22:30:03.942] [minifi log] [info] result of writing resource name is 22 +[2017-05-25 22:30:03.943] [minifi log] [info] status code is 20 +[2017-05-25 22:30:03.943] [minifi log] [info] Site2Site Protocol Negotiate protocol version OK +[2017-05-25 22:30:03.943] [minifi log] [info] Site2Site socket established +[2017-05-25 22:30:03.943] [minifi log] [info] Site2Site Protocol Perform hand shake with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:30:03.943] [minifi log] [info] Site2Site Protocol Send handshake properties GZIP false +[2017-05-25 22:30:03.943] [minifi log] [info] Site2Site Protocol Send handshake properties PORT_IDENTIFIER 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:30:03.943] [minifi log] [info] Site2Site Protocol Send handshake properties REQUEST_EXPIRATION_MILLIS 30000 +[2017-05-25 22:30:03.943] [minifi log] [debug] Enqueue flow file UUID 01bac7f1-ac06-447d-879b-c3065ad5d689 to connection GenerateFlowFileS2S +[2017-05-25 22:30:03.943] [minifi log] [debug] NiFi FlowFile Store event 01bac7f1-ac06-447d-879b-c3065ad5d689 size 337 success +[2017-05-25 22:30:03.943] [minifi log] [debug] Delete SnapShot FlowFile UUID 01bac7f1-ac06-447d-879b-c3065ad5d689 +[2017-05-25 22:30:03.943] [minifi log] [debug] NiFi Provenance Store event 56419305-b78d-4af1-8f71-604a71e2e625 size 471 success +[2017-05-25 22:30:03.943] [minifi log] [debug] NiFi Provenance Store event 0a35c034-6625-4594-b0b2-e2f2b0da517b size 345 success +[2017-05-25 22:30:03.952] [minifi log] [info] Processor Host Name property name Host Name value localhost +[2017-05-25 22:30:03.952] [minifi log] [info] Processor Port property name Port value 10001 +[2017-05-25 22:30:03.952] [minifi log] [info] Processor Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:30:03.954] [minifi log] [info] Site2Site HandShake Completed +[2017-05-25 22:30:03.954] [minifi log] [info] Site2Site Protocol Negotiate Codec with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:30:03.954] [minifi log] [info] Negotiate Codec version with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 current version 1 +[2017-05-25 22:30:03.954] [minifi log] [info] Negotiate protocol version with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 current version 5 +[2017-05-25 22:30:03.954] [minifi log] [info] result of writing resource name is 22 +[2017-05-25 22:30:03.955] [minifi log] [info] status code is 20 +[2017-05-25 22:30:03.955] [minifi log] [info] Site2Site Protocol Negotiate protocol version OK +[2017-05-25 22:30:03.955] [minifi log] [info] Site2Site socket established +[2017-05-25 22:30:03.955] [minifi log] [info] Site2Site Protocol Perform hand shake with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:30:03.955] [minifi log] [info] Site2Site Protocol Send handshake properties GZIP false +[2017-05-25 22:30:03.955] [minifi log] [info] Site2Site Protocol Send handshake properties PORT_IDENTIFIER 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:30:03.955] [minifi log] [info] Site2Site Protocol Send handshake properties REQUEST_EXPIRATION_MILLIS 30000 +[2017-05-25 22:30:03.967] [minifi log] [info] Site2Site Codec Negotiate version OK +[2017-05-25 22:30:03.967] [minifi log] [info] Site2Site Codec Completed and move to READY state for data transfer +[2017-05-25 22:30:03.967] [minifi log] [info] Site2Site Ready For data transaction +[2017-05-25 22:30:03.967] [minifi log] [info] Site2Site create transaction 4562b6cc-567e-4438-932b-b40843af2be9 +[2017-05-25 22:30:03.967] [minifi log] [info] Site2Site transaction 4562b6cc-567e-4438-932b-b40843af2be9 send flow record 1, total length 1904 +[2017-05-25 22:30:03.967] [minifi log] [info] Site2Site transaction 4562b6cc-567e-4438-932b-b40843af2be9 send bytes length 1904 +[2017-05-25 22:30:03.967] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 4562b6cc-567e-4438-932b-b40843af2be9 +[2017-05-25 22:30:03.967] [minifi log] [info] Site2Site HandShake Completed +[2017-05-25 22:30:03.967] [minifi log] [info] Site2Site Protocol Negotiate Codec with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:30:03.968] [minifi log] [info] Negotiate Codec version with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 current version 1 +[2017-05-25 22:30:03.979] [minifi log] [info] Site2Site transaction 4562b6cc-567e-4438-932b-b40843af2be9 peer confirm transaction with CRC 1611766123 +[2017-05-25 22:30:03.979] [minifi log] [info] Site2Site transaction 4562b6cc-567e-4438-932b-b40843af2be9 CRC matched +[2017-05-25 22:30:03.979] [minifi log] [info] Site2Site Codec Negotiate version OK +[2017-05-25 22:30:03.979] [minifi log] [info] Site2Site Codec Completed and move to READY state for data transfer +[2017-05-25 22:30:03.979] [minifi log] [info] Site2Site Ready For data transaction +[2017-05-25 22:30:03.979] [minifi log] [debug] Dequeue flow file UUID 01bac7f1-ac06-447d-879b-c3065ad5d689 from connection GenerateFlowFileS2S +[2017-05-25 22:30:03.979] [minifi log] [debug] Create Snapshot FlowFile with UUID 321e7825-9318-4633-9fab-0927c2f5f3c3 +[2017-05-25 22:30:03.979] [minifi log] [debug] Delete FlowFile UUID 321e7825-9318-4633-9fab-0927c2f5f3c3 +[2017-05-25 22:30:03.979] [minifi log] [info] Site2Site create transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c +[2017-05-25 22:30:03.979] [minifi log] [info] Site2Site transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c send attribute key absolute.path value /tmp/getfile/minifi-app.log.txt +[2017-05-25 22:30:03.979] [minifi log] [info] Site2Site transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c send attribute key filename value minifi-app.log.txt +[2017-05-25 22:30:03.979] [minifi log] [info] Site2Site transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c send attribute key path value /tmp/getfile +[2017-05-25 22:30:03.979] [minifi log] [info] Site2Site transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c send attribute key uuid value 01bac7f1-ac06-447d-879b-c3065ad5d689 +[2017-05-25 22:30:03.980] [minifi log] [info] Site2Site transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c send flow record 1, total length 36219 +[2017-05-25 22:30:03.980] [minifi log] [info] Site2Site transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c send flow record 01bac7f1-ac06-447d-879b-c3065ad5d689 +[2017-05-25 22:30:03.980] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c +[2017-05-25 22:30:03.993] [minifi log] [info] Site2Site transaction 4562b6cc-567e-4438-932b-b40843af2be9 peer finished transaction +[2017-05-25 22:30:03.993] [minifi log] [info] Site2Site transaction 4562b6cc-567e-4438-932b-b40843af2be9 successfully send flow record 1, content bytes 1904 +[2017-05-25 22:30:03.993] [minifi log] [info] Site2Site delete transaction 4562b6cc-567e-4438-932b-b40843af2be9 +[2017-05-25 22:30:03.994] [minifi log] [info] Site2Site transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c peer confirm transaction with CRC 3636446315 +[2017-05-25 22:30:03.994] [minifi log] [info] Site2Site transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c CRC matched +[2017-05-25 22:30:04.005] [minifi log] [info] Site2Site transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c peer finished transaction +[2017-05-25 22:30:04.005] [minifi log] [info] Site2Site transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c successfully send flow record 1, content bytes 36219 +[2017-05-25 22:30:04.005] [minifi log] [info] Site2Site delete transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c +[2017-05-25 22:30:04.005] [minifi log] [debug] Delete FlowFile UUID 01bac7f1-ac06-447d-879b-c3065ad5d689 +[2017-05-25 22:30:04.005] [minifi log] [debug] Delete Resource Claim ./content_repository/ea1adede-b3e3-4870-ae2a-b3ca33b3314e +[2017-05-25 22:30:04.005] [minifi log] [debug] NiFi Provenance Store event bfb69abc-84e6-42a5-9924-0bf5399ce05f size 504 success +[2017-05-25 22:30:04.005] [minifi log] [debug] NiFi Provenance Store event 8d2ad741-7659-476a-8f1f-6e8a1277252d size 550 success +[2017-05-25 22:30:04.948] [minifi log] [info] Is listing empty 1 +[2017-05-25 22:30:04.948] [minifi log] [info] Performing file listing against /tmp/getfile +[2017-05-25 22:30:04.948] [minifi log] [info] Performing file listing against /tmp/getfile +[2017-05-25 22:30:04.948] [minifi log] [info] Is listing empty 0 +[2017-05-25 22:30:04.948] [minifi log] [info] GetFile process /tmp/getfile/minifi-app.log.txt +[2017-05-25 22:30:04.948] [minifi log] [debug] Create FlowFile with UUID 9aac58de-d513-48c5-b2c1-a601ec67f656 +[2017-05-25 22:30:04.948] [minifi log] [debug] Resource Claim created ./content_repository/c06703c6-b16d-4ba6-afa2-ac7057b9e261 +[2017-05-25 22:30:04.948] [minifi log] [debug] Import offset 0 length 36219 into content ./content_repository/c06703c6-b16d-4ba6-afa2-ac7057b9e261 for FlowFile UUID 9aac58de-d513-48c5-b2c1-a601ec67f656 +[2017-05-25 22:30:04.951] [minifi log] [debug] Enqueue flow file UUID 9aac58de-d513-48c5-b2c1-a601ec67f656 to connection GenerateFlowFileS2S +[2017-05-25 22:30:04.951] [minifi log] [debug] NiFi FlowFile Store event 9aac58de-d513-48c5-b2c1-a601ec67f656 size 337 success +[2017-05-25 22:30:04.951] [minifi log] [debug] Delete SnapShot FlowFile UUID 9aac58de-d513-48c5-b2c1-a601ec67f656 +[2017-05-25 22:30:04.951] [minifi log] [debug] NiFi Provenance Store event b0396327-de9a-4403-aa7e-4464586feaf2 size 345 success +[2017-05-25 22:30:04.951] [minifi log] [debug] NiFi Provenance Store event 7316b430-e266-41df-9133-3498ebba5f4b size 471 success +[2017-05-25 22:30:04.959] [minifi log] [info] Processor Host Name property name Host Name value localhost +[2017-05-25 22:30:04.960] [minifi log] [info] Processor Port property name Port value 10001 +[2017-05-25 22:30:04.960] [minifi log] [info] Processor Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:30:04.960] [minifi log] [debug] Dequeue flow file UUID 9aac58de-d513-48c5-b2c1-a601ec67f656 from connection GenerateFlowFileS2S +[2017-05-25 22:30:04.960] [minifi log] [debug] Create Snapshot FlowFile with UUID ecdc657d-db14-4150-a477-06249ee357a7 +[2017-05-25 22:30:04.960] [minifi log] [debug] Delete FlowFile UUID ecdc657d-db14-4150-a477-06249ee357a7 +[2017-05-25 22:30:04.960] [minifi log] [info] Site2Site create transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be +[2017-05-25 22:30:04.960] [minifi log] [info] Site2Site transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be send attribute key absolute.path value /tmp/getfile/minifi-app.log.txt +[2017-05-25 22:30:04.960] [minifi log] [info] Site2Site transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be send attribute key filename value minifi-app.log.txt +[2017-05-25 22:30:04.960] [minifi log] [info] Site2Site transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be send attribute key path value /tmp/getfile +[2017-05-25 22:30:04.960] [minifi log] [info] Site2Site transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be send attribute key uuid value 9aac58de-d513-48c5-b2c1-a601ec67f656 +[2017-05-25 22:30:04.960] [minifi log] [info] Site2Site transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be send flow record 1, total length 36219 +[2017-05-25 22:30:04.960] [minifi log] [info] Site2Site transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be send flow record 9aac58de-d513-48c5-b2c1-a601ec67f656 +[2017-05-25 22:30:04.960] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be +[2017-05-25 22:30:04.967] [minifi log] [info] Site2Site transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be peer confirm transaction with CRC 872125207 +[2017-05-25 22:30:04.967] [minifi log] [info] Site2Site transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be CRC matched +[2017-05-25 22:30:04.979] [minifi log] [info] Site2Site transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be peer finished transaction +[2017-05-25 22:30:04.979] [minifi log] [info] Site2Site transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be successfully send flow record 1, content bytes 36219 +[2017-05-25 22:30:04.979] [minifi log] [info] Site2Site delete transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be +[2017-05-25 22:30:04.979] [minifi log] [debug] Delete FlowFile UUID 9aac58de-d513-48c5-b2c1-a601ec67f656 +[2017-05-25 22:30:04.979] [minifi log] [debug] Delete Resource Claim ./content_repository/c06703c6-b16d-4ba6-afa2-ac7057b9e261 +[2017-05-25 22:30:04.979] [minifi log] [debug] NiFi Provenance Store event 93a745dc-a4ef-48c3-b587-6ccc6db15194 size 550 success +[2017-05-25 22:30:04.979] [minifi log] [debug] NiFi Provenance Store event ba3cf4db-83ff-42a0-a3c3-be09ad69dbc3 size 504 success +[2017-05-25 22:30:04.983] [minifi log] [debug] HttpConfigurationListener load Client Certificates OK +[2017-05-25 22:30:04.998] [minifi log] [info] Site2Site create transaction 35489d64-f920-4be7-b101-a3307c6c6ea4 +[2017-05-25 22:30:04.998] [minifi log] [info] Site2Site transaction 35489d64-f920-4be7-b101-a3307c6c6ea4 send flow record 1, total length 6002 +[2017-05-25 22:30:04.998] [minifi log] [info] Site2Site transaction 35489d64-f920-4be7-b101-a3307c6c6ea4 send bytes length 6002 +[2017-05-25 22:30:04.998] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 35489d64-f920-4be7-b101-a3307c6c6ea4 +[2017-05-25 22:30:05.005] [minifi log] [info] Site2Site transaction 35489d64-f920-4be7-b101-a3307c6c6ea4 peer confirm transaction with CRC 3543481630 +[2017-05-25 22:30:05.005] [minifi log] [info] Site2Site transaction 35489d64-f920-4be7-b101-a3307c6c6ea4 CRC matched +[2017-05-25 22:30:05.017] [minifi log] [info] Site2Site transaction 35489d64-f920-4be7-b101-a3307c6c6ea4 peer finished transaction +[2017-05-25 22:30:05.017] [minifi log] [info] Site2Site transaction 35489d64-f920-4be7-b101-a3307c6c6ea4 successfully send flow record 1, content bytes 6002 +[2017-05-25 22:30:05.017] [minifi log] [info] Site2Site delete transaction 35489d64-f920-4be7-b101-a3307c6c6ea4 +[2017-05-25 22:30:05.088] [minifi log] [debug] HttpConfigurationListener -- curl successful to https://bbende:443 +[2017-05-25 22:30:05.088] [minifi log] [debug] config # Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Flow Controller: + id: 471deef6-2a6e-4a7d-912a-81cc17e3a205 + name: MiNiFi Flow + version: 1 + +Processors: + - name: GetFile + id: 471deef6-2a6e-4a7d-912a-81cc17e3a206 + class: org.apache.nifi.processors.standard.Ge +[2017-05-25 22:30:05.954] [minifi log] [info] Is listing empty 1 +[2017-05-25 22:30:05.954] [minifi log] [info] Performing file listing against /tmp/getfile +[2017-05-25 22:30:05.954] [minifi log] [info] Performing file listing against /tmp/getfile +[2017-05-25 22:30:05.955] [minifi log] [info] Is listing empty 0 +[2017-05-25 22:30:05.955] [minifi log] [info] GetFile process /tmp/getfile/minifi-app.log.txt +[2017-05-25 22:30:05.955] [minifi log] [debug] Create FlowFile with UUID 868a723a-db4f-4ead-b3f4-11450c4cc2d6 +[2017-05-25 22:30:05.955] [minifi log] [debug] Resource Claim created ./content_repository/cd8fa82c-f934-4811-a66b-deba027aa5fb +[2017-05-25 22:30:05.955] [minifi log] [debug] Import offset 0 length 36219 into content ./content_repository/cd8fa82c-f934-4811-a66b-deba027aa5fb for FlowFile UUID 868a723a-db4f-4ead-b3f4-11450c4cc2d6 +[2017-05-25 22:30:05.958] [minifi log] [debug] Enqueue flow file UUID 868a723a-db4f-4ead-b3f4-11450c4cc2d6 to connection GenerateFlowFileS2S +[2017-05-25 22:30:05.958] [minifi log] [debug] NiFi FlowFile Store event 868a723a-db4f-4ead-b3f4-11450c4cc2d6 size 337 success +[2017-05-25 22:30:05.958] [minifi log] [debug] Delete SnapShot FlowFile UUID 868a723a-db4f-4ead-b3f4-11450c4cc2d6 +[2017-05-25 22:30:05.958] [minifi log] [debug] NiFi Provenance Store event df952f23-5eba-444a-b33f-7d756b17c81f size 471 success +[2017-05-25 22:30:05.958] [minifi log] [debug] NiFi Provenance Store event 0b2939fe-6411-428c-954c-72bf5ef7da1c size 345 success +[2017-05-25 22:30:05.965] [minifi log] [info] Processor Host Name property name Host Name value localhost +[2017-05-25 22:30:05.965] [minifi log] [info] Processor Port property name Port value 10001 +[2017-05-25 22:30:05.965] [minifi log] [info] Processor Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:30:05.965] [minifi log] [debug] Dequeue flow file UUID 868a723a-db4f-4ead-b3f4-11450c4cc2d6 from connection GenerateFlowFileS2S +[2017-05-25 22:30:05.965] [minifi log] [debug] Create Snapshot FlowFile with UUID 6480c609-d2db-43fb-a6eb-5e81fd9491ce +[2017-05-25 22:30:05.965] [minifi log] [debug] Delete FlowFile UUID 6480c609-d2db-43fb-a6eb-5e81fd9491ce +[2017-05-25 22:30:05.965] [minifi log] [info] Site2Site create transaction 76819c0c-ecb0-4db8-af40-5b0183710336 +[2017-05-25 22:30:05.965] [minifi log] [info] Site2Site transaction 76819c0c-ecb0-4db8-af40-5b0183710336 send attribute key absolute.path value /tmp/getfile/minifi-app.log.txt +[2017-05-25 22:30:05.965] [minifi log] [info] Site2Site transaction 76819c0c-ecb0-4db8-af40-5b0183710336 send attribute key filename value minifi-app.log.txt +[2017-05-25 22:30:05.965] [minifi log] [info] Site2Site transaction 76819c0c-ecb0-4db8-af40-5b0183710336 send attribute key path value /tmp/getfile +[2017-05-25 22:30:05.965] [minifi log] [info] Site2Site transaction 76819c0c-ecb0-4db8-af40-5b0183710336 send attribute key uuid value 868a723a-db4f-4ead-b3f4-11450c4cc2d6 +[2017-05-25 22:30:05.966] [minifi log] [info] Site2Site transaction 76819c0c-ecb0-4db8-af40-5b0183710336 send flow record 1, total length 36219 +[2017-05-25 22:30:05.966] [minifi log] [info] Site2Site transaction 76819c0c-ecb0-4db8-af40-5b0183710336 send flow record 868a723a-db4f-4ead-b3f4-11450c4cc2d6 +[2017-05-25 22:30:05.966] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 76819c0c-ecb0-4db8-af40-5b0183710336 +[2017-05-25 22:30:05.977] [minifi log] [info] Site2Site transaction 76819c0c-ecb0-4db8-af40-5b0183710336 peer confirm transaction with CRC 2714741500 +[2017-05-25 22:30:05.977] [minifi log] [info] Site2Site transaction 76819c0c-ecb0-4db8-af40-5b0183710336 CRC matched +[2017-05-25 22:30:05.990] [minifi log] [info] Site2Site transaction 76819c0c-ecb0-4db8-af40-5b0183710336 peer finished transaction +[2017-05-25 22:30:05.990] [minifi log] [info] Site2Site transaction 76819c0c-ecb0-4db8-af40-5b0183710336 successfully send flow record 1, content bytes 36219 +[2017-05-25 22:30:05.990] [minifi log] [info] Site2Site delete transaction 76819c0c-ecb0-4db8-af40-5b0183710336 +[2017-05-25 22:30:05.990] [minifi log] [debug] Delete FlowFile UUID 868a723a-db4f-4ead-b3f4-11450c4cc2d6 +[2017-05-25 22:30:05.990] [minifi log] [debug] Delete Resource Claim ./content_repository/cd8fa82c-f934-4811-a66b-deba027aa5fb +[2017-05-25 22:30:05.990] [minifi log] [debug] NiFi Provenance Store event 94bf13c4-fe1d-4408-a827-e41764c5e669 size 550 success +[2017-05-25 22:30:05.990] [minifi log] [debug] NiFi Provenance Store event 8c3d17f4-1b4c-41bd-99e0-0a94a213593d size 504 success +[2017-05-25 22:30:06.021] [minifi log] [info] Site2Site create transaction d1d58f6e-a4cb-4a84-85a4-0ea9715a39c7 +[2017-05-25 22:30:06.021] [minifi log] [info] Site2Site transaction d1d58f6e-a4cb-4a84-85a4-0ea9715a39c7 send flow record 1, total length 3932 +[2017-05-25 22:30:06.021] [minifi log] [info] Site2Site transaction d1d58f6e-a4cb-4a84-85a4-0ea9715a39c7 send bytes length 3932 +[2017-05-25 22:30:06.021] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction d1d58f6e-a4cb-4a84-85a4-0ea9715a39c7 +[2017-05-25 22:30:06.024] [minifi log] [info] Site2Site transaction d1d58f6e-a4cb-4a84-85a4-0ea9715a39c7 peer confirm transaction with CRC 1622701313 +[2017-05-25 22:30:06.024] [minifi log] [info] Site2Site transaction d1d58f6e-a4cb-4a84-85a4-0ea9715a39c7 CRC matched +[2017-05-25 22:30:06.035] [minifi log] [info] Site2Site transaction d1d58f6e-a4cb-4a84-85a4-0ea9715a39c7 peer finished transaction +[2017-05-25 22:30:06.035] [minifi log] [info] Site2Site transaction d1d58f6e-a4cb-4a84-85a4-0ea9715a39c7 successfully send flow record 1, content bytes 3932 +[2017-05-25 22:30:06.035] [minifi log] [info] Site2Site delete transaction d1d58f6e-a4cb-4a84-85a4-0ea9715a39c7 +[2017-05-25 22:30:06.127] [minifi log] [debug] HttpConfigurationListener load Client Certificates OK +[2017-05-25 22:30:06.227] [minifi log] [debug] HttpConfigurationListener -- curl successful to https://bbende:443 +[2017-05-25 22:30:06.227] [minifi log] [debug] config # Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Flow Controller: + id: 471deef6-2a6e-4a7d-912a-81cc17e3a205 + name: MiNiFi Flow + version: 1 + +Processors: + - name: GetFile + id: 471deef6-2a6e-4a7d-912a-81cc17e3a206 + class: org.apache.nifi.processors.standard.Ge +[2017-05-25 22:30:06.963] [minifi log] [info] Is listing empty 1 +[2017-05-25 22:30:06.963] [minifi log] [info] Performing file listing against /tmp/getfile +[2017-05-25 22:30:06.963] [minifi log] [info] Performing file listing against /tmp/getfile +[2017-05-25 22:30:06.963] [minifi log] [info] Is listing empty 0 +[2017-05-25 22:30:06.963] [minifi log] [info] GetFile process /tmp/getfile/minifi-app.log.txt +[2017-05-25 22:30:06.963] [minifi log] [debug] Create FlowFile with UUID 868411c5-96ac-4e81-8dfb-03ba20dcc6bb +[2017-05-25 22:30:06.963] [minifi log] [debug] Resource Claim created ./content_repository/0376d019-f552-432c-b113-b1d4a751a1e7 +[2017-05-25 22:30:06.963] [minifi log] [debug] Import offset 0 length 36219 into content ./content_repository/0376d019-f552-432c-b113-b1d4a751a1e7 for FlowFile UUID 868411c5-96ac-4e81-8dfb-03ba20dcc6bb +[2017-05-25 22:30:06.966] [minifi log] [debug] Enqueue flow file UUID 868411c5-96ac-4e81-8dfb-03ba20dcc6bb to connection GenerateFlowFileS2S +[2017-05-25 22:30:06.966] [minifi log] [debug] NiFi FlowFile Store event 868411c5-96ac-4e81-8dfb-03ba20dcc6bb size 337 success +[2017-05-25 22:30:06.966] [minifi log] [debug] Delete SnapShot FlowFile UUID 868411c5-96ac-4e81-8dfb-03ba20dcc6bb +[2017-05-25 22:30:06.966] [minifi log] [debug] NiFi Provenance Store event 9772dc3c-0710-4a86-96ae-4e892a96162e size 345 success +[2017-05-25 22:30:06.966] [minifi log] [debug] NiFi Provenance Store event 67b9e157-70ff-42bb-b831-296e64766867 size 471 success +[2017-05-25 22:30:06.970] [minifi log] [info] Processor Host Name property name Host Name value localhost +[2017-05-25 22:30:06.970] [minifi log] [info] Processor Port property name Port value 10001 +[2017-05-25 22:30:06.970] [minifi log] [info] Processor Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:30:06.970] [minifi log] [debug] Dequeue flow file UUID 868411c5-96ac-4e81-8dfb-03ba20dcc6bb from connection GenerateFlowFileS2S +[2017-05-25 22:30:06.970] [minifi log] [debug] Create Snapshot FlowFile with UUID 8c8852db-f5b4-4733-b3c4-b2d76f97eebf +[2017-05-25 22:30:06.970] [minifi log] [debug] Delete FlowFile UUID 8c8852db-f5b4-4733-b3c4-b2d76f97eebf +[2017-05-25 22:30:06.970] [minifi log] [info] Site2Site create transaction c52fa4a2-8131-450f-b4f8-3203353622e2 +[2017-05-25 22:30:06.970] [minifi log] [info] Site2Site transaction c52fa4a2-8131-450f-b4f8-3203353622e2 send attribute key absolute.path value /tmp/getfile/minifi-app.log.txt +[2017-05-25 22:30:06.970] [minifi log] [info] Site2Site transaction c52fa4a2-8131-450f-b4f8-3203353622e2 send attribute key filename value minifi-app.log.txt +[2017-05-25 22:30:06.970] [minifi log] [info] Site2Site transaction c52fa4a2-8131-450f-b4f8-3203353622e2 send attribute key path value /tmp/getfile +[2017-05-25 22:30:06.970] [minifi log] [info] Site2Site transaction c52fa4a2-8131-450f-b4f8-3203353622e2 send attribute key uuid value 868411c5-96ac-4e81-8dfb-03ba20dcc6bb +[2017-05-25 22:30:06.971] [minifi log] [info] Site2Site transaction c52fa4a2-8131-450f-b4f8-3203353622e2 send flow record 1, total length 36219 +[2017-05-25 22:30:06.971] [minifi log] [info] Site2Site transaction c52fa4a2-8131-450f-b4f8-3203353622e2 send flow record 868411c5-96ac-4e81-8dfb-03ba20dcc6bb +[2017-05-25 22:30:06.971] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction c52fa4a2-8131-450f-b4f8-3203353622e2 +[2017-05-25 22:30:06.982] [minifi log] [info] Site2Site transaction c52fa4a2-8131-450f-b4f8-3203353622e2 peer confirm transaction with CRC 1979837776 +[2017-05-25 22:30:06.982] [minifi log] [info] Site2Site transaction c52fa4a2-8131-450f-b4f8-3203353622e2 CRC matched +[2017-05-25 22:30:06.993] [minifi log] [info] Site2Site transaction c52fa4a2-8131-450f-b4f8-3203353622e2 peer finished transaction +[2017-05-25 22:30:06.993] [minifi log] [info] Site2Site transaction c52fa4a2-8131-450f-b4f8-3203353622e2 successfully send flow record 1, content bytes 36219 +[2017-05-25 22:30:06.993] [minifi log] [info] Site2Site delete transaction c52fa4a2-8131-450f-b4f8-3203353622e2 +[2017-05-25 22:30:06.993] [minifi log] [debug] Delete FlowFile UUID 868411c5-96ac-4e81-8dfb-03ba20dcc6bb +[2017-05-25 22:30:06.993] [minifi log] [debug] Delete Resource Claim ./content_repository/0376d019-f552-432c-b113-b1d4a751a1e7 +[2017-05-25 22:30:06.993] [minifi log] [debug] NiFi Provenance Store event 32487f33-189f-4d07-bfdc-579e9d3ac8ef size 550 success +[2017-05-25 22:30:06.994] [minifi log] [debug] NiFi Provenance Store event fa13f8c9-b1f1-4ac7-bf67-370937e4c744 size 504 success +[2017-05-25 22:30:07.040] [minifi log] [info] Site2Site create transaction 37ab3c8f-aaba-4d05-a689-4541850320d0 +[2017-05-25 22:30:07.040] [minifi log] [info] Site2Site transaction 37ab3c8f-aaba-4d05-a689-4541850320d0 send flow record 1, total length 3933 +[2017-05-25 22:30:07.040] [minifi log] [info] Site2Site transaction 37ab3c8f-aaba-4d05-a689-4541850320d0 send bytes length 3933 +[2017-05-25 22:30:07.040] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 37ab3c8f-aaba-4d05-a689-4541850320d0 +[2017-05-25 22:30:07.048] [minifi log] [info] Site2Site transaction 37ab3c8f-aaba-4d05-a689-4541850320d0 peer confirm transaction with CRC 3894313933 +[2017-05-25 22:30:07.048] [minifi log] [info] Site2Site transaction 37ab3c8f-aaba-4d05-a689-4541850320d0 CRC matched +[2017-05-25 22:30:07.059] [minifi log] [info] Site2Site transaction 37ab3c8f-aaba-4d05-a689-4541850320d0 peer finished transaction +[2017-05-25 22:30:07.059] [minifi log] [info] Site2Site transaction 37ab3c8f-aaba-4d05-a689-4541850320d0 successfully send flow record 1, content bytes 3933 +[2017-05-25 22:30:07.059] [minifi log] [info] Site2Site delete transaction 37ab3c8f-aaba-4d05-a689-4541850320d0 +[2017-05-25 22:30:07.267] [minifi log] [debug] HttpConfigurationListener load Client Certificates OK +[2017-05-25 22:30:07.367] [minifi log] [debug] HttpConfigurationListener -- curl successful to https://bbende:443 +[2017-05-25 22:30:07.367] [minifi log] [debug] config # Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Flow Controller: + id: 471deef6-2a6e-4a7d-912a-81cc17e3a205 + name: MiNiFi Flow + version: 1 + +Processors: + - name: GetFile + id: 471deef6-2a6e-4a7d-912a-81cc17e3a206 + class: org.apache.nifi.processors.standard.Ge +[2017-05-25 22:30:07.971] [minifi log] [info] Is listing empty 1 +[2017-05-25 22:30:07.971] [minifi log] [info] Performing file listing against /tmp/getfile +[2017-05-25 22:30:07.971] [minifi log] [info] Performing file listing against /tmp/getfile +[2017-05-25 22:30:07.971] [minifi log] [info] Is listing empty 0 +[2017-05-25 22:30:07.971] [minifi log] [info] GetFile process /tmp/getfile/minifi-app.log.txt +[2017-05-25 22:30:07.971] [minifi log] [debug] Create FlowFile with UUID 7a832ebb-d8df-4f20-afad-9eea78c23972 +[2017-05-25 22:30:07.971] [minifi log] [debug] Resource Claim created ./content_repository/4ba970f2-2a03-43a6-8f42-ca153b5690e8 +[2017-05-25 22:30:07.972] [minifi log] [debug] Import offset 0 length 36219 into content ./content_repository/4ba970f2-2a03-43a6-8f42-ca153b5690e8 for FlowFile UUID 7a832ebb-d8df-4f20-afad-9eea78c23972 +[2017-05-25 22:30:07.974] [minifi log] [debug] Enqueue flow file UUID 7a832ebb-d8df-4f20-afad-9eea78c23972 to connection GenerateFlowFileS2S +[2017-05-25 22:30:07.975] [minifi log] [debug] NiFi FlowFile Store event 7a832ebb-d8df-4f20-afad-9eea78c23972 size 337 success +[2017-05-25 22:30:07.975] [minifi log] [debug] Delete SnapShot FlowFile UUID 7a832ebb-d8df-4f20-afad-9eea78c23972 +[2017-05-25 22:30:07.975] [minifi log] [debug] NiFi Provenance Store event cfdee20d-75fb-4070-bd54-6d5393bfaf5a size 471 success +[2017-05-25 22:30:07.975] [minifi log] [debug] NiFi Provenance Store event d82c39d6-ddc8-4522-bfb3-751b11e43d31 size 345 success +[2017-05-25 22:30:07.983] [minifi log] [info] Processor Host Name property name Host Name value localhost +[2017-05-25 22:30:07.983] [minifi log] [info] Processor Port property name Port value 10001 +[2017-05-25 22:30:07.983] [minifi log] [info] Processor Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:30:07.983] [minifi log] [debug] Dequeue flow file UUID 7a832ebb-d8df-4f20-afad-9eea78c23972 from connection GenerateFlowFileS2S +[2017-05-25 22:30:07.984] [minifi log] [debug] Create Snapshot FlowFile with UUID 3221f7e0-c3b4-4d24-9df7-6a7a26868b61 +[2017-05-25 22:30:07.984] [minifi log] [debug] Delete FlowFile UUID 3221f7e0-c3b4-4d24-9df7-6a7a26868b61 +[2017-05-25 22:30:07.984] [minifi log] [info] Site2Site create transaction 998d3839-5b7a-42c3-99a1-696080eece46 +[2017-05-25 22:30:07.984] [minifi log] [info] Site2Site transaction 998d3839-5b7a-42c3-99a1-696080eece46 send attribute key absolute.path value /tmp/getfile/minifi-app.log.txt +[2017-05-25 22:30:07.984] [minifi log] [info] Site2Site transaction 998d3839-5b7a-42c3-99a1-696080eece46 send attribute key filename value minifi-app.log.txt +[2017-05-25 22:30:07.984] [minifi log] [info] Site2Site transaction 998d3839-5b7a-42c3-99a1-696080eece46 send attribute key path value /tmp/getfile +[2017-05-25 22:30:07.984] [minifi log] [info] Site2Site transaction 998d3839-5b7a-42c3-99a1-696080eece46 send attribute key uuid value 7a832ebb-d8df-4f20-afad-9eea78c23972 +[2017-05-25 22:30:07.984] [minifi log] [info] Site2Site transaction 998d3839-5b7a-42c3-99a1-696080eece46 send flow record 1, total length 36219 +[2017-05-25 22:30:07.984] [minifi log] [info] Site2Site transaction 998d3839-5b7a-42c3-99a1-696080eece46 send flow record 7a832ebb-d8df-4f20-afad-9eea78c23972 +[2017-05-25 22:30:07.985] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 998d3839-5b7a-42c3-99a1-696080eece46 +[2017-05-25 22:30:07.992] [minifi log] [info] Site2Site transaction 998d3839-5b7a-42c3-99a1-696080eece46 peer confirm transaction with CRC 2009366010 +[2017-05-25 22:30:07.992] [minifi log] [info] Site2Site transaction 998d3839-5b7a-42c3-99a1-696080eece46 CRC matched +[2017-05-25 22:30:08.003] [minifi log] [info] Site2Site transaction 998d3839-5b7a-42c3-99a1-696080eece46 peer finished transaction +[2017-05-25 22:30:08.003] [minifi log] [info] Site2Site transaction 998d3839-5b7a-42c3-99a1-696080eece46 successfully send flow record 1, content bytes 36219 +[2017-05-25 22:30:08.003] [minifi log] [info] Site2Site delete transaction 998d3839-5b7a-42c3-99a1-696080eece46 +[2017-05-25 22:30:08.003] [minifi log] [debug] Delete FlowFile UUID 7a832ebb-d8df-4f20-afad-9eea78c23972 +[2017-05-25 22:30:08.003] [minifi log] [debug] Delete Resource Claim ./content_repository/4ba970f2-2a03-43a6-8f42-ca153b5690e8 +[2017-05-25 22:30:08.003] [minifi log] [debug] NiFi Provenance Store event b75d9212-7bea-4d14-b933-43bb6056dd6d size 504 success +[2017-05-25 22:30:08.003] [minifi log] [debug] NiFi Provenance Store event 20487c58-481d-44cd-8a21-5814372c774b size 550 success +[2017-05-25 22:30:08.063] [minifi log] [info] Site2Site create transaction d47eb1bc-7f0e-41a2-8b72-8559d4eabcf7 +[2017-05-25 22:30:08.063] [minifi log] [info] Site2Site transaction d47eb1bc-7f0e-41a2-8b72-8559d4eabcf7 send flow record 1, total length 3937 +[2017-05-25 22:30:08.063] [minifi log] [info] Site2Site transaction d47eb1bc-7f0e-41a2-8b72-8559d4eabcf7 send bytes length 3937 +[2017-05-25 22:30:08.063] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction d47eb1bc-7f0e-41a2-8b72-8559d4eabcf7 +[2017-05-25 22:30:08.064] [minifi log] [info] Site2Site transaction d47eb1bc-7f0e-41a2-8b72-8559d4eabcf7 peer confirm transaction with CRC 2316116091 +[2017-05-25 22:30:08.064] [minifi log] [info] Site2Site transaction d47eb1bc-7f0e-41a2-8b72-8559d4eabcf7 CRC matched +[2017-05-25 22:30:08.075] [minifi log] [info] Site2Site transaction d47eb1bc-7f0e-41a2-8b72-8559d4eabcf7 peer finished transaction +[2017-05-25 22:30:08.075] [minifi log] [info] Site2Site transaction d47eb1bc-7f0e-41a2-8b72-8559d4eabcf7 successfully send flow record 1, content bytes 3937 +[2017-05-25 22:30:08.075] [minifi log] [info] Site2Site delete transaction d47eb1bc-7f0e-41a2-8b72-8559d4eabcf7 +[2017-05-25 22:30:08.404] [minifi log] [debug] HttpConfigurationListener load Client Certificates OK +[2017-05-25 22:30:08.501] [minifi log] [debug] HttpConfigurationListener -- curl successful to https://bbende:443 +[2017-05-25 22:30:08.501] [minifi log] [debug] config # Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Flow Controller: + id: 471deef6-2a6e-4a7d-912a-81cc17e3a205 + name: MiNiFi Flow + version: 1 + +Processors: + - name: GetFile + id: 471deef6-2a6e-4a7d-912a-81cc17e3a206 + class: org.apache.nifi.processors.standard.Ge +[2017-05-25 22:30:08.979] [minifi log] [info] Is listing empty 1 +[2017-05-25 22:30:08.979] [minifi log] [info] Performing file listing against /tmp/getfile +[2017-05-25 22:30:08.979] [minifi log] [info] Performing file listing against /tmp/getfile +[2017-05-25 22:30:08.979] [minifi log] [info] Is listing empty 0 +[2017-05-25 22:30:08.979] [minifi log] [info] GetFile process /tmp/getfile/minifi-app.log.txt +[2017-05-25 22:30:08.979] [minifi log] [debug] Create FlowFile with UUID fc099429-0b17-4763-b6fb-d377c19e8e5b +[2017-05-25 22:30:08.979] [minifi log] [debug] Resource Claim created ./content_repository/ea2bbaae-bb8b-418c-8d1f-977acedcebf9 +[2017-05-25 22:30:08.980] [minifi log] [debug] Import offset 0 length 36219 into content ./content_repository/ea2bbaae-bb8b-418c-8d1f-977acedcebf9 for FlowFile UUID fc099429-0b17-4763-b6fb-d377c19e8e5b +[2017-05-25 22:30:08.982] [minifi log] [debug] Enqueue flow file UUID fc099429-0b17-4763-b6fb-d377c19e8e5b to connection GenerateFlowFileS2S +[2017-05-25 22:30:08.982] [minifi log] [debug] NiFi FlowFile Store event fc099429-0b17-4763-b6fb-d377c19e8e5b size 337 success +[2017-05-25 22:30:08.982] [minifi log] [debug] Delete SnapShot FlowFile UUID fc099429-0b17-4763-b6fb-d377c19e8e5b +[2017-05-25 22:30:08.982] [minifi log] [debug] NiFi Provenance Store event 3331f098-290f-4cb2-ba47-931551024568 size 471 success +[2017-05-25 22:30:08.983] [minifi log] [debug] NiFi Provenance Store event fa8337e2-2b83-49b1-b48e-b4db01aacb6b size 345 success +[2017-05-25 22:30:08.989] [minifi log] [info] Processor Host Name property name Host Name value localhost +[2017-05-25 22:30:08.989] [minifi log] [info] Processor Port property name Port value 10001 +[2017-05-25 22:30:08.989] [minifi log] [info] Processor Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:30:08.989] [minifi log] [debug] Dequeue flow file UUID fc099429-0b17-4763-b6fb-d377c19e8e5b from connection GenerateFlowFileS2S +[2017-05-25 22:30:08.990] [minifi log] [debug] Create Snapshot FlowFile with UUID e56d9e2b-b27d-40cd-bd97-3a423e7d3b9f +[2017-05-25 22:30:08.990] [minifi log] [debug] Delete FlowFile UUID e56d9e2b-b27d-40cd-bd97-3a423e7d3b9f +[2017-05-25 22:30:08.990] [minifi log] [info] Site2Site create transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 +[2017-05-25 22:30:08.990] [minifi log] [info] Site2Site transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 send attribute key absolute.path value /tmp/getfile/minifi-app.log.txt +[2017-05-25 22:30:08.990] [minifi log] [info] Site2Site transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 send attribute key filename value minifi-app.log.txt +[2017-05-25 22:30:08.990] [minifi log] [info] Site2Site transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 send attribute key path value /tmp/getfile +[2017-05-25 22:30:08.990] [minifi log] [info] Site2Site transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 send attribute key uuid value fc099429-0b17-4763-b6fb-d377c19e8e5b +[2017-05-25 22:30:08.990] [minifi log] [info] Site2Site transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 send flow record 1, total length 36219 +[2017-05-25 22:30:08.990] [minifi log] [info] Site2Site transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 send flow record fc099429-0b17-4763-b6fb-d377c19e8e5b +[2017-05-25 22:30:08.990] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 +[2017-05-25 22:30:09.001] [minifi log] [info] Site2Site transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 peer confirm transaction with CRC 208752716 +[2017-05-25 22:30:09.001] [minifi log] [info] Site2Site transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 CRC matched +[2017-05-25 22:30:09.013] [minifi log] [info] Site2Site transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 peer finished transaction +[2017-05-25 22:30:09.013] [minifi log] [info] Site2Site transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 successfully send flow record 1, content bytes 36219 +[2017-05-25 22:30:09.013] [minifi log] [info] Site2Site delete transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 +[2017-05-25 22:30:09.013] [minifi log] [debug] Delete FlowFile UUID fc099429-0b17-4763-b6fb-d377c19e8e5b +[2017-05-25 22:30:09.013] [minifi log] [debug] Delete Resource Claim ./content_repository/ea2bbaae-bb8b-418c-8d1f-977acedcebf9 +[2017-05-25 22:30:09.013] [minifi log] [debug] NiFi Provenance Store event 3fb7354f-9f92-4dee-8534-0551e0feecc0 size 550 success +[2017-05-25 22:30:09.013] [minifi log] [debug] NiFi Provenance Store event 6a47d995-0dda-4e7e-9f70-3f77fb5a6358 size 504 success +[2017-05-25 22:30:09.079] [minifi log] [info] Site2Site create transaction dd8dcf42-3a82-4ca8-a19e-dbffc1480fe8 +[2017-05-25 22:30:09.079] [minifi log] [info] Site2Site transaction dd8dcf42-3a82-4ca8-a19e-dbffc1480fe8 send flow record 1, total length 3918 +[2017-05-25 22:30:09.079] [minifi log] [info] Site2Site transaction dd8dcf42-3a82-4ca8-a19e-dbffc1480fe8 send bytes length 3918 +[2017-05-25 22:30:09.079] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction dd8dcf42-3a82-4ca8-a19e-dbffc1480fe8 +[2017-05-25 22:30:09.088] [minifi log] [info] Site2Site transaction dd8dcf42-3a82-4ca8-a19e-dbffc1480fe8 peer confirm transaction with CRC 2743550546 +[2017-05-25 22:30:09.088] [minifi log] [info] Site2Site transaction dd8dcf42-3a82-4ca8-a19e-dbffc1480fe8 CRC matched +[2017-05-25 22:30:09.099] [minifi log] [info] Site2Site transaction dd8dcf42-3a82-4ca8-a19e-dbffc1480fe8 peer finished transaction +[2017-05-25 22:30:09.099] [minifi log] [info] Site2Site transaction dd8dcf42-3a82-4ca8-a19e-dbffc1480fe8 successfully send flow record 1, content bytes 3918 +[2017-05-25 22:30:09.099] [minifi log] [info] Site2Site delete transaction dd8dcf42-3a82-4ca8-a19e-dbffc1480fe8 +[2017-05-25 22:30:09.537] [minifi log] [debug] HttpConfigurationListener load Client Certificates OK +[2017-05-25 22:30:09.629] [minifi log] [info] Stop Flow Controller +[2017-05-25 22:30:09.629] [minifi log] [info] org::apache::nifi::minifi::core::repository::FlowFileRepository Repository Monitor Thread Stop +[2017-05-25 22:30:09.629] [minifi log] [info] org::apache::nifi::minifi::provenance::ProvenanceRepository Repository Monitor Thread Stop +[2017-05-25 22:30:09.639] [minifi log] [debug] HttpConfigurationListener -- curl successful to https://bbende:443 +[2017-05-25 22:30:09.639] [minifi log] [debug] config # Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Flow Controller: + id: 471deef6-2a6e-4a7d-912a-81cc17e3a205 + name: MiNiFi Flow + version: 1 + +Processors: + - name: GetFile + id: 471deef6-2a6e-4a7d-912a-81cc17e3a206 + class: org.apache.nifi.processors.standard.Ge +[2017-05-25 22:30:10.678] [minifi log] [debug] HttpConfigurationListener load Client Certificates OK +[2017-05-25 22:30:10.782] [minifi log] [debug] HttpConfigurationListener -- curl successful to https://bbende:443 +[2017-05-25 22:30:10.782] [minifi log] [debug] config # Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Flow Controller: + id: 471deef6-2a6e-4a7d-912a-81cc17e3a205 + name: MiNiFi Flow + version: 1 + +Processors: + - name: GetFile + id: 471deef6-2a6e-4a7d-912a-81cc17e3a206 + class: org.apache.nifi.processors.standard.Ge +[2017-05-25 22:30:11.813] [minifi log] [debug] HttpConfigurationListener load Client Certificates OK +[2017-05-25 22:30:11.929] [minifi log] [debug] HttpConfigurationListener -- curl successful to https://bbende:443 +[2017-05-25 22:30:11.929] [minifi log] [debug] config # Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Flow Controller: + id: 471deef6-2a6e-4a7d-912a-81cc17e3a205 + name: MiNiFi Flow + version: 1 + +Processors: + - name: GetFile + id: 471deef6-2a6e-4a7d-912a-81cc17e3a206 + class: org.apache.nifi.processors.standard.Ge +[2017-05-25 22:30:12.632] [minifi log] [info] Shutting down threads for processor GetFile/471deef6-2a6e-4a7d-912a-81cc17e3a206 +[2017-05-25 22:30:12.632] [minifi log] [info] Scheduled thread 0 deleted for process GetFile +[2017-05-25 22:30:12.632] [minifi log] [info] Shutting down threads for processor SiteToSiteProvenanceReportingTask/d4a7865a-c352-4fff-a918-4c837749a705 +[2017-05-25 22:30:12.632] [minifi log] [info] Scheduled thread 0 deleted for process SiteToSiteProvenanceReportingTask +[2017-05-25 22:30:12.632] [minifi log] [info] Shutting down threads for processor From Node A/471deef6-2a6e-4a7d-912a-81cc17e3a204 +[2017-05-25 22:30:12.632] [minifi log] [info] Scheduled thread 0 deleted for process From Node A +[2017-05-25 22:30:12.632] [minifi log] [info] Unload Flow Controller +[2017-05-25 22:30:12.632] [minifi log] [debug] Drain connection GenerateFlowFileS2S +[2017-05-25 22:30:12.632] [minifi log] [info] Site2Site Protocol tearDown +[2017-05-25 22:30:12.633] [minifi log] [info] MiNiFi exit From c6e9b716bc1f291d1822c9b75b6381281ba208cc Mon Sep 17 00:00:00 2001 From: Bin Qiu Date: Sun, 4 Jun 2017 10:15:25 -0700 Subject: [PATCH 07/11] Address review comments --- libminifi/include/ConfigurationListener.h | 6 +- libminifi/include/FlowControlProtocol.h | 4 - libminifi/include/FlowController.h | 28 ++---- libminifi/include/HttpConfigurationListener.h | 24 +---- libminifi/include/processors/InvokeHTTP.h | 58 +---------- libminifi/include/utils/HTTPUtils.h | 97 +++++++++++++++++++ libminifi/src/ConfigurationListener.cpp | 8 +- libminifi/src/FlowController.cpp | 8 +- libminifi/src/HttpConfigurationListener.cpp | 4 +- libminifi/src/processors/InvokeHTTP.cpp | 16 +-- 10 files changed, 127 insertions(+), 126 deletions(-) create mode 100644 libminifi/include/utils/HTTPUtils.h diff --git a/libminifi/include/ConfigurationListener.h b/libminifi/include/ConfigurationListener.h index e381fafb30..0ca438a943 100644 --- a/libminifi/include/ConfigurationListener.h +++ b/libminifi/include/ConfigurationListener.h @@ -47,7 +47,7 @@ class ConfigurationListener { /*! * Create a new processor */ - ConfigurationListener(FlowController *controller, + ConfigurationListener(std::shared_ptr controller, std::shared_ptr configure, std::string type) : connect_timeout_(20000), read_timeout_(20000), type_(type), configure_( configure), controller_(controller), need_client_certificate_(false) { @@ -87,7 +87,7 @@ class ConfigurationListener { // thread std::thread thread_; // whether the thread is running - bool running_; + std::atomic running_; // url std::string url_; @@ -104,7 +104,7 @@ class ConfigurationListener { std::shared_ptr configure_; std::shared_ptr logger_; - FlowController *controller_; + std::shared_ptr controller_; bool need_client_certificate_; std::string certificate_; diff --git a/libminifi/include/FlowControlProtocol.h b/libminifi/include/FlowControlProtocol.h index 8992049af8..c0781b87b5 100644 --- a/libminifi/include/FlowControlProtocol.h +++ b/libminifi/include/FlowControlProtocol.h @@ -218,10 +218,6 @@ class FlowControlProtocol { } // Run function for the thread static void run(FlowControlProtocol *protocol); - // set 8 bytes SerialNumber - void setSerialNumber(uint8_t *number) { - memcpy(_serialNumber, number, 8); - } protected: diff --git a/libminifi/include/FlowController.h b/libminifi/include/FlowController.h index 8450182aaa..6ea802cd39 100644 --- a/libminifi/include/FlowController.h +++ b/libminifi/include/FlowController.h @@ -130,28 +130,14 @@ class FlowController : public core::controller::ControllerServiceProvider, publi root_->updatePropertyValue(processorName, propertyName, propertyValue); } - // set 6 bytes SerialNumber - void setSerialNumber(uint8_t *number) { - memcpy(serial_number_, number, 6); - } - - //get serial number - uint8_t *getSerialNumber() { - return serial_number_; + // set SerialNumber + void setSerialNumber(std::string number) { + serial_number_ = number; } // get serial number as string - std::string getSerialNumberStr() { - char str[18]; - char *strPtr = &str[0]; - - for (int i = 0; i < 6; i++) { - strPtr += sprintf(strPtr, "%02x", serial_number_[i]); - if (i != 5) - strPtr += sprintf(strPtr, ":"); - } - std::string value(str); - return value; + std::string getSerialNumber() { + return serial_number_; } // validate and apply passing yaml configuration payload @@ -338,9 +324,7 @@ class FlowController : public core::controller::ControllerServiceProvider, publi std::shared_ptr logger_; // http configuration listener object. std::unique_ptr http_configuration_listener_; - - // Serial Number - uint8_t serial_number_[6]; + std::string serial_number_; }; } /* namespace minifi */ diff --git a/libminifi/include/HttpConfigurationListener.h b/libminifi/include/HttpConfigurationListener.h index 3d3a52afa1..72d47284aa 100644 --- a/libminifi/include/HttpConfigurationListener.h +++ b/libminifi/include/HttpConfigurationListener.h @@ -23,32 +23,12 @@ #include "core/Core.h" #include "core/Property.h" #include "ConfigurationListener.h" +#include "utils/HTTPUtils.h" namespace org { namespace apache { namespace nifi { namespace minifi { -/** - * HTTP Response object - */ -struct HTTPRequestResponse { - std::vector data; - - /** - * Receive HTTP Response. - */ - static size_t recieve_write(char * data, size_t size, size_t nmemb, - void * p) { - return static_cast(p)->write_content(data, size, - nmemb); - } - - size_t write_content(char* ptr, size_t size, size_t nmemb) { - data.insert(data.end(), ptr, ptr + size * nmemb); - return size * nmemb; - } - -}; // HttpConfigurationListener Class class HttpConfigurationListener: public ConfigurationListener { @@ -58,7 +38,7 @@ class HttpConfigurationListener: public ConfigurationListener { /*! * Create a new processor */ - HttpConfigurationListener(FlowController *controller, + HttpConfigurationListener(std::shared_ptr controller, std::shared_ptr configure) : minifi::ConfigurationListener(controller, configure, "http") { std::string value; diff --git a/libminifi/include/processors/InvokeHTTP.h b/libminifi/include/processors/InvokeHTTP.h index ab78fd5347..c8e0c106fc 100644 --- a/libminifi/include/processors/InvokeHTTP.h +++ b/libminifi/include/processors/InvokeHTTP.h @@ -32,6 +32,7 @@ #include "controllers/SSLContextService.h" #include "utils/ByteInputCallBack.h" #include "core/logging/LoggerConfiguration.h" +#include "utils/HTTPUtils.h" namespace org { namespace apache { @@ -39,63 +40,6 @@ namespace nifi { namespace minifi { namespace processors { -struct CallBackPosition { - utils::ByteInputCallBack *ptr; - size_t pos; -}; - -/** - * HTTP Response object - */ -struct HTTPRequestResponse { - std::vector data; - - /** - * Receive HTTP Response. - */ - static size_t recieve_write(char * data, size_t size, size_t nmemb, void * p) { - return static_cast(p)->write_content(data, size, nmemb); - } - - /** - * Callback for post, put, and patch operations - * @param buffer - * @param size size of buffer - * @param nitems items to add - * @param insteam input stream object. - */ - - static size_t send_write(char * data, size_t size, size_t nmemb, void * p) { - if (p != 0) { - CallBackPosition *callback = (CallBackPosition*) p; - if (callback->pos <= callback->ptr->getBufferSize()) { - char *ptr = callback->ptr->getBuffer(); - int len = callback->ptr->getBufferSize() - callback->pos; - if (len <= 0) { - delete callback->ptr; - delete callback; - return 0; - } - if (len > size * nmemb) - len = size * nmemb; - memcpy(data, callback->ptr->getBuffer() + callback->pos, len); - callback->pos += len; - return len; - } - } else { - return CURL_READFUNC_ABORT; - } - - return 0; - } - - size_t write_content(char* ptr, size_t size, size_t nmemb) { - data.insert(data.end(), ptr, ptr + size * nmemb); - return size * nmemb; - } - -}; - // InvokeHTTP Class class InvokeHTTP : public core::Processor { public: diff --git a/libminifi/include/utils/HTTPUtils.h b/libminifi/include/utils/HTTPUtils.h new file mode 100644 index 0000000000..3f20f5e77c --- /dev/null +++ b/libminifi/include/utils/HTTPUtils.h @@ -0,0 +1,97 @@ +/** + * HTTPUtils class declaration + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __HTTP_UTILS_H__ +#define __HTTP_UTILS_H__ + +#include +#include +#include "ByteInputCallBack.h" + +namespace org { +namespace apache { +namespace nifi { +namespace minifi { +namespace utils { + +struct CallBackPosition { + ByteInputCallBack *ptr; + size_t pos; +}; + +/** + * HTTP Response object + */ +struct HTTPRequestResponse { + std::vector data; + + /** + * Receive HTTP Response. + */ + static size_t recieve_write(char * data, size_t size, size_t nmemb, + void * p) { + return static_cast(p)->write_content(data, size, + nmemb); + } + + /** + * Callback for post, put, and patch operations + * @param buffer + * @param size size of buffer + * @param nitems items to add + * @param insteam input stream object. + */ + + static size_t send_write(char * data, size_t size, size_t nmemb, void * p) { + if (p != 0) { + CallBackPosition *callback = (CallBackPosition*) p; + if (callback->pos <= callback->ptr->getBufferSize()) { + char *ptr = callback->ptr->getBuffer(); + int len = callback->ptr->getBufferSize() - callback->pos; + if (len <= 0) { + delete callback->ptr; + delete callback; + return 0; + } + if (len > size * nmemb) + len = size * nmemb; + memcpy(data, callback->ptr->getBuffer() + callback->pos, len); + callback->pos += len; + return len; + } + } else { + return CURL_READFUNC_ABORT; + } + + return 0; + } + + size_t write_content(char* ptr, size_t size, size_t nmemb) { + data.insert(data.end(), ptr, ptr + size * nmemb); + return size * nmemb; + } + +}; + +} /* namespace utils */ +} /* namespace minifi */ +} /* namespace nifi */ +} /* namespace apache */ +} /* namespace org */ + +#endif diff --git a/libminifi/src/ConfigurationListener.cpp b/libminifi/src/ConfigurationListener.cpp index b024a74069..b60c050050 100644 --- a/libminifi/src/ConfigurationListener.cpp +++ b/libminifi/src/ConfigurationListener.cpp @@ -55,7 +55,7 @@ void ConfigurationListener::start() { Configure::nifi_configuration_listener_client_ca_certificate, this->ca_certificate_)) { logger_->log_info("Configuration Listener CA certificates: [%s]", - this->ca_certificate_.c_str()); + this->ca_certificate_); } if (this->need_client_certificate_) { @@ -83,13 +83,13 @@ void ConfigurationListener::start() { } logger_->log_info("Configuration Listener certificate: [%s], private key: [%s], passphrase file: [%s]", - this->certificate_.c_str(), this->private_key_.c_str(), passphrase_file.c_str()); + this->certificate_, this->private_key_, passphrase_file); } thread_ = std::thread(&ConfigurationListener::threadExecutor, this); thread_.detach(); running_ = true; - logger_->log_info("%s ConfigurationListener Thread Start", type_.c_str()); + logger_->log_info("%s ConfigurationListener Thread Start", type_); } void ConfigurationListener::stop() { @@ -98,7 +98,7 @@ void ConfigurationListener::stop() { running_ = false; if (thread_.joinable()) thread_.join(); - logger_->log_info("%s ConfigurationListener Thread Stop", type_.c_str()); + logger_->log_info("%s ConfigurationListener Thread Stop", type_); } void ConfigurationListener::run() { diff --git a/libminifi/src/FlowController.cpp b/libminifi/src/FlowController.cpp index acd364ec16..d09f031f49 100644 --- a/libminifi/src/FlowController.cpp +++ b/libminifi/src/FlowController.cpp @@ -85,12 +85,7 @@ FlowController::FlowController(std::shared_ptr provenance_repo max_timer_driven_threads_ = DEFAULT_MAX_TIMER_DRIVEN_THREAD; running_ = false; initialized_ = false; -<<<<<<< HEAD root_ = nullptr; -======= - root_ = NULL; - memset(serial_number_, 0, 6); ->>>>>>> More configuration listener protocol_ = new FlowControlProtocol(this, configure); @@ -98,7 +93,8 @@ FlowController::FlowController(std::shared_ptr provenance_repo // grab the value for configuration if (configure->get(Configure::nifi_configuration_listener_type, listenerType)) { if (listenerType == "http") { - this->http_configuration_listener_ = std::unique_ptr(new minifi::HttpConfigurationListener(this, configure)); + this->http_configuration_listener_ = + std::unique_ptr(new minifi::HttpConfigurationListener(shared_from_this(), configure)); } } diff --git a/libminifi/src/HttpConfigurationListener.cpp b/libminifi/src/HttpConfigurationListener.cpp index 7dad6ed358..70d5793251 100644 --- a/libminifi/src/HttpConfigurationListener.cpp +++ b/libminifi/src/HttpConfigurationListener.cpp @@ -120,9 +120,9 @@ bool HttpConfigurationListener::pullConfiguration(std::string &configuration) { configureSecureConnection(http_session); } - HTTPRequestResponse content; + utils::HTTPRequestResponse content; curl_easy_setopt(http_session, CURLOPT_WRITEFUNCTION, - &HTTPRequestResponse::recieve_write); + &utils::HTTPRequestResponse::recieve_write); curl_easy_setopt(http_session, CURLOPT_WRITEDATA, static_cast(&content)); diff --git a/libminifi/src/processors/InvokeHTTP.cpp b/libminifi/src/processors/InvokeHTTP.cpp index fd39a64b97..c636201016 100644 --- a/libminifi/src/processors/InvokeHTTP.cpp +++ b/libminifi/src/processors/InvokeHTTP.cpp @@ -315,8 +315,9 @@ void InvokeHTTP::onTrigger(core::ProcessContext *context, core::ProcessSession * if (read_timeout_ > 0) { curl_easy_setopt(http_session, CURLOPT_TIMEOUT, read_timeout_); } - HTTPRequestResponse content; - curl_easy_setopt(http_session, CURLOPT_WRITEFUNCTION, &HTTPRequestResponse::recieve_write); + utils::HTTPRequestResponse content; + curl_easy_setopt(http_session, CURLOPT_WRITEFUNCTION, + &utils::HTTPRequestResponse::recieve_write); curl_easy_setopt(http_session, CURLOPT_WRITEDATA, static_cast(&content)); @@ -326,14 +327,17 @@ void InvokeHTTP::onTrigger(core::ProcessContext *context, core::ProcessSession * if (claim) { utils::ByteInputCallBack *callback = new utils::ByteInputCallBack(); session->read(flowFile, callback); - CallBackPosition *callbackObj = new CallBackPosition; + utils::CallBackPosition *callbackObj = new utils::CallBackPosition; callbackObj->ptr = callback; callbackObj->pos = 0; logger_->log_info("InvokeHTTP -- Setting callback"); curl_easy_setopt(http_session, CURLOPT_UPLOAD, 1L); - curl_easy_setopt(http_session, CURLOPT_INFILESIZE_LARGE, (curl_off_t)callback->getBufferSize()); - curl_easy_setopt(http_session, CURLOPT_READFUNCTION, &HTTPRequestResponse::send_write); - curl_easy_setopt(http_session, CURLOPT_READDATA, static_cast(callbackObj)); + curl_easy_setopt(http_session, CURLOPT_INFILESIZE_LARGE, + (curl_off_t)callback->getBufferSize()); + curl_easy_setopt(http_session, CURLOPT_READFUNCTION, + &utils::HTTPRequestResponse::send_write); + curl_easy_setopt(http_session, CURLOPT_READDATA, + static_cast(callbackObj)); } else { logger_->log_error("InvokeHTTP -- no resource claim"); } From 337378a162d91c6c43c821297dddfa7b374673cf Mon Sep 17 00:00:00 2001 From: Bin Qiu Date: Sun, 4 Jun 2017 11:10:27 -0700 Subject: [PATCH 08/11] merge --- libminifi/include/ConfigurationListener.h | 3 ++- libminifi/include/properties/Configure.h | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/libminifi/include/ConfigurationListener.h b/libminifi/include/ConfigurationListener.h index 0ca438a943..5574226025 100644 --- a/libminifi/include/ConfigurationListener.h +++ b/libminifi/include/ConfigurationListener.h @@ -32,6 +32,7 @@ #include "core/Property.h" #include "properties/Configure.h" #include "core/logging/Logger.h" +#include "core/logging/LoggerConfiguration.h" namespace org { namespace apache { @@ -51,7 +52,7 @@ class ConfigurationListener { std::shared_ptr configure, std::string type) : connect_timeout_(20000), read_timeout_(20000), type_(type), configure_( configure), controller_(controller), need_client_certificate_(false) { - logger_ = logging::Logger::getLogger(); + logger_ = logging::LoggerFactory::getLogger(); running_ = false; } // Destructor diff --git a/libminifi/include/properties/Configure.h b/libminifi/include/properties/Configure.h index 422f51992e..fa19a187fb 100644 --- a/libminifi/include/properties/Configure.h +++ b/libminifi/include/properties/Configure.h @@ -63,6 +63,12 @@ class Configure : public Properties { static const char *nifi_configuration_listener_http_url; static const char *nifi_configuration_listener_rest_url; static const char *nifi_configuration_listener_type; // http or rest + // configuration listener security config + static const char *nifi_configuration_listener_need_ClientAuth; + static const char *nifi_configuration_listener_client_certificate; + static const char *nifi_configuration_listener_private_key; + static const char *nifi_configuration_listener_client_pass_phrase; + static const char *nifi_configuration_listener_client_ca_certificate; }; } /* namespace minifi */ From 380cef43a378909e1d35f1666c81c75aaae425d7 Mon Sep 17 00:00:00 2001 From: Bin Qiu Date: Sun, 4 Jun 2017 11:18:32 -0700 Subject: [PATCH 09/11] remove uncessary file --- libminifi/test/minifi-app.log.txt | 0 minifi-app.log.txt | 856 ------------------------------ 2 files changed, 856 deletions(-) delete mode 100644 libminifi/test/minifi-app.log.txt delete mode 100644 minifi-app.log.txt diff --git a/libminifi/test/minifi-app.log.txt b/libminifi/test/minifi-app.log.txt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/minifi-app.log.txt b/minifi-app.log.txt deleted file mode 100644 index 3b2a4d3af4..0000000000 --- a/minifi-app.log.txt +++ /dev/null @@ -1,856 +0,0 @@ -[2017-05-25 22:29:59.723] [minifi log] [info] Using configuration file located at /Users/binqiu/today/nifi-minifi-cpp/conf/minifi.properties -[2017-05-25 22:29:59.724] [minifi log] [debug] nifi.flowcontroller.graceful.shutdown.period not set, defaulting to 30000 -[2017-05-25 22:29:59.724] [minifi log] [info] NiFi Provenance Repository Directory ./provenance_repository -[2017-05-25 22:29:59.724] [minifi log] [info] NiFi Provenance Max Partition Bytes 1048576 -[2017-05-25 22:29:59.724] [minifi log] [info] NiFi Provenance Max Storage Time: [60000] ms -[2017-05-25 22:29:59.746] [minifi log] [info] NiFi Provenance Repository database open ./provenance_repository success -[2017-05-25 22:29:59.746] [minifi log] [info] NiFi FlowFile Repository Directory ./flowfile_repository -[2017-05-25 22:29:59.746] [minifi log] [info] NiFi FlowFile Max Partition Bytes 10485760 -[2017-05-25 22:29:59.746] [minifi log] [info] NiFi FlowFile Max Storage Time: [600000] ms -[2017-05-25 22:29:59.754] [minifi log] [info] NiFi FlowFile Repository database open ./flowfile_repository success -[2017-05-25 22:29:59.754] [minifi log] [info] Http configuration listener URL https://bbende:443 -[2017-05-25 22:29:59.758] [minifi log] [info] Configuration Listener pull interval: [1000] ms -[2017-05-25 22:29:59.758] [minifi log] [info] Configuration Listener CA certificates: [./conf/nifi-cert.pem] -[2017-05-25 22:29:59.758] [minifi log] [info] Configuration Listener certificate: [./conf/client.pem], private key: [./conf/client.key], passphrase file: [./conf/password] -[2017-05-25 22:29:59.758] [minifi log] [info] http ConfigurationListener Thread Start -[2017-05-25 22:29:59.758] [minifi log] [info] FlowController NiFi Configuration file /Users/binqiu/today/nifi-minifi-cpp/conf/config.yml -[2017-05-25 22:29:59.758] [minifi log] [info] FlowController content directory /Users/binqiu/today/nifi-minifi-cpp/content_repository -[2017-05-25 22:29:59.758] [minifi log] [info] Loading FlowController -[2017-05-25 22:29:59.758] [minifi log] [info] Initializing timers -[2017-05-25 22:29:59.759] [minifi log] [info] Load Flow Controller from file /Users/binqiu/today/nifi-minifi-cpp/conf/config.yml -[2017-05-25 22:29:59.761] [minifi log] [debug] parseRootProcessGroup: id => [471deef6-2a6e-4a7d-912a-81cc17e3a205], name => [MiNiFi Flow] -[2017-05-25 22:29:59.761] [minifi log] [info] ProcessGroup MiNiFi Flow created -[2017-05-25 22:29:59.761] [minifi log] [debug] parseProcessorNode: name => [GetFile] id => [471deef6-2a6e-4a7d-912a-81cc17e3a206] -[2017-05-25 22:29:59.761] [minifi log] [debug] parseProcessorNode: class => [org.apache.nifi.processors.standard.GetFile] -[2017-05-25 22:29:59.761] [minifi log] [info] Processor GetFile created UUID 471deef6-2a6e-4a7d-912a-81cc17e3a206 -[2017-05-25 22:29:59.762] [minifi log] [info] Processor GetFile supported relationship name success -[2017-05-25 22:29:59.762] [minifi log] [debug] parseProcessorNode: scheduling strategy => [TIMER_DRIVEN] -[2017-05-25 22:29:59.762] [minifi log] [debug] parseProcessorNode: scheduling period => [1 sec] -[2017-05-25 22:29:59.762] [minifi log] [debug] parseProcessorNode: max concurrent tasks => [1] -[2017-05-25 22:29:59.762] [minifi log] [debug] parseProcessorNode: penalization period => [30 sec] -[2017-05-25 22:29:59.762] [minifi log] [debug] parseProcessorNode: yield period => [10 sec] -[2017-05-25 22:29:59.762] [minifi log] [debug] parseProcessorNode: run duration nanos => [] -[2017-05-25 22:29:59.762] [minifi log] [info] Component Input Directory property name Input Directory value /tmp/getfile -[2017-05-25 22:29:59.762] [minifi log] [info] Component Keep Source File property name Keep Source File value true -[2017-05-25 22:29:59.762] [minifi log] [debug] convert: parseProcessorNode: schedulingPeriod => [1000000000] ns -[2017-05-25 22:29:59.762] [minifi log] [debug] convert: parseProcessorNode: penalizationPeriod => [30000] ms -[2017-05-25 22:29:59.762] [minifi log] [debug] setting scheduling strategy as TIMER_DRIVEN -[2017-05-25 22:29:59.762] [minifi log] [debug] parseProcessorNode: maxConcurrentTasks => [1] -[2017-05-25 22:29:59.762] [minifi log] [info] Add processor GetFile into process group MiNiFi Flow -[2017-05-25 22:29:59.762] [minifi log] [debug] parseRemoteProcessGroupYaml: name => [NiFi Flow], id => [471deef6-2a6e-4a7d-912a-81cc17e3a208] -[2017-05-25 22:29:59.762] [minifi log] [debug] parseRemoteProcessGroupYaml: url => [https://localhost:8081/nifi] -[2017-05-25 22:29:59.762] [minifi log] [info] ProcessGroup NiFi Flow created -[2017-05-25 22:29:59.762] [minifi log] [info] Add child process group NiFi Flow into process group MiNiFi Flow -[2017-05-25 22:29:59.762] [minifi log] [debug] parseRemoteProcessGroupYaml: yield period => [10 sec] -[2017-05-25 22:29:59.762] [minifi log] [debug] parseRemoteProcessGroupYaml: yieldPeriod => [10000] ms -[2017-05-25 22:29:59.762] [minifi log] [debug] parseRemoteProcessGroupYaml: timeout => [30 secs] -[2017-05-25 22:29:59.762] [minifi log] [debug] parseRemoteProcessGroupYaml: timeoutValue => [30000] ms -[2017-05-25 22:29:59.762] [minifi log] [debug] Got a current port, iterating... -[2017-05-25 22:29:59.762] [minifi log] [info] Processor From Node A created UUID 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:29:59.762] [minifi log] [info] Processor From Node A supported relationship name undefined -[2017-05-25 22:29:59.762] [minifi log] [info] Component Port property name Port value 10001 -[2017-05-25 22:29:59.762] [minifi log] [info] Component Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:29:59.762] [minifi log] [info] Component Host Name property name Host Name value localhost -[2017-05-25 22:29:59.762] [minifi log] [info] Add processor From Node A into process group NiFi Flow -[2017-05-25 22:29:59.762] [minifi log] [debug] parseProcessorNode: maxConcurrentTasks => [1] -[2017-05-25 22:29:59.762] [minifi log] [info] Connection GenerateFlowFileS2S created -[2017-05-25 22:29:59.762] [minifi log] [debug] Created connection with UUID 471deef6-2a6e-4a7d-912a-81cc17e3a207 and name GenerateFlowFileS2S -[2017-05-25 22:29:59.762] [minifi log] [debug] parseConnection: relationship => [success] -[2017-05-25 22:29:59.762] [minifi log] [debug] Using 'source id' to match source with same id for connection 'GenerateFlowFileS2S': source id => [471deef6-2a6e-4a7d-912a-81cc17e3a206] -[2017-05-25 22:29:59.762] [minifi log] [debug] Using 'destination id' to match destination with same id for connection 'GenerateFlowFileS2S': destination id => [471deef6-2a6e-4a7d-912a-81cc17e3a204] -[2017-05-25 22:29:59.762] [minifi log] [info] Add connection GenerateFlowFileS2S into process group MiNiFi Flow -[2017-05-25 22:29:59.762] [minifi log] [info] find processor GetFile -[2017-05-25 22:29:59.762] [minifi log] [info] Add connection GenerateFlowFileS2S into Processor GetFile outgoing connection for relationship success -[2017-05-25 22:29:59.762] [minifi log] [info] find processor GetFile -[2017-05-25 22:29:59.762] [minifi log] [info] find processor child NiFi Flow -[2017-05-25 22:29:59.762] [minifi log] [info] find processor From Node A -[2017-05-25 22:29:59.762] [minifi log] [info] Add connection GenerateFlowFileS2S into Processor From Node A incoming connection -[2017-05-25 22:29:59.762] [minifi log] [info] Processor SiteToSiteProvenanceReportingTask created UUID 8c70d581-18f1-46ff-af18-a5957798f2f0 -[2017-05-25 22:29:59.762] [minifi log] [info] Add processor SiteToSiteProvenanceReportingTask into process group MiNiFi Flow -[2017-05-25 22:29:59.762] [minifi log] [debug] ProvenanceReportingTask schedulingPeriod 1000000000 ns -[2017-05-25 22:29:59.762] [minifi log] [debug] ProvenanceReportingTask scheduling strategy TIMER_DRIVEN -[2017-05-25 22:29:59.763] [minifi log] [debug] ProvenanceReportingTask host localhost -[2017-05-25 22:29:59.763] [minifi log] [debug] ProvenanceReportingTask port 10001 -[2017-05-25 22:29:59.763] [minifi log] [debug] ProvenanceReportingTask port uuid 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:29:59.763] [minifi log] [info] Loaded root processor Group -[2017-05-25 22:29:59.763] [minifi log] [info] Loaded controller service provider -[2017-05-25 22:29:59.763] [minifi log] [debug] Getting connection map -[2017-05-25 22:29:59.763] [minifi log] [debug] Number of connections from connectionMap 1 -[2017-05-25 22:29:59.763] [minifi log] [info] Loaded flow repository -[2017-05-25 22:29:59.763] [minifi log] [info] Starting Flow Controller -[2017-05-25 22:29:59.763] [minifi log] [info] Enabling 0 controller services -[2017-05-25 22:29:59.763] [minifi log] [debug] Starting GetFile -[2017-05-25 22:29:59.763] [minifi log] [debug] nifi_administrative_yield_duration: [30000] ms -[2017-05-25 22:29:59.763] [minifi log] [debug] nifi_bored_yield_duration: [10] ms -[2017-05-25 22:29:59.763] [minifi log] [info] Processor Input Directory property name Input Directory value /tmp/getfile -[2017-05-25 22:29:59.763] [minifi log] [info] Processor Batch Size property name Batch Size value 10 -[2017-05-25 22:29:59.763] [minifi log] [info] Processor Ignore Hidden Files property name Ignore Hidden Files value true -[2017-05-25 22:29:59.763] [minifi log] [info] Processor Keep Source File property name Keep Source File value true -[2017-05-25 22:29:59.763] [minifi log] [info] Processor Maximum File Age property name Maximum File Age value 0 sec -[2017-05-25 22:29:59.763] [minifi log] [debug] successfully applied _maxAge -[2017-05-25 22:29:59.763] [minifi log] [info] Processor Minimum File Age property name Minimum File Age value 0 sec -[2017-05-25 22:29:59.763] [minifi log] [debug] successfully applied _minAge -[2017-05-25 22:29:59.763] [minifi log] [info] Processor Maximum File Size property name Maximum File Size value 0 B -[2017-05-25 22:29:59.763] [minifi log] [info] Processor Minimum File Size property name Minimum File Size value 0 B -[2017-05-25 22:29:59.763] [minifi log] [info] Processor Polling Interval property name Polling Interval value 0 sec -[2017-05-25 22:29:59.763] [minifi log] [debug] successfully applied _pollInterval -[2017-05-25 22:29:59.763] [minifi log] [info] Processor Recurse Subdirectories property name Recurse Subdirectories value true -[2017-05-25 22:29:59.763] [minifi log] [info] Processor File Filter property name File Filter value [^\.].* -[2017-05-25 22:29:59.763] [minifi log] [info] Scheduled thread 0 running for process GetFile -[2017-05-25 22:29:59.763] [minifi log] [debug] Starting SiteToSiteProvenanceReportingTask -[2017-05-25 22:29:59.763] [minifi log] [debug] nifi_administrative_yield_duration: [30000] ms -[2017-05-25 22:29:59.763] [minifi log] [debug] nifi_bored_yield_duration: [10] ms -[2017-05-25 22:29:59.763] [minifi log] [info] Is listing empty 1 -[2017-05-25 22:29:59.763] [minifi log] [info] Performing file listing against /tmp/getfile -[2017-05-25 22:29:59.763] [minifi log] [info] Scheduled thread 0 running for process SiteToSiteProvenanceReportingTask -[2017-05-25 22:29:59.763] [minifi log] [debug] Starting From Node A -[2017-05-25 22:29:59.763] [minifi log] [info] Performing file listing against /tmp/getfile -[2017-05-25 22:29:59.763] [minifi log] [debug] nifi_administrative_yield_duration: [30000] ms -[2017-05-25 22:29:59.763] [minifi log] [debug] nifi_bored_yield_duration: [10] ms -[2017-05-25 22:29:59.764] [minifi log] [info] Processor Host Name property name Host Name value localhost -[2017-05-25 22:29:59.764] [minifi log] [info] Processor Port property name Port value 10001 -[2017-05-25 22:29:59.764] [minifi log] [info] Processor Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:29:59.764] [minifi log] [info] Scheduled thread 0 running for process From Node A -[2017-05-25 22:29:59.764] [minifi log] [info] org::apache::nifi::minifi::provenance::ProvenanceRepository Repository Monitor Thread Start -[2017-05-25 22:29:59.764] [minifi log] [info] org::apache::nifi::minifi::core::repository::FlowFileRepository Repository Monitor Thread Start -[2017-05-25 22:29:59.764] [minifi log] [info] Started Flow Controller -[2017-05-25 22:29:59.764] [minifi log] [info] MiNiFi started -[2017-05-25 22:29:59.764] [minifi log] [info] Is listing empty 0 -[2017-05-25 22:29:59.764] [minifi log] [info] GetFile process /tmp/getfile/minifi-app.log.txt -[2017-05-25 22:29:59.764] [minifi log] [debug] Create FlowFile with UUID 06e4a308-a065-460e-97a1-75ba8b46efaa -[2017-05-25 22:29:59.764] [minifi log] [debug] Resource Claim created ./content_repository/ce638c86-375d-4213-9be7-1b6e33f8cf0b -[2017-05-25 22:29:59.765] [minifi log] [debug] Import offset 0 length 36219 into content ./content_repository/ce638c86-375d-4213-9be7-1b6e33f8cf0b for FlowFile UUID 06e4a308-a065-460e-97a1-75ba8b46efaa -[2017-05-25 22:29:59.766] [minifi log] [info] Negotiate protocol version with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 current version 5 -[2017-05-25 22:29:59.766] [minifi log] [info] result of writing resource name is 22 -[2017-05-25 22:29:59.767] [minifi log] [info] status code is 20 -[2017-05-25 22:29:59.767] [minifi log] [info] Site2Site Protocol Negotiate protocol version OK -[2017-05-25 22:29:59.767] [minifi log] [info] Site2Site socket established -[2017-05-25 22:29:59.767] [minifi log] [info] Site2Site Protocol Perform hand shake with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:29:59.767] [minifi log] [info] Site2Site Protocol Send handshake properties GZIP false -[2017-05-25 22:29:59.767] [minifi log] [info] Site2Site Protocol Send handshake properties PORT_IDENTIFIER 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:29:59.767] [minifi log] [info] Site2Site Protocol Send handshake properties REQUEST_EXPIRATION_MILLIS 30000 -[2017-05-25 22:29:59.768] [minifi log] [debug] Enqueue flow file UUID 06e4a308-a065-460e-97a1-75ba8b46efaa to connection GenerateFlowFileS2S -[2017-05-25 22:29:59.768] [minifi log] [debug] NiFi FlowFile Store event 06e4a308-a065-460e-97a1-75ba8b46efaa size 337 success -[2017-05-25 22:29:59.768] [minifi log] [debug] Delete SnapShot FlowFile UUID 06e4a308-a065-460e-97a1-75ba8b46efaa -[2017-05-25 22:29:59.768] [minifi log] [debug] NiFi Provenance Store event 6ba5ea25-689f-4654-85c4-2875f28ca15d size 345 success -[2017-05-25 22:29:59.768] [minifi log] [debug] NiFi Provenance Store event 68fec583-55bc-43c9-98bd-06453b0179ef size 471 success -[2017-05-25 22:29:59.776] [minifi log] [info] Processor Host Name property name Host Name value localhost -[2017-05-25 22:29:59.776] [minifi log] [info] Processor Port property name Port value 10001 -[2017-05-25 22:29:59.776] [minifi log] [info] Processor Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:29:59.778] [minifi log] [info] Site2Site HandShake Completed -[2017-05-25 22:29:59.778] [minifi log] [info] Negotiate protocol version with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 current version 5 -[2017-05-25 22:29:59.778] [minifi log] [info] Site2Site Protocol Negotiate Codec with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:29:59.778] [minifi log] [info] result of writing resource name is 22 -[2017-05-25 22:29:59.778] [minifi log] [info] Negotiate Codec version with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 current version 1 -[2017-05-25 22:29:59.779] [minifi log] [info] status code is 20 -[2017-05-25 22:29:59.779] [minifi log] [info] Site2Site Protocol Negotiate protocol version OK -[2017-05-25 22:29:59.779] [minifi log] [info] Site2Site socket established -[2017-05-25 22:29:59.779] [minifi log] [info] Site2Site Protocol Perform hand shake with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:29:59.779] [minifi log] [info] Site2Site Protocol Send handshake properties GZIP false -[2017-05-25 22:29:59.779] [minifi log] [info] Site2Site Protocol Send handshake properties PORT_IDENTIFIER 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:29:59.779] [minifi log] [info] Site2Site Protocol Send handshake properties REQUEST_EXPIRATION_MILLIS 30000 -[2017-05-25 22:29:59.790] [minifi log] [info] Site2Site HandShake Completed -[2017-05-25 22:29:59.791] [minifi log] [info] Site2Site Codec Negotiate version OK -[2017-05-25 22:29:59.791] [minifi log] [info] Site2Site Protocol Negotiate Codec with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:29:59.791] [minifi log] [info] Site2Site Codec Completed and move to READY state for data transfer -[2017-05-25 22:29:59.791] [minifi log] [info] Site2Site Ready For data transaction -[2017-05-25 22:29:59.791] [minifi log] [info] Negotiate Codec version with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 current version 1 -[2017-05-25 22:29:59.791] [minifi log] [info] Site2Site create transaction ea140105-e7a4-464b-8784-ab8b270bfa5f -[2017-05-25 22:29:59.791] [minifi log] [info] Site2Site transaction ea140105-e7a4-464b-8784-ab8b270bfa5f send flow record 1, total length 1902 -[2017-05-25 22:29:59.791] [minifi log] [info] Site2Site transaction ea140105-e7a4-464b-8784-ab8b270bfa5f send bytes length 1902 -[2017-05-25 22:29:59.791] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction ea140105-e7a4-464b-8784-ab8b270bfa5f -[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site Codec Negotiate version OK -[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site Codec Completed and move to READY state for data transfer -[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site Ready For data transaction -[2017-05-25 22:29:59.803] [minifi log] [debug] Dequeue flow file UUID 06e4a308-a065-460e-97a1-75ba8b46efaa from connection GenerateFlowFileS2S -[2017-05-25 22:29:59.803] [minifi log] [debug] Create Snapshot FlowFile with UUID 45ba7411-95e9-402a-a1b1-5196c831dcdf -[2017-05-25 22:29:59.803] [minifi log] [debug] Delete FlowFile UUID 45ba7411-95e9-402a-a1b1-5196c831dcdf -[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site create transaction 08d59ed6-daf6-4250-bb91-be486de22108 -[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site transaction 08d59ed6-daf6-4250-bb91-be486de22108 send attribute key absolute.path value /tmp/getfile/minifi-app.log.txt -[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site transaction 08d59ed6-daf6-4250-bb91-be486de22108 send attribute key filename value minifi-app.log.txt -[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site transaction 08d59ed6-daf6-4250-bb91-be486de22108 send attribute key path value /tmp/getfile -[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site transaction 08d59ed6-daf6-4250-bb91-be486de22108 send attribute key uuid value 06e4a308-a065-460e-97a1-75ba8b46efaa -[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site transaction ea140105-e7a4-464b-8784-ab8b270bfa5f peer confirm transaction with CRC 239634877 -[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site transaction ea140105-e7a4-464b-8784-ab8b270bfa5f CRC matched -[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site transaction 08d59ed6-daf6-4250-bb91-be486de22108 send flow record 1, total length 36219 -[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site transaction 08d59ed6-daf6-4250-bb91-be486de22108 send flow record 06e4a308-a065-460e-97a1-75ba8b46efaa -[2017-05-25 22:29:59.803] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 08d59ed6-daf6-4250-bb91-be486de22108 -[2017-05-25 22:29:59.815] [minifi log] [info] Site2Site transaction ea140105-e7a4-464b-8784-ab8b270bfa5f peer finished transaction -[2017-05-25 22:29:59.815] [minifi log] [info] Site2Site transaction ea140105-e7a4-464b-8784-ab8b270bfa5f successfully send flow record 1, content bytes 1902 -[2017-05-25 22:29:59.815] [minifi log] [info] Site2Site delete transaction ea140105-e7a4-464b-8784-ab8b270bfa5f -[2017-05-25 22:29:59.816] [minifi log] [info] Site2Site transaction 08d59ed6-daf6-4250-bb91-be486de22108 peer confirm transaction with CRC 2452005643 -[2017-05-25 22:29:59.816] [minifi log] [info] Site2Site transaction 08d59ed6-daf6-4250-bb91-be486de22108 CRC matched -[2017-05-25 22:29:59.827] [minifi log] [info] Site2Site transaction 08d59ed6-daf6-4250-bb91-be486de22108 peer finished transaction -[2017-05-25 22:29:59.827] [minifi log] [info] Site2Site transaction 08d59ed6-daf6-4250-bb91-be486de22108 successfully send flow record 1, content bytes 36219 -[2017-05-25 22:29:59.827] [minifi log] [info] Site2Site delete transaction 08d59ed6-daf6-4250-bb91-be486de22108 -[2017-05-25 22:29:59.827] [minifi log] [debug] Delete FlowFile UUID 06e4a308-a065-460e-97a1-75ba8b46efaa -[2017-05-25 22:29:59.827] [minifi log] [debug] Delete Resource Claim ./content_repository/ce638c86-375d-4213-9be7-1b6e33f8cf0b -[2017-05-25 22:29:59.828] [minifi log] [debug] NiFi Provenance Store event d6c555da-ff88-406e-930d-f216570e7464 size 504 success -[2017-05-25 22:29:59.828] [minifi log] [debug] NiFi Provenance Store event c85bc571-8882-43fa-99ab-3eb29e274b7a size 550 success -[2017-05-25 22:30:00.770] [minifi log] [info] Is listing empty 1 -[2017-05-25 22:30:00.770] [minifi log] [info] Performing file listing against /tmp/getfile -[2017-05-25 22:30:00.770] [minifi log] [info] Performing file listing against /tmp/getfile -[2017-05-25 22:30:00.770] [minifi log] [info] Is listing empty 0 -[2017-05-25 22:30:00.770] [minifi log] [info] GetFile process /tmp/getfile/minifi-app.log.txt -[2017-05-25 22:30:00.770] [minifi log] [debug] Create FlowFile with UUID 03ce4edc-c0a9-45f7-9793-cbcc12f3e8c6 -[2017-05-25 22:30:00.770] [minifi log] [debug] Resource Claim created ./content_repository/157415c6-ef77-4004-9f9a-b1df8c6ce54c -[2017-05-25 22:30:00.771] [minifi log] [debug] Import offset 0 length 36219 into content ./content_repository/157415c6-ef77-4004-9f9a-b1df8c6ce54c for FlowFile UUID 03ce4edc-c0a9-45f7-9793-cbcc12f3e8c6 -[2017-05-25 22:30:00.774] [minifi log] [debug] Enqueue flow file UUID 03ce4edc-c0a9-45f7-9793-cbcc12f3e8c6 to connection GenerateFlowFileS2S -[2017-05-25 22:30:00.774] [minifi log] [debug] NiFi FlowFile Store event 03ce4edc-c0a9-45f7-9793-cbcc12f3e8c6 size 337 success -[2017-05-25 22:30:00.774] [minifi log] [debug] Delete SnapShot FlowFile UUID 03ce4edc-c0a9-45f7-9793-cbcc12f3e8c6 -[2017-05-25 22:30:00.774] [minifi log] [debug] NiFi Provenance Store event 042b7639-c864-46c2-a8b3-33e076c343c7 size 471 success -[2017-05-25 22:30:00.774] [minifi log] [debug] NiFi Provenance Store event 0c8d2130-7c79-4ea8-97be-9f3c92b93e38 size 345 success -[2017-05-25 22:30:00.778] [minifi log] [info] Processor Host Name property name Host Name value localhost -[2017-05-25 22:30:00.778] [minifi log] [info] Processor Port property name Port value 10001 -[2017-05-25 22:30:00.778] [minifi log] [info] Processor Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:30:00.778] [minifi log] [debug] Dequeue flow file UUID 03ce4edc-c0a9-45f7-9793-cbcc12f3e8c6 from connection GenerateFlowFileS2S -[2017-05-25 22:30:00.778] [minifi log] [debug] Create Snapshot FlowFile with UUID 5f216db1-abeb-4edc-aab5-7db36d33f5c3 -[2017-05-25 22:30:00.778] [minifi log] [debug] Delete FlowFile UUID 5f216db1-abeb-4edc-aab5-7db36d33f5c3 -[2017-05-25 22:30:00.778] [minifi log] [info] Site2Site create transaction 2307ba45-6195-40ed-9a32-83ac59b16609 -[2017-05-25 22:30:00.778] [minifi log] [info] Site2Site transaction 2307ba45-6195-40ed-9a32-83ac59b16609 send attribute key absolute.path value /tmp/getfile/minifi-app.log.txt -[2017-05-25 22:30:00.778] [minifi log] [info] Site2Site transaction 2307ba45-6195-40ed-9a32-83ac59b16609 send attribute key filename value minifi-app.log.txt -[2017-05-25 22:30:00.778] [minifi log] [info] Site2Site transaction 2307ba45-6195-40ed-9a32-83ac59b16609 send attribute key path value /tmp/getfile -[2017-05-25 22:30:00.778] [minifi log] [info] Site2Site transaction 2307ba45-6195-40ed-9a32-83ac59b16609 send attribute key uuid value 03ce4edc-c0a9-45f7-9793-cbcc12f3e8c6 -[2017-05-25 22:30:00.778] [minifi log] [info] Site2Site transaction 2307ba45-6195-40ed-9a32-83ac59b16609 send flow record 1, total length 36219 -[2017-05-25 22:30:00.778] [minifi log] [info] Site2Site transaction 2307ba45-6195-40ed-9a32-83ac59b16609 send flow record 03ce4edc-c0a9-45f7-9793-cbcc12f3e8c6 -[2017-05-25 22:30:00.778] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 2307ba45-6195-40ed-9a32-83ac59b16609 -[2017-05-25 22:30:00.790] [minifi log] [info] Site2Site transaction 2307ba45-6195-40ed-9a32-83ac59b16609 peer confirm transaction with CRC 1532199305 -[2017-05-25 22:30:00.790] [minifi log] [info] Site2Site transaction 2307ba45-6195-40ed-9a32-83ac59b16609 CRC matched -[2017-05-25 22:30:00.802] [minifi log] [info] Site2Site transaction 2307ba45-6195-40ed-9a32-83ac59b16609 peer finished transaction -[2017-05-25 22:30:00.802] [minifi log] [info] Site2Site transaction 2307ba45-6195-40ed-9a32-83ac59b16609 successfully send flow record 1, content bytes 36219 -[2017-05-25 22:30:00.802] [minifi log] [info] Site2Site delete transaction 2307ba45-6195-40ed-9a32-83ac59b16609 -[2017-05-25 22:30:00.802] [minifi log] [debug] Delete FlowFile UUID 03ce4edc-c0a9-45f7-9793-cbcc12f3e8c6 -[2017-05-25 22:30:00.802] [minifi log] [debug] Delete Resource Claim ./content_repository/157415c6-ef77-4004-9f9a-b1df8c6ce54c -[2017-05-25 22:30:00.802] [minifi log] [debug] NiFi Provenance Store event a84e3e3c-f886-4ea3-9a28-b04388e08e88 size 550 success -[2017-05-25 22:30:00.803] [minifi log] [debug] NiFi Provenance Store event 6d959d3e-a18f-4964-9d3b-b8816909af10 size 504 success -[2017-05-25 22:30:00.819] [minifi log] [info] Site2Site create transaction 3758fa47-3a50-49f6-98b1-869ab08e0a6b -[2017-05-25 22:30:00.819] [minifi log] [info] Site2Site transaction 3758fa47-3a50-49f6-98b1-869ab08e0a6b send flow record 1, total length 5982 -[2017-05-25 22:30:00.819] [minifi log] [info] Site2Site transaction 3758fa47-3a50-49f6-98b1-869ab08e0a6b send bytes length 5982 -[2017-05-25 22:30:00.819] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 3758fa47-3a50-49f6-98b1-869ab08e0a6b -[2017-05-25 22:30:00.824] [minifi log] [info] Site2Site transaction 3758fa47-3a50-49f6-98b1-869ab08e0a6b peer confirm transaction with CRC 2217791887 -[2017-05-25 22:30:00.824] [minifi log] [info] Site2Site transaction 3758fa47-3a50-49f6-98b1-869ab08e0a6b CRC matched -[2017-05-25 22:30:00.833] [minifi log] [debug] HttpConfigurationListener load Client Certificates OK -[2017-05-25 22:30:00.837] [minifi log] [info] Site2Site transaction 3758fa47-3a50-49f6-98b1-869ab08e0a6b peer finished transaction -[2017-05-25 22:30:00.837] [minifi log] [info] Site2Site transaction 3758fa47-3a50-49f6-98b1-869ab08e0a6b successfully send flow record 1, content bytes 5982 -[2017-05-25 22:30:00.837] [minifi log] [info] Site2Site delete transaction 3758fa47-3a50-49f6-98b1-869ab08e0a6b -[2017-05-25 22:30:00.933] [minifi log] [debug] HttpConfigurationListener -- curl successful to https://bbende:443 -[2017-05-25 22:30:00.933] [minifi log] [debug] config # Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Flow Controller: - id: 471deef6-2a6e-4a7d-912a-81cc17e3a205 - name: MiNiFi Flow - version: 1 - -Processors: - - name: GetFile - id: 471deef6-2a6e-4a7d-912a-81cc17e3a206 - class: org.apache.nifi.processors.standard.Ge -[2017-05-25 22:30:00.934] [minifi log] [debug] parseRootProcessorGroup: version => [1] -[2017-05-25 22:30:00.934] [minifi log] [debug] parseRootProcessGroup: id => [471deef6-2a6e-4a7d-912a-81cc17e3a205], name => [MiNiFi Flow] -[2017-05-25 22:30:00.934] [minifi log] [info] ProcessGroup MiNiFi Flow created -[2017-05-25 22:30:00.934] [minifi log] [debug] parseProcessorNode: name => [GetFile] id => [471deef6-2a6e-4a7d-912a-81cc17e3a206] -[2017-05-25 22:30:00.934] [minifi log] [debug] parseProcessorNode: class => [org.apache.nifi.processors.standard.GetFile] -[2017-05-25 22:30:00.934] [minifi log] [info] Processor GetFile created UUID 471deef6-2a6e-4a7d-912a-81cc17e3a206 -[2017-05-25 22:30:00.935] [minifi log] [info] Processor GetFile supported relationship name success -[2017-05-25 22:30:00.935] [minifi log] [debug] parseProcessorNode: scheduling strategy => [TIMER_DRIVEN] -[2017-05-25 22:30:00.935] [minifi log] [debug] parseProcessorNode: scheduling period => [1 sec] -[2017-05-25 22:30:00.935] [minifi log] [debug] parseProcessorNode: max concurrent tasks => [1] -[2017-05-25 22:30:00.935] [minifi log] [debug] parseProcessorNode: penalization period => [30 sec] -[2017-05-25 22:30:00.935] [minifi log] [debug] parseProcessorNode: yield period => [10 sec] -[2017-05-25 22:30:00.935] [minifi log] [debug] parseProcessorNode: run duration nanos => [] -[2017-05-25 22:30:00.935] [minifi log] [info] Component Input Directory property name Input Directory value /tmp/getfile -[2017-05-25 22:30:00.935] [minifi log] [info] Component Keep Source File property name Keep Source File value true -[2017-05-25 22:30:00.935] [minifi log] [debug] convert: parseProcessorNode: schedulingPeriod => [1000000000] ns -[2017-05-25 22:30:00.935] [minifi log] [debug] convert: parseProcessorNode: penalizationPeriod => [30000] ms -[2017-05-25 22:30:00.935] [minifi log] [debug] setting scheduling strategy as TIMER_DRIVEN -[2017-05-25 22:30:00.935] [minifi log] [debug] parseProcessorNode: maxConcurrentTasks => [1] -[2017-05-25 22:30:00.935] [minifi log] [info] Add processor GetFile into process group MiNiFi Flow -[2017-05-25 22:30:00.935] [minifi log] [debug] parseRemoteProcessGroupYaml: name => [NiFi Flow], id => [471deef6-2a6e-4a7d-912a-81cc17e3a208] -[2017-05-25 22:30:00.935] [minifi log] [debug] parseRemoteProcessGroupYaml: url => [https://localhost:8081/nifi] -[2017-05-25 22:30:00.935] [minifi log] [info] ProcessGroup NiFi Flow created -[2017-05-25 22:30:00.935] [minifi log] [info] Add child process group NiFi Flow into process group MiNiFi Flow -[2017-05-25 22:30:00.935] [minifi log] [debug] parseRemoteProcessGroupYaml: yield period => [10 sec] -[2017-05-25 22:30:00.935] [minifi log] [debug] parseRemoteProcessGroupYaml: yieldPeriod => [10000] ms -[2017-05-25 22:30:00.935] [minifi log] [debug] parseRemoteProcessGroupYaml: timeout => [30 secs] -[2017-05-25 22:30:00.935] [minifi log] [debug] parseRemoteProcessGroupYaml: timeoutValue => [30000] ms -[2017-05-25 22:30:00.935] [minifi log] [debug] Got a current port, iterating... -[2017-05-25 22:30:00.935] [minifi log] [info] Processor From Node A created UUID 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:30:00.935] [minifi log] [info] Processor From Node A supported relationship name undefined -[2017-05-25 22:30:00.935] [minifi log] [info] Component Port property name Port value 10001 -[2017-05-25 22:30:00.935] [minifi log] [info] Component Host Name property name Host Name value localhost -[2017-05-25 22:30:00.935] [minifi log] [info] Component Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:30:00.935] [minifi log] [info] Add processor From Node A into process group NiFi Flow -[2017-05-25 22:30:00.935] [minifi log] [debug] parseProcessorNode: maxConcurrentTasks => [1] -[2017-05-25 22:30:00.935] [minifi log] [info] Connection GenerateFlowFileS2S created -[2017-05-25 22:30:00.935] [minifi log] [debug] Created connection with UUID 471deef6-2a6e-4a7d-912a-81cc17e3a207 and name GenerateFlowFileS2S -[2017-05-25 22:30:00.935] [minifi log] [debug] parseConnection: relationship => [success] -[2017-05-25 22:30:00.935] [minifi log] [debug] Using 'source id' to match source with same id for connection 'GenerateFlowFileS2S': source id => [471deef6-2a6e-4a7d-912a-81cc17e3a206] -[2017-05-25 22:30:00.935] [minifi log] [debug] Using 'destination id' to match destination with same id for connection 'GenerateFlowFileS2S': destination id => [471deef6-2a6e-4a7d-912a-81cc17e3a204] -[2017-05-25 22:30:00.935] [minifi log] [info] Add connection GenerateFlowFileS2S into process group MiNiFi Flow -[2017-05-25 22:30:00.935] [minifi log] [info] find processor GetFile -[2017-05-25 22:30:00.935] [minifi log] [info] Add connection GenerateFlowFileS2S into Processor GetFile outgoing connection for relationship success -[2017-05-25 22:30:00.935] [minifi log] [info] find processor GetFile -[2017-05-25 22:30:00.935] [minifi log] [info] find processor child NiFi Flow -[2017-05-25 22:30:00.935] [minifi log] [info] find processor From Node A -[2017-05-25 22:30:00.935] [minifi log] [info] Add connection GenerateFlowFileS2S into Processor From Node A incoming connection -[2017-05-25 22:30:00.935] [minifi log] [info] Processor SiteToSiteProvenanceReportingTask created UUID d4a7865a-c352-4fff-a918-4c837749a705 -[2017-05-25 22:30:00.935] [minifi log] [info] Add processor SiteToSiteProvenanceReportingTask into process group MiNiFi Flow -[2017-05-25 22:30:00.935] [minifi log] [debug] ProvenanceReportingTask schedulingPeriod 1000000000 ns -[2017-05-25 22:30:00.935] [minifi log] [debug] ProvenanceReportingTask scheduling strategy TIMER_DRIVEN -[2017-05-25 22:30:00.935] [minifi log] [debug] ProvenanceReportingTask host localhost -[2017-05-25 22:30:00.935] [minifi log] [debug] ProvenanceReportingTask port 10001 -[2017-05-25 22:30:00.935] [minifi log] [debug] ProvenanceReportingTask port uuid 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:30:00.935] [minifi log] [info] Starting to reload Flow Controller with flow control name MiNiFi Flow, version 1 -[2017-05-25 22:30:00.935] [minifi log] [info] Stop Flow Controller -[2017-05-25 22:30:00.935] [minifi log] [info] org::apache::nifi::minifi::core::repository::FlowFileRepository Repository Monitor Thread Stop -[2017-05-25 22:30:00.935] [minifi log] [info] org::apache::nifi::minifi::provenance::ProvenanceRepository Repository Monitor Thread Stop -[2017-05-25 22:30:03.939] [minifi log] [info] Shutting down threads for processor GetFile/471deef6-2a6e-4a7d-912a-81cc17e3a206 -[2017-05-25 22:30:03.939] [minifi log] [info] Scheduled thread 0 deleted for process GetFile -[2017-05-25 22:30:03.939] [minifi log] [info] Shutting down threads for processor SiteToSiteProvenanceReportingTask/8c70d581-18f1-46ff-af18-a5957798f2f0 -[2017-05-25 22:30:03.939] [minifi log] [info] Scheduled thread 0 deleted for process SiteToSiteProvenanceReportingTask -[2017-05-25 22:30:03.939] [minifi log] [info] Shutting down threads for processor From Node A/471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:30:03.939] [minifi log] [info] Scheduled thread 0 deleted for process From Node A -[2017-05-25 22:30:03.939] [minifi log] [debug] Getting connection map -[2017-05-25 22:30:03.939] [minifi log] [debug] Number of connections from connectionMap 1 -[2017-05-25 22:30:03.939] [minifi log] [info] Starting Flow Controller -[2017-05-25 22:30:03.939] [minifi log] [info] Enabling 0 controller services -[2017-05-25 22:30:03.939] [minifi log] [debug] Starting GetFile -[2017-05-25 22:30:03.939] [minifi log] [debug] nifi_administrative_yield_duration: [30000] ms -[2017-05-25 22:30:03.939] [minifi log] [debug] nifi_bored_yield_duration: [10] ms -[2017-05-25 22:30:03.939] [minifi log] [info] Processor Input Directory property name Input Directory value /tmp/getfile -[2017-05-25 22:30:03.939] [minifi log] [info] Processor Batch Size property name Batch Size value 10 -[2017-05-25 22:30:03.939] [minifi log] [info] Processor Ignore Hidden Files property name Ignore Hidden Files value true -[2017-05-25 22:30:03.939] [minifi log] [info] Processor Keep Source File property name Keep Source File value true -[2017-05-25 22:30:03.939] [minifi log] [info] Processor Maximum File Age property name Maximum File Age value 0 sec -[2017-05-25 22:30:03.939] [minifi log] [debug] successfully applied _maxAge -[2017-05-25 22:30:03.939] [minifi log] [info] Processor Minimum File Age property name Minimum File Age value 0 sec -[2017-05-25 22:30:03.939] [minifi log] [debug] successfully applied _minAge -[2017-05-25 22:30:03.939] [minifi log] [info] Processor Maximum File Size property name Maximum File Size value 0 B -[2017-05-25 22:30:03.939] [minifi log] [info] Processor Minimum File Size property name Minimum File Size value 0 B -[2017-05-25 22:30:03.939] [minifi log] [info] Processor Polling Interval property name Polling Interval value 0 sec -[2017-05-25 22:30:03.939] [minifi log] [debug] successfully applied _pollInterval -[2017-05-25 22:30:03.939] [minifi log] [info] Processor Recurse Subdirectories property name Recurse Subdirectories value true -[2017-05-25 22:30:03.939] [minifi log] [info] Processor File Filter property name File Filter value [^\.].* -[2017-05-25 22:30:03.939] [minifi log] [info] Is listing empty 1 -[2017-05-25 22:30:03.939] [minifi log] [info] Scheduled thread 0 running for process GetFile -[2017-05-25 22:30:03.939] [minifi log] [info] Performing file listing against /tmp/getfile -[2017-05-25 22:30:03.939] [minifi log] [debug] Starting SiteToSiteProvenanceReportingTask -[2017-05-25 22:30:03.939] [minifi log] [debug] nifi_administrative_yield_duration: [30000] ms -[2017-05-25 22:30:03.939] [minifi log] [debug] nifi_bored_yield_duration: [10] ms -[2017-05-25 22:30:03.939] [minifi log] [info] Performing file listing against /tmp/getfile -[2017-05-25 22:30:03.940] [minifi log] [info] Is listing empty 0 -[2017-05-25 22:30:03.940] [minifi log] [info] GetFile process /tmp/getfile/minifi-app.log.txt -[2017-05-25 22:30:03.940] [minifi log] [info] Scheduled thread 0 running for process SiteToSiteProvenanceReportingTask -[2017-05-25 22:30:03.940] [minifi log] [debug] Starting From Node A -[2017-05-25 22:30:03.940] [minifi log] [debug] nifi_administrative_yield_duration: [30000] ms -[2017-05-25 22:30:03.940] [minifi log] [debug] nifi_bored_yield_duration: [10] ms -[2017-05-25 22:30:03.940] [minifi log] [debug] Create FlowFile with UUID 01bac7f1-ac06-447d-879b-c3065ad5d689 -[2017-05-25 22:30:03.940] [minifi log] [info] Processor Host Name property name Host Name value localhost -[2017-05-25 22:30:03.940] [minifi log] [debug] Resource Claim created ./content_repository/ea1adede-b3e3-4870-ae2a-b3ca33b3314e -[2017-05-25 22:30:03.940] [minifi log] [info] Processor Port property name Port value 10001 -[2017-05-25 22:30:03.940] [minifi log] [info] Processor Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:30:03.940] [minifi log] [info] Scheduled thread 0 running for process From Node A -[2017-05-25 22:30:03.940] [minifi log] [info] org::apache::nifi::minifi::provenance::ProvenanceRepository Repository Monitor Thread Start -[2017-05-25 22:30:03.940] [minifi log] [info] org::apache::nifi::minifi::core::repository::FlowFileRepository Repository Monitor Thread Start -[2017-05-25 22:30:03.940] [minifi log] [info] Started Flow Controller -[2017-05-25 22:30:03.940] [minifi log] [debug] Import offset 0 length 36219 into content ./content_repository/ea1adede-b3e3-4870-ae2a-b3ca33b3314e for FlowFile UUID 01bac7f1-ac06-447d-879b-c3065ad5d689 -[2017-05-25 22:30:03.942] [minifi log] [info] Negotiate protocol version with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 current version 5 -[2017-05-25 22:30:03.942] [minifi log] [info] result of writing resource name is 22 -[2017-05-25 22:30:03.943] [minifi log] [info] status code is 20 -[2017-05-25 22:30:03.943] [minifi log] [info] Site2Site Protocol Negotiate protocol version OK -[2017-05-25 22:30:03.943] [minifi log] [info] Site2Site socket established -[2017-05-25 22:30:03.943] [minifi log] [info] Site2Site Protocol Perform hand shake with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:30:03.943] [minifi log] [info] Site2Site Protocol Send handshake properties GZIP false -[2017-05-25 22:30:03.943] [minifi log] [info] Site2Site Protocol Send handshake properties PORT_IDENTIFIER 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:30:03.943] [minifi log] [info] Site2Site Protocol Send handshake properties REQUEST_EXPIRATION_MILLIS 30000 -[2017-05-25 22:30:03.943] [minifi log] [debug] Enqueue flow file UUID 01bac7f1-ac06-447d-879b-c3065ad5d689 to connection GenerateFlowFileS2S -[2017-05-25 22:30:03.943] [minifi log] [debug] NiFi FlowFile Store event 01bac7f1-ac06-447d-879b-c3065ad5d689 size 337 success -[2017-05-25 22:30:03.943] [minifi log] [debug] Delete SnapShot FlowFile UUID 01bac7f1-ac06-447d-879b-c3065ad5d689 -[2017-05-25 22:30:03.943] [minifi log] [debug] NiFi Provenance Store event 56419305-b78d-4af1-8f71-604a71e2e625 size 471 success -[2017-05-25 22:30:03.943] [minifi log] [debug] NiFi Provenance Store event 0a35c034-6625-4594-b0b2-e2f2b0da517b size 345 success -[2017-05-25 22:30:03.952] [minifi log] [info] Processor Host Name property name Host Name value localhost -[2017-05-25 22:30:03.952] [minifi log] [info] Processor Port property name Port value 10001 -[2017-05-25 22:30:03.952] [minifi log] [info] Processor Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:30:03.954] [minifi log] [info] Site2Site HandShake Completed -[2017-05-25 22:30:03.954] [minifi log] [info] Site2Site Protocol Negotiate Codec with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:30:03.954] [minifi log] [info] Negotiate Codec version with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 current version 1 -[2017-05-25 22:30:03.954] [minifi log] [info] Negotiate protocol version with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 current version 5 -[2017-05-25 22:30:03.954] [minifi log] [info] result of writing resource name is 22 -[2017-05-25 22:30:03.955] [minifi log] [info] status code is 20 -[2017-05-25 22:30:03.955] [minifi log] [info] Site2Site Protocol Negotiate protocol version OK -[2017-05-25 22:30:03.955] [minifi log] [info] Site2Site socket established -[2017-05-25 22:30:03.955] [minifi log] [info] Site2Site Protocol Perform hand shake with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:30:03.955] [minifi log] [info] Site2Site Protocol Send handshake properties GZIP false -[2017-05-25 22:30:03.955] [minifi log] [info] Site2Site Protocol Send handshake properties PORT_IDENTIFIER 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:30:03.955] [minifi log] [info] Site2Site Protocol Send handshake properties REQUEST_EXPIRATION_MILLIS 30000 -[2017-05-25 22:30:03.967] [minifi log] [info] Site2Site Codec Negotiate version OK -[2017-05-25 22:30:03.967] [minifi log] [info] Site2Site Codec Completed and move to READY state for data transfer -[2017-05-25 22:30:03.967] [minifi log] [info] Site2Site Ready For data transaction -[2017-05-25 22:30:03.967] [minifi log] [info] Site2Site create transaction 4562b6cc-567e-4438-932b-b40843af2be9 -[2017-05-25 22:30:03.967] [minifi log] [info] Site2Site transaction 4562b6cc-567e-4438-932b-b40843af2be9 send flow record 1, total length 1904 -[2017-05-25 22:30:03.967] [minifi log] [info] Site2Site transaction 4562b6cc-567e-4438-932b-b40843af2be9 send bytes length 1904 -[2017-05-25 22:30:03.967] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 4562b6cc-567e-4438-932b-b40843af2be9 -[2017-05-25 22:30:03.967] [minifi log] [info] Site2Site HandShake Completed -[2017-05-25 22:30:03.967] [minifi log] [info] Site2Site Protocol Negotiate Codec with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:30:03.968] [minifi log] [info] Negotiate Codec version with destination port 471deef6-2a6e-4a7d-912a-81cc17e3a204 current version 1 -[2017-05-25 22:30:03.979] [minifi log] [info] Site2Site transaction 4562b6cc-567e-4438-932b-b40843af2be9 peer confirm transaction with CRC 1611766123 -[2017-05-25 22:30:03.979] [minifi log] [info] Site2Site transaction 4562b6cc-567e-4438-932b-b40843af2be9 CRC matched -[2017-05-25 22:30:03.979] [minifi log] [info] Site2Site Codec Negotiate version OK -[2017-05-25 22:30:03.979] [minifi log] [info] Site2Site Codec Completed and move to READY state for data transfer -[2017-05-25 22:30:03.979] [minifi log] [info] Site2Site Ready For data transaction -[2017-05-25 22:30:03.979] [minifi log] [debug] Dequeue flow file UUID 01bac7f1-ac06-447d-879b-c3065ad5d689 from connection GenerateFlowFileS2S -[2017-05-25 22:30:03.979] [minifi log] [debug] Create Snapshot FlowFile with UUID 321e7825-9318-4633-9fab-0927c2f5f3c3 -[2017-05-25 22:30:03.979] [minifi log] [debug] Delete FlowFile UUID 321e7825-9318-4633-9fab-0927c2f5f3c3 -[2017-05-25 22:30:03.979] [minifi log] [info] Site2Site create transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c -[2017-05-25 22:30:03.979] [minifi log] [info] Site2Site transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c send attribute key absolute.path value /tmp/getfile/minifi-app.log.txt -[2017-05-25 22:30:03.979] [minifi log] [info] Site2Site transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c send attribute key filename value minifi-app.log.txt -[2017-05-25 22:30:03.979] [minifi log] [info] Site2Site transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c send attribute key path value /tmp/getfile -[2017-05-25 22:30:03.979] [minifi log] [info] Site2Site transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c send attribute key uuid value 01bac7f1-ac06-447d-879b-c3065ad5d689 -[2017-05-25 22:30:03.980] [minifi log] [info] Site2Site transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c send flow record 1, total length 36219 -[2017-05-25 22:30:03.980] [minifi log] [info] Site2Site transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c send flow record 01bac7f1-ac06-447d-879b-c3065ad5d689 -[2017-05-25 22:30:03.980] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c -[2017-05-25 22:30:03.993] [minifi log] [info] Site2Site transaction 4562b6cc-567e-4438-932b-b40843af2be9 peer finished transaction -[2017-05-25 22:30:03.993] [minifi log] [info] Site2Site transaction 4562b6cc-567e-4438-932b-b40843af2be9 successfully send flow record 1, content bytes 1904 -[2017-05-25 22:30:03.993] [minifi log] [info] Site2Site delete transaction 4562b6cc-567e-4438-932b-b40843af2be9 -[2017-05-25 22:30:03.994] [minifi log] [info] Site2Site transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c peer confirm transaction with CRC 3636446315 -[2017-05-25 22:30:03.994] [minifi log] [info] Site2Site transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c CRC matched -[2017-05-25 22:30:04.005] [minifi log] [info] Site2Site transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c peer finished transaction -[2017-05-25 22:30:04.005] [minifi log] [info] Site2Site transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c successfully send flow record 1, content bytes 36219 -[2017-05-25 22:30:04.005] [minifi log] [info] Site2Site delete transaction 2aacbe92-6b3c-40d5-9481-bc87ac42450c -[2017-05-25 22:30:04.005] [minifi log] [debug] Delete FlowFile UUID 01bac7f1-ac06-447d-879b-c3065ad5d689 -[2017-05-25 22:30:04.005] [minifi log] [debug] Delete Resource Claim ./content_repository/ea1adede-b3e3-4870-ae2a-b3ca33b3314e -[2017-05-25 22:30:04.005] [minifi log] [debug] NiFi Provenance Store event bfb69abc-84e6-42a5-9924-0bf5399ce05f size 504 success -[2017-05-25 22:30:04.005] [minifi log] [debug] NiFi Provenance Store event 8d2ad741-7659-476a-8f1f-6e8a1277252d size 550 success -[2017-05-25 22:30:04.948] [minifi log] [info] Is listing empty 1 -[2017-05-25 22:30:04.948] [minifi log] [info] Performing file listing against /tmp/getfile -[2017-05-25 22:30:04.948] [minifi log] [info] Performing file listing against /tmp/getfile -[2017-05-25 22:30:04.948] [minifi log] [info] Is listing empty 0 -[2017-05-25 22:30:04.948] [minifi log] [info] GetFile process /tmp/getfile/minifi-app.log.txt -[2017-05-25 22:30:04.948] [minifi log] [debug] Create FlowFile with UUID 9aac58de-d513-48c5-b2c1-a601ec67f656 -[2017-05-25 22:30:04.948] [minifi log] [debug] Resource Claim created ./content_repository/c06703c6-b16d-4ba6-afa2-ac7057b9e261 -[2017-05-25 22:30:04.948] [minifi log] [debug] Import offset 0 length 36219 into content ./content_repository/c06703c6-b16d-4ba6-afa2-ac7057b9e261 for FlowFile UUID 9aac58de-d513-48c5-b2c1-a601ec67f656 -[2017-05-25 22:30:04.951] [minifi log] [debug] Enqueue flow file UUID 9aac58de-d513-48c5-b2c1-a601ec67f656 to connection GenerateFlowFileS2S -[2017-05-25 22:30:04.951] [minifi log] [debug] NiFi FlowFile Store event 9aac58de-d513-48c5-b2c1-a601ec67f656 size 337 success -[2017-05-25 22:30:04.951] [minifi log] [debug] Delete SnapShot FlowFile UUID 9aac58de-d513-48c5-b2c1-a601ec67f656 -[2017-05-25 22:30:04.951] [minifi log] [debug] NiFi Provenance Store event b0396327-de9a-4403-aa7e-4464586feaf2 size 345 success -[2017-05-25 22:30:04.951] [minifi log] [debug] NiFi Provenance Store event 7316b430-e266-41df-9133-3498ebba5f4b size 471 success -[2017-05-25 22:30:04.959] [minifi log] [info] Processor Host Name property name Host Name value localhost -[2017-05-25 22:30:04.960] [minifi log] [info] Processor Port property name Port value 10001 -[2017-05-25 22:30:04.960] [minifi log] [info] Processor Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:30:04.960] [minifi log] [debug] Dequeue flow file UUID 9aac58de-d513-48c5-b2c1-a601ec67f656 from connection GenerateFlowFileS2S -[2017-05-25 22:30:04.960] [minifi log] [debug] Create Snapshot FlowFile with UUID ecdc657d-db14-4150-a477-06249ee357a7 -[2017-05-25 22:30:04.960] [minifi log] [debug] Delete FlowFile UUID ecdc657d-db14-4150-a477-06249ee357a7 -[2017-05-25 22:30:04.960] [minifi log] [info] Site2Site create transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be -[2017-05-25 22:30:04.960] [minifi log] [info] Site2Site transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be send attribute key absolute.path value /tmp/getfile/minifi-app.log.txt -[2017-05-25 22:30:04.960] [minifi log] [info] Site2Site transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be send attribute key filename value minifi-app.log.txt -[2017-05-25 22:30:04.960] [minifi log] [info] Site2Site transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be send attribute key path value /tmp/getfile -[2017-05-25 22:30:04.960] [minifi log] [info] Site2Site transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be send attribute key uuid value 9aac58de-d513-48c5-b2c1-a601ec67f656 -[2017-05-25 22:30:04.960] [minifi log] [info] Site2Site transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be send flow record 1, total length 36219 -[2017-05-25 22:30:04.960] [minifi log] [info] Site2Site transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be send flow record 9aac58de-d513-48c5-b2c1-a601ec67f656 -[2017-05-25 22:30:04.960] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be -[2017-05-25 22:30:04.967] [minifi log] [info] Site2Site transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be peer confirm transaction with CRC 872125207 -[2017-05-25 22:30:04.967] [minifi log] [info] Site2Site transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be CRC matched -[2017-05-25 22:30:04.979] [minifi log] [info] Site2Site transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be peer finished transaction -[2017-05-25 22:30:04.979] [minifi log] [info] Site2Site transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be successfully send flow record 1, content bytes 36219 -[2017-05-25 22:30:04.979] [minifi log] [info] Site2Site delete transaction 63ed3f1f-9964-4b5a-92e7-a070f9e8d8be -[2017-05-25 22:30:04.979] [minifi log] [debug] Delete FlowFile UUID 9aac58de-d513-48c5-b2c1-a601ec67f656 -[2017-05-25 22:30:04.979] [minifi log] [debug] Delete Resource Claim ./content_repository/c06703c6-b16d-4ba6-afa2-ac7057b9e261 -[2017-05-25 22:30:04.979] [minifi log] [debug] NiFi Provenance Store event 93a745dc-a4ef-48c3-b587-6ccc6db15194 size 550 success -[2017-05-25 22:30:04.979] [minifi log] [debug] NiFi Provenance Store event ba3cf4db-83ff-42a0-a3c3-be09ad69dbc3 size 504 success -[2017-05-25 22:30:04.983] [minifi log] [debug] HttpConfigurationListener load Client Certificates OK -[2017-05-25 22:30:04.998] [minifi log] [info] Site2Site create transaction 35489d64-f920-4be7-b101-a3307c6c6ea4 -[2017-05-25 22:30:04.998] [minifi log] [info] Site2Site transaction 35489d64-f920-4be7-b101-a3307c6c6ea4 send flow record 1, total length 6002 -[2017-05-25 22:30:04.998] [minifi log] [info] Site2Site transaction 35489d64-f920-4be7-b101-a3307c6c6ea4 send bytes length 6002 -[2017-05-25 22:30:04.998] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 35489d64-f920-4be7-b101-a3307c6c6ea4 -[2017-05-25 22:30:05.005] [minifi log] [info] Site2Site transaction 35489d64-f920-4be7-b101-a3307c6c6ea4 peer confirm transaction with CRC 3543481630 -[2017-05-25 22:30:05.005] [minifi log] [info] Site2Site transaction 35489d64-f920-4be7-b101-a3307c6c6ea4 CRC matched -[2017-05-25 22:30:05.017] [minifi log] [info] Site2Site transaction 35489d64-f920-4be7-b101-a3307c6c6ea4 peer finished transaction -[2017-05-25 22:30:05.017] [minifi log] [info] Site2Site transaction 35489d64-f920-4be7-b101-a3307c6c6ea4 successfully send flow record 1, content bytes 6002 -[2017-05-25 22:30:05.017] [minifi log] [info] Site2Site delete transaction 35489d64-f920-4be7-b101-a3307c6c6ea4 -[2017-05-25 22:30:05.088] [minifi log] [debug] HttpConfigurationListener -- curl successful to https://bbende:443 -[2017-05-25 22:30:05.088] [minifi log] [debug] config # Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Flow Controller: - id: 471deef6-2a6e-4a7d-912a-81cc17e3a205 - name: MiNiFi Flow - version: 1 - -Processors: - - name: GetFile - id: 471deef6-2a6e-4a7d-912a-81cc17e3a206 - class: org.apache.nifi.processors.standard.Ge -[2017-05-25 22:30:05.954] [minifi log] [info] Is listing empty 1 -[2017-05-25 22:30:05.954] [minifi log] [info] Performing file listing against /tmp/getfile -[2017-05-25 22:30:05.954] [minifi log] [info] Performing file listing against /tmp/getfile -[2017-05-25 22:30:05.955] [minifi log] [info] Is listing empty 0 -[2017-05-25 22:30:05.955] [minifi log] [info] GetFile process /tmp/getfile/minifi-app.log.txt -[2017-05-25 22:30:05.955] [minifi log] [debug] Create FlowFile with UUID 868a723a-db4f-4ead-b3f4-11450c4cc2d6 -[2017-05-25 22:30:05.955] [minifi log] [debug] Resource Claim created ./content_repository/cd8fa82c-f934-4811-a66b-deba027aa5fb -[2017-05-25 22:30:05.955] [minifi log] [debug] Import offset 0 length 36219 into content ./content_repository/cd8fa82c-f934-4811-a66b-deba027aa5fb for FlowFile UUID 868a723a-db4f-4ead-b3f4-11450c4cc2d6 -[2017-05-25 22:30:05.958] [minifi log] [debug] Enqueue flow file UUID 868a723a-db4f-4ead-b3f4-11450c4cc2d6 to connection GenerateFlowFileS2S -[2017-05-25 22:30:05.958] [minifi log] [debug] NiFi FlowFile Store event 868a723a-db4f-4ead-b3f4-11450c4cc2d6 size 337 success -[2017-05-25 22:30:05.958] [minifi log] [debug] Delete SnapShot FlowFile UUID 868a723a-db4f-4ead-b3f4-11450c4cc2d6 -[2017-05-25 22:30:05.958] [minifi log] [debug] NiFi Provenance Store event df952f23-5eba-444a-b33f-7d756b17c81f size 471 success -[2017-05-25 22:30:05.958] [minifi log] [debug] NiFi Provenance Store event 0b2939fe-6411-428c-954c-72bf5ef7da1c size 345 success -[2017-05-25 22:30:05.965] [minifi log] [info] Processor Host Name property name Host Name value localhost -[2017-05-25 22:30:05.965] [minifi log] [info] Processor Port property name Port value 10001 -[2017-05-25 22:30:05.965] [minifi log] [info] Processor Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:30:05.965] [minifi log] [debug] Dequeue flow file UUID 868a723a-db4f-4ead-b3f4-11450c4cc2d6 from connection GenerateFlowFileS2S -[2017-05-25 22:30:05.965] [minifi log] [debug] Create Snapshot FlowFile with UUID 6480c609-d2db-43fb-a6eb-5e81fd9491ce -[2017-05-25 22:30:05.965] [minifi log] [debug] Delete FlowFile UUID 6480c609-d2db-43fb-a6eb-5e81fd9491ce -[2017-05-25 22:30:05.965] [minifi log] [info] Site2Site create transaction 76819c0c-ecb0-4db8-af40-5b0183710336 -[2017-05-25 22:30:05.965] [minifi log] [info] Site2Site transaction 76819c0c-ecb0-4db8-af40-5b0183710336 send attribute key absolute.path value /tmp/getfile/minifi-app.log.txt -[2017-05-25 22:30:05.965] [minifi log] [info] Site2Site transaction 76819c0c-ecb0-4db8-af40-5b0183710336 send attribute key filename value minifi-app.log.txt -[2017-05-25 22:30:05.965] [minifi log] [info] Site2Site transaction 76819c0c-ecb0-4db8-af40-5b0183710336 send attribute key path value /tmp/getfile -[2017-05-25 22:30:05.965] [minifi log] [info] Site2Site transaction 76819c0c-ecb0-4db8-af40-5b0183710336 send attribute key uuid value 868a723a-db4f-4ead-b3f4-11450c4cc2d6 -[2017-05-25 22:30:05.966] [minifi log] [info] Site2Site transaction 76819c0c-ecb0-4db8-af40-5b0183710336 send flow record 1, total length 36219 -[2017-05-25 22:30:05.966] [minifi log] [info] Site2Site transaction 76819c0c-ecb0-4db8-af40-5b0183710336 send flow record 868a723a-db4f-4ead-b3f4-11450c4cc2d6 -[2017-05-25 22:30:05.966] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 76819c0c-ecb0-4db8-af40-5b0183710336 -[2017-05-25 22:30:05.977] [minifi log] [info] Site2Site transaction 76819c0c-ecb0-4db8-af40-5b0183710336 peer confirm transaction with CRC 2714741500 -[2017-05-25 22:30:05.977] [minifi log] [info] Site2Site transaction 76819c0c-ecb0-4db8-af40-5b0183710336 CRC matched -[2017-05-25 22:30:05.990] [minifi log] [info] Site2Site transaction 76819c0c-ecb0-4db8-af40-5b0183710336 peer finished transaction -[2017-05-25 22:30:05.990] [minifi log] [info] Site2Site transaction 76819c0c-ecb0-4db8-af40-5b0183710336 successfully send flow record 1, content bytes 36219 -[2017-05-25 22:30:05.990] [minifi log] [info] Site2Site delete transaction 76819c0c-ecb0-4db8-af40-5b0183710336 -[2017-05-25 22:30:05.990] [minifi log] [debug] Delete FlowFile UUID 868a723a-db4f-4ead-b3f4-11450c4cc2d6 -[2017-05-25 22:30:05.990] [minifi log] [debug] Delete Resource Claim ./content_repository/cd8fa82c-f934-4811-a66b-deba027aa5fb -[2017-05-25 22:30:05.990] [minifi log] [debug] NiFi Provenance Store event 94bf13c4-fe1d-4408-a827-e41764c5e669 size 550 success -[2017-05-25 22:30:05.990] [minifi log] [debug] NiFi Provenance Store event 8c3d17f4-1b4c-41bd-99e0-0a94a213593d size 504 success -[2017-05-25 22:30:06.021] [minifi log] [info] Site2Site create transaction d1d58f6e-a4cb-4a84-85a4-0ea9715a39c7 -[2017-05-25 22:30:06.021] [minifi log] [info] Site2Site transaction d1d58f6e-a4cb-4a84-85a4-0ea9715a39c7 send flow record 1, total length 3932 -[2017-05-25 22:30:06.021] [minifi log] [info] Site2Site transaction d1d58f6e-a4cb-4a84-85a4-0ea9715a39c7 send bytes length 3932 -[2017-05-25 22:30:06.021] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction d1d58f6e-a4cb-4a84-85a4-0ea9715a39c7 -[2017-05-25 22:30:06.024] [minifi log] [info] Site2Site transaction d1d58f6e-a4cb-4a84-85a4-0ea9715a39c7 peer confirm transaction with CRC 1622701313 -[2017-05-25 22:30:06.024] [minifi log] [info] Site2Site transaction d1d58f6e-a4cb-4a84-85a4-0ea9715a39c7 CRC matched -[2017-05-25 22:30:06.035] [minifi log] [info] Site2Site transaction d1d58f6e-a4cb-4a84-85a4-0ea9715a39c7 peer finished transaction -[2017-05-25 22:30:06.035] [minifi log] [info] Site2Site transaction d1d58f6e-a4cb-4a84-85a4-0ea9715a39c7 successfully send flow record 1, content bytes 3932 -[2017-05-25 22:30:06.035] [minifi log] [info] Site2Site delete transaction d1d58f6e-a4cb-4a84-85a4-0ea9715a39c7 -[2017-05-25 22:30:06.127] [minifi log] [debug] HttpConfigurationListener load Client Certificates OK -[2017-05-25 22:30:06.227] [minifi log] [debug] HttpConfigurationListener -- curl successful to https://bbende:443 -[2017-05-25 22:30:06.227] [minifi log] [debug] config # Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Flow Controller: - id: 471deef6-2a6e-4a7d-912a-81cc17e3a205 - name: MiNiFi Flow - version: 1 - -Processors: - - name: GetFile - id: 471deef6-2a6e-4a7d-912a-81cc17e3a206 - class: org.apache.nifi.processors.standard.Ge -[2017-05-25 22:30:06.963] [minifi log] [info] Is listing empty 1 -[2017-05-25 22:30:06.963] [minifi log] [info] Performing file listing against /tmp/getfile -[2017-05-25 22:30:06.963] [minifi log] [info] Performing file listing against /tmp/getfile -[2017-05-25 22:30:06.963] [minifi log] [info] Is listing empty 0 -[2017-05-25 22:30:06.963] [minifi log] [info] GetFile process /tmp/getfile/minifi-app.log.txt -[2017-05-25 22:30:06.963] [minifi log] [debug] Create FlowFile with UUID 868411c5-96ac-4e81-8dfb-03ba20dcc6bb -[2017-05-25 22:30:06.963] [minifi log] [debug] Resource Claim created ./content_repository/0376d019-f552-432c-b113-b1d4a751a1e7 -[2017-05-25 22:30:06.963] [minifi log] [debug] Import offset 0 length 36219 into content ./content_repository/0376d019-f552-432c-b113-b1d4a751a1e7 for FlowFile UUID 868411c5-96ac-4e81-8dfb-03ba20dcc6bb -[2017-05-25 22:30:06.966] [minifi log] [debug] Enqueue flow file UUID 868411c5-96ac-4e81-8dfb-03ba20dcc6bb to connection GenerateFlowFileS2S -[2017-05-25 22:30:06.966] [minifi log] [debug] NiFi FlowFile Store event 868411c5-96ac-4e81-8dfb-03ba20dcc6bb size 337 success -[2017-05-25 22:30:06.966] [minifi log] [debug] Delete SnapShot FlowFile UUID 868411c5-96ac-4e81-8dfb-03ba20dcc6bb -[2017-05-25 22:30:06.966] [minifi log] [debug] NiFi Provenance Store event 9772dc3c-0710-4a86-96ae-4e892a96162e size 345 success -[2017-05-25 22:30:06.966] [minifi log] [debug] NiFi Provenance Store event 67b9e157-70ff-42bb-b831-296e64766867 size 471 success -[2017-05-25 22:30:06.970] [minifi log] [info] Processor Host Name property name Host Name value localhost -[2017-05-25 22:30:06.970] [minifi log] [info] Processor Port property name Port value 10001 -[2017-05-25 22:30:06.970] [minifi log] [info] Processor Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:30:06.970] [minifi log] [debug] Dequeue flow file UUID 868411c5-96ac-4e81-8dfb-03ba20dcc6bb from connection GenerateFlowFileS2S -[2017-05-25 22:30:06.970] [minifi log] [debug] Create Snapshot FlowFile with UUID 8c8852db-f5b4-4733-b3c4-b2d76f97eebf -[2017-05-25 22:30:06.970] [minifi log] [debug] Delete FlowFile UUID 8c8852db-f5b4-4733-b3c4-b2d76f97eebf -[2017-05-25 22:30:06.970] [minifi log] [info] Site2Site create transaction c52fa4a2-8131-450f-b4f8-3203353622e2 -[2017-05-25 22:30:06.970] [minifi log] [info] Site2Site transaction c52fa4a2-8131-450f-b4f8-3203353622e2 send attribute key absolute.path value /tmp/getfile/minifi-app.log.txt -[2017-05-25 22:30:06.970] [minifi log] [info] Site2Site transaction c52fa4a2-8131-450f-b4f8-3203353622e2 send attribute key filename value minifi-app.log.txt -[2017-05-25 22:30:06.970] [minifi log] [info] Site2Site transaction c52fa4a2-8131-450f-b4f8-3203353622e2 send attribute key path value /tmp/getfile -[2017-05-25 22:30:06.970] [minifi log] [info] Site2Site transaction c52fa4a2-8131-450f-b4f8-3203353622e2 send attribute key uuid value 868411c5-96ac-4e81-8dfb-03ba20dcc6bb -[2017-05-25 22:30:06.971] [minifi log] [info] Site2Site transaction c52fa4a2-8131-450f-b4f8-3203353622e2 send flow record 1, total length 36219 -[2017-05-25 22:30:06.971] [minifi log] [info] Site2Site transaction c52fa4a2-8131-450f-b4f8-3203353622e2 send flow record 868411c5-96ac-4e81-8dfb-03ba20dcc6bb -[2017-05-25 22:30:06.971] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction c52fa4a2-8131-450f-b4f8-3203353622e2 -[2017-05-25 22:30:06.982] [minifi log] [info] Site2Site transaction c52fa4a2-8131-450f-b4f8-3203353622e2 peer confirm transaction with CRC 1979837776 -[2017-05-25 22:30:06.982] [minifi log] [info] Site2Site transaction c52fa4a2-8131-450f-b4f8-3203353622e2 CRC matched -[2017-05-25 22:30:06.993] [minifi log] [info] Site2Site transaction c52fa4a2-8131-450f-b4f8-3203353622e2 peer finished transaction -[2017-05-25 22:30:06.993] [minifi log] [info] Site2Site transaction c52fa4a2-8131-450f-b4f8-3203353622e2 successfully send flow record 1, content bytes 36219 -[2017-05-25 22:30:06.993] [minifi log] [info] Site2Site delete transaction c52fa4a2-8131-450f-b4f8-3203353622e2 -[2017-05-25 22:30:06.993] [minifi log] [debug] Delete FlowFile UUID 868411c5-96ac-4e81-8dfb-03ba20dcc6bb -[2017-05-25 22:30:06.993] [minifi log] [debug] Delete Resource Claim ./content_repository/0376d019-f552-432c-b113-b1d4a751a1e7 -[2017-05-25 22:30:06.993] [minifi log] [debug] NiFi Provenance Store event 32487f33-189f-4d07-bfdc-579e9d3ac8ef size 550 success -[2017-05-25 22:30:06.994] [minifi log] [debug] NiFi Provenance Store event fa13f8c9-b1f1-4ac7-bf67-370937e4c744 size 504 success -[2017-05-25 22:30:07.040] [minifi log] [info] Site2Site create transaction 37ab3c8f-aaba-4d05-a689-4541850320d0 -[2017-05-25 22:30:07.040] [minifi log] [info] Site2Site transaction 37ab3c8f-aaba-4d05-a689-4541850320d0 send flow record 1, total length 3933 -[2017-05-25 22:30:07.040] [minifi log] [info] Site2Site transaction 37ab3c8f-aaba-4d05-a689-4541850320d0 send bytes length 3933 -[2017-05-25 22:30:07.040] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 37ab3c8f-aaba-4d05-a689-4541850320d0 -[2017-05-25 22:30:07.048] [minifi log] [info] Site2Site transaction 37ab3c8f-aaba-4d05-a689-4541850320d0 peer confirm transaction with CRC 3894313933 -[2017-05-25 22:30:07.048] [minifi log] [info] Site2Site transaction 37ab3c8f-aaba-4d05-a689-4541850320d0 CRC matched -[2017-05-25 22:30:07.059] [minifi log] [info] Site2Site transaction 37ab3c8f-aaba-4d05-a689-4541850320d0 peer finished transaction -[2017-05-25 22:30:07.059] [minifi log] [info] Site2Site transaction 37ab3c8f-aaba-4d05-a689-4541850320d0 successfully send flow record 1, content bytes 3933 -[2017-05-25 22:30:07.059] [minifi log] [info] Site2Site delete transaction 37ab3c8f-aaba-4d05-a689-4541850320d0 -[2017-05-25 22:30:07.267] [minifi log] [debug] HttpConfigurationListener load Client Certificates OK -[2017-05-25 22:30:07.367] [minifi log] [debug] HttpConfigurationListener -- curl successful to https://bbende:443 -[2017-05-25 22:30:07.367] [minifi log] [debug] config # Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Flow Controller: - id: 471deef6-2a6e-4a7d-912a-81cc17e3a205 - name: MiNiFi Flow - version: 1 - -Processors: - - name: GetFile - id: 471deef6-2a6e-4a7d-912a-81cc17e3a206 - class: org.apache.nifi.processors.standard.Ge -[2017-05-25 22:30:07.971] [minifi log] [info] Is listing empty 1 -[2017-05-25 22:30:07.971] [minifi log] [info] Performing file listing against /tmp/getfile -[2017-05-25 22:30:07.971] [minifi log] [info] Performing file listing against /tmp/getfile -[2017-05-25 22:30:07.971] [minifi log] [info] Is listing empty 0 -[2017-05-25 22:30:07.971] [minifi log] [info] GetFile process /tmp/getfile/minifi-app.log.txt -[2017-05-25 22:30:07.971] [minifi log] [debug] Create FlowFile with UUID 7a832ebb-d8df-4f20-afad-9eea78c23972 -[2017-05-25 22:30:07.971] [minifi log] [debug] Resource Claim created ./content_repository/4ba970f2-2a03-43a6-8f42-ca153b5690e8 -[2017-05-25 22:30:07.972] [minifi log] [debug] Import offset 0 length 36219 into content ./content_repository/4ba970f2-2a03-43a6-8f42-ca153b5690e8 for FlowFile UUID 7a832ebb-d8df-4f20-afad-9eea78c23972 -[2017-05-25 22:30:07.974] [minifi log] [debug] Enqueue flow file UUID 7a832ebb-d8df-4f20-afad-9eea78c23972 to connection GenerateFlowFileS2S -[2017-05-25 22:30:07.975] [minifi log] [debug] NiFi FlowFile Store event 7a832ebb-d8df-4f20-afad-9eea78c23972 size 337 success -[2017-05-25 22:30:07.975] [minifi log] [debug] Delete SnapShot FlowFile UUID 7a832ebb-d8df-4f20-afad-9eea78c23972 -[2017-05-25 22:30:07.975] [minifi log] [debug] NiFi Provenance Store event cfdee20d-75fb-4070-bd54-6d5393bfaf5a size 471 success -[2017-05-25 22:30:07.975] [minifi log] [debug] NiFi Provenance Store event d82c39d6-ddc8-4522-bfb3-751b11e43d31 size 345 success -[2017-05-25 22:30:07.983] [minifi log] [info] Processor Host Name property name Host Name value localhost -[2017-05-25 22:30:07.983] [minifi log] [info] Processor Port property name Port value 10001 -[2017-05-25 22:30:07.983] [minifi log] [info] Processor Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:30:07.983] [minifi log] [debug] Dequeue flow file UUID 7a832ebb-d8df-4f20-afad-9eea78c23972 from connection GenerateFlowFileS2S -[2017-05-25 22:30:07.984] [minifi log] [debug] Create Snapshot FlowFile with UUID 3221f7e0-c3b4-4d24-9df7-6a7a26868b61 -[2017-05-25 22:30:07.984] [minifi log] [debug] Delete FlowFile UUID 3221f7e0-c3b4-4d24-9df7-6a7a26868b61 -[2017-05-25 22:30:07.984] [minifi log] [info] Site2Site create transaction 998d3839-5b7a-42c3-99a1-696080eece46 -[2017-05-25 22:30:07.984] [minifi log] [info] Site2Site transaction 998d3839-5b7a-42c3-99a1-696080eece46 send attribute key absolute.path value /tmp/getfile/minifi-app.log.txt -[2017-05-25 22:30:07.984] [minifi log] [info] Site2Site transaction 998d3839-5b7a-42c3-99a1-696080eece46 send attribute key filename value minifi-app.log.txt -[2017-05-25 22:30:07.984] [minifi log] [info] Site2Site transaction 998d3839-5b7a-42c3-99a1-696080eece46 send attribute key path value /tmp/getfile -[2017-05-25 22:30:07.984] [minifi log] [info] Site2Site transaction 998d3839-5b7a-42c3-99a1-696080eece46 send attribute key uuid value 7a832ebb-d8df-4f20-afad-9eea78c23972 -[2017-05-25 22:30:07.984] [minifi log] [info] Site2Site transaction 998d3839-5b7a-42c3-99a1-696080eece46 send flow record 1, total length 36219 -[2017-05-25 22:30:07.984] [minifi log] [info] Site2Site transaction 998d3839-5b7a-42c3-99a1-696080eece46 send flow record 7a832ebb-d8df-4f20-afad-9eea78c23972 -[2017-05-25 22:30:07.985] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 998d3839-5b7a-42c3-99a1-696080eece46 -[2017-05-25 22:30:07.992] [minifi log] [info] Site2Site transaction 998d3839-5b7a-42c3-99a1-696080eece46 peer confirm transaction with CRC 2009366010 -[2017-05-25 22:30:07.992] [minifi log] [info] Site2Site transaction 998d3839-5b7a-42c3-99a1-696080eece46 CRC matched -[2017-05-25 22:30:08.003] [minifi log] [info] Site2Site transaction 998d3839-5b7a-42c3-99a1-696080eece46 peer finished transaction -[2017-05-25 22:30:08.003] [minifi log] [info] Site2Site transaction 998d3839-5b7a-42c3-99a1-696080eece46 successfully send flow record 1, content bytes 36219 -[2017-05-25 22:30:08.003] [minifi log] [info] Site2Site delete transaction 998d3839-5b7a-42c3-99a1-696080eece46 -[2017-05-25 22:30:08.003] [minifi log] [debug] Delete FlowFile UUID 7a832ebb-d8df-4f20-afad-9eea78c23972 -[2017-05-25 22:30:08.003] [minifi log] [debug] Delete Resource Claim ./content_repository/4ba970f2-2a03-43a6-8f42-ca153b5690e8 -[2017-05-25 22:30:08.003] [minifi log] [debug] NiFi Provenance Store event b75d9212-7bea-4d14-b933-43bb6056dd6d size 504 success -[2017-05-25 22:30:08.003] [minifi log] [debug] NiFi Provenance Store event 20487c58-481d-44cd-8a21-5814372c774b size 550 success -[2017-05-25 22:30:08.063] [minifi log] [info] Site2Site create transaction d47eb1bc-7f0e-41a2-8b72-8559d4eabcf7 -[2017-05-25 22:30:08.063] [minifi log] [info] Site2Site transaction d47eb1bc-7f0e-41a2-8b72-8559d4eabcf7 send flow record 1, total length 3937 -[2017-05-25 22:30:08.063] [minifi log] [info] Site2Site transaction d47eb1bc-7f0e-41a2-8b72-8559d4eabcf7 send bytes length 3937 -[2017-05-25 22:30:08.063] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction d47eb1bc-7f0e-41a2-8b72-8559d4eabcf7 -[2017-05-25 22:30:08.064] [minifi log] [info] Site2Site transaction d47eb1bc-7f0e-41a2-8b72-8559d4eabcf7 peer confirm transaction with CRC 2316116091 -[2017-05-25 22:30:08.064] [minifi log] [info] Site2Site transaction d47eb1bc-7f0e-41a2-8b72-8559d4eabcf7 CRC matched -[2017-05-25 22:30:08.075] [minifi log] [info] Site2Site transaction d47eb1bc-7f0e-41a2-8b72-8559d4eabcf7 peer finished transaction -[2017-05-25 22:30:08.075] [minifi log] [info] Site2Site transaction d47eb1bc-7f0e-41a2-8b72-8559d4eabcf7 successfully send flow record 1, content bytes 3937 -[2017-05-25 22:30:08.075] [minifi log] [info] Site2Site delete transaction d47eb1bc-7f0e-41a2-8b72-8559d4eabcf7 -[2017-05-25 22:30:08.404] [minifi log] [debug] HttpConfigurationListener load Client Certificates OK -[2017-05-25 22:30:08.501] [minifi log] [debug] HttpConfigurationListener -- curl successful to https://bbende:443 -[2017-05-25 22:30:08.501] [minifi log] [debug] config # Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Flow Controller: - id: 471deef6-2a6e-4a7d-912a-81cc17e3a205 - name: MiNiFi Flow - version: 1 - -Processors: - - name: GetFile - id: 471deef6-2a6e-4a7d-912a-81cc17e3a206 - class: org.apache.nifi.processors.standard.Ge -[2017-05-25 22:30:08.979] [minifi log] [info] Is listing empty 1 -[2017-05-25 22:30:08.979] [minifi log] [info] Performing file listing against /tmp/getfile -[2017-05-25 22:30:08.979] [minifi log] [info] Performing file listing against /tmp/getfile -[2017-05-25 22:30:08.979] [minifi log] [info] Is listing empty 0 -[2017-05-25 22:30:08.979] [minifi log] [info] GetFile process /tmp/getfile/minifi-app.log.txt -[2017-05-25 22:30:08.979] [minifi log] [debug] Create FlowFile with UUID fc099429-0b17-4763-b6fb-d377c19e8e5b -[2017-05-25 22:30:08.979] [minifi log] [debug] Resource Claim created ./content_repository/ea2bbaae-bb8b-418c-8d1f-977acedcebf9 -[2017-05-25 22:30:08.980] [minifi log] [debug] Import offset 0 length 36219 into content ./content_repository/ea2bbaae-bb8b-418c-8d1f-977acedcebf9 for FlowFile UUID fc099429-0b17-4763-b6fb-d377c19e8e5b -[2017-05-25 22:30:08.982] [minifi log] [debug] Enqueue flow file UUID fc099429-0b17-4763-b6fb-d377c19e8e5b to connection GenerateFlowFileS2S -[2017-05-25 22:30:08.982] [minifi log] [debug] NiFi FlowFile Store event fc099429-0b17-4763-b6fb-d377c19e8e5b size 337 success -[2017-05-25 22:30:08.982] [minifi log] [debug] Delete SnapShot FlowFile UUID fc099429-0b17-4763-b6fb-d377c19e8e5b -[2017-05-25 22:30:08.982] [minifi log] [debug] NiFi Provenance Store event 3331f098-290f-4cb2-ba47-931551024568 size 471 success -[2017-05-25 22:30:08.983] [minifi log] [debug] NiFi Provenance Store event fa8337e2-2b83-49b1-b48e-b4db01aacb6b size 345 success -[2017-05-25 22:30:08.989] [minifi log] [info] Processor Host Name property name Host Name value localhost -[2017-05-25 22:30:08.989] [minifi log] [info] Processor Port property name Port value 10001 -[2017-05-25 22:30:08.989] [minifi log] [info] Processor Port UUID property name Port UUID value 471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:30:08.989] [minifi log] [debug] Dequeue flow file UUID fc099429-0b17-4763-b6fb-d377c19e8e5b from connection GenerateFlowFileS2S -[2017-05-25 22:30:08.990] [minifi log] [debug] Create Snapshot FlowFile with UUID e56d9e2b-b27d-40cd-bd97-3a423e7d3b9f -[2017-05-25 22:30:08.990] [minifi log] [debug] Delete FlowFile UUID e56d9e2b-b27d-40cd-bd97-3a423e7d3b9f -[2017-05-25 22:30:08.990] [minifi log] [info] Site2Site create transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 -[2017-05-25 22:30:08.990] [minifi log] [info] Site2Site transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 send attribute key absolute.path value /tmp/getfile/minifi-app.log.txt -[2017-05-25 22:30:08.990] [minifi log] [info] Site2Site transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 send attribute key filename value minifi-app.log.txt -[2017-05-25 22:30:08.990] [minifi log] [info] Site2Site transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 send attribute key path value /tmp/getfile -[2017-05-25 22:30:08.990] [minifi log] [info] Site2Site transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 send attribute key uuid value fc099429-0b17-4763-b6fb-d377c19e8e5b -[2017-05-25 22:30:08.990] [minifi log] [info] Site2Site transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 send flow record 1, total length 36219 -[2017-05-25 22:30:08.990] [minifi log] [info] Site2Site transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 send flow record fc099429-0b17-4763-b6fb-d377c19e8e5b -[2017-05-25 22:30:08.990] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 -[2017-05-25 22:30:09.001] [minifi log] [info] Site2Site transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 peer confirm transaction with CRC 208752716 -[2017-05-25 22:30:09.001] [minifi log] [info] Site2Site transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 CRC matched -[2017-05-25 22:30:09.013] [minifi log] [info] Site2Site transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 peer finished transaction -[2017-05-25 22:30:09.013] [minifi log] [info] Site2Site transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 successfully send flow record 1, content bytes 36219 -[2017-05-25 22:30:09.013] [minifi log] [info] Site2Site delete transaction 30cfc2e8-5371-4894-b366-3ff6542701e9 -[2017-05-25 22:30:09.013] [minifi log] [debug] Delete FlowFile UUID fc099429-0b17-4763-b6fb-d377c19e8e5b -[2017-05-25 22:30:09.013] [minifi log] [debug] Delete Resource Claim ./content_repository/ea2bbaae-bb8b-418c-8d1f-977acedcebf9 -[2017-05-25 22:30:09.013] [minifi log] [debug] NiFi Provenance Store event 3fb7354f-9f92-4dee-8534-0551e0feecc0 size 550 success -[2017-05-25 22:30:09.013] [minifi log] [debug] NiFi Provenance Store event 6a47d995-0dda-4e7e-9f70-3f77fb5a6358 size 504 success -[2017-05-25 22:30:09.079] [minifi log] [info] Site2Site create transaction dd8dcf42-3a82-4ca8-a19e-dbffc1480fe8 -[2017-05-25 22:30:09.079] [minifi log] [info] Site2Site transaction dd8dcf42-3a82-4ca8-a19e-dbffc1480fe8 send flow record 1, total length 3918 -[2017-05-25 22:30:09.079] [minifi log] [info] Site2Site transaction dd8dcf42-3a82-4ca8-a19e-dbffc1480fe8 send bytes length 3918 -[2017-05-25 22:30:09.079] [minifi log] [info] Site2Site Send FINISH TRANSACTION for transaction dd8dcf42-3a82-4ca8-a19e-dbffc1480fe8 -[2017-05-25 22:30:09.088] [minifi log] [info] Site2Site transaction dd8dcf42-3a82-4ca8-a19e-dbffc1480fe8 peer confirm transaction with CRC 2743550546 -[2017-05-25 22:30:09.088] [minifi log] [info] Site2Site transaction dd8dcf42-3a82-4ca8-a19e-dbffc1480fe8 CRC matched -[2017-05-25 22:30:09.099] [minifi log] [info] Site2Site transaction dd8dcf42-3a82-4ca8-a19e-dbffc1480fe8 peer finished transaction -[2017-05-25 22:30:09.099] [minifi log] [info] Site2Site transaction dd8dcf42-3a82-4ca8-a19e-dbffc1480fe8 successfully send flow record 1, content bytes 3918 -[2017-05-25 22:30:09.099] [minifi log] [info] Site2Site delete transaction dd8dcf42-3a82-4ca8-a19e-dbffc1480fe8 -[2017-05-25 22:30:09.537] [minifi log] [debug] HttpConfigurationListener load Client Certificates OK -[2017-05-25 22:30:09.629] [minifi log] [info] Stop Flow Controller -[2017-05-25 22:30:09.629] [minifi log] [info] org::apache::nifi::minifi::core::repository::FlowFileRepository Repository Monitor Thread Stop -[2017-05-25 22:30:09.629] [minifi log] [info] org::apache::nifi::minifi::provenance::ProvenanceRepository Repository Monitor Thread Stop -[2017-05-25 22:30:09.639] [minifi log] [debug] HttpConfigurationListener -- curl successful to https://bbende:443 -[2017-05-25 22:30:09.639] [minifi log] [debug] config # Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Flow Controller: - id: 471deef6-2a6e-4a7d-912a-81cc17e3a205 - name: MiNiFi Flow - version: 1 - -Processors: - - name: GetFile - id: 471deef6-2a6e-4a7d-912a-81cc17e3a206 - class: org.apache.nifi.processors.standard.Ge -[2017-05-25 22:30:10.678] [minifi log] [debug] HttpConfigurationListener load Client Certificates OK -[2017-05-25 22:30:10.782] [minifi log] [debug] HttpConfigurationListener -- curl successful to https://bbende:443 -[2017-05-25 22:30:10.782] [minifi log] [debug] config # Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Flow Controller: - id: 471deef6-2a6e-4a7d-912a-81cc17e3a205 - name: MiNiFi Flow - version: 1 - -Processors: - - name: GetFile - id: 471deef6-2a6e-4a7d-912a-81cc17e3a206 - class: org.apache.nifi.processors.standard.Ge -[2017-05-25 22:30:11.813] [minifi log] [debug] HttpConfigurationListener load Client Certificates OK -[2017-05-25 22:30:11.929] [minifi log] [debug] HttpConfigurationListener -- curl successful to https://bbende:443 -[2017-05-25 22:30:11.929] [minifi log] [debug] config # Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Flow Controller: - id: 471deef6-2a6e-4a7d-912a-81cc17e3a205 - name: MiNiFi Flow - version: 1 - -Processors: - - name: GetFile - id: 471deef6-2a6e-4a7d-912a-81cc17e3a206 - class: org.apache.nifi.processors.standard.Ge -[2017-05-25 22:30:12.632] [minifi log] [info] Shutting down threads for processor GetFile/471deef6-2a6e-4a7d-912a-81cc17e3a206 -[2017-05-25 22:30:12.632] [minifi log] [info] Scheduled thread 0 deleted for process GetFile -[2017-05-25 22:30:12.632] [minifi log] [info] Shutting down threads for processor SiteToSiteProvenanceReportingTask/d4a7865a-c352-4fff-a918-4c837749a705 -[2017-05-25 22:30:12.632] [minifi log] [info] Scheduled thread 0 deleted for process SiteToSiteProvenanceReportingTask -[2017-05-25 22:30:12.632] [minifi log] [info] Shutting down threads for processor From Node A/471deef6-2a6e-4a7d-912a-81cc17e3a204 -[2017-05-25 22:30:12.632] [minifi log] [info] Scheduled thread 0 deleted for process From Node A -[2017-05-25 22:30:12.632] [minifi log] [info] Unload Flow Controller -[2017-05-25 22:30:12.632] [minifi log] [debug] Drain connection GenerateFlowFileS2S -[2017-05-25 22:30:12.632] [minifi log] [info] Site2Site Protocol tearDown -[2017-05-25 22:30:12.633] [minifi log] [info] MiNiFi exit From 4bf2c5d56e72ad790ae3390e2fdeda6aaadbb4dd Mon Sep 17 00:00:00 2001 From: Bin Qiu Date: Sun, 4 Jun 2017 16:54:36 -0700 Subject: [PATCH 10/11] fix for shared_from_this in construtor --- libminifi/src/FlowController.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/libminifi/src/FlowController.cpp b/libminifi/src/FlowController.cpp index d09f031f49..fd75fddc0b 100644 --- a/libminifi/src/FlowController.cpp +++ b/libminifi/src/FlowController.cpp @@ -89,15 +89,6 @@ FlowController::FlowController(std::shared_ptr provenance_repo protocol_ = new FlowControlProtocol(this, configure); - std::string listenerType; - // grab the value for configuration - if (configure->get(Configure::nifi_configuration_listener_type, listenerType)) { - if (listenerType == "http") { - this->http_configuration_listener_ = - std::unique_ptr(new minifi::HttpConfigurationListener(shared_from_this(), configure)); - } - } - if (!headless_mode) { std::string rawConfigFileString; configure->get(Configure::nifi_flow_configuration_file, rawConfigFileString); @@ -249,6 +240,15 @@ void FlowController::load() { stop(true); } if (!initialized_) { + std::string listenerType; + // grab the value for configuration + if (this->http_configuration_listener_ == nullptr && configuration_->get(Configure::nifi_configuration_listener_type, listenerType)) { + if (listenerType == "http") { + this->http_configuration_listener_ = + std::unique_ptr(new minifi::HttpConfigurationListener(shared_from_this(), configuration_)); + } + } + logger_->log_info("Initializing timers"); if (nullptr == timer_scheduler_) { timer_scheduler_ = std::make_shared(std::static_pointer_cast(shared_from_this()), provenance_repo_, configuration_); From 61035a4cec6e964116164bbc443661c4353c77f1 Mon Sep 17 00:00:00 2001 From: Bin Qiu Date: Tue, 6 Jun 2017 21:37:53 -0700 Subject: [PATCH 11/11] Add test case --- cmake/BuildTests.cmake | 3 + libminifi/src/ConfigurationListener.cpp | 5 +- .../HttpConfigurationListenerTest.cpp | 144 ++++++++++++++++++ 3 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 libminifi/test/integration/HttpConfigurationListenerTest.cpp diff --git a/cmake/BuildTests.cmake b/cmake/BuildTests.cmake index bad01d3563..aedae105db 100644 --- a/cmake/BuildTests.cmake +++ b/cmake/BuildTests.cmake @@ -36,6 +36,7 @@ function(createTests testName) target_include_directories(${testName} PRIVATE BEFORE "thirdparty/spdlog-0.13.0/include") target_include_directories(${testName} PRIVATE BEFORE "thirdparty/yaml-cpp-yaml-cpp-0.5.3/include") target_include_directories(${testName} PRIVATE BEFORE "thirdparty/jsoncpp/include") + target_include_directories(${testName} PRIVATE BEFORE "thirdparty/civetweb-1.9.1/include") target_include_directories(${testName} PRIVATE BEFORE ${LEVELDB_INCLUDE_DIRS}) target_include_directories(${testName} PRIVATE BEFORE "include") target_include_directories(${testName} PRIVATE BEFORE "libminifi/include/") @@ -87,6 +88,8 @@ add_test(NAME ControllerServiceIntegrationTests COMMAND ControllerServiceIntegra add_test(NAME HttpGetIntegrationTest COMMAND HttpGetIntegrationTest "${TEST_RESOURCES}/TestHTTPGet.yml" "${TEST_RESOURCES}/") +add_test(NAME HttpConfigurationListenerTest COMMAND HttpConfigurationListenerTest "${TEST_RESOURCES}/TestHTTPGet.yml" "${TEST_RESOURCES}/") + add_test(NAME HttpGetIntegrationTestSecure COMMAND HttpGetIntegrationTest "${TEST_RESOURCES}/TestHTTPGetSecure.yml" "${TEST_RESOURCES}/") add_test(NAME HttpPostIntegrationTest COMMAND HttpPostIntegrationTest "${TEST_RESOURCES}/TestHTTPPost.yml" ) diff --git a/libminifi/src/ConfigurationListener.cpp b/libminifi/src/ConfigurationListener.cpp index b60c050050..d52a08838a 100644 --- a/libminifi/src/ConfigurationListener.cpp +++ b/libminifi/src/ConfigurationListener.cpp @@ -102,9 +102,10 @@ void ConfigurationListener::stop() { } void ConfigurationListener::run() { + std::unique_lock lk(mutex_); + std::condition_variable cv; int64_t interval = 0; - while (running_) { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + while (!cv.wait_for(lk, std::chrono::milliseconds(100), [this] {return (running_ == false);})) { interval += 100; if (interval >= pull_interval_) { std::string payload; diff --git a/libminifi/test/integration/HttpConfigurationListenerTest.cpp b/libminifi/test/integration/HttpConfigurationListenerTest.cpp new file mode 100644 index 0000000000..a86b884a31 --- /dev/null +++ b/libminifi/test/integration/HttpConfigurationListenerTest.cpp @@ -0,0 +1,144 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../TestBase.h" +#include "utils/StringUtils.h" +#include "core/Core.h" +#include "../include/core/logging/Logger.h" +#include "core/ProcessGroup.h" +#include "core/yaml/YamlConfiguration.h" +#include "HttpConfigurationListener.h" +#include "FlowController.h" +#include "properties/Configure.h" +#include "../unit/ProvenanceTestHelper.h" +#include "io/StreamFactory.h" +#include "CivetServer.h" +#include + +void waitToVerifyProcessor() { + std::this_thread::sleep_for(std::chrono::seconds(10)); +} + +class ConfigHandler: public CivetHandler { + public: + bool handleGet(CivetServer *server, struct mg_connection *conn) { + std::ifstream myfile(test_file_location_.c_str()); + + if (myfile.is_open()) { + std::stringstream buffer; + buffer << myfile.rdbuf(); + std::string str = buffer.str(); + myfile.close(); + mg_printf(conn, "HTTP/1.1 200 OK\r\nContent-Type: " + "text/plain\r\nContent-Length: %lu\r\nConnection: close\r\n\r\n", + str.length()); + mg_printf(conn, "%s", str.c_str()); + } else { + mg_printf(conn, "HTTP/1.1 500 Internal Server Error\r\n"); + } + + return true; + } + std::string test_file_location_; +}; + +int main(int argc, char **argv) { + LogTestController::getInstance().setInfo(); + LogTestController::getInstance().setInfo(); + LogTestController::getInstance().setInfo(); + + const char *options[] = { "document_root", ".", "listening_ports", "9090", 0 }; + std::vector < std::string > cpp_options; + for (int i = 0; i < (sizeof(options) / sizeof(options[0]) - 1); i++) { + cpp_options.push_back(options[i]); + } + + CivetServer server(cpp_options); + ConfigHandler h_ex; + server.addHandler("/config", h_ex); + LogTestController::getInstance().setDebug(); + std::string key_dir, test_file_location; + if (argc > 1) { + h_ex.test_file_location_ = test_file_location = argv[1]; + key_dir = argv[2]; + } + std::shared_ptr configuration = std::make_shared< + minifi::Configure>(); + configuration->set(minifi::Configure::nifi_default_directory, key_dir); + configuration->set(minifi::Configure::nifi_configuration_listener_type, + "http"); + configuration->set( + minifi::Configure::nifi_configuration_listener_pull_interval, "1 sec"); + configuration->set(minifi::Configure::nifi_configuration_listener_http_url, + "http://localhost:9090/config"); + mkdir("content_repository", S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); + + std::shared_ptr test_repo = + std::make_shared(); + std::shared_ptr test_flow_repo = std::make_shared< + TestFlowRepository>(); + + configuration->set(minifi::Configure::nifi_flow_configuration_file, + test_file_location); + + std::shared_ptr stream_factory = std::make_shared + < minifi::io::StreamFactory > (configuration); + std::unique_ptr yaml_ptr = std::unique_ptr + < core::YamlConfiguration + > (new core::YamlConfiguration(test_repo, test_repo, stream_factory, + configuration, test_file_location)); + std::shared_ptr repo = std::static_pointer_cast + < TestRepository > (test_repo); + + std::shared_ptr controller = + std::make_shared < minifi::FlowController + > (test_repo, test_flow_repo, configuration, std::move(yaml_ptr), DEFAULT_ROOT_GROUP_NAME, true); + + core::YamlConfiguration yaml_config(test_repo, test_repo, stream_factory, + configuration, test_file_location); + + std::unique_ptr ptr = yaml_config.getRoot( + test_file_location); + std::shared_ptr pg = std::shared_ptr < core::ProcessGroup + > (ptr.get()); + ptr.release(); + + controller->load(); + controller->start(); + waitToVerifyProcessor(); + + controller->waitUnload(60000); + std::string logs = LogTestController::getInstance().log_output.str(); + assert(logs.find("HttpConfigurationListener -- curl successful to http://localhost:9090/config") != std::string::npos); + assert(logs.find("Starting to reload Flow Controller with flow control name MiNiFi Flow, version 0") != std::string::npos); + LogTestController::getInstance().reset(); + rmdir("./content_repository"); + return 0; +}