From 6a345c9e94f2adfdfea4c4bc97dd9c3532d5f789 Mon Sep 17 00:00:00 2001 From: Marc Parisi Date: Sat, 4 Aug 2018 14:26:54 -0400 Subject: [PATCH 1/2] MINIFICPP-590: Fix zero length files at startup MINIFICPP-589: Institute regex filtering for interfaces MINIFICPP-589: Limit Pcap to 4.9.4+ --- extensions/pcap/CapturePacket.cpp | 42 ++++++++++++++++++++++--- extensions/pcap/CapturePacket.h | 9 ++++-- libminifi/include/core/ProcessContext.h | 2 +- libminifi/include/core/Property.h | 1 - libminifi/test/pcap-tests/PcapTest.cpp | 3 ++ linux.sh | 10 ++++++ 6 files changed, 59 insertions(+), 8 deletions(-) diff --git a/extensions/pcap/CapturePacket.cpp b/extensions/pcap/CapturePacket.cpp index ba88280418..8091709099 100644 --- a/extensions/pcap/CapturePacket.cpp +++ b/extensions/pcap/CapturePacket.cpp @@ -16,7 +16,7 @@ * limitations under the License. */ -#include +#include #include #include #include @@ -54,6 +54,7 @@ namespace processors { std::shared_ptr CapturePacket::id_generator_ = utils::IdGenerator::getIdGenerator(); core::Property CapturePacket::BaseDir("Base Directory", "Scratch directory for PCAP files", "/tmp/"); core::Property CapturePacket::BatchSize("Batch Size", "The number of packets to combine within a given PCAP", "50"); +core::Property CapturePacket::NetworkControllers("Network Controllers", "List of network controllers to attach to -- each may be a regex", ".*"); core::Property CapturePacket::CaptureBluetooth("Capture Bluetooth", "True indicates that we support bluetooth interfaces", "false"); const char *CapturePacket::ProcessorName = "CapturePacket"; @@ -61,6 +62,7 @@ const char *CapturePacket::ProcessorName = "CapturePacket"; std::string CapturePacket::generate_new_pcap(const std::string &base_path) { std::string path = base_path; // can use relaxed for a counter + //int cnt = num_.fetch_add(1, std::memory_order_relaxed); int cnt = num_.fetch_add(1, std::memory_order_relaxed); std::string filename = std::to_string(cnt); path.append(filename); @@ -84,9 +86,11 @@ void CapturePacket::packet_callback(pcpp::RawPacket* packet, pcpp::PcapLiveDevic capture->writer_->close(); + auto new_capture = create_new_capture(capture->getBasePath(), capture->getMaxSize()); + capture_mechanism->sink.enqueue(capture); - capture_mechanism->source.enqueue(create_new_capture(capture->getBasePath(), capture->getMaxSize())); + capture_mechanism->source.enqueue(new_capture); } else { capture_mechanism->source.enqueue(capture); } @@ -112,6 +116,7 @@ void CapturePacket::initialize() { // Set the supported properties std::set properties; properties.insert(BatchSize); + properties.insert(NetworkControllers); properties.insert(BaseDir); properties.insert(CaptureBluetooth); setSupportedProperties(properties); @@ -126,6 +131,7 @@ void CapturePacket::onSchedule(const std::shared_ptr &cont if (context->getProperty(BatchSize.getName(), value)) { core::Property::StringToInt(value, pcap_batch_size_); } + value = ""; if (context->getProperty(BaseDir.getName(), value)) { base_dir_ = value; @@ -135,6 +141,13 @@ void CapturePacket::onSchedule(const std::shared_ptr &cont if (context->getProperty(CaptureBluetooth.getName(), value)) { utils::StringUtils::StringToBool(value, capture_bluetooth_); } + + core::Property attached_controllers("Network Controllers", "List of network controllers to attach to -- each may be a regex", ".*"); + + getProperty(attached_controllers.getName(), attached_controllers); + + std::vector allowed_interfaces = attached_controllers.getValues(); + if (IsNullOrEmpty(base_dir_)) { base_dir_ = "/tmp/"; } @@ -151,6 +164,26 @@ void CapturePacket::onSchedule(const std::shared_ptr &cont for (auto iter : devList) { const std::string name = iter->getName(); + if (!allowed_interfaces.empty()) { + bool found_match = false; + std::string matching_regex = ""; + for (const auto &filter : allowed_interfaces) { + std::regex r(filter); + std::smatch m; + if (std::regex_match(name, m, r)) { + matching_regex = filter; + found_match = true; + break; + } + } + if (!found_match) { + logger_->log_debug("Skipping %s because it does not match any regex", name); + continue; + } else { + logger_->log_trace("Accepting %s because it matches %s", name, matching_regex); + } + } + if (!iter->open()) { logger_->log_error("Could not open device %s", name); continue; @@ -189,9 +222,10 @@ CapturePacket::~CapturePacket() { void CapturePacket::onTrigger(const std::shared_ptr &context, const std::shared_ptr &session) { CapturePacketMechanism *capture; if (mover->sink.try_dequeue(capture)) { - logger_->log_debug("Received packet capture in file %s", capture->getFile()); + logger_->log_debug("Received packet capture in file %s %d", capture->getFile(), capture->getSize()); auto ff = session->create(); - session->import(capture->getFile(), ff, false, 0); + session->import(capture->getFile(), ff, true, 0); + logger_->log_debug("Received packet capture in file %s %d for %s", capture->getFile(), capture->getSize(), ff->getResourceClaim()->getContentFullPath()); session->transfer(ff, Success); delete capture; } else { diff --git a/extensions/pcap/CapturePacket.h b/extensions/pcap/CapturePacket.h index 08b71b4d52..623b13166c 100644 --- a/extensions/pcap/CapturePacket.h +++ b/extensions/pcap/CapturePacket.h @@ -56,7 +56,7 @@ class CapturePacketMechanism { } bool inline incrementAndCheck() { - return atomic_count_++ >= *max_size_; + return ++atomic_count_ >= *max_size_; } int64_t *getMaxSize() { @@ -72,6 +72,10 @@ class CapturePacketMechanism { const std::string &getFile() { return file_; } + + long getSize() const{ + return atomic_count_; + } protected: CapturePacketMechanism &operator=(const CapturePacketMechanism &other) = delete; std::string path_; @@ -98,7 +102,6 @@ class CapturePacket : public core::Processor { capture_bluetooth_(false), pcap_batch_size_(50), logger_(logging::LoggerFactory::getLogger()) { - num_ = 0; mover = std::unique_ptr(new PacketMovers()); } // Destructor @@ -106,6 +109,7 @@ class CapturePacket : public core::Processor { // Processor Name static const char *ProcessorName; static core::Property BatchSize; + static core::Property NetworkControllers; static core::Property BaseDir; static core::Property CaptureBluetooth; // Supported Relationships @@ -151,6 +155,7 @@ class CapturePacket : public core::Processor { } bool capture_bluetooth_; std::string base_dir_; + std::vector attached_controllers_; std::string base_path_; int64_t pcap_batch_size_; std::unique_ptr mover; diff --git a/libminifi/include/core/ProcessContext.h b/libminifi/include/core/ProcessContext.h index e7f70b79d2..e1c42e488c 100644 --- a/libminifi/include/core/ProcessContext.h +++ b/libminifi/include/core/ProcessContext.h @@ -80,7 +80,7 @@ class ProcessContext : public controller::ControllerServiceLookup { // Sets the property value using the property's string name bool setProperty(const std::string &name, std::string value) { return processor_node_->setProperty(name, value); - } // Sets the dynamic property value using the property's string name + } // Sets the dynamic property value using the property's string name bool setDynamicProperty(const std::string &name, std::string value) { return processor_node_->setDynamicProperty(name, value); } diff --git a/libminifi/include/core/Property.h b/libminifi/include/core/Property.h index 50f589650c..a59a53d5da 100644 --- a/libminifi/include/core/Property.h +++ b/libminifi/include/core/Property.h @@ -90,7 +90,6 @@ class Property { description_(""), is_required_(false), is_collection_(false) { - } virtual ~Property() = default; diff --git a/libminifi/test/pcap-tests/PcapTest.cpp b/libminifi/test/pcap-tests/PcapTest.cpp index 7a68295441..aa8d214060 100644 --- a/libminifi/test/pcap-tests/PcapTest.cpp +++ b/libminifi/test/pcap-tests/PcapTest.cpp @@ -56,6 +56,7 @@ class PcapTestHarness : public IntegrationBase { LogTestController::getInstance().setDebug(); LogTestController::getInstance().setDebug(); LogTestController::getInstance().setDebug(); + LogTestController::getInstance().setDebug(); LogTestController::getInstance().setDebug(); } @@ -67,6 +68,7 @@ class PcapTestHarness : public IntegrationBase { assert(LogTestController::getInstance().contains("Starting capture") == true); assert(LogTestController::getInstance().contains("Stopping capture") == true); assert(LogTestController::getInstance().contains("Stopped device capture. clearing queues") == true); + assert(LogTestController::getInstance().contains("Accepting ") == true && LogTestController::getInstance().contains("because it matches .*") ); } void queryRootProcessGroup(std::shared_ptr pg) { @@ -77,6 +79,7 @@ class PcapTestHarness : public IntegrationBase { assert(inv != nullptr); configuration->set(minifi::processors::CapturePacket::BaseDir.getName(), dir); + configuration->set(minifi::processors::CapturePacket::NetworkControllers.getName(), ".*"); configuration->set("nifi.c2.enable", "false"); } diff --git a/linux.sh b/linux.sh index e279c840c7..54730ba0f9 100644 --- a/linux.sh +++ b/linux.sh @@ -42,6 +42,16 @@ verify_gcc_enable(){ else echo "false" fi + elif [ "$feature" = "PCAP_ENABLED" ]; then + if (( COMPILER_MAJOR >= 4 )); then + if (( COMPILER_MAJOR > 4 || COMPILER_MINOR >= 9 )); then + echo "true" + else + echo "false" + fi + else + echo "false" + fi else echo "true" fi From 81b55ea5882f6985551fda6ac206da9930084d73 Mon Sep 17 00:00:00 2001 From: Marc Parisi Date: Mon, 6 Aug 2018 19:20:53 -0400 Subject: [PATCH 2/2] MINIFICPP-590: Improve readability of PCAP option --- extensions/pcap/CapturePacket.cpp | 5 ++--- extensions/pcap/CapturePacket.h | 2 +- libminifi/test/pcap-tests/PcapTest.cpp | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/extensions/pcap/CapturePacket.cpp b/extensions/pcap/CapturePacket.cpp index 8091709099..bdd0c63381 100644 --- a/extensions/pcap/CapturePacket.cpp +++ b/extensions/pcap/CapturePacket.cpp @@ -54,7 +54,7 @@ namespace processors { std::shared_ptr CapturePacket::id_generator_ = utils::IdGenerator::getIdGenerator(); core::Property CapturePacket::BaseDir("Base Directory", "Scratch directory for PCAP files", "/tmp/"); core::Property CapturePacket::BatchSize("Batch Size", "The number of packets to combine within a given PCAP", "50"); -core::Property CapturePacket::NetworkControllers("Network Controllers", "List of network controllers to attach to -- each may be a regex", ".*"); +core::Property CapturePacket::NetworkController("Network Controller", "Regular expression network controller(s) to which we will attach", ".*"); core::Property CapturePacket::CaptureBluetooth("Capture Bluetooth", "True indicates that we support bluetooth interfaces", "false"); const char *CapturePacket::ProcessorName = "CapturePacket"; @@ -62,7 +62,6 @@ const char *CapturePacket::ProcessorName = "CapturePacket"; std::string CapturePacket::generate_new_pcap(const std::string &base_path) { std::string path = base_path; // can use relaxed for a counter - //int cnt = num_.fetch_add(1, std::memory_order_relaxed); int cnt = num_.fetch_add(1, std::memory_order_relaxed); std::string filename = std::to_string(cnt); path.append(filename); @@ -116,7 +115,7 @@ void CapturePacket::initialize() { // Set the supported properties std::set properties; properties.insert(BatchSize); - properties.insert(NetworkControllers); + properties.insert(NetworkController); properties.insert(BaseDir); properties.insert(CaptureBluetooth); setSupportedProperties(properties); diff --git a/extensions/pcap/CapturePacket.h b/extensions/pcap/CapturePacket.h index 623b13166c..3084eed047 100644 --- a/extensions/pcap/CapturePacket.h +++ b/extensions/pcap/CapturePacket.h @@ -109,7 +109,7 @@ class CapturePacket : public core::Processor { // Processor Name static const char *ProcessorName; static core::Property BatchSize; - static core::Property NetworkControllers; + static core::Property NetworkController; static core::Property BaseDir; static core::Property CaptureBluetooth; // Supported Relationships diff --git a/libminifi/test/pcap-tests/PcapTest.cpp b/libminifi/test/pcap-tests/PcapTest.cpp index aa8d214060..3c6fd27eb7 100644 --- a/libminifi/test/pcap-tests/PcapTest.cpp +++ b/libminifi/test/pcap-tests/PcapTest.cpp @@ -79,7 +79,7 @@ class PcapTestHarness : public IntegrationBase { assert(inv != nullptr); configuration->set(minifi::processors::CapturePacket::BaseDir.getName(), dir); - configuration->set(minifi::processors::CapturePacket::NetworkControllers.getName(), ".*"); + configuration->set(minifi::processors::CapturePacket::NetworkController.getName(), ".*"); configuration->set("nifi.c2.enable", "false"); }