KEP-1206: Infrastructure for reporting daemon statistics

bluzelle · Mar 1, 2019 · 02eaa41 · 02eaa41
1 parent 49e683e
commit 02eaa41
Show file tree

Hide file tree

Showing 29 changed files with 673 additions and 28 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -84,6 +84,7 @@ add_subdirectory(audit)
 add_subdirectory(pbft)
 add_subdirectory(chaos)
 add_subdirectory(crypto)
+add_subdirectory(monitor)
 
 include(cmake/static_analysis.cmake)
 

diff --git a/crypto/crypto.cpp b/crypto/crypto.cpp
@@ -26,8 +26,9 @@ namespace
     const std::string PEM_SUFFIX = "\n-----END PUBLIC KEY-----\n";
 }
 
-crypto::crypto(std::shared_ptr<bzn::options_base> options)
+crypto::crypto(std::shared_ptr<bzn::options_base> options, std::shared_ptr<bzn::monitor_base> monitor)
         : options(std::move(options))
+        , monitor(std::move(monitor))
 {
     LOG(info) << "Using " << SSLeay_version(SSLEAY_VERSION);
     if (this->options->get_simple_options().get<bool>(bzn::option_names::CRYPTO_ENABLED_OUTGOING))
@@ -154,6 +155,13 @@ crypto::verify(const bzn_envelope& msg)
      */
     ERR_clear_error();
 
+    this->monitor->send_counter(bzn::statistic::signature_verified);
+    this->monitor->send_counter(bzn::statistic::signature_verified_bytes, msg_text.length());
+    if (!result)
+    {
+        this->monitor->send_counter(bzn::statistic::signature_rejected);
+    }
+
     return result;
 }
 
@@ -219,6 +227,9 @@ crypto::sign(bzn_envelope& msg)
 
     }
 
+    this->monitor->send_counter(bzn::statistic::signature_computed);
+    this->monitor->send_counter(bzn::statistic::signature_computed_bytes, msg_text.length());
+
     return result;
 }
 
@@ -266,6 +277,9 @@ crypto::hash(const std::string& msg)
         throw std::runtime_error(std::string("\nfailed to compute message hash ") + msg);
     }
 
+    this->monitor->send_counter(bzn::statistic::hash_computed);
+    this->monitor->send_counter(bzn::statistic::hash_computed_bytes, msg.length());
+
     return std::string(reinterpret_cast<char*>(hash_buffer.get()), md_size);
 }
 

diff --git a/crypto/crypto.hpp b/crypto/crypto.hpp
@@ -20,14 +20,15 @@
 #include <proto/bluzelle.pb.h>
 #include <openssl/evp.h>
 #include <openssl/ec.h>
+#include <monitor/monitor_base.hpp>
 
 namespace bzn
 {
     class crypto : public bzn::crypto_base
     {
     public:
 
-        crypto(std::shared_ptr<bzn::options_base> options);
+        crypto(std::shared_ptr<bzn::options_base> options, std::shared_ptr<bzn::monitor_base> monitor);
 
         bool sign(bzn_envelope& msg) override;
 
@@ -53,6 +54,7 @@ namespace bzn
         const std::string deterministic_serialize(const bzn_envelope& msg);
 
         std::shared_ptr<bzn::options_base> options;
+        std::shared_ptr<bzn::monitor_base> monitor;
 
         EVP_PKEY_ptr_t private_key_EVP = EVP_PKEY_ptr_t(nullptr, &EVP_PKEY_free);
         EC_KEY_ptr_t private_key_EC = EC_KEY_ptr_t(nullptr, &EC_KEY_free);

diff --git a/crypto/test/crypto_test.cpp b/crypto/test/crypto_test.cpp
@@ -14,6 +14,7 @@
 
 #include <crypto/crypto.hpp>
 #include <mocks/mock_options_base.hpp>
+#include <mocks/mock_monitor.hpp>
 #include <options/options.hpp>
 #include <gtest/gtest.h>
 #include <proto/bluzelle.pb.h>
@@ -26,6 +27,7 @@ class crypto_test : public Test
 {
 public:
     std::shared_ptr<bzn::options_base> options = std::make_shared<bzn::options>();
+    std::shared_ptr<bzn::mock_monitor> monitor = std::make_shared<NiceMock<bzn::mock_monitor>>();
     std::shared_ptr<bzn::crypto> crypto;
 
     const std::string private_key_file = "test_private_key.pem";
@@ -63,7 +65,7 @@ class crypto_test : public Test
         this->options->get_mutable_simple_options().set(bzn::option_names::CRYPTO_ENABLED_INCOMING, std::to_string(true));
         this->options->get_mutable_simple_options().set(bzn::option_names::CRYPTO_ENABLED_OUTGOING, std::to_string(true));
 
-        this->crypto = std::make_shared<bzn::crypto>(this->options);
+        this->crypto = std::make_shared<bzn::crypto>(this->options, this->monitor);
 
     }
 

diff --git a/include/system_clock.hpp b/include/system_clock.hpp
@@ -0,0 +1,37 @@
+// Copyright (C) 2018 Bluzelle
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License, version 3,
+// as published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+#pragma once
+#include <chrono>
+#include <ratio>
+
+namespace bzn
+{
+    class system_clock_base
+    {
+    public:
+        virtual uint64_t microseconds_since_epoch() = 0;
+
+        virtual ~system_clock_base() = default;
+    };
+
+    class system_clock : public system_clock_base
+    {
+        uint64_t microseconds_since_epoch()
+        {
+            auto res = std::chrono::time_point_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now());
+            return res.time_since_epoch().count();
+        }
+    };
+}
diff --git a/mocks/mock_monitor.hpp b/mocks/mock_monitor.hpp
@@ -0,0 +1,32 @@
+// Copyright (C) 2018 Bluzelle
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License, version 3,
+// as published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+#pragma once
+
+#include <monitor/monitor_base.hpp>
+#include <gmock/gmock.h>
+
+namespace bzn {
+
+    class mock_monitor : public monitor_base {
+    public:
+        MOCK_METHOD1(start_timer,
+            void(std::string instance_id));
+        MOCK_METHOD2(finish_timer,
+            void(statistic stat, std::string instance_id));
+        MOCK_METHOD2(send_counter,
+            void(statistic, uint64_t));
+    };
+
+}  // namespace bzn
diff --git a/mocks/mock_system_clock.hpp b/mocks/mock_system_clock.hpp
@@ -0,0 +1,28 @@
+// Copyright (C) 2018 Bluzelle
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License, version 3,
+// as published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+#pragma once
+
+#include <include/system_clock.hpp>
+#include <gmock/gmock.h>
+
+namespace bzn
+{
+    class mock_system_clock : public system_clock_base
+    {
+    public:
+        MOCK_METHOD0(microseconds_since_epoch, uint64_t());
+    };
+}
+
diff --git a/monitor/CMakeLists.txt b/monitor/CMakeLists.txt
@@ -0,0 +1,8 @@
+add_library(monitor
+        monitor_base.hpp
+        monitor.hpp
+        monitor.cpp
+        )
+
+target_link_libraries(utils)
+add_subdirectory(test)
diff --git a/monitor/monitor.cpp b/monitor/monitor.cpp
@@ -0,0 +1,145 @@
+// Copyright (C) 2018 Bluzelle
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License, version 3,
+// as published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+#include <monitor/monitor.hpp>
+#include <unordered_map>
+#include <boost/format.hpp>
+#include <ratio>
+
+using namespace bzn;
+
+namespace
+{
+    std::map<statistic, std::string> statistic_names{
+            {
+                    {statistic::hash_computed, "crypto.hashes_computed"},
+                    {statistic::hash_computed_bytes, "crypto.bytes_hashed"},
+                    {statistic::signature_computed, "crypto.signatures_computed"},
+                    {statistic::signature_computed_bytes, "crypto.bytes_signed"},
+                    {statistic::signature_verified, "crypto.signatures_verified"},
+                    {statistic::signature_verified_bytes, "crypto.bytes_verified"},
+                    {statistic::signature_rejected, "crypto.signatures_rejected"},
+
+                    {statistic::session_opened, "node.sessions_opened"},
+                    {statistic::message_sent, "node.messages_sent"},
+                    {statistic::message_sent_bytes, "node.bytes_sent"},
+
+                    {statistic::pbft_no_primary, "pbft.liveness.no_primary"},
+                    {statistic::pbft_failure_detected, "pbft.liveness.failure_detected"},
+                    {statistic::pbft_commit_conflict, "pbft.safety.commit_conflict"},
+                    {statistic::pbft_primary_conflict, "pbft.safety.primary_conflict"},
+
+                    {statistic::request_latency, "total-server-latency"}
+            }
+    };
+}
+
+monitor::monitor(std::shared_ptr<bzn::options_base> options, std::shared_ptr<bzn::asio::io_context_base> context, std::shared_ptr<bzn::system_clock_base> clock)
+        : options(std::move(options))
+        , context(std::move(context))
+        , clock(std::move(clock))
+        , socket(this->context->make_unique_udp_socket())
+        , monitor_endpoint(this->options->get_monitor_endpoint(this->context))
+        , scope_prefix("com.bluzelle.swarm.singleton.node." + this->options->get_uuid())
+{
+    if (this->monitor_endpoint)
+    {
+        LOG(info) << boost::format("Will send stats to %1%:%2%")
+                     % this->monitor_endpoint->address().to_string()
+                     % this->monitor_endpoint->port();
+    }
+    else
+    {
+        LOG(info) << "No monitor is configured; stats will not be collected";
+    }
+
+}
+
+void
+monitor::start_timer(std::string timer_id)
+{
+    if (!this->monitor_endpoint)
+    {
+        return;
+    }
+
+    std::lock_guard<std::mutex> lock(this->timers_lock);
+    if (this->start_times.find(timer_id) != this->start_times.end())
+    {
+        return;
+    }
+
+    this->start_times.emplace(std::make_pair(timer_id, this->clock->microseconds_since_epoch()));
+    this->ordered_timers.push_back(timer_id);
+
+    while (this->ordered_timers.size() > this->options->get_simple_options().get<uint64_t>(bzn::option_names::MONITOR_MAX_TIMERS))
+    {
+        this->start_times.erase(this->ordered_timers.front());
+        this->ordered_timers.pop_front();
+    }
+}
+
+void
+monitor::finish_timer(bzn::statistic stat, std::string timer_id)
+{
+    if (!this->monitor_endpoint)
+    {
+        return;
+    }
+
+    uint64_t result;
+
+    {
+        std::lock_guard<std::mutex> lock(this->timers_lock);
+        if (this->start_times.find(timer_id) == this->start_times.end())
+        {
+            return;
+        }
+
+        result = this->clock->microseconds_since_epoch() - start_times.at(timer_id);
+        this->start_times.erase(timer_id);
+    }
+
+    auto stat_string = this->scope_prefix + "." + statistic_names.at(stat) + ":" + std::to_string(result) + "|us";
+    LOG(debug) << stat_string;
+    this->send(stat_string);
+}
+
+void
+monitor::send_counter(bzn::statistic stat, uint64_t amount)
+{
+    if (!this->monitor_endpoint)
+    {
+        return;
+    }
+
+    auto stat_string = this->scope_prefix + "." + statistic_names.at(stat) + ":" + std::to_string(amount) + "|c";
+    LOG(debug) << stat_string;
+
+    this->send(stat_string);
+}
+
+void
+monitor::send(const std::string& stat)
+{
+    std::shared_ptr<boost::asio::const_buffer> buffer = std::make_shared<boost::asio::const_buffer>(stat.c_str(), stat.size());
+    this->socket->async_send_to(*buffer, *(this->monitor_endpoint),
+            [buffer](const boost::system::error_code& ec, std::size_t /*bytes*/)
+            {
+                if (ec)
+                {
+                    LOG(error) << boost::format("UDP send failed");
+                }
+            });
+}