consolidated invariant failures into single metric

stellar · Jan 7, 2019 · 22c2c09 · 22c2c09 · latobarita · Jan 7, 2019
1 parent 9defa61
commit 22c2c09
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 23 deletions.
diff --git a/docs/metrics.md b/docs/metrics.md
@@ -54,7 +54,7 @@ history.download-<X>.success      | meter     | download of <X> completed succes
 history.download-<X>.failure      | meter     | download of <X> failed
 history.verify-<X>.success        | meter     | verification of <X> succeeded
 history.verify-<X>.failure        | meter     | verification of <X> failed
-invariant.does-not-hold.count.<X> | counter   | number of times invariant <X> failed
+ledger.invariant.failure          | counter   | number of times invariants failed
 ledger.transaction.apply          | timer     | time to apply one transaction
 ledger.transaction.count          | histogram | number of transactions per ledger
 ledger.transaction.internal-error | counter   | number of internal errors since start

diff --git a/src/invariant/InvariantManagerImpl.cpp b/src/invariant/InvariantManagerImpl.cpp
@@ -32,27 +32,26 @@ InvariantManager::create(Application& app)
 }
 
 InvariantManagerImpl::InvariantManagerImpl(medida::MetricsRegistry& registry)
-    : mMetricsRegistry(registry)
+    : mInvariantFailureCount(
+          registry.NewCounter({"ledger", "invariant", "failure"}))
 {
 }
 
 Json::Value
 InvariantManagerImpl::getJsonInfo()
 {
     Json::Value failures;
-    for (auto const& invariant : mInvariants)
-    {
-        auto& counter = mMetricsRegistry.NewCounter(
-            {"invariant", "does-not-hold", "count", invariant.first});
-        if (counter.count() > 0)
-        {
-            auto const& info = mFailureInformation.at(invariant.first);
 
-            auto& fail = failures[invariant.first];
-            fail["count"] = (Json::Int64)counter.count();
-            fail["last_failed_on_ledger"] = info.lastFailedOnLedger;
-            fail["last_failed_with_message"] = info.lastFailedWithMessage;
-        }
+    for (auto const& fi : mFailureInformation)
+    {
+        auto& fail = failures[fi.first];
+        auto& info = fi.second;
+        fail["last_failed_on_ledger"] = info.lastFailedOnLedger;
+        fail["last_failed_with_message"] = info.lastFailedWithMessage;
+    }
+    if (!failures.empty())
+    {
+        failures["count"] = (Json::Int64)mInvariantFailureCount.count();
     }
     return failures;
 }
@@ -131,8 +130,6 @@ InvariantManagerImpl::registerInvariant(std::shared_ptr<Invariant> invariant)
     if (iter == mInvariants.end())
     {
         mInvariants[name] = invariant;
-        mMetricsRegistry.NewCounter(
-            {"invariant", "does-not-hold", "count", invariant->getName()});
     }
     else
     {
@@ -208,12 +205,8 @@ InvariantManagerImpl::onInvariantFailure(std::shared_ptr<Invariant> invariant,
                                          std::string const& message,
                                          uint32_t ledger)
 {
-    mMetricsRegistry
-        .NewCounter(
-            {"invariant", "does-not-hold", "count", invariant->getName()})
-        .inc();
-    mFailureInformation[invariant->getName()].lastFailedOnLedger = ledger;
-    mFailureInformation[invariant->getName()].lastFailedWithMessage = message;
+    mInvariantFailureCount.inc();
+    mFailureInformation[invariant->getName()] = {ledger, message};
     handleInvariantFailure(invariant, message);
 }
 

diff --git a/src/invariant/InvariantManagerImpl.h b/src/invariant/InvariantManagerImpl.h
@@ -11,6 +11,7 @@
 namespace medida
 {
 class MetricsRegistry;
+class Counter;
 }
 
 namespace stellar
@@ -20,7 +21,7 @@ class InvariantManagerImpl : public InvariantManager
 {
     std::map<std::string, std::shared_ptr<Invariant>> mInvariants;
     std::vector<std::shared_ptr<Invariant>> mEnabled;
-    medida::MetricsRegistry& mMetricsRegistry;
+    medida::Counter& mInvariantFailureCount;
 
     struct InvariantFailureInformation
     {

diff --git a/src/ledger/LedgerManagerImpl.cpp b/src/ledger/LedgerManagerImpl.cpp
@@ -1007,6 +1007,9 @@ LedgerManagerImpl::applyTransactions(std::vector<TransactionFramePtr>& txs,
         }
         catch (InvariantDoesNotHold&)
         {
+            CLOG(ERROR, "Ledger")
+                << "Invariant failure during tx->apply for tx "
+                << tx->getFullHash();
             throw;
         }
         catch (std::runtime_error& e)