Add upstream_cx_overflow stat (#113)

envoyproxy · Oct 3, 2016 · c950a55 · c950a55
1 parent 9d97f94
commit c950a55
Show file tree

Hide file tree

Showing 6 changed files with 15 additions and 2 deletions.
diff --git a/docs/configuration/cluster_manager/cluster_stats.rst b/docs/configuration/cluster_manager/cluster_stats.rst
@@ -15,6 +15,7 @@ Every cluster has a statistics tree rooted at *cluster.<name>.* with the followi
   upstream_cx_http2_total, Counter, Total HTTP/2 connections
   upstream_cx_connect_fail, Counter, Total connection failures
   upstream_cx_connect_timeout, Counter, Total connection timeouts
+  upstream_cx_overflow, Counter, Total times that the cluster's connection circuit breaker overflowed
   upstream_cx_connect_ms, Timer, Connection establishment milliseconds
   upstream_cx_length_ms, Timer, Connection length milliseconds
   upstream_cx_destroy, Counter, Total destroyed connections

diff --git a/include/envoy/upstream/upstream.h b/include/envoy/upstream/upstream.h
@@ -140,6 +140,7 @@ class HostSet {
   COUNTER(upstream_cx_http2_total)                                                                 \
   COUNTER(upstream_cx_connect_fail)                                                                \
   COUNTER(upstream_cx_connect_timeout)                                                             \
+  COUNTER(upstream_cx_overflow)                                                                    \
   TIMER  (upstream_cx_connect_ms)                                                                  \
   TIMER  (upstream_cx_length_ms)                                                                   \
   COUNTER(upstream_cx_destroy)                                                                     \

diff --git a/source/common/filter/tcp_proxy.cc b/source/common/filter/tcp_proxy.cc
@@ -51,6 +51,7 @@ void TcpProxy::initializeUpstreamConnection() {
           ->resourceManager(Upstream::ResourcePriority::Default);
 
   if (!upstream_cluster_resource_manager.connections().canCreate()) {
+    cluster_manager_.get(config_->clusterName())->stats().upstream_cx_overflow_.inc();
     read_callbacks_->connection().close(Network::ConnectionCloseType::NoFlush);
     return;
   }

diff --git a/source/common/http/http1/conn_pool.cc b/source/common/http/http1/conn_pool.cc
@@ -72,9 +72,14 @@ ConnectionPool::Cancellable* ConnPoolImpl::newStream(StreamDecoder& response_dec
   }
 
   if (host_->cluster().resourceManager(priority_).pendingRequests().canCreate()) {
+    bool can_create_connection =
+        host_->cluster().resourceManager(priority_).connections().canCreate();
+    if (!can_create_connection) {
+      host_->cluster().stats().upstream_cx_overflow_.inc();
+    }
+
     // If we have no connections at all, make one no matter what so we don't starve.
-    if ((ready_clients_.size() == 0 && busy_clients_.size() == 0) ||
-        host_->cluster().resourceManager(priority_).connections().canCreate()) {
+    if ((ready_clients_.size() == 0 && busy_clients_.size() == 0) || can_create_connection) {
       createNewConnection();
     }
 

diff --git a/test/common/filter/tcp_proxy_test.cc b/test/common/filter/tcp_proxy_test.cc
@@ -170,6 +170,10 @@ TEST_F(TcpProxyTest, UpstreamConnectionLimit) {
   // The downstream connection closes if the proxy can't make an upstream connection.
   EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::NoFlush));
   ASSERT_EQ(Network::FilterStatus::StopIteration, filter_->onData(buffer));
+  EXPECT_EQ(
+      1U,
+      cluster_manager_.cluster_.stats_store_.counter("cluster.fake_cluster.upstream_cx_overflow")
+          .value());
 }
 
 } // Filter
diff --git a/test/common/http/http1/conn_pool_test.cc b/test/common/http/http1/conn_pool_test.cc
@@ -349,6 +349,7 @@ TEST_F(Http1ConnPoolImplTest, MaxConnections) {
   NiceMock<Http::MockStreamDecoder> outer_decoder2;
   ConnPoolCallbacks callbacks2;
   handle = conn_pool_.newStream(outer_decoder2, callbacks2);
+  EXPECT_EQ(1U, cluster_.stats_.upstream_cx_overflow_.value());
 
   EXPECT_NE(nullptr, handle);