ENG-2824 Increase tcmalloc local thread cache size for tservers.

Summary:
I've increased TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES for tservers to be 256MB. The default for
TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES is about 32MB for our tservers and this was not sufficient for
all the threads that we have. TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES is the maximum sum of all
thread caches. As a result, if this is not sufficient, we end up seeing issues where threads don't
have enough memory in their caches and this causes slowness since one thread tries to steal memory
from the cache of another thread.

Test Plan:
1) Reproduced the issue on a tserver.
2) Verified that with these changes, we no longer see the issue consistently anymore. We do see a
few threads that get stuck at process startup, but once everything stabilizes, there are no issues.

Reviewers: amitanand, karthik, mikhail, kannan

Reviewed By: mikhail

Subscribers: ybase

Differential Revision: https://phabricator.dev.yugabyte.com/D4112

src/yb/tserver/tablet_server.h

@@ -58,6 +58,8 @@ class MaintenanceManager;
 
 namespace tserver {
 
+constexpr const char* const kTcMallocMaxThreadCacheBytes = "tcmalloc.max_total_thread_cache_bytes";
+
 class Heartbeater;
 class TabletServerPathHandlers;
 class TSTabletManager;

src/yb/tserver/tablet_server_main.cc

@@ -35,6 +35,7 @@
 
 #include <boost/optional/optional.hpp>
 #include <glog/logging.h>
+#include <gperftools/malloc_extension.h>
 
 #include "yb/gutil/strings/substitute.h"
 #include "yb/yql/redis/redisserver/redis_server.h"
@@ -47,12 +48,15 @@
 #include "yb/util/init.h"
 #include "yb/util/logging.h"
 #include "yb/util/main_util.h"
+#include "yb/util/size_literals.h"
 
 using yb::redisserver::RedisServer;
 using yb::redisserver::RedisServerOptions;
 using yb::cqlserver::CQLServer;
 using yb::cqlserver::CQLServerOptions;
 
+using namespace yb::size_literals;
+
 DEFINE_bool(start_redis_proxy, true, "Starts a redis proxy along with the tablet server");
 DEFINE_string(redis_proxy_bind_address, "", "Address to bind the redis proxy to");
 DEFINE_int32(redis_proxy_webserver_port, 0, "Webserver port for redis proxy");
@@ -63,6 +67,8 @@ DEFINE_string(cql_proxy_broadcast_rpc_address, "",
                   " system.local table");
 DEFINE_string(cql_proxy_bind_address, "", "Address to bind the CQL proxy to");
 DEFINE_int32(cql_proxy_webserver_port, 0, "Webserver port for CQL proxy");
+DEFINE_int64(tserver_tcmalloc_max_total_thread_cache_bytes, 256_MB, "Total number of bytes to "
+             "use for the thread cache for tcmalloc across all threads in the tserver.");
 
 DECLARE_string(rpc_bind_addresses);
 DECLARE_bool(callhome_enabled);
@@ -81,6 +87,13 @@ static int TabletServerMain(int argc, char** argv) {
   FLAGS_cql_proxy_bind_address = strings::Substitute("0.0.0.0:$0", CQLServer::kDefaultPort);
   FLAGS_cql_proxy_webserver_port = CQLServer::kDefaultWebPort;
 
+#ifdef TCMALLOC_ENABLED
+  if (!MallocExtension::instance()->SetNumericProperty(kTcMallocMaxThreadCacheBytes,
+      FLAGS_tserver_tcmalloc_max_total_thread_cache_bytes)) {
+    LOG(FATAL) << "Failed to set Tcmalloc property: " << kTcMallocMaxThreadCacheBytes;
+  }
+#endif
+
   ParseCommandLineFlags(&argc, &argv, true);
   if (argc != 1) {
     std::cerr << "usage: " << argv[0] << std::endl;