mds: release pool allocator memory after exceeding size limit

src/common/config_opts.h

@@ -491,6 +491,7 @@ OPTION(mds_recall_state_timeout, OPT_FLOAT, 60)    // detect clients which aren'
 OPTION(mds_freeze_tree_timeout, OPT_FLOAT, 30)    // detecting freeze tree deadlock
 OPTION(mds_session_autoclose, OPT_FLOAT, 300) // autoclose idle session
 OPTION(mds_health_summarize_threshold, OPT_INT, 10) // collapse N-client health metrics to a single 'many'
+OPTION(mds_health_cache_threshold, OPT_FLOAT, 1.5) // warn on cache size if it exceeds mds_cache_size by this factor
 OPTION(mds_reconnect_timeout, OPT_FLOAT, 45)  // seconds to wait for clients during mds restart
 	      //  make it (mds_session_timeout - mds_beacon_grace)
 OPTION(mds_tick_interval, OPT_FLOAT, 5)

src/mds/Beacon.cc

@@ -482,7 +482,8 @@ void Beacon::notify_health(MDSRank const *mds)
   }
 
   // Report if we have significantly exceeded our cache size limit
-  if (mds->mdcache->get_num_inodes() > g_conf->mds_cache_size * 1.5) {
+  if (mds->mdcache->get_num_inodes() >
+        g_conf->mds_cache_size * g_conf->mds_health_cache_threshold) {
     std::ostringstream oss;
     oss << "Too many inodes in cache (" << mds->mdcache->get_num_inodes()
         << "/" << g_conf->mds_cache_size << "), "

src/mds/MDCache.cc

@@ -179,6 +179,7 @@ class MDCacheLogContext : public virtual MDSLogContextBase {
 MDCache::MDCache(MDSRank *m) :
   mds(m),
   filer(m->objecter, m->finisher),
+  exceeded_size_limit(false),
   recovery_queue(m),
   stray_manager(m)
 {
@@ -280,6 +281,11 @@ void MDCache::add_inode(CInode *in)
     if (in->is_base())
       base_inodes.insert(in);
   }
+
+  if (get_num_inodes() >
+        g_conf->mds_cache_size * g_conf->mds_health_cache_threshold) {
+    exceeded_size_limit = true;
+  }
 }
 
 void MDCache::remove_inode(CInode *o) 
@@ -7378,6 +7384,22 @@ void MDCache::check_memory_usage()
       mds->server->recall_client_state(ratio);
     }
   }
+
+  // If the cache size had exceeded its limit, but we're back in bounds
+  // now, free any unused pool memory so that our memory usage isn't
+  // permanently bloated.
+  if (exceeded_size_limit
+      && get_num_inodes() <=
+        g_conf->mds_cache_size * g_conf->mds_health_cache_threshold) {
+    // Only do this once we are back in bounds: otherwise the releases would
+    // slow down whatever process caused us to exceed bounds to begin with
+    dout(2) << "check_memory_usage: releasing unused space from pool allocators"
+            << dendl;
+    CInode::pool.release_memory();
+    CDir::pool.release_memory();
+    CDentry::pool.release_memory();
+    exceeded_size_limit = false;
+  }
 }
 
 

src/mds/MDCache.h

@@ -139,6 +139,8 @@ class MDCache {
 
   Filer filer;
 
+  bool exceeded_size_limit;
+
 public:
   void advance_stray() {
     stray_index = (stray_index+1)%NUM_STRAY;