mds: decay counter for tracking request load average

Signed-off-by: Venky Shankar <vshankar@redhat.com>

src/common/options.cc

@@ -7173,6 +7173,10 @@ std::vector<Option> get_mds_options() {
 
     Option("mds_inject_migrator_session_race", Option::TYPE_BOOL, Option::LEVEL_DEV)
      .set_default(false),
+
+    Option("mds_request_load_average_decay_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
+    .set_default(60)
+    .set_description("rate of decay in seconds for calculating request load average"),
   });
 }
 

src/mds/Server.cc

@@ -1540,6 +1540,9 @@ void Server::early_reply(MDRequestRef& mdr, CInode *tracei, CDentry *tracedn)
   mds->logger->inc(l_mds_reply);
   utime_t lat = ceph_clock_now() - req->get_recv_stamp();
   mds->logger->tinc(l_mds_reply_latency, lat);
+  if (client_inst.name.is_client()) {
+    mds->sessionmap.hit_session(mdr->session);
+  }
   perf_gather_op_latency(req, lat);
   dout(20) << "lat " << lat << dendl;
 
@@ -1597,6 +1600,9 @@ void Server::reply_client_request(MDRequestRef& mdr, MClientReply *reply)
     mds->logger->inc(l_mds_reply);
     utime_t lat = ceph_clock_now() - mdr->client_request->get_recv_stamp();
     mds->logger->tinc(l_mds_reply_latency, lat);
+    if (client_inst.name.is_client()) {
+      mds->sessionmap.hit_session(session);
+    }
     perf_gather_op_latency(req, lat);
     dout(20) << "lat " << lat << dendl;
 

src/mds/SessionMap.cc

@@ -21,6 +21,7 @@
 
 #include "common/config.h"
 #include "common/errno.h"
+#include "common/DecayCounter.h"
 #include "include/assert.h"
 #include "include/stringify.h"
 
@@ -56,6 +57,8 @@ void SessionMap::register_perfcounters()
               "Sessions currently open");
   plb.add_u64(l_mdssm_session_stale, "sessions_stale",
               "Sessions currently stale");
+  plb.add_u64(l_mdssm_total_load, "total_load", "Total Load");
+  plb.add_u64(l_mdssm_avg_load, "average_load", "Average Load");
   logger = plb.create_perf_counters();
   g_ceph_context->get_perfcounters_collection()->add(logger);
 }
@@ -148,8 +151,10 @@ void SessionMapStore::decode_values(std::map<std::string, bufferlist> &session_v
     }
 
     Session *s = get_or_add_session(inst);
-    if (s->is_closed())
+    if (s->is_closed()) {
       s->set_state(Session::STATE_OPEN);
+      s->set_load_avg_decay_rate(decay_rate);
+    }
     auto q = i->second.cbegin();
     s->decode(q);
   }
@@ -488,6 +493,10 @@ uint64_t SessionMap::set_state(Session *session, int s) {
       by_state_entry = by_state.emplace(s, new xlist<Session*>).first;
     by_state_entry->second->push_back(&session->item_session_list);
 
+    if (session->is_open() || session->is_stale()) {
+      session->set_load_avg_decay_rate(decay_rate);
+    }
+
     // refresh number of sessions for states which have perf
     // couters associated
     logger->set(l_mdssm_session_open,
@@ -514,8 +523,10 @@ void SessionMapStore::decode_legacy(bufferlist::const_iterator& p)
       entity_inst_t inst;
       decode(inst.name, p);
       Session *s = get_or_add_session(inst);
-      if (s->is_closed())
+      if (s->is_closed()) {
         s->set_state(Session::STATE_OPEN);
+        s->set_load_avg_decay_rate(decay_rate);
+      }
       s->decode(p);
     }
 
@@ -544,6 +555,7 @@ void SessionMapStore::decode_legacy(bufferlist::const_iterator& p)
 	session_map[s->info.inst.name] = s;
       }
       s->set_state(Session::STATE_OPEN);
+      s->set_load_avg_decay_rate(decay_rate);
       s->last_cap_renew = now;
     }
   }
@@ -949,6 +961,21 @@ int Session::check_access(CInode *in, unsigned mask,
   return 0;
 }
 
+// track total and per session load
+void SessionMap::hit_session(Session *session) {
+  uint64_t sessions = get_session_count_in_state(Session::STATE_OPEN) +
+                      get_session_count_in_state(Session::STATE_STALE);
+  assert(sessions != 0);
+
+  double total_load = total_load_avg.hit();
+  double avg_load = total_load / sessions;
+
+  logger->set(l_mdssm_total_load, (uint64_t)total_load);
+  logger->set(l_mdssm_avg_load, (uint64_t)avg_load);
+
+  session->hit_session();
+}
+
 int SessionFilter::parse(
     const std::vector<std::string> &args,
     std::stringstream *ss)

src/mds/SessionMap.h

@@ -30,6 +30,7 @@ using std::set;
 
 class CInode;
 struct MDRequestImpl;
+class DecayCounter;
 
 #include "CInode.h"
 #include "Capability.h"
@@ -43,6 +44,8 @@ enum {
   l_mdssm_session_remove,
   l_mdssm_session_open,
   l_mdssm_session_stale,
+  l_mdssm_total_load,
+  l_mdssm_avg_load,
   l_mdssm_last,
 };
 
@@ -101,7 +104,8 @@ class Session : public RefCountedObject {
   // that appropriate mark_dirty calls follow.
   std::deque<version_t> projected;
 
-
+  // request load average for this session
+  DecayCounter load_avg;
 
 public:
 
@@ -204,6 +208,14 @@ class Session : public RefCountedObject {
   }
   bool is_importing() const { return importing_count > 0; }
 
+  void set_load_avg_decay_rate(double rate) {
+    assert(is_open() || is_stale());
+    load_avg = DecayCounter(rate);
+  }
+  void hit_session() {
+    load_avg.adjust();
+  }
+
   // -- caps --
 private:
   version_t cap_push_seq;        // cap push seq #
@@ -413,6 +425,11 @@ class SessionMapStore {
   version_t version;
   ceph::unordered_map<entity_name_t, Session*> session_map;
   PerfCounters *logger;
+
+  // total request load avg
+  double decay_rate;
+  DecayCounter total_load_avg;
+
 public:
   mds_rank_t rank;
 
@@ -454,7 +471,11 @@ class SessionMapStore {
     session_map.clear();
   }
 
-  SessionMapStore() : version(0), logger(nullptr), rank(MDS_RANK_NONE) {}
+  SessionMapStore()
+    : version(0), logger(nullptr),
+      decay_rate(g_conf().get_val<double>("mds_request_load_average_decay_rate")),
+      total_load_avg(decay_rate), rank(MDS_RANK_NONE) {
+  }
   virtual ~SessionMapStore() {};
 };
 
@@ -691,6 +712,9 @@ class SessionMap : public SessionMapStore {
   uint64_t get_session_count_in_state(int state) {
     return !is_any_state(state) ? 0 : by_state[state]->size();
   }
+
+public:
+  void hit_session(Session *session);
 };
 
 std::ostream& operator<<(std::ostream &out, const Session &s);