[mysql-5.6][PR] Add stall counters to 'SHOW GLOBAL STATUS' output.

Summary:
@update-submodule: rocksdb

The stall counters are already shown in 'SHOW ENGINE ROCKSDB STATUS
output' but this command is a bit expensive to run and you have to parse
the output to monitor these values. This branch adds these counters to
structured output of SHOW GLOBAL STATUS command.
Also fixed the bug in INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS  which showed only integer values but not float.

The rocksdb module was updated to bring [this](facebook/rocksdb#2794 (review)) change  which has the following changes to GetMapProperty method:
1. Changed values from double to string
2. Change existing keys to start with "compaction." prefix for all keys returned by old version of GetMapProperty.
3. Added stall values with keys starting with "io_stall." prefix
Closes #695
GitHub Author: Artem Danilov <rtm@fb.com>

Github PR Sync: {sync, type="child", parent="github", parentrepo="facebook/mysql-5.6", parentprnum="695", parentprfbid="136088867001144"}

Test Plan:
* Check that compaction level metrics are still published to
INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS properly
* Check the output of the command:
```
mysql> show status like 'rocksdb_stall_%';
+-----------------------------------------------------+-------+
| Variable_name                                       | Value |
+-----------------------------------------------------+-------+
| rocksdb_stall_level0_slowdown                       | 2     |
| rocksdb_stall_level0_slowdown_with_compaction       | 4     |
| rocksdb_stall_level0_numfiles                       | 10    |
| rocksdb_stall_level0_numfiles_with_compaction       | 12    |
| rocksdb_stall_stop_for_pending_compaction_bytes     | 16    |
| rocksdb_stall_slowdown_for_pending_compaction_bytes | 14    |
| rocksdb_stall_memtable_compaction                   | 6     |
| rocksdb_stall_memtable_slowdown                     | 8     |
| rocksdb_stall_total_count                           | 56    |
| rocksdb_stall_micros                                | 0     |
+-----------------------------------------------------+-------+
```
(In order to have some non-zero numbers, I've initialized internal
RocksDB stall stats with 1-8 numbers. They all doubled in output as
there are two column families in my instance: default and system.
The stall_micros has been introduced previously. total_count doesn't
include level0_slowdown_with_compaction and numfiles_with_compaction
as only their total counterparts are included in this counter.)

Reviewers: mung, herman, svcscm

Reviewed By: svcscm

Subscribers: svcscm, webscalesql-eng@fb.com

Differential Revision: https://phabricator.intern.facebook.com/D5723305

Signature: 5723305:1505173795:9b3e4d835d5fc9f2fa1c165b137c2d4cdf1efdae

mysql-test/suite/rocksdb/r/rocksdb.result

@@ -1507,6 +1507,16 @@ rocksdb_number_superversion_cleanups	#
 rocksdb_number_superversion_releases	#
 rocksdb_rate_limit_delay_millis	#
 rocksdb_snapshot_conflict_errors	#
+rocksdb_stall_l0_file_count_limit_slowdowns	#
+rocksdb_stall_locked_l0_file_count_limit_slowdowns	#
+rocksdb_stall_l0_file_count_limit_stops	#
+rocksdb_stall_locked_l0_file_count_limit_stops	#
+rocksdb_stall_pending_compaction_limit_stops	#
+rocksdb_stall_pending_compaction_limit_slowdowns	#
+rocksdb_stall_memtable_limit_stops	#
+rocksdb_stall_memtable_limit_slowdowns	#
+rocksdb_stall_total_stops	#
+rocksdb_stall_total_slowdowns	#
 rocksdb_stall_micros	#
 rocksdb_wal_bytes	#
 rocksdb_wal_group_syncs	#
@@ -1584,6 +1594,16 @@ ROCKSDB_NUMBER_SUPERVERSION_CLEANUPS
 ROCKSDB_NUMBER_SUPERVERSION_RELEASES
 ROCKSDB_RATE_LIMIT_DELAY_MILLIS
 ROCKSDB_SNAPSHOT_CONFLICT_ERRORS
+ROCKSDB_STALL_L0_FILE_COUNT_LIMIT_SLOWDOWNS
+ROCKSDB_STALL_LOCKED_L0_FILE_COUNT_LIMIT_SLOWDOWNS
+ROCKSDB_STALL_L0_FILE_COUNT_LIMIT_STOPS
+ROCKSDB_STALL_LOCKED_L0_FILE_COUNT_LIMIT_STOPS
+ROCKSDB_STALL_PENDING_COMPACTION_LIMIT_STOPS
+ROCKSDB_STALL_PENDING_COMPACTION_LIMIT_SLOWDOWNS
+ROCKSDB_STALL_MEMTABLE_LIMIT_STOPS
+ROCKSDB_STALL_MEMTABLE_LIMIT_SLOWDOWNS
+ROCKSDB_STALL_TOTAL_STOPS
+ROCKSDB_STALL_TOTAL_SLOWDOWNS
 ROCKSDB_STALL_MICROS
 ROCKSDB_WAL_BYTES
 ROCKSDB_WAL_GROUP_SYNCS
@@ -1663,6 +1683,16 @@ ROCKSDB_NUMBER_SUPERVERSION_CLEANUPS
 ROCKSDB_NUMBER_SUPERVERSION_RELEASES
 ROCKSDB_RATE_LIMIT_DELAY_MILLIS
 ROCKSDB_SNAPSHOT_CONFLICT_ERRORS
+ROCKSDB_STALL_L0_FILE_COUNT_LIMIT_SLOWDOWNS
+ROCKSDB_STALL_LOCKED_L0_FILE_COUNT_LIMIT_SLOWDOWNS
+ROCKSDB_STALL_L0_FILE_COUNT_LIMIT_STOPS
+ROCKSDB_STALL_LOCKED_L0_FILE_COUNT_LIMIT_STOPS
+ROCKSDB_STALL_PENDING_COMPACTION_LIMIT_STOPS
+ROCKSDB_STALL_PENDING_COMPACTION_LIMIT_SLOWDOWNS
+ROCKSDB_STALL_MEMTABLE_LIMIT_STOPS
+ROCKSDB_STALL_MEMTABLE_LIMIT_SLOWDOWNS
+ROCKSDB_STALL_TOTAL_STOPS
+ROCKSDB_STALL_TOTAL_SLOWDOWNS
 ROCKSDB_STALL_MICROS
 ROCKSDB_WAL_BYTES
 ROCKSDB_WAL_GROUP_SYNCS

storage/rocksdb/ha_rocksdb.cc

@@ -103,6 +103,7 @@ namespace myrocks {
 static st_global_stats global_stats;
 static st_export_stats export_stats;
 static st_memory_stats memory_stats;
+static st_io_stall_stats io_stall_stats;
 
 const std::string DEFAULT_CF_NAME("default");
 const std::string DEFAULT_SYSTEM_CF_NAME("__system__");
@@ -11328,6 +11329,91 @@ static void show_myrocks_vars(THD *thd, SHOW_VAR *var, char *buff) {
   var->value = reinterpret_cast<char *>(&myrocks_status_variables);
 }
 
+static ulonglong
+io_stall_prop_value(const std::map<std::string, std::string> &props,
+                    const std::string &key) {
+  std::map<std::string, std::string>::const_iterator iter =
+      props.find("io_stalls." + key);
+  if (iter != props.end()) {
+    return std::stoull(iter->second);
+  } else {
+    DBUG_PRINT("warning",
+               ("RocksDB GetMapPropery hasn't returned key=%s", key.c_str()));
+    DBUG_ASSERT(0);
+    return 0;
+  }
+}
+
+static void update_rocksdb_stall_status() {
+  st_io_stall_stats local_io_stall_stats;
+  for (const auto &cf_name : cf_manager.get_cf_names()) {
+    rocksdb::ColumnFamilyHandle *cfh = cf_manager.get_cf(cf_name);
+    if (cfh == nullptr) {
+      continue;
+    }
+
+    std::map<std::string, std::string> props;
+    if (!rdb->GetMapProperty(cfh, "rocksdb.cfstats", &props)) {
+      continue;
+    }
+
+    local_io_stall_stats.level0_slowdown +=
+        io_stall_prop_value(props, "level0_slowdown");
+    local_io_stall_stats.level0_slowdown_with_compaction +=
+        io_stall_prop_value(props, "level0_slowdown_with_compaction");
+    local_io_stall_stats.level0_numfiles +=
+        io_stall_prop_value(props, "level0_numfiles");
+    local_io_stall_stats.level0_numfiles_with_compaction +=
+        io_stall_prop_value(props, "level0_numfiles_with_compaction");
+    local_io_stall_stats.stop_for_pending_compaction_bytes +=
+        io_stall_prop_value(props, "stop_for_pending_compaction_bytes");
+    local_io_stall_stats.slowdown_for_pending_compaction_bytes +=
+        io_stall_prop_value(props, "slowdown_for_pending_compaction_bytes");
+    local_io_stall_stats.memtable_compaction +=
+        io_stall_prop_value(props, "memtable_compaction");
+    local_io_stall_stats.memtable_slowdown +=
+        io_stall_prop_value(props, "memtable_slowdown");
+    local_io_stall_stats.total_stop += io_stall_prop_value(props, "total_stop");
+    local_io_stall_stats.total_slowdown +=
+        io_stall_prop_value(props, "total_slowdown");
+  }
+  io_stall_stats = local_io_stall_stats;
+}
+
+static SHOW_VAR rocksdb_stall_status_variables[] = {
+    DEF_STATUS_VAR_FUNC("l0_file_count_limit_slowdowns",
+                        &io_stall_stats.level0_slowdown, SHOW_LONGLONG),
+    DEF_STATUS_VAR_FUNC("locked_l0_file_count_limit_slowdowns",
+                        &io_stall_stats.level0_slowdown_with_compaction,
+                        SHOW_LONGLONG),
+    DEF_STATUS_VAR_FUNC("l0_file_count_limit_stops",
+                        &io_stall_stats.level0_numfiles, SHOW_LONGLONG),
+    DEF_STATUS_VAR_FUNC("locked_l0_file_count_limit_stops",
+                        &io_stall_stats.level0_numfiles_with_compaction,
+                        SHOW_LONGLONG),
+    DEF_STATUS_VAR_FUNC("pending_compaction_limit_stops",
+                        &io_stall_stats.stop_for_pending_compaction_bytes,
+                        SHOW_LONGLONG),
+    DEF_STATUS_VAR_FUNC("pending_compaction_limit_slowdowns",
+                        &io_stall_stats.slowdown_for_pending_compaction_bytes,
+                        SHOW_LONGLONG),
+    DEF_STATUS_VAR_FUNC("memtable_limit_stops",
+                        &io_stall_stats.memtable_compaction, SHOW_LONGLONG),
+    DEF_STATUS_VAR_FUNC("memtable_limit_slowdowns",
+                        &io_stall_stats.memtable_slowdown, SHOW_LONGLONG),
+    DEF_STATUS_VAR_FUNC("total_stops", &io_stall_stats.total_stop,
+                        SHOW_LONGLONG),
+    DEF_STATUS_VAR_FUNC("total_slowdowns", &io_stall_stats.total_slowdown,
+                        SHOW_LONGLONG),
+    // end of the array marker
+    {NullS, NullS, SHOW_LONG}};
+
+static void show_rocksdb_stall_vars(THD *thd, SHOW_VAR *var, char *buff) {
+  update_rocksdb_stall_status();
+  var->type = SHOW_ARRAY;
+  var->value = reinterpret_cast<char *>(&rocksdb_stall_status_variables);
+}
+
 static SHOW_VAR rocksdb_status_vars[] = {
     DEF_STATUS_VAR(block_cache_miss),
     DEF_STATUS_VAR(block_cache_hit),
@@ -11395,7 +11481,12 @@ static SHOW_VAR rocksdb_status_vars[] = {
                        SHOW_LONGLONG),
     DEF_STATUS_VAR_PTR("number_sst_entry_other", &rocksdb_num_sst_entry_other,
                        SHOW_LONGLONG),
+    // the variables generated by SHOW_FUNC are sorted only by prefix (first
+    // arg in the tuple below), so make sure it is unique to make sorting
+    // deterministic as quick sort is not stable
     {"rocksdb", reinterpret_cast<char *>(&show_myrocks_vars), SHOW_FUNC},
+    {"rocksdb_stall", reinterpret_cast<char *>(&show_rocksdb_stall_vars),
+     SHOW_FUNC},
     {NullS, NullS, SHOW_LONG}};
 
 /*

storage/rocksdb/ha_rocksdb.h

@@ -405,6 +405,27 @@ struct st_memory_stats {
   ulonglong memtable_unflushed;
 };
 
+/* Struct used for exporting RocksDB IO stalls stats */
+struct st_io_stall_stats {
+  ulonglong level0_slowdown;
+  ulonglong level0_slowdown_with_compaction;
+  ulonglong level0_numfiles;
+  ulonglong level0_numfiles_with_compaction;
+  ulonglong stop_for_pending_compaction_bytes;
+  ulonglong slowdown_for_pending_compaction_bytes;
+  ulonglong memtable_compaction;
+  ulonglong memtable_slowdown;
+  ulonglong total_stop;
+  ulonglong total_slowdown;
+
+  st_io_stall_stats()
+      : level0_slowdown(0), level0_slowdown_with_compaction(0),
+        level0_numfiles(0), level0_numfiles_with_compaction(0),
+        stop_for_pending_compaction_bytes(0),
+        slowdown_for_pending_compaction_bytes(0), memtable_compaction(0),
+        memtable_slowdown(0), total_stop(0), total_slowdown(0) {}
+};
+
 } // namespace myrocks
 
 #include "./rdb_buff.h"

storage/rocksdb/rdb_i_s.cc

@@ -845,17 +845,22 @@ static int rdb_i_s_compact_stats_fill_table(
       continue;
     }
 
-    std::map<std::string, double> props;
+    std::map<std::string, std::string> props;
     bool bool_ret MY_ATTRIBUTE((__unused__));
     bool_ret = rdb->GetMapProperty(cfh, "rocksdb.cfstats", &props);
     DBUG_ASSERT(bool_ret);
 
+    const std::string prop_name_prefix = "compaction.";
     for (auto const &prop_ent : props) {
       std::string prop_name = prop_ent.first;
-      double value = prop_ent.second;
-      std::size_t del_pos = prop_name.find('.');
+      if (prop_name.find(prop_name_prefix) != 0) {
+        continue;
+      }
+      std::string value = prop_ent.second;
+      std::size_t del_pos = prop_name.find('.', prop_name_prefix.size());
       DBUG_ASSERT(del_pos != std::string::npos);
-      std::string level_str = prop_name.substr(0, del_pos);
+      std::string level_str = prop_name.substr(
+          prop_name_prefix.size(), del_pos - prop_name_prefix.size());
       std::string type_str = prop_name.substr(del_pos + 1);
 
       Field **field = tables->table->field;
@@ -864,7 +869,7 @@ static int rdb_i_s_compact_stats_fill_table(
       field[0]->store(cf_name.c_str(), cf_name.size(), system_charset_info);
       field[1]->store(level_str.c_str(), level_str.size(), system_charset_info);
       field[2]->store(type_str.c_str(), type_str.size(), system_charset_info);
-      field[3]->store(value, true);
+      field[3]->store(std::stod(value));
 
       ret |= static_cast<int>(
           my_core::schema_table_store_record(thd, tables->table));