[#9063] Support metrics filtering for /prometheus-metrics endpoint

Summary:
Metrics filtering is supported by the /metrics end point. This
diff allows /prometheus-metrics to use the same mechanism

For example:
  /prometheus-metrics?metrics=<metric-substring1>,<metric-substring2>

Test Plan:
Launch a local yb-ctl and curl prometheus-metrics with ?metrics=<param1>,<param2>

```
dev-server-amitanand2:~/code/yugabyte-perf [:194aa24|✚ 5…4⚑ 22]
05:28 $ curl http://127.0.0.1:7000/prometheus-metrics?metrics=transactions_running
transactions_running{exported_instance="dev-server-amitanand2:7000",table_id="sys.catalog.uuid",namespace_name="",table_name="sys.catalog",metric_type="tablet"} 0 1625203749777
dev-server-amitanand2:~/code/yugabyte-perf [:194aa24|✚ 5…4⚑ 22]
05:29 $ curl http://127.0.0.1:7000/prometheus-metrics?metrics=transactions_running,write_operations_inflight
transactions_running{exported_instance="dev-server-amitanand2:7000",table_id="sys.catalog.uuid",namespace_name="",table_name="sys.catalog",metric_type="tablet"} 0 1625203761186
write_operations_inflight{exported_instance="dev-server-amitanand2:7000",table_id="sys.catalog.uuid",namespace_name="",table_name="sys.catalog",metric_type="tablet"} 0 1625203761186
dev-server-amitanand2:~/code/yugabyte-perf [:194aa24|✚ 5…4⚑ 22]

```
with no-params/empty list of params:
```
dev-server-amitanand2:~/code/yugabyte-perf [:194aa24|✚ 4…4⚑ 22]
18:53 $ curl -q http://127.0.0.1:7000/prometheus-metrics 2>/dev/null | wc -l
1245
dev-server-amitanand2:~/code/yugabyte-perf [:194aa24|✚ 4…4⚑ 22]
18:53 $ curl -q http://127.0.0.1:7000/prometheus-metrics?metrics= 2>/dev/null | wc -l
0
```

Reviewers: bogdan

Reviewed By: bogdan

Subscribers: ybase

Differential Revision: https://phabricator.dev.yugabyte.com/D12152

src/yb/server/default-path-handlers.cc

@@ -237,57 +237,69 @@ static MetricLevel MetricLevelFromName(const std::string& level) {
   }
 }
 
-static void WriteMetricsAsJson(const MetricRegistry* const metrics,
-                               const Webserver::WebRequest& req, Webserver::WebResponse* resp) {
-  std::stringstream *output = &resp->output;
-  const string* requested_metrics_param = FindOrNull(req.parsed_args, "metrics");
-  vector<string> requested_metrics;
-  MetricJsonOptions opts;
+static void ParseRequestOptions(const Webserver::WebRequest& req,
+                                vector<string> *requested_metrics,
+                                MetricPrometheusOptions *promethus_opts,
+                                MetricJsonOptions *json_opts = nullptr,
+                                JsonWriter::Mode *json_mode = nullptr) {
+
+  if (requested_metrics) {
+    const string* requested_metrics_param = FindOrNull(req.parsed_args, "metrics");
+    if (requested_metrics_param != nullptr) {
+      SplitStringUsing(*requested_metrics_param, ",", requested_metrics);
+    } else {
+      // Default to including all metrics.
+      requested_metrics->push_back("*");
+    }
+  }
+
+  string arg;
+  if (json_opts) {
+    arg = FindWithDefault(req.parsed_args, "include_raw_histograms", "false");
+    json_opts->include_raw_histograms = ParseLeadingBoolValue(arg.c_str(), false);
 
-  {
-    string arg = FindWithDefault(req.parsed_args, "include_raw_histograms", "false");
-    opts.include_raw_histograms = ParseLeadingBoolValue(arg.c_str(), false);
+    arg = FindWithDefault(req.parsed_args, "include_schema", "false");
+    json_opts->include_schema_info = ParseLeadingBoolValue(arg.c_str(), false);
+
+    arg = FindWithDefault(req.parsed_args, "level", "debug");
+    json_opts->level = MetricLevelFromName(arg);
   }
-  {
-    string arg = FindWithDefault(req.parsed_args, "include_schema", "false");
-    opts.include_schema_info = ParseLeadingBoolValue(arg.c_str(), false);
+
+  if (promethus_opts) {
+    arg = FindWithDefault(req.parsed_args, "level", "debug");
+    promethus_opts->level = MetricLevelFromName(arg);
   }
-  {
-    string arg = FindWithDefault(req.parsed_args, "level", "debug");
-    opts.level = MetricLevelFromName(arg);
+
+  if (json_mode) {
+    arg = FindWithDefault(req.parsed_args, "compact", "false");
+    *json_mode =
+        ParseLeadingBoolValue(arg.c_str(), false) ? JsonWriter::COMPACT : JsonWriter::PRETTY;
   }
+}
+
+static void WriteMetricsAsJson(const MetricRegistry* const metrics,
+                               const Webserver::WebRequest& req, Webserver::WebResponse* resp) {
+  vector<string> requested_metrics;
+  MetricJsonOptions opts;
   JsonWriter::Mode json_mode;
-  {
-    string arg = FindWithDefault(req.parsed_args, "compact", "false");
-    json_mode = ParseLeadingBoolValue(arg.c_str(), false) ?
-      JsonWriter::COMPACT : JsonWriter::PRETTY;
-  }
+  ParseRequestOptions(req, &requested_metrics, /* prometheus opts */ nullptr, &opts, &json_mode);
 
+  std::stringstream* output = &resp->output;
   JsonWriter writer(output, json_mode);
 
-  if (requested_metrics_param != nullptr) {
-    SplitStringUsing(*requested_metrics_param, ",", &requested_metrics);
-  } else {
-    // Default to including all metrics.
-    requested_metrics.push_back("*");
-  }
-
   WARN_NOT_OK(metrics->WriteAsJson(&writer, requested_metrics, opts),
               "Couldn't write JSON metrics over HTTP");
 }
 
 static void WriteMetricsForPrometheus(const MetricRegistry* const metrics,
                                const Webserver::WebRequest& req, Webserver::WebResponse* resp) {
-  std::stringstream *output = &resp->output;
+  vector<string> requested_metrics;
   MetricPrometheusOptions opts;
+  ParseRequestOptions(req, &requested_metrics, &opts);
 
-  {
-    string arg = FindWithDefault(req.parsed_args, "level", "debug");
-    opts.level = MetricLevelFromName(arg);
-  }
-
+  std::stringstream *output = &resp->output;
   PrometheusWriter writer(output);
-  WARN_NOT_OK(metrics->WriteForPrometheus(&writer, opts),
+  WARN_NOT_OK(metrics->WriteForPrometheus(&writer, requested_metrics, opts),
               "Couldn't write text metrics for Prometheus");
 }
 

src/yb/tserver/metrics_snapshotter.cc

@@ -320,7 +320,8 @@ Status MetricsSnapshotter::Thread::DoMetricsSnapshot() {
   NMSWriter::EntityMetricsMap table_metrics;
   NMSWriter::MetricsMap server_metrics;
   NMSWriter nmswriter{&table_metrics, &server_metrics};
-  WARN_NOT_OK(server_->metric_registry()->WriteForPrometheus(&nmswriter, MetricPrometheusOptions()),
+  WARN_NOT_OK(
+      server_->metric_registry()->WriteForPrometheus(&nmswriter, MetricPrometheusOptions()),
       "Couldn't write metrics for native metrics storage");
   for (const auto& kv : server_metrics) {
     if (tserver_metrics_whitelist_.find(kv.first) != tserver_metrics_whitelist_.end()) {

src/yb/util/metrics.cc

@@ -224,11 +224,13 @@ scoped_refptr<Metric> MetricEntity::FindOrNull(const MetricPrototype& prototype)
 
 namespace {
 
+const string kWildCardString = "*";
+
 bool MatchMetricInList(const string& metric_name,
                        const vector<string>& match_params) {
   for (const string& param : match_params) {
     // Handle wildcard.
-    if (param == "*") return true;
+    if (param == kWildCardString) return true;
     // The parameter is a substring match of the metric name.
     if (metric_name.find(param) != std::string::npos) {
       return true;
@@ -307,7 +309,10 @@ Status MetricEntity::WriteAsJson(JsonWriter* writer,
 }
 
 CHECKED_STATUS MetricEntity::WriteForPrometheus(PrometheusWriter* writer,
+                                                const vector<string>& requested_metrics,
                                                 const MetricPrometheusOptions& opts) const {
+  bool select_all = MatchMetricInList(id(), requested_metrics);
+
   // We want the keys to be in alphabetical order when printing, so we use an ordered map here.
   typedef std::map<const char*, scoped_refptr<Metric> > OrderedMetricMap;
   OrderedMetricMap metrics;
@@ -323,9 +328,20 @@ CHECKED_STATUS MetricEntity::WriteForPrometheus(PrometheusWriter* writer,
       const MetricPrototype* prototype = val.first;
       const scoped_refptr<Metric>& metric = val.second;
 
-      InsertOrDie(&metrics, prototype->name(), metric);
+      if (select_all || MatchMetricInList(prototype->name(), requested_metrics)) {
+        InsertOrDie(&metrics, prototype->name(), metric);
+      }
     }
   }
+
+  // If we had a filter, and we didn't either match this entity or any metrics inside
+  // it, don't print the entity at all.
+  // If metrics is empty, we'd still call the callbacks if the entity matches,
+  // i.e. requested_metrics and select_all is true.
+  if (!requested_metrics.empty() && !select_all && metrics.empty()) {
+    return Status::OK();
+  }
+
   AttributeMap prometheus_attr;
   // Per tablet metrics come with tablet_id, as well as table_id and table_name attributes.
   // We ignore the tablet part to squash at the table level.
@@ -480,6 +496,12 @@ Status MetricRegistry::WriteAsJson(JsonWriter* writer,
 
 CHECKED_STATUS MetricRegistry::WriteForPrometheus(PrometheusWriter* writer,
                                                   const MetricPrometheusOptions& opts) const {
+  return WriteForPrometheus(writer, {kWildCardString}, opts);  // Include all metrics.
+}
+
+CHECKED_STATUS MetricRegistry::WriteForPrometheus(PrometheusWriter* writer,
+                                                  const vector<string>& requested_metrics,
+                                                  const MetricPrometheusOptions& opts) const {
   EntityMap entities;
   {
     std::lock_guard<simple_spinlock> l(lock_);
@@ -491,7 +513,7 @@ CHECKED_STATUS MetricRegistry::WriteForPrometheus(PrometheusWriter* writer,
       continue;
     }
 
-    WARN_NOT_OK(e.second->WriteForPrometheus(writer, opts),
+    WARN_NOT_OK(e.second->WriteForPrometheus(writer, requested_metrics, opts),
                 Substitute("Failed to write entity $0 as Prometheus", e.second->id()));
   }
   RETURN_NOT_OK(writer->FlushAggregatedValues());

src/yb/util/metrics.h

@@ -592,6 +592,7 @@ class MetricEntity : public RefCountedThreadSafe<MetricEntity> {
                      const MetricJsonOptions& opts) const;
 
   CHECKED_STATUS WriteForPrometheus(PrometheusWriter* writer,
+                     const std::vector<std::string>& requested_metrics,
                      const MetricPrometheusOptions& opts) const;
 
   const MetricMap& UnsafeMetricsMapForTests() const { return metric_map_; }
@@ -855,7 +856,24 @@ class MetricRegistry {
                      const std::vector<std::string>& requested_metrics,
                      const MetricJsonOptions& opts) const;
 
+  // Writes metrics in this registry to 'writer'.
+  //
+  // See the MetricPrometheusOptions struct definition above for options changing the
+  // output of this function.
+  CHECKED_STATUS WriteForPrometheus(PrometheusWriter* writer,
+                     const MetricPrometheusOptions& opts) const;
+  // Writes metrics in this registry to 'writer'.
+  //
+  // 'requested_metrics' is a set of substrings to match metric names against,
+  // where '*' matches all metrics.
+  //
+  // The string matching can either match an entity ID or a metric name.
+  // If it matches an entity ID, then all metrics for that entity will be printed.
+  //
+  // See the MetricPrometheusOptions struct definition above for options changing the
+  // output of this function.
   CHECKED_STATUS WriteForPrometheus(PrometheusWriter* writer,
+                     const std::vector<std::string>& requested_metrics,
                      const MetricPrometheusOptions& opts) const;
 
   // For each registered entity, retires orphaned metrics. If an entity has no more