feature/prom docker

centralized_collection/README.md

@@ -0,0 +1,11 @@
+# Centralized Collection
+
+This directory contains resources for centrally collecting and visualizing the metrics from pktvisor.
+
+Because pktvisor exposes its metrics over a generic JSON interface, it is able to work with any time series database.
+This directory contains resources for interacting with common databases.
+
+See the individual READMEs for more information:
+
+* [Prometheus](prometheus/README.md)
+* [Elasticsearch](elastic/README.md)

centralized_collection/elastic/README.md

@@ -0,0 +1,6 @@
+# Centralized Elasticsearch Collection
+
+This directory contains resources for building a docker container that contains pktvisord and
+the [telegraf](https://github.com/influxdata/telegraf) for collecting and sending metrics to Elasticsearch.
+
+It also contains an example [Grafana dashboard](grafana-dashboard-elk.json).

centralized_collection/prometheus/README.md

@@ -0,0 +1,30 @@
+# Centralized Prometheus Collection
+
+This directory contains resources for building a docker container that it includes pktvisord and
+the [Grafana Agent](https://github.com/grafana/agent) for collecting and sending metrics to Prometheus through
+[remote write](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage), including cloud
+providers.
+
+There is also a sample [Grafana dashboard](grafana-dashboard-prometheus.json).
+
+Example:
+
+```shell
+$ docker pull ns1labs/pktvisor-prom-write
+$ docker run -d --mount type=bind,source=/usr/local/geo,target=/geo --net=host --env PKTVISORD_ARGS="--prom-instance <INSTANCE>
+--geo-city /geo/GeoIP2-City.mmdb --geo-asn /geo/GeoIP2-ISP.mmdb <INTERFACE>" --env
+REMOTE_URL="https://<REMOTEHOST>/api/prom/push" --env USERNAME="<USERNAME>" --env PASSWORD="<PASSWORD>"
+ns1labs/pktvisor-prom-write
+```
+
+There are a few pieces of information you need to substitute above:
+
+* `<INSTANCE>`: The prometheus "instance" label for all metrics, e.g. "myhost"
+* `<INTERFACE>`: The ethernet interface to capture on, e.g. "eth0"
+* `<REMOTEHOST>`: The remote host to remote_write the prometheus metric to
+* `<USERNAME>`: If required by your prometheus setup, the user name to connect. If not required, leave off this
+  environment variable.
+* `<PASSWORD>`: If required by your prometheus setup, the password to connect. If not required, leave off this
+  environment variable.
+
+Other pktvisor arguments may be passed in the PKTVISORD_ARGS environment variable.

reporting/prometheus/docker-grafana-agent/files/entrypoint → centralized_collection/prometheus/docker-grafana-agent/files/entrypoint

@@ -2,26 +2,33 @@
 
 (
   cat <<END
+server:
+  log_level: info
+  http_listen_port: 20853
 prometheus:
   global:
     scrape_interval: 1m
-  remote_write:
-      - url: $REMOTE_URL
-        basic_auth:
-          username: $USERNAME
-          password: $PASSWORD
   configs:
-    - name: agent
+    - name: pktvisor
       scrape_configs:
+        - job_name: grafana_agent
+          static_configs:
+            - targets: ['127.0.0.1:20853']
         - job_name: pktvisor
           honor_timestamps: true
+          honor_labels: true
           scrape_interval: 1m
           scrape_timeout: 5s
           metrics_path: /metrics
           scheme: http
           static_configs:
             - targets:
                 - localhost:10853
+      remote_write:
+          - url: $REMOTE_URL
+            basic_auth:
+              username: $USERNAME
+              password: $PASSWORD
 END
 ) >/etc/agent/agent.yaml
 

cmd/pktvisord/main.cpp

@@ -31,25 +31,26 @@ static const char USAGE[] =
 
     IFACE, if specified, is either a network interface or an IP address (4 or 6). If this is specified,
     a "pcap" input stream will be automatically created, with "net" and "dns" handler modules attached.
-    ** Note that this is deprecated; you should instead use --admin-api and create the pcap input stream via API.
 
     Base Options:
       -l HOST               Run webserver on the given host or IP [default: localhost]
       -p PORT               Run webserver on the given port [default: 10853]
       --admin-api           Enable admin REST API giving complete control plane functionality [default: false]
                             When not specified, the exposed API is read-only access to summarized metrics.
                             When specified, write access is enabled for all modules.
-      --prometheus          Enable native Prometheus metrics at path /metrics
       -h --help             Show this screen
       -v                    Verbose log output
       --no-track            Don't send lightweight, anonymous usage metrics.
       --version             Show version
       --geo-city FILE       GeoLite2 City database to use for IP to Geo mapping (if enabled)
       --geo-asn FILE        GeoLite2 ASN database to use for IP to ASN mapping (if enabled)
+    Prometheus Options:
+      --prometheus          Enable native Prometheus metrics at path /metrics
+      --prom-instance ID    Optionally set the 'instance' tag to ID
     Handler Module Defaults:
       --max-deep-sample N   Never deep sample more than N% of streams (an int between 0 and 100) [default: 100]
       --periods P           Hold this many 60 second time periods of history in memory [default: 5]
-    pcap Input Module Options (deprecated, use admin-api instead):
+    pcap Input Module Options:
       -b BPF                Filter packets using the given BPF string
       -H HOSTSPEC           Specify subnets (comma separated) to consider HOST, in CIDR form. In live capture this /may/ be detected automatically
                             from capture device but /must/ be specified for pcaps. Example: "10.0.1.0/24,10.0.2.1/32,2001:db8::/64"
@@ -89,11 +90,14 @@ int main(int argc, char *argv[])
         logger->set_level(spdlog::level::debug);
     }
 
-    std::string prometheus_path;
+    PrometheusConfig prom_config;
     if (args["--prometheus"].asBool()) {
-        prometheus_path = "/metrics";
+        prom_config.path = "/metrics";
+        if (args["--prom-instance"]) {
+            prom_config.instance = args["--prom-instance"].asString();
+        }
     }
-    CoreServer svr(!args["--admin-api"].asBool(), logger, prometheus_path);
+    CoreServer svr(!args["--admin-api"].asBool(), logger, prom_config);
     svr.set_http_logger([&logger](const auto &req, const auto &res) {
         logger->info("REQUEST: {} {} {}", req.method, req.path, res.status);
         if (res.status == 500) {

src/CoreServer.cpp

@@ -3,12 +3,13 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
 #include "CoreServer.h"
+#include "Metrics.h"
 #include "visor_config.h"
 #include <chrono>
 #include <spdlog/stopwatch.h>
 #include <vector>
 
-visor::CoreServer::CoreServer(bool read_only, std::shared_ptr<spdlog::logger> logger, const std::string &prometheus_path)
+visor::CoreServer::CoreServer(bool read_only, std::shared_ptr<spdlog::logger> logger, const PrometheusConfig &prom_config)
     : _svr(read_only)
     , _logger(logger)
     , _start_time(std::chrono::system_clock::now())
@@ -36,7 +37,10 @@ visor::CoreServer::CoreServer(bool read_only, std::shared_ptr<spdlog::logger> lo
         _handler_plugins.emplace_back(std::move(mod));
     }
 
-    _setup_routes(prometheus_path);
+    _setup_routes(prom_config);
+    if (!prom_config.instance.empty()) {
+        Metric::add_base_label("instance", prom_config.instance);
+    }
 }
 void visor::CoreServer::start(const std::string &host, int port)
 {
@@ -72,7 +76,7 @@ visor::CoreServer::~CoreServer()
 {
     stop();
 }
-void visor::CoreServer::_setup_routes(const std::string &prometheus_path)
+void visor::CoreServer::_setup_routes(const PrometheusConfig &prom_config)
 {
 
     _logger->info("Initialize server control plane");
@@ -158,9 +162,9 @@ void visor::CoreServer::_setup_routes(const std::string &prometheus_path)
             res.set_content(e.what(), "text/plain");
         }
     });
-    if (!prometheus_path.empty()) {
-        _logger->info("enabling prometheus metrics on: {}", prometheus_path);
-        _svr.Get(prometheus_path.c_str(), [&]([[maybe_unused]] const httplib::Request &req, httplib::Response &res) {
+    if (!prom_config.path.empty()) {
+        _logger->info("enabling prometheus metrics on: {}", prom_config.path);
+        _svr.Get(prom_config.path.c_str(), [&]([[maybe_unused]] const httplib::Request &req, httplib::Response &res) {
             std::stringstream output;
             try {
                 auto [handler_modules, hm_lock] = _handler_manager->module_get_all_locked();

src/CoreServer.h

@@ -16,6 +16,11 @@
 
 namespace visor {
 
+struct PrometheusConfig {
+    std::string path;
+    std::string instance;
+};
+
 class CoreServer
 {
 public:
@@ -39,10 +44,10 @@ class CoreServer
     std::shared_ptr<spdlog::logger> _logger;
     std::chrono::system_clock::time_point _start_time;
 
-    void _setup_routes(const std::string &prometheus_path);
+    void _setup_routes(const PrometheusConfig &prom_config);
 
 public:
-    CoreServer(bool read_only, std::shared_ptr<spdlog::logger> logger, const std::string &prometheus_path);
+    CoreServer(bool read_only, std::shared_ptr<spdlog::logger> logger, const PrometheusConfig &prom_config);
     ~CoreServer();
 
     void start(const std::string &host, int port);

src/Metrics.cpp

@@ -14,8 +14,8 @@ void Counter::to_json(json &j) const
 
 void Counter::to_prometheus(std::stringstream &out) const
 {
-    out << "# HELP " << name_snake() << ' ' << _desc << std::endl;
-    out << "# TYPE " << name_snake() << " gauge" << std::endl;
+    out << "# HELP " << base_name_snake() << ' ' << _desc << std::endl;
+    out << "# TYPE " << base_name_snake() << " gauge" << std::endl;
     out << name_snake() << ' ' << _value << std::endl;
 }
 
@@ -50,14 +50,14 @@ void Rate::to_prometheus(std::stringstream &out) const
     auto quantiles = _quantile.get_quantiles(fractions, 4);
 
     if (quantiles.size()) {
-        out << "# HELP " << name_snake() << ' ' << _desc << std::endl;
-        out << "# TYPE " << name_snake() << " summary" << std::endl;
-        out << name_snake() << "{quantile=\"0.5\"} " << quantiles[0] << std::endl;
-        out << name_snake() << "{quantile=\"0.9\"} " << quantiles[1] << std::endl;
-        out << name_snake() << "{quantile=\"0.95\"} " << quantiles[2] << std::endl;
-        out << name_snake() << "{quantile=\"0.99\"} " << quantiles[3] << std::endl;
-        out << name_snake() << "_sum " << _quantile.get_max_value() << std::endl;
-        out << name_snake() << "_count " << _quantile.get_n() << std::endl;
+        out << "# HELP " << base_name_snake() << ' ' << _desc << std::endl;
+        out << "# TYPE " << base_name_snake() << " summary" << std::endl;
+        out << name_snake({}, {{"quantile", "0.5"}}) << ' ' << quantiles[0] << std::endl;
+        out << name_snake({}, {{"quantile", "0.9"}}) << ' ' << quantiles[1] << std::endl;
+        out << name_snake({}, {{"quantile", "0.95"}}) << ' ' << quantiles[2] << std::endl;
+        out << name_snake({}, {{"quantile", "0.99"}}) << ' ' << quantiles[3] << std::endl;
+        out << name_snake({"sum"}) << ' ' << _quantile.get_max_value() << std::endl;
+        out << name_snake({"count"}) << ' ' << _quantile.get_n() << std::endl;
     }
 }
 
@@ -74,11 +74,14 @@ void Cardinality::to_json(json &j) const
 }
 void Cardinality::to_prometheus(std::stringstream &out) const
 {
-    out << "# HELP " << name_snake() << ' ' << _desc << std::endl;
-    out << "# TYPE " << name_snake() << " gauge" << std::endl;
+    out << "# HELP " << base_name_snake() << ' ' << _desc << std::endl;
+    out << "# TYPE " << base_name_snake() << " gauge" << std::endl;
     out << name_snake() << ' ' << lround(_set.get_estimate()) << std::endl;
 }
 
+// static storage for base labels
+Metric::LabelMap Metric::_base_labels;
+
 void Metric::name_json_assign(json &j, const json &val) const
 {
     json *j_part = &j;
@@ -98,5 +101,41 @@ void Metric::name_json_assign(json &j, std::initializer_list<std::string> add_na
     }
     (*j_part) = val;
 }
+std::string Metric::base_name_snake() const
+{
+    auto snake = [](const std::string &ss, const std::string &s) {
+        return ss.empty() ? s : ss + "_" + s;
+    };
+    std::string name_text = _schema_key + "_" + std::accumulate(std::begin(_name), std::end(_name), std::string(), snake);
+    return name_text;
+}
+
+std::string Metric::name_snake(std::initializer_list<std::string> add_names, Metric::LabelMap add_labels) const
+{
+    std::string label_text{"{"};
+    if (!_base_labels.empty()) {
+        for (const auto &[key, value] : _base_labels) {
+            label_text.append(key + "=\"" + value + "\",");
+        }
+    }
+    if (add_labels.size()) {
+        for (const auto &[key, value] : add_labels) {
+            label_text.append(key + "=\"" + value + "\",");
+        }
+    }
+    if (label_text.back() == ',') {
+        label_text.pop_back();
+    }
+    label_text.push_back('}');
+    auto snake = [](const std::string &ss, const std::string &s) {
+        return ss.empty() ? s : ss + "_" + s;
+    };
+    std::string name_text = _schema_key + "_" + std::accumulate(std::begin(_name), std::end(_name), std::string(), snake);
+    if (add_names.size()) {
+        name_text.push_back('_');
+        name_text.append(std::accumulate(std::begin(add_names), std::end(add_names), std::string(), snake));
+    }
+    return name_text + label_text;
+}
 
 }

src/Metrics.h

@@ -26,6 +26,15 @@ using namespace std::chrono;
 
 class Metric
 {
+public:
+    typedef std::map<std::string, std::string> LabelMap;
+
+private:
+    /**
+     * labels which will be applied to all metrics
+     */
+    static LabelMap _base_labels;
+
 protected:
     std::vector<std::string> _name;
     std::string _desc;
@@ -47,15 +56,16 @@ class Metric
         _schema_key = schema_key;
     }
 
+    static void add_base_label(const std::string &label, const std::string &value)
+    {
+        _base_labels.emplace(label, value);
+    }
+
     void name_json_assign(json &j, const json &val) const;
     void name_json_assign(json &j, std::initializer_list<std::string> add_names, const json &val) const;
 
-    [[nodiscard]] std::string name_snake() const
-    {
-        return _schema_key + "_" + std::accumulate(std::begin(_name), std::end(_name), std::string(), [](const std::string &ss, const std::string &s) {
-            return ss.empty() ? s : ss + "_" + s;
-        });
-    }
+    [[nodiscard]] std::string base_name_snake() const;
+    [[nodiscard]] std::string name_snake(std::initializer_list<std::string> add_names = {}, LabelMap add_labels = {}) const;
 
     virtual void to_json(json &j) const = 0;
     virtual void to_prometheus(std::stringstream &out) const = 0;
@@ -163,14 +173,14 @@ class Quantile final : public Metric
         auto quantiles = _quantile.get_quantiles(fractions, 4);
 
         if (quantiles.size()) {
-            out << "# HELP " << name_snake() << ' ' << _desc << std::endl;
-            out << "# TYPE " << name_snake() << " summary" << std::endl;
-            out << name_snake() << "{quantile=\"0.5\"} " << quantiles[0] << std::endl;
-            out << name_snake() << "{quantile=\"0.9\"} " << quantiles[1] << std::endl;
-            out << name_snake() << "{quantile=\"0.95\"} " << quantiles[2] << std::endl;
-            out << name_snake() << "{quantile=\"0.99\"} " << quantiles[3] << std::endl;
-            out << name_snake() << "_sum " << _quantile.get_max_value() << std::endl;
-            out << name_snake() << "_count " << _quantile.get_n() << std::endl;
+            out << "# HELP " << base_name_snake() << ' ' << _desc << std::endl;
+            out << "# TYPE " << base_name_snake() << " summary" << std::endl;
+            out << name_snake({}, {{"quantile", "0.5"}}) << ' ' << quantiles[0] << std::endl;
+            out << name_snake({}, {{"quantile", "0.9"}}) << ' ' << quantiles[1] << std::endl;
+            out << name_snake({}, {{"quantile", "0.95"}}) << ' ' << quantiles[2] << std::endl;
+            out << name_snake({}, {{"quantile", "0.99"}}) << ' ' << quantiles[3] << std::endl;
+            out << name_snake({"sum"}) << ' ' << _quantile.get_max_value() << std::endl;
+            out << name_snake({"count"}) << ' ' << _quantile.get_n() << std::endl;
         }
     }
 };
@@ -241,9 +251,9 @@ class TopN final : public Metric
     {
         auto items = _fi.get_frequent_items(datasketches::frequent_items_error_type::NO_FALSE_NEGATIVES);
         for (uint64_t i = 0; i < std::min(_top_count, items.size()); i++) {
-            out << "# HELP " << name_snake() << ' ' << _desc << std::endl;
-            out << "# TYPE " << name_snake() << " gauge" << std::endl;
-            out << name_snake() << "{name=\"" << formatter(items[i].get_item()) << "\"} " << items[i].get_estimate() << std::endl;
+            out << "# HELP " << base_name_snake() << ' ' << _desc << std::endl;
+            out << "# TYPE " << base_name_snake() << " gauge" << std::endl;
+            out << name_snake({}, {{"name", formatter(items[i].get_item())}}) << ' ' << items[i].get_estimate() << std::endl;
         }
     }
 
@@ -263,9 +273,11 @@ class TopN final : public Metric
     {
         auto items = _fi.get_frequent_items(datasketches::frequent_items_error_type::NO_FALSE_NEGATIVES);
         for (uint64_t i = 0; i < std::min(_top_count, items.size()); i++) {
-            out << "# HELP " << name_snake() << ' ' << _desc << std::endl;
-            out << "# TYPE " << name_snake() << " gauge" << std::endl;
-            out << name_snake() << "{name=\"" << items[i].get_item() << "\"} " << items[i].get_estimate() << std::endl;
+            out << "# HELP " << base_name_snake() << ' ' << _desc << std::endl;
+            out << "# TYPE " << base_name_snake() << " gauge" << std::endl;
+            std::stringstream name_text;
+            name_text << items[i].get_item();
+            out << name_snake({}, {{"name", name_text.str()}}) << ' ' << items[i].get_estimate() << std::endl;
         }
     }
 };