From e45121dac59ed9727041ad186a4840ab3cdef464 Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Tue, 6 Apr 2021 17:16:16 -0400 Subject: [PATCH 1/3] prom config --- .../docker-grafana-agent/files/entrypoint | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/reporting/prometheus/docker-grafana-agent/files/entrypoint b/reporting/prometheus/docker-grafana-agent/files/entrypoint index bcd36360b..ff27cf0a7 100755 --- a/reporting/prometheus/docker-grafana-agent/files/entrypoint +++ b/reporting/prometheus/docker-grafana-agent/files/entrypoint @@ -2,17 +2,18 @@ ( cat </etc/agent/agent.yaml From a02dfd873707ecce13c8d48a3a8eacd3a28f1e9a Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Wed, 7 Apr 2021 11:26:49 -0400 Subject: [PATCH 2/3] add proper support for labels in prometheus metrics, and add support for setting prometheus instance label --- cmd/pktvisord/main.cpp | 16 +++-- .../docker-grafana-agent/files/entrypoint | 1 + src/CoreServer.cpp | 16 +++-- src/CoreServer.h | 9 ++- src/Metrics.cpp | 63 +++++++++++++++---- src/Metrics.h | 52 +++++++++------ 6 files changed, 111 insertions(+), 46 deletions(-) diff --git a/cmd/pktvisord/main.cpp b/cmd/pktvisord/main.cpp index ea6f0dc09..6e8bf2b27 100644 --- a/cmd/pktvisord/main.cpp +++ b/cmd/pktvisord/main.cpp @@ -31,7 +31,6 @@ static const char USAGE[] = IFACE, if specified, is either a network interface or an IP address (4 or 6). If this is specified, a "pcap" input stream will be automatically created, with "net" and "dns" handler modules attached. - ** Note that this is deprecated; you should instead use --admin-api and create the pcap input stream via API. Base Options: -l HOST Run webserver on the given host or IP [default: localhost] @@ -39,17 +38,19 @@ static const char USAGE[] = --admin-api Enable admin REST API giving complete control plane functionality [default: false] When not specified, the exposed API is read-only access to summarized metrics. When specified, write access is enabled for all modules. - --prometheus Enable native Prometheus metrics at path /metrics -h --help Show this screen -v Verbose log output --no-track Don't send lightweight, anonymous usage metrics. --version Show version --geo-city FILE GeoLite2 City database to use for IP to Geo mapping (if enabled) --geo-asn FILE GeoLite2 ASN database to use for IP to ASN mapping (if enabled) + Prometheus Options: + --prometheus Enable native Prometheus metrics at path /metrics + --prom-instance ID Optionally set the 'instance' tag to ID Handler Module Defaults: --max-deep-sample N Never deep sample more than N% of streams (an int between 0 and 100) [default: 100] --periods P Hold this many 60 second time periods of history in memory [default: 5] - pcap Input Module Options (deprecated, use admin-api instead): + pcap Input Module Options: -b BPF Filter packets using the given BPF string -H HOSTSPEC Specify subnets (comma separated) to consider HOST, in CIDR form. In live capture this /may/ be detected automatically from capture device but /must/ be specified for pcaps. Example: "10.0.1.0/24,10.0.2.1/32,2001:db8::/64" @@ -89,11 +90,14 @@ int main(int argc, char *argv[]) logger->set_level(spdlog::level::debug); } - std::string prometheus_path; + PrometheusConfig prom_config; if (args["--prometheus"].asBool()) { - prometheus_path = "/metrics"; + prom_config.path = "/metrics"; + if (args["--prom-instance"]) { + prom_config.instance = args["--prom-instance"].asString(); + } } - CoreServer svr(!args["--admin-api"].asBool(), logger, prometheus_path); + CoreServer svr(!args["--admin-api"].asBool(), logger, prom_config); svr.set_http_logger([&logger](const auto &req, const auto &res) { logger->info("REQUEST: {} {} {}", req.method, req.path, res.status); if (res.status == 500) { diff --git a/reporting/prometheus/docker-grafana-agent/files/entrypoint b/reporting/prometheus/docker-grafana-agent/files/entrypoint index ff27cf0a7..9f2bc7b5c 100755 --- a/reporting/prometheus/docker-grafana-agent/files/entrypoint +++ b/reporting/prometheus/docker-grafana-agent/files/entrypoint @@ -16,6 +16,7 @@ prometheus: - targets: ['127.0.0.1:20853'] - job_name: pktvisor honor_timestamps: true + honor_labels: true scrape_interval: 1m scrape_timeout: 5s metrics_path: /metrics diff --git a/src/CoreServer.cpp b/src/CoreServer.cpp index 035a4fb74..8490ff101 100644 --- a/src/CoreServer.cpp +++ b/src/CoreServer.cpp @@ -3,12 +3,13 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ #include "CoreServer.h" +#include "Metrics.h" #include "visor_config.h" #include #include #include -visor::CoreServer::CoreServer(bool read_only, std::shared_ptr logger, const std::string &prometheus_path) +visor::CoreServer::CoreServer(bool read_only, std::shared_ptr logger, const PrometheusConfig &prom_config) : _svr(read_only) , _logger(logger) , _start_time(std::chrono::system_clock::now()) @@ -36,7 +37,10 @@ visor::CoreServer::CoreServer(bool read_only, std::shared_ptr lo _handler_plugins.emplace_back(std::move(mod)); } - _setup_routes(prometheus_path); + _setup_routes(prom_config); + if (!prom_config.instance.empty()) { + Metric::add_base_label("instance", prom_config.instance); + } } void visor::CoreServer::start(const std::string &host, int port) { @@ -72,7 +76,7 @@ visor::CoreServer::~CoreServer() { stop(); } -void visor::CoreServer::_setup_routes(const std::string &prometheus_path) +void visor::CoreServer::_setup_routes(const PrometheusConfig &prom_config) { _logger->info("Initialize server control plane"); @@ -158,9 +162,9 @@ void visor::CoreServer::_setup_routes(const std::string &prometheus_path) res.set_content(e.what(), "text/plain"); } }); - if (!prometheus_path.empty()) { - _logger->info("enabling prometheus metrics on: {}", prometheus_path); - _svr.Get(prometheus_path.c_str(), [&]([[maybe_unused]] const httplib::Request &req, httplib::Response &res) { + if (!prom_config.path.empty()) { + _logger->info("enabling prometheus metrics on: {}", prom_config.path); + _svr.Get(prom_config.path.c_str(), [&]([[maybe_unused]] const httplib::Request &req, httplib::Response &res) { std::stringstream output; try { auto [handler_modules, hm_lock] = _handler_manager->module_get_all_locked(); diff --git a/src/CoreServer.h b/src/CoreServer.h index 68091519c..66c34ce20 100644 --- a/src/CoreServer.h +++ b/src/CoreServer.h @@ -16,6 +16,11 @@ namespace visor { +struct PrometheusConfig { + std::string path; + std::string instance; +}; + class CoreServer { public: @@ -39,10 +44,10 @@ class CoreServer std::shared_ptr _logger; std::chrono::system_clock::time_point _start_time; - void _setup_routes(const std::string &prometheus_path); + void _setup_routes(const PrometheusConfig &prom_config); public: - CoreServer(bool read_only, std::shared_ptr logger, const std::string &prometheus_path); + CoreServer(bool read_only, std::shared_ptr logger, const PrometheusConfig &prom_config); ~CoreServer(); void start(const std::string &host, int port); diff --git a/src/Metrics.cpp b/src/Metrics.cpp index 21ad1cb79..d67e8540c 100644 --- a/src/Metrics.cpp +++ b/src/Metrics.cpp @@ -14,8 +14,8 @@ void Counter::to_json(json &j) const void Counter::to_prometheus(std::stringstream &out) const { - out << "# HELP " << name_snake() << ' ' << _desc << std::endl; - out << "# TYPE " << name_snake() << " gauge" << std::endl; + out << "# HELP " << base_name_snake() << ' ' << _desc << std::endl; + out << "# TYPE " << base_name_snake() << " gauge" << std::endl; out << name_snake() << ' ' << _value << std::endl; } @@ -50,14 +50,14 @@ void Rate::to_prometheus(std::stringstream &out) const auto quantiles = _quantile.get_quantiles(fractions, 4); if (quantiles.size()) { - out << "# HELP " << name_snake() << ' ' << _desc << std::endl; - out << "# TYPE " << name_snake() << " summary" << std::endl; - out << name_snake() << "{quantile=\"0.5\"} " << quantiles[0] << std::endl; - out << name_snake() << "{quantile=\"0.9\"} " << quantiles[1] << std::endl; - out << name_snake() << "{quantile=\"0.95\"} " << quantiles[2] << std::endl; - out << name_snake() << "{quantile=\"0.99\"} " << quantiles[3] << std::endl; - out << name_snake() << "_sum " << _quantile.get_max_value() << std::endl; - out << name_snake() << "_count " << _quantile.get_n() << std::endl; + out << "# HELP " << base_name_snake() << ' ' << _desc << std::endl; + out << "# TYPE " << base_name_snake() << " summary" << std::endl; + out << name_snake({}, {{"quantile", "0.5"}}) << ' ' << quantiles[0] << std::endl; + out << name_snake({}, {{"quantile", "0.9"}}) << ' ' << quantiles[1] << std::endl; + out << name_snake({}, {{"quantile", "0.95"}}) << ' ' << quantiles[2] << std::endl; + out << name_snake({}, {{"quantile", "0.99"}}) << ' ' << quantiles[3] << std::endl; + out << name_snake({"sum"}) << ' ' << _quantile.get_max_value() << std::endl; + out << name_snake({"count"}) << ' ' << _quantile.get_n() << std::endl; } } @@ -74,11 +74,14 @@ void Cardinality::to_json(json &j) const } void Cardinality::to_prometheus(std::stringstream &out) const { - out << "# HELP " << name_snake() << ' ' << _desc << std::endl; - out << "# TYPE " << name_snake() << " gauge" << std::endl; + out << "# HELP " << base_name_snake() << ' ' << _desc << std::endl; + out << "# TYPE " << base_name_snake() << " gauge" << std::endl; out << name_snake() << ' ' << lround(_set.get_estimate()) << std::endl; } +// static storage for base labels +Metric::LabelMap Metric::_base_labels; + void Metric::name_json_assign(json &j, const json &val) const { json *j_part = &j; @@ -98,5 +101,41 @@ void Metric::name_json_assign(json &j, std::initializer_list add_na } (*j_part) = val; } +std::string Metric::base_name_snake() const +{ + auto snake = [](const std::string &ss, const std::string &s) { + return ss.empty() ? s : ss + "_" + s; + }; + std::string name_text = _schema_key + "_" + std::accumulate(std::begin(_name), std::end(_name), std::string(), snake); + return name_text; +} + +std::string Metric::name_snake(std::initializer_list add_names, Metric::LabelMap add_labels) const +{ + std::string label_text{"{"}; + if (!_base_labels.empty()) { + for (const auto &[key, value] : _base_labels) { + label_text.append(key + "=\"" + value + "\","); + } + } + if (add_labels.size()) { + for (const auto &[key, value] : add_labels) { + label_text.append(key + "=\"" + value + "\","); + } + } + if (label_text.back() == ',') { + label_text.pop_back(); + } + label_text.push_back('}'); + auto snake = [](const std::string &ss, const std::string &s) { + return ss.empty() ? s : ss + "_" + s; + }; + std::string name_text = _schema_key + "_" + std::accumulate(std::begin(_name), std::end(_name), std::string(), snake); + if (add_names.size()) { + name_text.push_back('_'); + name_text.append(std::accumulate(std::begin(add_names), std::end(add_names), std::string(), snake)); + } + return name_text + label_text; +} } \ No newline at end of file diff --git a/src/Metrics.h b/src/Metrics.h index 39baec2b4..fd5bfa081 100644 --- a/src/Metrics.h +++ b/src/Metrics.h @@ -26,6 +26,15 @@ using namespace std::chrono; class Metric { +public: + typedef std::map LabelMap; + +private: + /** + * labels which will be applied to all metrics + */ + static LabelMap _base_labels; + protected: std::vector _name; std::string _desc; @@ -47,15 +56,16 @@ class Metric _schema_key = schema_key; } + static void add_base_label(const std::string &label, const std::string &value) + { + _base_labels.emplace(label, value); + } + void name_json_assign(json &j, const json &val) const; void name_json_assign(json &j, std::initializer_list add_names, const json &val) const; - [[nodiscard]] std::string name_snake() const - { - return _schema_key + "_" + std::accumulate(std::begin(_name), std::end(_name), std::string(), [](const std::string &ss, const std::string &s) { - return ss.empty() ? s : ss + "_" + s; - }); - } + [[nodiscard]] std::string base_name_snake() const; + [[nodiscard]] std::string name_snake(std::initializer_list add_names = {}, LabelMap add_labels = {}) const; virtual void to_json(json &j) const = 0; virtual void to_prometheus(std::stringstream &out) const = 0; @@ -163,14 +173,14 @@ class Quantile final : public Metric auto quantiles = _quantile.get_quantiles(fractions, 4); if (quantiles.size()) { - out << "# HELP " << name_snake() << ' ' << _desc << std::endl; - out << "# TYPE " << name_snake() << " summary" << std::endl; - out << name_snake() << "{quantile=\"0.5\"} " << quantiles[0] << std::endl; - out << name_snake() << "{quantile=\"0.9\"} " << quantiles[1] << std::endl; - out << name_snake() << "{quantile=\"0.95\"} " << quantiles[2] << std::endl; - out << name_snake() << "{quantile=\"0.99\"} " << quantiles[3] << std::endl; - out << name_snake() << "_sum " << _quantile.get_max_value() << std::endl; - out << name_snake() << "_count " << _quantile.get_n() << std::endl; + out << "# HELP " << base_name_snake() << ' ' << _desc << std::endl; + out << "# TYPE " << base_name_snake() << " summary" << std::endl; + out << name_snake({}, {{"quantile", "0.5"}}) << ' ' << quantiles[0] << std::endl; + out << name_snake({}, {{"quantile", "0.9"}}) << ' ' << quantiles[1] << std::endl; + out << name_snake({}, {{"quantile", "0.95"}}) << ' ' << quantiles[2] << std::endl; + out << name_snake({}, {{"quantile", "0.99"}}) << ' ' << quantiles[3] << std::endl; + out << name_snake({"sum"}) << ' ' << _quantile.get_max_value() << std::endl; + out << name_snake({"count"}) << ' ' << _quantile.get_n() << std::endl; } } }; @@ -241,9 +251,9 @@ class TopN final : public Metric { auto items = _fi.get_frequent_items(datasketches::frequent_items_error_type::NO_FALSE_NEGATIVES); for (uint64_t i = 0; i < std::min(_top_count, items.size()); i++) { - out << "# HELP " << name_snake() << ' ' << _desc << std::endl; - out << "# TYPE " << name_snake() << " gauge" << std::endl; - out << name_snake() << "{name=\"" << formatter(items[i].get_item()) << "\"} " << items[i].get_estimate() << std::endl; + out << "# HELP " << base_name_snake() << ' ' << _desc << std::endl; + out << "# TYPE " << base_name_snake() << " gauge" << std::endl; + out << name_snake({}, {{"name", formatter(items[i].get_item())}}) << ' ' << items[i].get_estimate() << std::endl; } } @@ -263,9 +273,11 @@ class TopN final : public Metric { auto items = _fi.get_frequent_items(datasketches::frequent_items_error_type::NO_FALSE_NEGATIVES); for (uint64_t i = 0; i < std::min(_top_count, items.size()); i++) { - out << "# HELP " << name_snake() << ' ' << _desc << std::endl; - out << "# TYPE " << name_snake() << " gauge" << std::endl; - out << name_snake() << "{name=\"" << items[i].get_item() << "\"} " << items[i].get_estimate() << std::endl; + out << "# HELP " << base_name_snake() << ' ' << _desc << std::endl; + out << "# TYPE " << base_name_snake() << " gauge" << std::endl; + std::stringstream name_text; + name_text << items[i].get_item(); + out << name_snake({}, {{"name", name_text.str()}}) << ' ' << items[i].get_estimate() << std::endl; } } }; From 5eadcb0970f7ec267a364781c79bde9d125520ea Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Wed, 7 Apr 2021 15:31:23 -0400 Subject: [PATCH 3/3] centralized collection organization and documentation --- centralized_collection/README.md | 11 +++++++ centralized_collection/elastic/README.md | 6 ++++ .../elastic/docker/README.md | 0 .../elastic/docker/config/telegraf-pop1.conf | 0 .../elastic/docker/config/telegraf-pop2.conf | 0 .../elastic/docker/datasource.json | 0 .../elastic/docker/docker-compose.yml | 0 .../elastic/docker/setup.sh | 0 .../elastic/docker/with_telegraf/Dockerfile | 0 .../docker/with_telegraf/misc/entrypoint | 0 .../docker/with_telegraf/misc/run-dig.sh | 0 .../with_telegraf/misc/run-pktvisord.sh | 0 .../docker/with_telegraf/misc/run-telegraf.sh | 0 .../elastic/grafana-dashboard-elk.json | 0 .../elastic/top_n.elk | 0 centralized_collection/prometheus/README.md | 30 +++++++++++++++++++ .../docker-grafana-agent/Dockerfile | 0 .../docker-grafana-agent/files/entrypoint | 0 .../files/run-grafana-agent.sh | 0 .../files/run-pktvisord.sh | 0 .../grafana-dashboard-prometheus.json | 0 reporting/README.md | 0 22 files changed, 47 insertions(+) create mode 100644 centralized_collection/README.md create mode 100644 centralized_collection/elastic/README.md rename {reporting => centralized_collection}/elastic/docker/README.md (100%) rename {reporting => centralized_collection}/elastic/docker/config/telegraf-pop1.conf (100%) rename {reporting => centralized_collection}/elastic/docker/config/telegraf-pop2.conf (100%) rename {reporting => centralized_collection}/elastic/docker/datasource.json (100%) rename {reporting => centralized_collection}/elastic/docker/docker-compose.yml (100%) rename {reporting => centralized_collection}/elastic/docker/setup.sh (100%) rename {reporting => centralized_collection}/elastic/docker/with_telegraf/Dockerfile (100%) rename {reporting => centralized_collection}/elastic/docker/with_telegraf/misc/entrypoint (100%) rename {reporting => centralized_collection}/elastic/docker/with_telegraf/misc/run-dig.sh (100%) rename {reporting => centralized_collection}/elastic/docker/with_telegraf/misc/run-pktvisord.sh (100%) rename {reporting => centralized_collection}/elastic/docker/with_telegraf/misc/run-telegraf.sh (100%) rename {reporting => centralized_collection}/elastic/grafana-dashboard-elk.json (100%) rename {reporting => centralized_collection}/elastic/top_n.elk (100%) create mode 100644 centralized_collection/prometheus/README.md rename {reporting => centralized_collection}/prometheus/docker-grafana-agent/Dockerfile (100%) rename {reporting => centralized_collection}/prometheus/docker-grafana-agent/files/entrypoint (100%) rename {reporting => centralized_collection}/prometheus/docker-grafana-agent/files/run-grafana-agent.sh (100%) rename {reporting => centralized_collection}/prometheus/docker-grafana-agent/files/run-pktvisord.sh (100%) rename {reporting => centralized_collection}/prometheus/grafana-dashboard-prometheus.json (100%) delete mode 100644 reporting/README.md diff --git a/centralized_collection/README.md b/centralized_collection/README.md new file mode 100644 index 000000000..5a610a494 --- /dev/null +++ b/centralized_collection/README.md @@ -0,0 +1,11 @@ +# Centralized Collection + +This directory contains resources for centrally collecting and visualizing the metrics from pktvisor. + +Because pktvisor exposes its metrics over a generic JSON interface, it is able to work with any time series database. +This directory contains resources for interacting with common databases. + +See the individual READMEs for more information: + +* [Prometheus](prometheus/README.md) +* [Elasticsearch](elastic/README.md) diff --git a/centralized_collection/elastic/README.md b/centralized_collection/elastic/README.md new file mode 100644 index 000000000..f26eb7337 --- /dev/null +++ b/centralized_collection/elastic/README.md @@ -0,0 +1,6 @@ +# Centralized Elasticsearch Collection + +This directory contains resources for building a docker container that contains pktvisord and +the [telegraf](https://github.com/influxdata/telegraf) for collecting and sending metrics to Elasticsearch. + +It also contains an example [Grafana dashboard](grafana-dashboard-elk.json). diff --git a/reporting/elastic/docker/README.md b/centralized_collection/elastic/docker/README.md similarity index 100% rename from reporting/elastic/docker/README.md rename to centralized_collection/elastic/docker/README.md diff --git a/reporting/elastic/docker/config/telegraf-pop1.conf b/centralized_collection/elastic/docker/config/telegraf-pop1.conf similarity index 100% rename from reporting/elastic/docker/config/telegraf-pop1.conf rename to centralized_collection/elastic/docker/config/telegraf-pop1.conf diff --git a/reporting/elastic/docker/config/telegraf-pop2.conf b/centralized_collection/elastic/docker/config/telegraf-pop2.conf similarity index 100% rename from reporting/elastic/docker/config/telegraf-pop2.conf rename to centralized_collection/elastic/docker/config/telegraf-pop2.conf diff --git a/reporting/elastic/docker/datasource.json b/centralized_collection/elastic/docker/datasource.json similarity index 100% rename from reporting/elastic/docker/datasource.json rename to centralized_collection/elastic/docker/datasource.json diff --git a/reporting/elastic/docker/docker-compose.yml b/centralized_collection/elastic/docker/docker-compose.yml similarity index 100% rename from reporting/elastic/docker/docker-compose.yml rename to centralized_collection/elastic/docker/docker-compose.yml diff --git a/reporting/elastic/docker/setup.sh b/centralized_collection/elastic/docker/setup.sh similarity index 100% rename from reporting/elastic/docker/setup.sh rename to centralized_collection/elastic/docker/setup.sh diff --git a/reporting/elastic/docker/with_telegraf/Dockerfile b/centralized_collection/elastic/docker/with_telegraf/Dockerfile similarity index 100% rename from reporting/elastic/docker/with_telegraf/Dockerfile rename to centralized_collection/elastic/docker/with_telegraf/Dockerfile diff --git a/reporting/elastic/docker/with_telegraf/misc/entrypoint b/centralized_collection/elastic/docker/with_telegraf/misc/entrypoint similarity index 100% rename from reporting/elastic/docker/with_telegraf/misc/entrypoint rename to centralized_collection/elastic/docker/with_telegraf/misc/entrypoint diff --git a/reporting/elastic/docker/with_telegraf/misc/run-dig.sh b/centralized_collection/elastic/docker/with_telegraf/misc/run-dig.sh similarity index 100% rename from reporting/elastic/docker/with_telegraf/misc/run-dig.sh rename to centralized_collection/elastic/docker/with_telegraf/misc/run-dig.sh diff --git a/reporting/elastic/docker/with_telegraf/misc/run-pktvisord.sh b/centralized_collection/elastic/docker/with_telegraf/misc/run-pktvisord.sh similarity index 100% rename from reporting/elastic/docker/with_telegraf/misc/run-pktvisord.sh rename to centralized_collection/elastic/docker/with_telegraf/misc/run-pktvisord.sh diff --git a/reporting/elastic/docker/with_telegraf/misc/run-telegraf.sh b/centralized_collection/elastic/docker/with_telegraf/misc/run-telegraf.sh similarity index 100% rename from reporting/elastic/docker/with_telegraf/misc/run-telegraf.sh rename to centralized_collection/elastic/docker/with_telegraf/misc/run-telegraf.sh diff --git a/reporting/elastic/grafana-dashboard-elk.json b/centralized_collection/elastic/grafana-dashboard-elk.json similarity index 100% rename from reporting/elastic/grafana-dashboard-elk.json rename to centralized_collection/elastic/grafana-dashboard-elk.json diff --git a/reporting/elastic/top_n.elk b/centralized_collection/elastic/top_n.elk similarity index 100% rename from reporting/elastic/top_n.elk rename to centralized_collection/elastic/top_n.elk diff --git a/centralized_collection/prometheus/README.md b/centralized_collection/prometheus/README.md new file mode 100644 index 000000000..4a3fd16b5 --- /dev/null +++ b/centralized_collection/prometheus/README.md @@ -0,0 +1,30 @@ +# Centralized Prometheus Collection + +This directory contains resources for building a docker container that it includes pktvisord and +the [Grafana Agent](https://github.com/grafana/agent) for collecting and sending metrics to Prometheus through +[remote write](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage), including cloud +providers. + +There is also a sample [Grafana dashboard](grafana-dashboard-prometheus.json). + +Example: + +```shell +$ docker pull ns1labs/pktvisor-prom-write +$ docker run -d --mount type=bind,source=/usr/local/geo,target=/geo --net=host --env PKTVISORD_ARGS="--prom-instance +--geo-city /geo/GeoIP2-City.mmdb --geo-asn /geo/GeoIP2-ISP.mmdb " --env +REMOTE_URL="https:///api/prom/push" --env USERNAME="" --env PASSWORD="" +ns1labs/pktvisor-prom-write +``` + +There are a few pieces of information you need to substitute above: + +* ``: The prometheus "instance" label for all metrics, e.g. "myhost" +* ``: The ethernet interface to capture on, e.g. "eth0" +* ``: The remote host to remote_write the prometheus metric to +* ``: If required by your prometheus setup, the user name to connect. If not required, leave off this + environment variable. +* ``: If required by your prometheus setup, the password to connect. If not required, leave off this + environment variable. + +Other pktvisor arguments may be passed in the PKTVISORD_ARGS environment variable. diff --git a/reporting/prometheus/docker-grafana-agent/Dockerfile b/centralized_collection/prometheus/docker-grafana-agent/Dockerfile similarity index 100% rename from reporting/prometheus/docker-grafana-agent/Dockerfile rename to centralized_collection/prometheus/docker-grafana-agent/Dockerfile diff --git a/reporting/prometheus/docker-grafana-agent/files/entrypoint b/centralized_collection/prometheus/docker-grafana-agent/files/entrypoint similarity index 100% rename from reporting/prometheus/docker-grafana-agent/files/entrypoint rename to centralized_collection/prometheus/docker-grafana-agent/files/entrypoint diff --git a/reporting/prometheus/docker-grafana-agent/files/run-grafana-agent.sh b/centralized_collection/prometheus/docker-grafana-agent/files/run-grafana-agent.sh similarity index 100% rename from reporting/prometheus/docker-grafana-agent/files/run-grafana-agent.sh rename to centralized_collection/prometheus/docker-grafana-agent/files/run-grafana-agent.sh diff --git a/reporting/prometheus/docker-grafana-agent/files/run-pktvisord.sh b/centralized_collection/prometheus/docker-grafana-agent/files/run-pktvisord.sh similarity index 100% rename from reporting/prometheus/docker-grafana-agent/files/run-pktvisord.sh rename to centralized_collection/prometheus/docker-grafana-agent/files/run-pktvisord.sh diff --git a/reporting/prometheus/grafana-dashboard-prometheus.json b/centralized_collection/prometheus/grafana-dashboard-prometheus.json similarity index 100% rename from reporting/prometheus/grafana-dashboard-prometheus.json rename to centralized_collection/prometheus/grafana-dashboard-prometheus.json diff --git a/reporting/README.md b/reporting/README.md deleted file mode 100644 index e69de29bb..000000000