From c62e418cf776f162242f6933a07653d99c444d0b Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Wed, 24 Mar 2021 14:11:56 -0400 Subject: [PATCH 1/8] adjust policy naming --- docs/POLICIES.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/POLICIES.md b/docs/POLICIES.md index 85ce8b086..328d31703 100644 --- a/docs/POLICIES.md +++ b/docs/POLICIES.md @@ -24,7 +24,7 @@ Line Examples below. ```yaml version: "1.0" -policy: +visor: # each tap has input module specific configuration options taps: # a pcap tap which uses eth0 and is referenced by the identifier "anycast" @@ -57,7 +57,7 @@ are not stored in a file, but rather in the control plane database). ```yaml version: "1.0" -policy: +visor: collection: # policy name and description anycast_dns: @@ -143,7 +143,7 @@ information such as region, pop, and node type. ```yaml version: "1.0" -policy: +orb: vitals: region: EU pop: ams02 @@ -171,12 +171,13 @@ Or instead combine them into a single file: ```yaml version: "1.0" -policy: +pktvisor: taps: anycast: type: pcap config: iface: eth0 +orb: vitals: region: EU pop: ams02 @@ -197,7 +198,7 @@ the control plane, not on the command line or in files. ```yaml version: "1.0" -policy: +orb: selectors: all_dns: node_type: dns @@ -215,7 +216,7 @@ on the command line or in files. ```yaml version: "1.0" -policy: +orb: sinks: default_prometheus: type: prometheus_exporter @@ -241,11 +242,11 @@ success or failure. Upon success, the sink will be created. ```yaml version: "1.0" -policy: - orb: +orb: + policy: selectors: - eu_dns - collection_policy: anycast_dns + collection: anycast_dns sinks: - default_prometheus ``` From 0d444806a092d9bdb0e58553f9521f842cc61381 Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Wed, 24 Mar 2021 14:12:12 -0400 Subject: [PATCH 2/8] go Release build in docker container --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index fb84acdd8..3147d85f5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,7 +15,7 @@ RUN \ conan profile update settings.compiler.libcxx=libstdc++11 default RUN \ - PKG_CONFIG_PATH=/local/lib/pkgconfig cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo /pktvisor-src && \ + PKG_CONFIG_PATH=/local/lib/pkgconfig cmake -DCMAKE_BUILD_TYPE=Release /pktvisor-src && \ make all test -j 4 FROM golang:latest AS gobuild From d81ed5adccfd21236ebef716dfc76939aec90050 Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Wed, 24 Mar 2021 14:12:41 -0400 Subject: [PATCH 3/8] add two nodes --- reporting/prometheus/config/prometheus.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/reporting/prometheus/config/prometheus.yml b/reporting/prometheus/config/prometheus.yml index dd46a370f..ec2b65727 100644 --- a/reporting/prometheus/config/prometheus.yml +++ b/reporting/prometheus/config/prometheus.yml @@ -28,3 +28,21 @@ scrape_configs: static_configs: - targets: - localhost:10853 + - job_name: pktvisor-gw + honor_timestamps: true + scrape_interval: 1m + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + static_configs: + - targets: + - 192.168.0.1:10853 + - job_name: pktvisor-dns + honor_timestamps: true + scrape_interval: 1m + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + static_configs: + - targets: + - 192.168.0.54:10853 From 8f44b07f6954fd04be2e2569b4ba455aedc80dae Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Wed, 24 Mar 2021 14:48:08 -0400 Subject: [PATCH 4/8] initial prometheus dashboard --- .../grafana-dashboard-prometheus.json | 226 ++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100644 reporting/prometheus/grafana-dashboard-prometheus.json diff --git a/reporting/prometheus/grafana-dashboard-prometheus.json b/reporting/prometheus/grafana-dashboard-prometheus.json new file mode 100644 index 000000000..e03b0ed84 --- /dev/null +++ b/reporting/prometheus/grafana-dashboard-prometheus.json @@ -0,0 +1,226 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 1, + "iteration": 1616611136727, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "interval": "1m", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "maxDataPoints": 100, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.0-pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:75", + "alias": "in (p0.99)", + "lines": false, + "pointradius": 1, + "points": true + }, + { + "$$hashKey": "object:305", + "alias": "out (p0.99)", + "lines": false, + "pointradius": 1, + "points": true + }, + { + "$$hashKey": "object:330", + "alias": "/^out.*/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "packets_rates_pps_in{job=~\"$job\",quantile=~\"0.95|0.99\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "in (p{{quantile}})", + "refId": "A" + }, + { + "exemplar": true, + "expr": "packets_rates_pps_out{job=~\"$job\",quantile=~\"0.95|0.99\"}", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "out (p{{quantile}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:156", + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:157", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": false, + "text": "pktvisor-gw", + "value": "pktvisor-gw" + }, + "datasource": null, + "definition": "label_values(job)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "job", + "options": [ + { + "selected": false, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "pktvisor", + "value": "pktvisor" + }, + { + "selected": false, + "text": "pktvisor-dns", + "value": "pktvisor-dns" + }, + { + "selected": true, + "text": "pktvisor-gw", + "value": "pktvisor-gw" + } + ], + "query": { + "query": "label_values(job)", + "refId": "StandardVariableQuery" + }, + "refresh": 0, + "regex": "/^pktvisor/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "pktvisor", + "uid": "2dObW-wGk", + "version": 11 +} \ No newline at end of file From 5919eecb7d55ae329f6b70705de2e663f01f4242 Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Thu, 25 Mar 2021 10:50:12 -0400 Subject: [PATCH 5/8] add grafana agent --- .../docker-grafana-agent/Dockerfile | 19 +++++++++++++ .../docker-grafana-agent/files/entrypoint | 28 +++++++++++++++++++ .../files/run-grafana-agent.sh | 3 ++ .../files/run-pktvisord.sh | 5 ++++ 4 files changed, 55 insertions(+) create mode 100644 reporting/prometheus/docker-grafana-agent/Dockerfile create mode 100755 reporting/prometheus/docker-grafana-agent/files/entrypoint create mode 100755 reporting/prometheus/docker-grafana-agent/files/run-grafana-agent.sh create mode 100755 reporting/prometheus/docker-grafana-agent/files/run-pktvisord.sh diff --git a/reporting/prometheus/docker-grafana-agent/Dockerfile b/reporting/prometheus/docker-grafana-agent/Dockerfile new file mode 100644 index 000000000..4c8aee45d --- /dev/null +++ b/reporting/prometheus/docker-grafana-agent/Dockerfile @@ -0,0 +1,19 @@ +FROM grafana/agent:latest as agent + +FROM ns1labs/pktvisor:develop + +COPY --from=agent /bin/agent /usr/local/bin/agent + +RUN apt-get update \ + && apt-get install -y runit-init \ + && rm -rf /var/lib/apt \ + && mkdir -p /etc/runit/ \ + && mkdir -p /etc/agent/ \ + && mkdir -p /etc/agent/data \ + && rm -rf /etc/service/* + +COPY files/run-grafana-agent.sh /etc/service/agent/run +COPY files/run-pktvisord.sh /etc/service/pktvisord/run +COPY files/entrypoint /usr/local/bin/entrypoint + +ENTRYPOINT /usr/local/bin/entrypoint diff --git a/reporting/prometheus/docker-grafana-agent/files/entrypoint b/reporting/prometheus/docker-grafana-agent/files/entrypoint new file mode 100755 index 000000000..a4ea90edd --- /dev/null +++ b/reporting/prometheus/docker-grafana-agent/files/entrypoint @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +( +cat < /etc/agent/agent.yaml + +exec runsvdir /etc/service/ diff --git a/reporting/prometheus/docker-grafana-agent/files/run-grafana-agent.sh b/reporting/prometheus/docker-grafana-agent/files/run-grafana-agent.sh new file mode 100755 index 000000000..30cc27256 --- /dev/null +++ b/reporting/prometheus/docker-grafana-agent/files/run-grafana-agent.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +exec agent --config.file=/etc/agent/agent.yaml --prometheus.wal-directory=/etc/agent/data diff --git a/reporting/prometheus/docker-grafana-agent/files/run-pktvisord.sh b/reporting/prometheus/docker-grafana-agent/files/run-pktvisord.sh new file mode 100755 index 000000000..ce39db3f9 --- /dev/null +++ b/reporting/prometheus/docker-grafana-agent/files/run-pktvisord.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +[[ "$PKTVISOR_ARGS" == "" ]] && PKTVISOR_ARGS="eth0" + +exec pktvisord --prometheus $PKTVISOR_ARGS From c8cc464a52e6628cebfa3ce8794fd6247acc1ec1 Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Thu, 25 Mar 2021 10:50:34 -0400 Subject: [PATCH 6/8] add grafana agent --- reporting/prometheus/config/prometheus.yml | 48 ---------------------- reporting/prometheus/docker-compose.yaml | 11 ----- 2 files changed, 59 deletions(-) delete mode 100644 reporting/prometheus/config/prometheus.yml delete mode 100644 reporting/prometheus/docker-compose.yaml diff --git a/reporting/prometheus/config/prometheus.yml b/reporting/prometheus/config/prometheus.yml deleted file mode 100644 index ec2b65727..000000000 --- a/reporting/prometheus/config/prometheus.yml +++ /dev/null @@ -1,48 +0,0 @@ -global: - scrape_interval: 15s - scrape_timeout: 10s - evaluation_interval: 15s -alerting: - alertmanagers: - - scheme: http - timeout: 10s - api_version: v1 - static_configs: - - targets: [ ] -scrape_configs: - - job_name: prometheus - honor_timestamps: true - scrape_interval: 15s - scrape_timeout: 10s - metrics_path: /metrics - scheme: http - static_configs: - - targets: - - localhost:9090 - - job_name: pktvisor - honor_timestamps: true - scrape_interval: 1m - scrape_timeout: 10s - metrics_path: /metrics - scheme: http - static_configs: - - targets: - - localhost:10853 - - job_name: pktvisor-gw - honor_timestamps: true - scrape_interval: 1m - scrape_timeout: 10s - metrics_path: /metrics - scheme: http - static_configs: - - targets: - - 192.168.0.1:10853 - - job_name: pktvisor-dns - honor_timestamps: true - scrape_interval: 1m - scrape_timeout: 10s - metrics_path: /metrics - scheme: http - static_configs: - - targets: - - 192.168.0.54:10853 diff --git a/reporting/prometheus/docker-compose.yaml b/reporting/prometheus/docker-compose.yaml deleted file mode 100644 index efd980723..000000000 --- a/reporting/prometheus/docker-compose.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -version: '3.3' -services: - prometheus: - image: prom/prometheus - network_mode: "host" - volumes: - - './config/prometheus.yml:/etc/prometheus/prometheus.yml:ro' - grafana: - image: grafana/grafana:7.3.0 - network_mode: "host" From 13f0d2214085f27e8b07bb230e452fe0db59f79d Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Fri, 26 Mar 2021 15:34:42 -0400 Subject: [PATCH 7/8] docker --- docker/Dockerfile | 6 ++++-- reporting/prometheus/docker-grafana-agent/files/entrypoint | 4 ++-- .../prometheus/docker-grafana-agent/files/run-pktvisord.sh | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 3147d85f5..e67271300 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,9 +1,10 @@ FROM debian:bullseye-slim AS cppbuild -ENV BUILD_DEPS "g++ cmake make git pkgconf jq python3-pip python3-setuptools" +ENV BUILD_DEPS "g++ cmake make git pkgconf jq python3-pip python3-setuptools ca-certificates" RUN \ apt-get update && \ + apt-get upgrade && \ apt-get install --yes --no-install-recommends ${BUILD_DEPS} && \ pip3 install conan @@ -25,10 +26,11 @@ RUN go build -o pktvisor-cli cmd/pktvisor-cli/main.go FROM debian:bullseye-slim AS runtime -ENV RUNTIME_DEPS "curl" +ENV RUNTIME_DEPS "curl ca-certificates" RUN \ apt-get update && \ + apt-get upgrade && \ apt-get install --yes --no-install-recommends ${RUNTIME_DEPS} && \ rm -rf /var/lib/apt diff --git a/reporting/prometheus/docker-grafana-agent/files/entrypoint b/reporting/prometheus/docker-grafana-agent/files/entrypoint index a4ea90edd..bcd36360b 100755 --- a/reporting/prometheus/docker-grafana-agent/files/entrypoint +++ b/reporting/prometheus/docker-grafana-agent/files/entrypoint @@ -1,7 +1,7 @@ #!/usr/bin/env bash ( -cat < /etc/agent/agent.yaml +) >/etc/agent/agent.yaml exec runsvdir /etc/service/ diff --git a/reporting/prometheus/docker-grafana-agent/files/run-pktvisord.sh b/reporting/prometheus/docker-grafana-agent/files/run-pktvisord.sh index ce39db3f9..aae60c15c 100755 --- a/reporting/prometheus/docker-grafana-agent/files/run-pktvisord.sh +++ b/reporting/prometheus/docker-grafana-agent/files/run-pktvisord.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -[[ "$PKTVISOR_ARGS" == "" ]] && PKTVISOR_ARGS="eth0" +[[ "$PKTVISORD_ARGS" == "" ]] && PKTVISORD_ARGS="eth0" -exec pktvisord --prometheus $PKTVISOR_ARGS +exec pktvisord --prometheus $PKTVISORD_ARGS From 4da3f699a967c21243502d5aed3285ee9ce115ec Mon Sep 17 00:00:00 2001 From: Shannon Weyrick Date: Fri, 26 Mar 2021 15:38:38 -0400 Subject: [PATCH 8/8] bug: fix client version display in CLI UI --- golang/cmd/pktvisor-cli/main.go | 4 ++-- golang/pkg/client/version.go.in | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/golang/cmd/pktvisor-cli/main.go b/golang/cmd/pktvisor-cli/main.go index c992d94b6..0c6615524 100644 --- a/golang/cmd/pktvisor-cli/main.go +++ b/golang/cmd/pktvisor-cli/main.go @@ -50,7 +50,7 @@ Options: flag.Parse() if *wantVersion { - fmt.Println(client.VizerVersionNum) + fmt.Println(client.VisorVersionNum) return } if *wantHelp { @@ -381,7 +381,7 @@ func layout(g *gocui.Gui) error { if err != gocui.ErrUnknownView { return err } - v.Title = fmt.Sprintf("pktvisor-cli (client: %s | server: %s)", client.VizerVersionNum, serverVersion) + v.Title = fmt.Sprintf("pktvisor-cli (client: %s | server: %s)", client.VisorVersionNum, serverVersion) } //if currentView == "main" { diff --git a/golang/pkg/client/version.go.in b/golang/pkg/client/version.go.in index 41627ec02..833c08d07 100644 --- a/golang/pkg/client/version.go.in +++ b/golang/pkg/client/version.go.in @@ -5,7 +5,7 @@ package client var ( - VizerVersionNum = "${VIZER_VERSION_NUM}" - VizerVersion = "${VIZER_VERSION}" + VisorVersionNum = "${VISOR_VERSION_NUM}" + VisorVersion = "${VISOR_VERSION}" )