-
Notifications
You must be signed in to change notification settings - Fork 0
Prometheus & Grafana
Jake Jarvis edited this page Dec 22, 2022
·
22 revisions
personal notes:
- public dashboard: https://fediverse.jarv.is/grafana/public-dashboards/b5ca7a7c8e844f90b0973d2ab02bad0a
- this is an experimental Grafana feature: see
[feature_toggles]
ingrafana.ini
below
- this is an experimental Grafana feature: see
- Grafana frontend is reverse proxied through nginx from a subdirectory (https://fediverse.jarv.is/grafana/) — see
sites-available/mastodon.conf
below - logins are via grafana.com OAuth: get tokens at https://grafana.com/orgs/ORGNAME/oauth2/clients
- https://grafana.pipe.fail/public-dashboards/b5ca7a7c8e844f90b0973d2ab02bad0a
- https://ipng.ch/s/articles/2022/11/27/mastodon-3.html
- https://ourcodeworld.com/articles/read/1686/how-to-install-prometheus-node-exporter-on-ubuntu-2004
- https://github.com/prometheus-community/postgres_exporter
- https://github.com/prometheus/statsd_exporter
- https://github.com/oliver006/redis_exporter
- https://github.com/prometheus/node_exporter
- https://github.com/nginxinc/nginx-prometheus-exporter
- https://github.com/prometheus-community/json_exporter
repeat for each exporter:
wget https://github.com/oliver006/redis_exporter/releases/download/v1.45.0/redis_exporter-v1.45.0.linux-amd64.tar.gz
tar xvf redis_exporter-v1.45.0.linux-amd64.tar.gz
cp redis_exporter-v1.45.0.linux-amd64/redis_exporter /usr/local/bin/
useradd --no-create-home --shell /bin/false redis_exporter
chown redis_exporter:redis_exporter /usr/local/bin/redis_exporter
nano /etc/system/systemd/redis-exporter.service # see below
systemctl daemon-reload
systemctl enable --now redis-exporter.service
systemctl status redis-exporter.service
STATSD_ADDR=localhost:9125
global:
scrape_interval: 15s
evaluation_interval: 15s
scrape_configs:
- job_name: "prometheus"
static_configs:
- targets: ["localhost:9090"]
- job_name: "node_exporter_metrics"
static_configs:
- targets: ["localhost:9100"]
- job_name: "redis_exporter_targets"
static_configs:
- targets: ["redis://localhost:6379"]
metrics_path: /scrape
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: localhost:9121
- job_name: "redis_exporter_metrics"
static_configs:
- targets: ["localhost:9121"]
- job_name: "postgres_exporter_metrics"
static_configs:
- targets: ["localhost:9187"]
- job_name: "nginx_exporter_metrics"
static_configs:
- targets: ["localhost:9113"]
- job_name: "statsd_exporter_metrics"
static_configs:
- targets: ["localhost:9102"]
- job_name: "elasticsearch_exporter_metrics"
static_configs:
- targets: ["localhost:9114"]
- job_name: "json_exporter_metrics"
static_configs:
- targets: ["localhost:9079"]
- job_name: "json_exporter_targets"
metrics_path: /probe
scrape_interval: 30s
params:
module: [linode_bucket]
static_configs:
- targets:
- https://api.linode.com/v4/object-storage/buckets/us-east-1/jarvis-mastodon
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: localhost:9079
- job_name: "json_exporter_targets"
metrics_path: /probe
scrape_interval: 30s
params:
module: [linode_transfer]
static_configs:
- targets:
- https://api.linode.com/v4/account/transfer
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: localhost:9079
modules:
linode_bucket:
headers:
# https://cloud.linode.com/profile/tokens
Authorization: "Bearer XXXXXX"
metrics:
- name: json_linode_size
path: "{.size}"
labels:
bucket: "{.label}"
zone: "{.cluster}"
hostname: "{.hostname}"
- name: json_linode_objects
path: "{.objects}"
labels:
bucket: "{.label}"
zone: "{.cluster}"
hostname: "{.hostname}"
linode_transfer:
headers:
# https://cloud.linode.com/profile/tokens
Authorization: "Bearer XXXXXX"
metrics:
- name: json_linode_transfer_used
path: "{.used}"
- name: json_linode_transfer_quota
path: "{.quota}"
## Prometheus Statsd Exporter mapping for Mastodon 4.0+
##
## Version 1.0, November 2022
##
## Documentation: https://ipng.ch/s/articles/2022/11/27/mastodon-3.html
mappings:
## Web collector
- match: Mastodon\.production\.web\.(.+)\.(.+)\.(.+)\.status\.(.+)
match_type: regex
name: "mastodon_controller_status"
labels:
controller: $1
action: $2
format: $3
status: $4
mastodon: "web"
- match: Mastodon\.production\.web\.(.+)\.(.+)\.(.+)\.db_time
match_type: regex
name: "mastodon_controller_db_time"
labels:
controller: $1
action: $2
format: $3
mastodon: "web"
- match: Mastodon\.production\.web\.(.+)\.(.+)\.(.+)\.view_time
match_type: regex
name: "mastodon_controller_view_time"
labels:
controller: $1
action: $2
format: $3
mastodon: "web"
- match: Mastodon\.production\.web\.(.+)\.(.+)\.(.+)\.total_duration
match_type: regex
name: "mastodon_controller_duration"
labels:
controller: $1
action: $2
format: $3
mastodon: "web"
## Database collector
- match: Mastodon\.production\.db\.tables\.(.+)\.queries\.(.+)\.duration
match_type: regex
name: "mastodon_db_operation"
labels:
table: "$1"
operation: "$2"
mastodon: "db"
## Cache collector
- match: Mastodon\.production\.cache\.(.+)\.duration
match_type: regex
name: "mastodon_cache_duration"
labels:
operation: "$1"
mastodon: "cache"
## Sidekiq collector
- match: Mastodon\.production\.sidekiq\.(.+)\.processing_time
match_type: regex
name: "mastodon_sidekiq_worker_processing_time"
labels:
worker: "$1"
mastodon: "sidekiq"
- match: Mastodon\.production\.sidekiq\.(.+)\.success
match_type: regex
name: "mastodon_sidekiq_worker_success_total"
labels:
worker: "$1"
mastodon: "sidekiq"
- match: Mastodon\.production\.sidekiq\.(.+)\.failure
match_type: regex
name: "mastodon_sidekiq_worker_failure_total"
labels:
worker: "$1"
mastodon: "sidekiq"
- match: Mastodon\.production\.sidekiq\.queues\.(.+)\.enqueued
match_type: regex
name: "mastodon_sidekiq_queue_enqueued"
labels:
queue: "$1"
mastodon: "sidekiq"
- match: Mastodon\.production\.sidekiq\.queues\.(.+)\.latency
match_type: regex
name: "mastodon_sidekiq_queue_latency"
labels:
queue: "$1"
mastodon: "sidekiq"
- match: Mastodon\.production\.sidekiq\.(.+)
match_type: regex
name: "mastodon_sidekiq_$1"
labels:
mastodon: "sidekiq"
[Unit]
Description=Redis Exporter
Wants=network-online.target
After=network-online.target
[Service]
User=redis_exporter
Group=redis_exporter
Type=simple
ExecStart=/usr/local/bin/redis_exporter
Restart=always
[Install]
WantedBy=multi-user.target
[server]
protocol = http
http_addr = 127.0.0.1
http_port = 5100
domain = fediverse.jarv.is
enforce_domain = false
root_url = https://%(domain)s/grafana/
serve_from_sub_path = true
[analytics]
reporting_enabled = false
check_for_updates = false
check_for_plugin_updates = false
feedback_links_enabled = false
[security]
disable_initial_admin_creation = true
disable_gravatar = true
cookie_secure = true
x_xss_protection = false
content_security_policy = true
content_security_policy_template = """default-src 'none'; script-src 'self' 'unsafe-eval' 'unsafe-inline' 'strict-dynamic' $NONCE; object-src 'none'; font-src 'self'; style-src 'self' 'unsafe-inline' blob:; img-src 'self' data:; base-uri 'self'; frame-ancestors 'none'; connect-src 'self' https://grafana.com wss://$ROOT_PATH; manifest-src 'self'; media-src 'none'; form-action 'self'"""
[snapshots]
external_enabled = false
[dashboards]
versions_to_keep = 100
[users]
allow_sign_up = false
default_theme = dark
[auth]
disable_login = true
disable_login_form = true
[auth.anonymous]
hide_version = true
[auth.grafana_com]
enabled = true
allow_sign_up = false
scopes = user:email
# https://grafana.com/orgs/XXXXXX/oauth2/clients
allowed_organizations = XXXXXX
client_id = XXXXXX
client_secret = XXXXXX
[smtp]
enabled = true
host = email-smtp.us-east-1.amazonaws.com:587
user = XXXXXX
password = XXXXXX
from_address = noreply@fediverse.jarv.is
from_name = Grafana
[metrics]
enabled = false
[live]
max_connections = 10
[feature_toggles]
publicDashboards = true
[date_formats]
default_timezone = America/New_York
Grafana under a subfolder: https://grafana.com/tutorials/run-grafana-behind-a-proxy/#configure-nginx
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
upstream grafana {
server 127.0.0.1:5100 fail_timeout=0;
}
# ...
server {
listen [::]:443 http2 ssl ipv6only=on;
listen 443 http2 ssl;
server_name fediverse.jarv.is;
# ...
location /grafana/ {
rewrite ^/grafana/(.*) /$1 break;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Proxy "";
proxy_pass http://grafana;
proxy_buffering on;
proxy_redirect off;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
add_header Strict-Transport-Security "max-age=63072000" always;
add_header Via "1.1 $proxy_host" always;
tcp_nodelay on;
}
location /grafana/api/live/ {
rewrite ^/grafana/(.*) /$1 break;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Proxy "";
proxy_pass http://grafana;
proxy_buffering off;
proxy_redirect off;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
add_header Strict-Transport-Security "max-age=63072000" always;
add_header Via "1.1 $proxy_host" always;
tcp_nodelay on;
}
# ...
}