Skip to content

Commit

Permalink
Update monitoring to use per node config
Browse files Browse the repository at this point in the history 
 allowing simpler control over scrape configs.
  • Loading branch information
@jbgi
jbgi committed Mar 2, 2021
1 parent 755be47 commit 704c9a3
Show file tree
Hide file tree
Showing 8 changed files with 64 additions and 81 deletions.
67 changes: 0 additions & 67 deletions clusters/cardano.nix
View file
Expand Up @@ -28,70 +28,6 @@ let
roles.isMonitor = true;
org = def.org or "IOHK";
};

services.prometheus = {
scrapeConfigs = (lib.optionals globals.withExplorer ([
# TODO: remove once explorer exports metrics at path `/metrics`
{
job_name = "explorer-exporter";
scrape_interval = "10s";
metrics_path = "/metrics2/exporter";
static_configs = [{
targets = [ "explorer-ip" ];
labels = { alias = "explorer-exporter"; };
}];
}
{
job_name = "cardano-graphql-exporter";
scrape_interval = "10s";
metrics_path = "/metrics2/cardano-graphql";
static_configs = [{
targets = [ "explorer-ip" ];
labels = { alias = "cardano-graphql-exporter"; };
}];
}
])) ++ (lib.optionals globals.withSmash [
{
job_name = "smash-exporter";
scrape_interval = "10s";
metrics_path = "/metrics2/exporter";
static_configs = [{
targets = [ "smash-ip" ];
labels = { alias = "smash-exporter"; };
}];
}
]) ++ (lib.optional globals.withFaucet (
{
job_name = "cardano-faucet";
scrape_interval = "10s";
metrics_path = "/metrics";
static_configs = [{
targets = [ "${globals.faucetHostname}.${globals.domain}" ];
labels = { alias = "cardano-faucet"; };
}];
}
));
#)) ++ (lib.optional globals.withMetadata (
#{
# job_name = "metadata";
# scrape_interval = "10s";
# metrics_path = "/metrics";
# static_configs = [{
# targets = [ "metadata-ip" ];
# labels = { alias = "metadata"; };
# }];
#}));
#})) ++
#[{
# job_name = "netdata";
# scrape_interval = "60s";
# metrics_path = "/api/v1/allmetrics?format=prometheus";
# static_configs = pkgs.lib.traceValFn (x: __toJSON x) (map (n: {
# targets = [ "${n.name}-ip:${toString globals.netdataExporterPort}" ];
# labels = { alias = "${n.name}"; };
# }) (coreNodes ++ relayNodes ++ (topology.testNodes or [])));
#}];
};
} def;
}) // (lib.optionalAttrs globals.withExplorer {
explorer = let def = (topology.explorer or {}); in mkNode {
Expand All @@ -105,9 +41,6 @@ let
cardano-ops.roles.explorer
];

services.monitoring-exporters.extraPrometheusExportersPorts =
[ globals.cardanoNodePrometheusExporterPort ];

services.cardano-node.allProducers = if (relayNodes != [])
then [ pkgs.globals.relaysNew ]
else (map (n: n.name) coreNodes);
Expand Down
8 changes: 7 additions & 1 deletion modules/base-service.nix
View file
Expand Up @@ -89,7 +89,13 @@ in
environment.variables = {
CARDANO_NODE_SOCKET_PATH = cfg.socketPath;
};
services.monitoring-exporters.extraPrometheusExportersPorts = genList (i: monitoringPort + i) cfg.instances;
services.monitoring-exporters.extraPrometheusExporters = genList (i: {
job_name = "cardano-node";
scrape_interval = "10s";
port = monitoringPort + i;
metrics_path = "/metrics";
labels = optionalAttrs (i > 0) { alias = "${name}.${toString i}"; };
}) cfg.instances;
services.custom-metrics = {
enable = true;
statsdExporter = "node";
Expand Down
8 changes: 8 additions & 0 deletions modules/custom-metrics.nix
View file
Expand Up @@ -361,5 +361,13 @@ in with pkgs; {
};
wantedBy = [ "timers.target" ];
};

services.monitoring-exporters.extraPrometheusExporters = lib.optional (cfg.enableNetdata)
{
job_name = "netdata";
scrape_interval = "60s";
metrics_path = "/api/v1/allmetrics?format=prometheus";
port = globals.netdataExporterPort;
};
};
}
8 changes: 4 additions & 4 deletions nix/sources.json
View file
Expand Up @@ -157,15 +157,15 @@
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"ops-lib": {
"branch": "master",
"branch": "extraPrometheusExporters",
"description": "NixOps deployment configuration library for IOHK devops ",
"homepage": null,
"owner": "input-output-hk",
"repo": "ops-lib",
"rev": "fd89385a37e7d37922c0f03ca7d26ec3bc626688",
"sha256": "00jx1q5ghzs0k4f2a5hn2jf97f4cbgal3b3jpwrshqpwlv2b1hcq",
"rev": "c88c1aea619855ff40de9a98e67493afdc5375e6",
"sha256": "0crj1mlz5sgc4qbzfb59b2r3qcqsifbrdzkyalwnibvz10l1s2i7",
"type": "tarball",
"url": "https://github.com/input-output-hk/ops-lib/archive/fd89385a37e7d37922c0f03ca7d26ec3bc626688.tar.gz",
"url": "https://github.com/input-output-hk/ops-lib/archive/c88c1aea619855ff40de9a98e67493afdc5375e6.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"smash": {
Expand Down
21 changes: 20 additions & 1 deletion roles/explorer.nix
View file
Expand Up @@ -58,7 +58,10 @@ in {

services.cardano-rosetta-server = {
enable = true;
topologyFilePath = nodeCfg.topology;
topologyFilePath = iohkNix.cardanoLib.mkEdgeTopology {
edgeNodes = map (p: p.addr) nodeCfg.producers;
port = nodeCfg.port;
};
cardanoCliPath = cardano-cli + /bin/cardano-cli;
genesisPath = nodeCfg.nodeConfig.ShelleyGenesisFile;
cardanoNodePath = cardano-node + /bin/cardano-node;
Expand Down Expand Up @@ -380,4 +383,20 @@ in {
};
};
};

services.monitoring-exporters.extraPrometheusExporters = [
# TODO: remove once explorer exports metrics at path `/metrics`
{
job_name = "explorer-exporter";
scrape_interval = "10s";
metrics_path = "/metrics2/exporter";
labels = { alias = "explorer-exporter"; };
}
{
job_name = "cardano-graphql-exporter";
scrape_interval = "10s";
metrics_path = "/metrics2/cardano-graphql";
labels = { alias = "cardano-graphql-exporter"; };
}
];
}
10 changes: 8 additions & 2 deletions roles/faucet.nix
View file
Expand Up @@ -28,14 +28,20 @@ in {
80
443
nodePort
monitoringPort
];

environment.systemPackages = with pkgs; [
sqlite-interactive
];

services.monitoring-exporters.extraPrometheusExportersPorts = [ monitoringPort ];
services.monitoring-exporters.extraPrometheusExporters = [{
job_name = "cardano-node";
scrape_interval = "10s";
port = monitoringPort;
metrics_path = "/metrics";
labels = { alias = "cardano-faucet"; };
}];

services.custom-metrics = {
enable = true;
statsdExporter = "node";
Expand Down
11 changes: 8 additions & 3 deletions roles/metadata.nix
View file
Expand Up @@ -278,11 +278,16 @@ in {
#"/metrics/metadata" = {
# proxyPass = "http://127.0.0.1:8080/";
#};
"/metrics/varnish" = {
proxyPass = "http://127.0.0.1:9131/metrics";
};
};
};
};
};

services.monitoring-exporters.extraPrometheusExporters = [
#{
# job_name = "metadata";
# scrape_interval = "10s";
# metrics_path = "/metrics/metadata";
#}
];
}
12 changes: 9 additions & 3 deletions roles/smash.nix
View file
Expand Up @@ -264,11 +264,17 @@ in {
"/metrics2/exporter" = {
proxyPass = "http://127.0.0.1:8080/";
};
"/metrics/varnish" = {
proxyPass = "http://127.0.0.1:9131/metrics";
};
};
};
};
};

services.monitoring-exporters.extraPrometheusExporters = [
{
job_name = "smash-exporter";
scrape_interval = "10s";
metrics_path = "/metrics2/exporter";
labels = { alias = "smash-exporter"; };
}
];
}

0 comments on commit 704c9a3

Please sign in to comment.