Skip to content
Permalink
Browse files

Additional prometheus labels from envvars (#61)

* Initial implementation of additional prometheus labels from envvars

* Unit tests..

* Catching up with refactored config field in master

* Style

* Type erasure ¯\_(ツ)_/¯

* Tests to prove fix in 58e093d
  • Loading branch information...
andlaz authored and erebe committed Oct 30, 2019
1 parent 9e20c58 commit 26de7ce87448811a25b24c7efe25596a92275517
@@ -115,6 +115,13 @@ user:
password:
listenAddress: 0.0.0.0
listenPort: 8080
# Regular expression to match environment variables that will be added
# as labels to all data points. The name of the label will be either
# $1 from the regex below, or the entire environment variable name if no match groups are defined
#
# Example:
# additionalLabelsFromEnvvars: "^ADDL\_(.*)$"
additionalLabelsFromEnvvars:
blacklist:
# Unaccessible metrics (not enough privilege)
- java:lang:memorypool:.*usagethreshold.*
@@ -4,6 +4,13 @@ user:
password:
listenAddress: 0.0.0.0
listenPort: 8080
# Regular expression to match environment variable names that will be added
# as labels to all data points. The name of the label will be either
# $1 from the regex below, or the entire environment variable name if no match groups are defined
#
# Example:
# additionalLabelsFromEnvvars: "^ADDL\_(.*)$"
additionalLabelsFromEnvvars:
blacklist:
# To profile the duration of jmx call you can start the program with the following options
# > java -Dorg.slf4j.simpleLogger.defaultLogLevel=trace -jar cassandra_exporter.jar config.yml --oneshot
@@ -11,6 +11,7 @@
import java.util.Map;
import java.util.Optional;
import java.util.SortedMap;
import java.util.regex.Pattern;

public final class Config {

@@ -23,6 +24,7 @@
private String user;
private String password;
private SortedMap<Integer, List<String>> maxScrapeFrequencyInSec;
private Pattern additionalLabelsFromEnvvars;

public static Optional<Config> fromFile(String filePath) {
Logger logger = LoggerFactory.getLogger(Config.class);
@@ -70,5 +72,7 @@ public String getHost() {
public boolean getSSL() {
return ssl;
}

public Optional<Pattern> getAdditionalLabelsFromEnvvars() { return additionalLabelsFromEnvvars == null ? Optional.empty() : Optional.of(additionalLabelsFromEnvvars); }
}

@@ -23,11 +23,7 @@


public class JmxScraper {
static final Gauge STATS = Gauge.build()
.name("cassandra_stats")
.help("node stats")
.labelNames("cluster", "datacenter", "keyspace", "table", "name")
.register();
private final Gauge stats;
private static final Logger logger = LoggerFactory.getLogger(JmxScraper.class);
private static final double[] offsetPercentiles = new double[]{0.5, 0.75, 0.95, 0.98, 0.99};
private static final String metricSeparator = ":";
@@ -39,13 +35,22 @@
private final TreeMap<Integer, List<Pattern>> scrapFrequencies;
private final Map<Integer, Long> lastScrapes;
private final Map<String, Object> jmxEnv;
private final String[] additionalLabelValues;


public JmxScraper(String jmxUrl, Optional<String> username, Optional<String> password, boolean ssl, List<String> blacklist, SortedMap<Integer, List<String>> scrapFrequencies) {
public JmxScraper(String jmxUrl, Optional<String> username, Optional<String> password, boolean ssl, List<String> blacklist, SortedMap<Integer, List<String>> scrapFrequencies, Map<String, String> additionalLabels) {
this.jmxUrl = jmxUrl;
this.blacklist = blacklist.stream().map(Pattern::compile).collect(toList());
this.scrapFrequencies = new TreeMap<>();
this.lastScrapes = new HashMap<>(scrapFrequencies.size());
String[] additionalLabelKeys = additionalLabels.keySet().stream().toArray(String[]::new);
this.additionalLabelValues = additionalLabels.values().stream().toArray(String[]::new);

this.stats = Gauge.build()
.name("cassandra_stats")
.help("node stats")
.labelNames(concat(new String[]{"cluster", "datacenter", "keyspace", "table", "name"}, additionalLabelKeys))
.register();

scrapFrequencies.forEach((k, v) -> {
this.scrapFrequencies.put(k * 1000, v.stream().map(Pattern::compile).collect(toList()));
@@ -97,15 +102,15 @@ public JmxScraper(String jmxUrl, Optional<String> username, Optional<String> pas
return result;
}

private static void updateStats(NodeInfo nodeInfo, String metricName, Double value) {
private void updateStats(NodeInfo nodeInfo, String metricName, Double value) {

if (metricName.startsWith("org:apache:cassandra:metrics:keyspace:")) {
int pathLength = "org:apache:cassandra:metrics:keyspace:".length();
int pos = metricName.indexOf(':', pathLength);
String keyspaceName = metricName.substring(pathLength, pos);

STATS.labels(nodeInfo.clusterName, nodeInfo.datacenterName,
nodeInfo.keyspaces.contains(keyspaceName) ? keyspaceName : "", "", metricName).set(value);
this.stats.labels(concat(new String[] {nodeInfo.clusterName, nodeInfo.datacenterName,
nodeInfo.keyspaces.contains(keyspaceName) ? keyspaceName : "", "", metricName}, this.additionalLabelValues)).set(value);
return;
}

@@ -118,7 +123,7 @@ private static void updateStats(NodeInfo nodeInfo, String metricName, Double val
String tableName = tablePos > 0 ? metricName.substring(keyspacePos + 1, tablePos) : "";

if (nodeInfo.keyspaces.contains(keyspaceName) && nodeInfo.tables.contains(tableName)) {
STATS.labels(nodeInfo.clusterName, nodeInfo.datacenterName, keyspaceName, tableName, metricName).set(value);
this.stats.labels(concat(new String[] {nodeInfo.clusterName, nodeInfo.datacenterName, keyspaceName, tableName, metricName}, additionalLabelValues)).set(value);
return;
}
}
@@ -132,17 +137,17 @@ private static void updateStats(NodeInfo nodeInfo, String metricName, Double val
String tableName = tablePos > 0 ? metricName.substring(keyspacePos + 1, tablePos) : "";

if (nodeInfo.keyspaces.contains(keyspaceName) && nodeInfo.tables.contains(tableName)) {
STATS.labels(nodeInfo.clusterName, nodeInfo.datacenterName, keyspaceName, tableName, metricName).set(value);
this.stats.labels(concat(new String[] {nodeInfo.clusterName, nodeInfo.datacenterName, keyspaceName, tableName, metricName}, additionalLabelValues)).set(value);
return;
}
}

STATS.labels(nodeInfo.clusterName, nodeInfo.datacenterName, "", "", metricName).set(value);
this.stats.labels(concat( new String[] { nodeInfo.clusterName, nodeInfo.datacenterName, "", "", metricName}, additionalLabelValues)).set(value);
}

public void run(final boolean forever) throws Exception {

STATS.clear();
this.stats.clear();
try (JMXConnector jmxc = JMXConnectorFactory.connect(new JMXServiceURL(jmxUrl), jmxEnv)) {
final MBeanServerConnection beanConn = jmxc.getMBeanServerConnection();

@@ -331,6 +336,12 @@ private void updateMetric(MBeanServerConnection beanConn, MBeanInfo mBeanInfo, N
logger.trace("Scrapping took {}ms for {}", (System.currentTimeMillis() - start), mBeanInfo.metricName);
}

public static <T> T[] concat(T[] a, T[] b) {
T[] finalArray = Arrays.copyOf(a, a.length + b.length);
System.arraycopy(b, 0, finalArray, a.length, b.length);
return finalArray;
}

/**
* POJO to hold information regarding a metric
*/
@@ -4,14 +4,17 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Arrays;
import java.util.Optional;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

public class Main {

private final static Logger logger = LoggerFactory.getLogger(Main.class);

public static void main(String[] args) throws Exception {
Logger logger = LoggerFactory.getLogger(Main.class);


String configPath = args.length > 0 ? args[0] : Config.DEFAULT_PATH;
Optional<Config> cfgO = Config.fromFile(configPath);
@@ -24,7 +27,7 @@ public static void main(String[] args) throws Exception {
Config cfg = cfgO.get();
boolean isOneShot = Arrays.asList(args).contains("--oneshot");
HTTPServer server = new HTTPServer(cfg.getListenAddress(), cfg.getListenPort());
JmxScraper scrapper = new JmxScraper(String.format("service:jmx:rmi:///jndi/rmi://%s/jmxrmi", cfg.getHost()), cfg.getUser(), cfg.getPassword(), cfg.getSSL(), cfg.getBlacklist(), cfg.getMaxScrapeFrequencyInSec());
JmxScraper scrapper = new JmxScraper(String.format("service:jmx:rmi:///jndi/rmi://%s/jmxrmi", cfg.getHost()), cfg.getUser(), cfg.getPassword(), cfg.getSSL(), cfg.getBlacklist(), cfg.getMaxScrapeFrequencyInSec(), findAdditionalLabelsInEnvironment(System.getenv(), cfg.getAdditionalLabelsFromEnvvars()));

if (isOneShot) {
scrapper.run(false);
@@ -46,4 +49,24 @@ public static void main(String[] args) throws Exception {
}
}

public static Map<String, String> findAdditionalLabelsInEnvironment(Map<String, String> environment, Optional<Pattern> matchNames) {
if (matchNames.isPresent()) {


return environment.entrySet().stream()
.filter(e -> matchNames.get().matcher(e.getKey()).matches())
.collect(Collectors.toMap(e -> {
Matcher m = matchNames.get().matcher(e.getKey());
m.matches(); // guaranteed to pass due to .filter above
if (m.groupCount() > 0) {
return m.group(1);
}
return e.getKey();
}, e -> e.getValue()));

}

return Collections.emptyMap();
}

}
@@ -0,0 +1,26 @@
package com.criteo.nosql.cassandra.exporter;

import org.junit.Test;

import java.util.Optional;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.junit.Assert.*;

public class ConfigTest {
Logger logger = LoggerFactory.getLogger(ConfigTest.class);

@Test
public void test_additional_envvars_parsed() {

Optional<Config> configWithAdditionalEnvvars = Config.fromFile("src/test/resources/config_tests/config_with_additional_envvars_regexp.yml");
assertTrue(configWithAdditionalEnvvars.isPresent());
assertTrue(configWithAdditionalEnvvars.get().getAdditionalLabelsFromEnvvars().isPresent());

assertEquals(Pattern.compile("^ADDL\\_(.*)$").pattern(), configWithAdditionalEnvvars.get().getAdditionalLabelsFromEnvvars().get().pattern());

}

}
@@ -0,0 +1,27 @@
package com.criteo.nosql.cassandra.exporter;

import org.junit.Test;

import static org.hamcrest.CoreMatchers.instanceOf;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertThat;

public class JmxScraperTest {

@Test
public void test_concat_concatenates_array_contents() {
String[] arrayA = new String[]{"foo","bar","1"};
String[] arrayB = new String[]{"baz","2"};

assertArrayEquals(new String[]{"foo","bar","1", "baz", "2"}, JmxScraper.concat(arrayA, arrayB));
}

@Test
public void test_concat_results_in_generic_array_of_expected_type_parameter() {
String[] arrayA = new String[]{"foo","bar","1"};
String[] arrayB = new String[]{"baz","2"};

assertThat(JmxScraper.concat(arrayA, arrayB), instanceOf(String[].class));
}

}
@@ -0,0 +1,59 @@
package com.criteo.nosql.cassandra.exporter;

import org.junit.Test;

import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.regex.Pattern;

import static org.junit.Assert.assertEquals;

public class MainTest {

private static class AdditionalEnvvars {

private final static Map<String, String> envWithMatchingVars = new HashMap<String, String>() {{
put("foo", "bar");
put("ADDL_relevant-envvar-1", "relevant-value-1");
put("ADDL_relevant-envvar-2", "relevant-value-2");
}};

private final static Map<String, String> envWithNoMatchingVars = new HashMap<String, String>() {{
put("foo", "bar");
put("IRRELEVANT-ENVVAR-1", "irrelevant-value-1");
put("IRRELEVANT-ENVVAR-2", "irrelevant-value-2");
}};

private final static Pattern findRelevantEnvvarsGroup = Pattern.compile("^ADDL\\_(.*)$");
private final static Pattern findRelevantEnvvarsNoGroup = Pattern.compile("^ADDL\\_.*$");
}


@Test
public void test_findAdditionalLabelsInEnvironment_with_group_pattern_present_and_matching_envvars() {

assertEquals(new HashMap<String, String>() {{
put("relevant-envvar-1", "relevant-value-1");
put("relevant-envvar-2", "relevant-value-2");
}}, Main.findAdditionalLabelsInEnvironment(AdditionalEnvvars.envWithMatchingVars, Optional.of(AdditionalEnvvars.findRelevantEnvvarsGroup)));

}

@Test
public void test_findAdditionalLabelsInEnvironment_with_pattern_present_and_no_matching_envvars() {

assertEquals(new HashMap<String, String>() {{
put("ADDL_relevant-envvar-1", "relevant-value-1");
put("ADDL_relevant-envvar-2", "relevant-value-2");
}}, Main.findAdditionalLabelsInEnvironment(AdditionalEnvvars.envWithMatchingVars, Optional.of(AdditionalEnvvars.findRelevantEnvvarsNoGroup)));
}

@Test
public void test_findAdditionalLabelsInEnvironment_with_no_pattern_present() {

assertEquals(Collections.emptyMap(), Main.findAdditionalLabelsInEnvironment(AdditionalEnvvars.envWithMatchingVars, Optional.empty()));
}

}
@@ -0,0 +1,66 @@
host: localhost:7199
ssl: False
user:
password:
listenAddress: 0.0.0.0
listenPort: 8080
# Regular expression to match environment variables that will be added
# as labels to all data points. The name of the label will be either
# $1 from the regex below, or the entire environment variable name if no match groups are defined
#
# Example:
# additionalLabelsFromEnvvars: "^ADDL\_(.*)$"
additionalLabelsFromEnvvars: "^ADDL\\_(.*)$"
blacklist:
# To profile the duration of jmx call you can start the program with the following options
# > java -Dorg.slf4j.simpleLogger.defaultLogLevel=trace -jar cassandra_exporter.jar config.yml --oneshot
#
# To get intuition of what is done by cassandra when something is called you can look in cassandra
# https://github.com/apache/cassandra/tree/trunk/src/java/org/apache/cassandra/metrics
# Please avoid to scrape frequently those calls that are iterating over all sstables

# Unaccessible metrics (not enough privilege)
- java:lang:memorypool:.*usagethreshold.*

# Leaf attributes not interesting for us but that are presents in many path
- .*:999thpercentile
- .*:95thpercentile
- .*:fifteenminuterate
- .*:fiveminuterate
- .*:durationunit
- .*:rateunit
- .*:stddev
- .*:meanrate
- .*:mean
- .*:min

# Path present in many metrics but uninterresting
- .*:viewlockacquiretime:.*
- .*:viewreadtime:.*
- .*:cas[a-z]+latency:.*
- .*:colupdatetimedeltahistogram:.*

# Mostly for RPC, do not scrap them
- org:apache:cassandra:db:.*

# columnfamily is an alias for Table metrics
# https://github.com/apache/cassandra/blob/8b3a60b9a7dbefeecc06bace617279612ec7092d/src/java/org/apache/cassandra/metrics/TableMetrics.java#L162
- org:apache:cassandra:metrics:columnfamily:.*

# Should we export metrics for system keyspaces/tables ?
- org:apache:cassandra:metrics:[^:]+:system[^:]*:.*

# Don't scrap us
- com:criteo:nosql:cassandra:exporter:.*

maxScrapeFrequencyInSec:
50:
- .*

# Refresh those metrics only every hour as it is costly for cassandra to retrieve them
3600:
- .*:snapshotssize:.*
- .*:estimated.*
- .*:totaldiskspaceused:.*


0 comments on commit 26de7ce

Please sign in to comment.
You can’t perform that action at this time.