Additional prometheus labels from envvars (#61)

* Initial implementation of additional prometheus labels from envvars

* Unit tests..

* Catching up with refactored config field in master

* Style

* Type erasure ¯\_(ツ)_/¯

* Tests to prove fix in 58e093d

README.md

@@ -115,6 +115,13 @@ user:
 password:
 listenAddress: 0.0.0.0
 listenPort: 8080
+# Regular expression to match environment variables that will be added
+# as labels to all data points. The name of the label will be either
+# $1 from the regex below, or the entire environment variable name if no match groups are defined
+#
+# Example:
+# additionalLabelsFromEnvvars: "^ADDL\_(.*)$"
+additionalLabelsFromEnvvars:
 blacklist:
    # Unaccessible metrics (not enough privilege)
    - java:lang:memorypool:.*usagethreshold.*

config.yml

@@ -4,6 +4,13 @@ user:
 password:
 listenAddress: 0.0.0.0
 listenPort: 8080
+# Regular expression to match environment variable names that will be added
+# as labels to all data points. The name of the label will be either
+# $1 from the regex below, or the entire environment variable name if no match groups are defined
+#
+# Example:
+# additionalLabelsFromEnvvars: "^ADDL\_(.*)$"
+additionalLabelsFromEnvvars:
 blacklist:
    # To profile the duration of jmx call you can start the program with the following options
    # > java -Dorg.slf4j.simpleLogger.defaultLogLevel=trace -jar cassandra_exporter.jar config.yml --oneshot

src/main/java/com/criteo/nosql/cassandra/exporter/Config.java

@@ -11,6 +11,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.SortedMap;
+import java.util.regex.Pattern;
 
 public final class Config {
 
@@ -23,6 +24,7 @@ public final class Config {
     private String user;
     private String password;
     private SortedMap<Integer, List<String>> maxScrapeFrequencyInSec;
+    private Pattern additionalLabelsFromEnvvars;
 
     public static Optional<Config> fromFile(String filePath) {
         Logger logger = LoggerFactory.getLogger(Config.class);
@@ -70,5 +72,7 @@ public List<String> getBlacklist() {
     public boolean getSSL() {
         return ssl;
     }
+
+    public Optional<Pattern> getAdditionalLabelsFromEnvvars() { return additionalLabelsFromEnvvars == null ? Optional.empty() : Optional.of(additionalLabelsFromEnvvars); }
 }
 

src/main/java/com/criteo/nosql/cassandra/exporter/JmxScraper.java

@@ -23,11 +23,7 @@
 
 
 public class JmxScraper {
-    static final Gauge STATS = Gauge.build()
-            .name("cassandra_stats")
-            .help("node stats")
-            .labelNames("cluster", "datacenter", "keyspace", "table", "name")
-            .register();
+    private final Gauge stats;
     private static final Logger logger = LoggerFactory.getLogger(JmxScraper.class);
     private static final double[] offsetPercentiles = new double[]{0.5, 0.75, 0.95, 0.98, 0.99};
     private static final String metricSeparator = ":";
@@ -39,13 +35,22 @@ public class JmxScraper {
     private final TreeMap<Integer, List<Pattern>> scrapFrequencies;
     private final Map<Integer, Long> lastScrapes;
     private final Map<String, Object> jmxEnv;
+    private final String[] additionalLabelValues;
 
 
-    public JmxScraper(String jmxUrl, Optional<String> username, Optional<String> password, boolean ssl, List<String> blacklist, SortedMap<Integer, List<String>> scrapFrequencies) {
+    public JmxScraper(String jmxUrl, Optional<String> username, Optional<String> password, boolean ssl, List<String> blacklist, SortedMap<Integer, List<String>> scrapFrequencies, Map<String, String> additionalLabels) {
         this.jmxUrl = jmxUrl;
         this.blacklist = blacklist.stream().map(Pattern::compile).collect(toList());
         this.scrapFrequencies = new TreeMap<>();
         this.lastScrapes = new HashMap<>(scrapFrequencies.size());
+        String[] additionalLabelKeys = additionalLabels.keySet().stream().toArray(String[]::new);
+        this.additionalLabelValues = additionalLabels.values().stream().toArray(String[]::new);
+
+        this.stats = Gauge.build()
+                .name("cassandra_stats")
+                .help("node stats")
+                .labelNames(concat(new String[]{"cluster", "datacenter", "keyspace", "table", "name"}, additionalLabelKeys))
+                .register();
 
         scrapFrequencies.forEach((k, v) -> {
             this.scrapFrequencies.put(k * 1000, v.stream().map(Pattern::compile).collect(toList()));
@@ -97,15 +102,15 @@ private static double[] metricPercentilesAsArray(long[] counts) {
         return result;
     }
 
-    private static void updateStats(NodeInfo nodeInfo, String metricName, Double value) {
+    private void updateStats(NodeInfo nodeInfo, String metricName, Double value) {
 
         if (metricName.startsWith("org:apache:cassandra:metrics:keyspace:")) {
             int pathLength = "org:apache:cassandra:metrics:keyspace:".length();
             int pos = metricName.indexOf(':', pathLength);
             String keyspaceName = metricName.substring(pathLength, pos);
 
-            STATS.labels(nodeInfo.clusterName, nodeInfo.datacenterName,
-                    nodeInfo.keyspaces.contains(keyspaceName) ? keyspaceName : "", "", metricName).set(value);
+            this.stats.labels(concat(new String[] {nodeInfo.clusterName, nodeInfo.datacenterName,
+                    nodeInfo.keyspaces.contains(keyspaceName) ? keyspaceName : "", "", metricName}, this.additionalLabelValues)).set(value);
             return;
         }
 
@@ -118,7 +123,7 @@ private static void updateStats(NodeInfo nodeInfo, String metricName, Double val
             String tableName = tablePos > 0 ? metricName.substring(keyspacePos + 1, tablePos) : "";
 
             if (nodeInfo.keyspaces.contains(keyspaceName) && nodeInfo.tables.contains(tableName)) {
-                STATS.labels(nodeInfo.clusterName, nodeInfo.datacenterName, keyspaceName, tableName, metricName).set(value);
+                this.stats.labels(concat(new String[] {nodeInfo.clusterName, nodeInfo.datacenterName, keyspaceName, tableName, metricName}, additionalLabelValues)).set(value);
                 return;
             }
         }
@@ -132,17 +137,17 @@ private static void updateStats(NodeInfo nodeInfo, String metricName, Double val
             String tableName = tablePos > 0 ? metricName.substring(keyspacePos + 1, tablePos) : "";
 
             if (nodeInfo.keyspaces.contains(keyspaceName) && nodeInfo.tables.contains(tableName)) {
-                STATS.labels(nodeInfo.clusterName, nodeInfo.datacenterName, keyspaceName, tableName, metricName).set(value);
+                this.stats.labels(concat(new String[] {nodeInfo.clusterName, nodeInfo.datacenterName, keyspaceName, tableName, metricName}, additionalLabelValues)).set(value);
                 return;
             }
         }
 
-        STATS.labels(nodeInfo.clusterName, nodeInfo.datacenterName, "", "", metricName).set(value);
+        this.stats.labels(concat( new String[] { nodeInfo.clusterName, nodeInfo.datacenterName, "", "", metricName}, additionalLabelValues)).set(value);
     }
 
     public void run(final boolean forever) throws Exception {
 
-        STATS.clear();
+        this.stats.clear();
         try (JMXConnector jmxc = JMXConnectorFactory.connect(new JMXServiceURL(jmxUrl), jmxEnv)) {
             final MBeanServerConnection beanConn = jmxc.getMBeanServerConnection();
 
@@ -331,6 +336,12 @@ private void updateMetric(MBeanServerConnection beanConn, MBeanInfo mBeanInfo, N
         logger.trace("Scrapping took {}ms for {}", (System.currentTimeMillis() - start), mBeanInfo.metricName);
     }
 
+    public static <T> T[] concat(T[] a, T[] b) {
+        T[] finalArray = Arrays.copyOf(a, a.length + b.length);
+        System.arraycopy(b, 0, finalArray, a.length, b.length);
+        return finalArray;
+    }
+
     /**
      * POJO to hold information regarding a metric
      */

src/main/java/com/criteo/nosql/cassandra/exporter/Main.java

@@ -4,14 +4,17 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.Arrays;
-import java.util.Optional;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 public class Main {
 
+    private final static Logger logger = LoggerFactory.getLogger(Main.class);
 
     public static void main(String[] args) throws Exception {
-        Logger logger = LoggerFactory.getLogger(Main.class);
+
 
         String configPath = args.length > 0 ? args[0] : Config.DEFAULT_PATH;
         Optional<Config> cfgO = Config.fromFile(configPath);
@@ -24,7 +27,7 @@ public static void main(String[] args) throws Exception {
         Config cfg = cfgO.get();
         boolean isOneShot = Arrays.asList(args).contains("--oneshot");
         HTTPServer server = new HTTPServer(cfg.getListenAddress(), cfg.getListenPort());
-        JmxScraper scrapper = new JmxScraper(String.format("service:jmx:rmi:///jndi/rmi://%s/jmxrmi", cfg.getHost()), cfg.getUser(), cfg.getPassword(), cfg.getSSL(), cfg.getBlacklist(), cfg.getMaxScrapeFrequencyInSec());
+        JmxScraper scrapper = new JmxScraper(String.format("service:jmx:rmi:///jndi/rmi://%s/jmxrmi", cfg.getHost()), cfg.getUser(), cfg.getPassword(), cfg.getSSL(), cfg.getBlacklist(), cfg.getMaxScrapeFrequencyInSec(), findAdditionalLabelsInEnvironment(System.getenv(), cfg.getAdditionalLabelsFromEnvvars()));
 
         if (isOneShot) {
             scrapper.run(false);
@@ -46,4 +49,24 @@ public static void main(String[] args) throws Exception {
         }
     }
 
+    public static Map<String, String> findAdditionalLabelsInEnvironment(Map<String, String> environment, Optional<Pattern> matchNames) {
+        if (matchNames.isPresent()) {
+
+
+            return environment.entrySet().stream()
+                    .filter(e -> matchNames.get().matcher(e.getKey()).matches())
+                    .collect(Collectors.toMap(e -> {
+                        Matcher m = matchNames.get().matcher(e.getKey());
+                        m.matches(); // guaranteed to pass due to .filter above
+                        if (m.groupCount() > 0) {
+                            return m.group(1);
+                        }
+                        return e.getKey();
+                    }, e -> e.getValue()));
+
+        }
+
+        return Collections.emptyMap();
+    }
+
 }

src/test/java/com/criteo/nosql/cassandra/exporter/ConfigTest.java

@@ -0,0 +1,26 @@
+package com.criteo.nosql.cassandra.exporter;
+
+import org.junit.Test;
+
+import java.util.Optional;
+import java.util.regex.Pattern;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.*;
+
+public class ConfigTest {
+    Logger logger = LoggerFactory.getLogger(ConfigTest.class);
+
+    @Test
+    public void test_additional_envvars_parsed() {
+
+        Optional<Config> configWithAdditionalEnvvars = Config.fromFile("src/test/resources/config_tests/config_with_additional_envvars_regexp.yml");
+        assertTrue(configWithAdditionalEnvvars.isPresent());
+        assertTrue(configWithAdditionalEnvvars.get().getAdditionalLabelsFromEnvvars().isPresent());
+
+        assertEquals(Pattern.compile("^ADDL\\_(.*)$").pattern(), configWithAdditionalEnvvars.get().getAdditionalLabelsFromEnvvars().get().pattern());
+
+    }
+
+}

src/test/java/com/criteo/nosql/cassandra/exporter/JmxScraperTest.java

@@ -0,0 +1,27 @@
+package com.criteo.nosql.cassandra.exporter;
+
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertThat;
+
+public class JmxScraperTest {
+
+    @Test
+    public void test_concat_concatenates_array_contents() {
+        String[] arrayA = new String[]{"foo","bar","1"};
+        String[] arrayB = new String[]{"baz","2"};
+
+        assertArrayEquals(new String[]{"foo","bar","1", "baz", "2"}, JmxScraper.concat(arrayA, arrayB));
+    }
+
+    @Test
+    public void test_concat_results_in_generic_array_of_expected_type_parameter() {
+        String[] arrayA = new String[]{"foo","bar","1"};
+        String[] arrayB = new String[]{"baz","2"};
+
+        assertThat(JmxScraper.concat(arrayA, arrayB), instanceOf(String[].class));
+    }
+
+}

src/test/java/com/criteo/nosql/cassandra/exporter/MainTest.java

@@ -0,0 +1,59 @@
+package com.criteo.nosql.cassandra.exporter;
+
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.regex.Pattern;
+
+import static org.junit.Assert.assertEquals;
+
+public class MainTest {
+
+    private static class AdditionalEnvvars {
+
+        private final static Map<String, String> envWithMatchingVars = new HashMap<String, String>() {{
+            put("foo", "bar");
+            put("ADDL_relevant-envvar-1", "relevant-value-1");
+            put("ADDL_relevant-envvar-2", "relevant-value-2");
+        }};
+
+        private final static Map<String, String> envWithNoMatchingVars = new HashMap<String, String>() {{
+            put("foo", "bar");
+            put("IRRELEVANT-ENVVAR-1", "irrelevant-value-1");
+            put("IRRELEVANT-ENVVAR-2", "irrelevant-value-2");
+        }};
+
+        private final static Pattern findRelevantEnvvarsGroup = Pattern.compile("^ADDL\\_(.*)$");
+        private final static Pattern findRelevantEnvvarsNoGroup = Pattern.compile("^ADDL\\_.*$");
+    }
+
+
+    @Test
+    public void test_findAdditionalLabelsInEnvironment_with_group_pattern_present_and_matching_envvars() {
+
+        assertEquals(new HashMap<String, String>() {{
+            put("relevant-envvar-1", "relevant-value-1");
+            put("relevant-envvar-2", "relevant-value-2");
+        }}, Main.findAdditionalLabelsInEnvironment(AdditionalEnvvars.envWithMatchingVars, Optional.of(AdditionalEnvvars.findRelevantEnvvarsGroup)));
+
+    }
+
+    @Test
+    public void test_findAdditionalLabelsInEnvironment_with_pattern_present_and_no_matching_envvars() {
+
+        assertEquals(new HashMap<String, String>() {{
+            put("ADDL_relevant-envvar-1", "relevant-value-1");
+            put("ADDL_relevant-envvar-2", "relevant-value-2");
+        }}, Main.findAdditionalLabelsInEnvironment(AdditionalEnvvars.envWithMatchingVars, Optional.of(AdditionalEnvvars.findRelevantEnvvarsNoGroup)));
+    }
+
+    @Test
+    public void test_findAdditionalLabelsInEnvironment_with_no_pattern_present() {
+
+        assertEquals(Collections.emptyMap(), Main.findAdditionalLabelsInEnvironment(AdditionalEnvvars.envWithMatchingVars, Optional.empty()));
+    }
+
+}

src/test/resources/config_tests/config_with_additional_envvars_regexp.yml

@@ -0,0 +1,66 @@
+host: localhost:7199
+ssl: False
+user:
+password:
+listenAddress: 0.0.0.0
+listenPort: 8080
+# Regular expression to match environment variables that will be added
+# as labels to all data points. The name of the label will be either
+# $1 from the regex below, or the entire environment variable name if no match groups are defined
+#
+# Example:
+# additionalLabelsFromEnvvars: "^ADDL\_(.*)$"
+additionalLabelsFromEnvvars: "^ADDL\\_(.*)$"
+blacklist:
+  # To profile the duration of jmx call you can start the program with the following options
+  # > java -Dorg.slf4j.simpleLogger.defaultLogLevel=trace -jar cassandra_exporter.jar config.yml --oneshot
+  #
+  # To get intuition of what is done by cassandra when something is called you can look in cassandra
+  # https://github.com/apache/cassandra/tree/trunk/src/java/org/apache/cassandra/metrics
+  # Please avoid to scrape frequently those calls that are iterating over all sstables
+
+  # Unaccessible metrics (not enough privilege)
+  - java:lang:memorypool:.*usagethreshold.*
+
+  # Leaf attributes not interesting for us but that are presents in many path
+  - .*:999thpercentile
+  - .*:95thpercentile
+  - .*:fifteenminuterate
+  - .*:fiveminuterate
+  - .*:durationunit
+  - .*:rateunit
+  - .*:stddev
+  - .*:meanrate
+  - .*:mean
+  - .*:min
+
+  # Path present in many metrics but uninterresting
+  - .*:viewlockacquiretime:.*
+  - .*:viewreadtime:.*
+  - .*:cas[a-z]+latency:.*
+  - .*:colupdatetimedeltahistogram:.*
+
+  # Mostly for RPC, do not scrap them
+  - org:apache:cassandra:db:.*
+
+  # columnfamily is an alias for Table metrics
+  # https://github.com/apache/cassandra/blob/8b3a60b9a7dbefeecc06bace617279612ec7092d/src/java/org/apache/cassandra/metrics/TableMetrics.java#L162
+  - org:apache:cassandra:metrics:columnfamily:.*
+
+  # Should we export metrics for system keyspaces/tables ?
+  - org:apache:cassandra:metrics:[^:]+:system[^:]*:.*
+
+  # Don't scrap us
+  - com:criteo:nosql:cassandra:exporter:.*
+
+maxScrapeFrequencyInSec:
+  50:
+    - .*
+
+  # Refresh those metrics only every hour as it is costly for cassandra to retrieve them
+  3600:
+    - .*:snapshotssize:.*
+    - .*:estimated.*
+    - .*:totaldiskspaceused:.*
+
+