From e2f0a3b322b1ffc90407808bec8a92f22a44cbec Mon Sep 17 00:00:00 2001 From: zentol Date: Mon, 12 Jun 2017 15:25:35 +0200 Subject: [PATCH 1/2] [FLINK-6900] [metrics] Limit size of metric name components --- docs/monitoring/metrics.md | 4 ++++ .../dropwizard/ScheduledDropwizardReporter.java | 10 +++++++++- .../ScheduledDropwizardReporterTest.java | 17 +++++++++++++++++ .../flink/metrics/statsd/StatsDReporter.java | 13 +++++++++++-- .../metrics/statsd/StatsDReporterTest.java | 14 ++++++++++++++ 5 files changed, 55 insertions(+), 3 deletions(-) diff --git a/docs/monitoring/metrics.md b/docs/monitoring/metrics.md index 8444c8adbdca9..ecfbbed43fb5f 100644 --- a/docs/monitoring/metrics.md +++ b/docs/monitoring/metrics.md @@ -632,6 +632,7 @@ Parameters: - `host` - the Graphite server host - `port` - the Graphite server port - `protocol` - protocol to use (TCP/UDP) +- `maxComponentLength` - limits the length of each scope component Example configuration: @@ -641,6 +642,7 @@ metrics.reporter.grph.class: org.apache.flink.metrics.graphite.GraphiteReporter metrics.reporter.grph.host: localhost metrics.reporter.grph.port: 2003 metrics.reporter.grph.protocol: TCP +metrics.reporter.grph.maxComponentLength: 80 {% endhighlight %} @@ -708,6 +710,7 @@ Parameters: - `host` - the StatsD server host - `port` - the StatsD server port +- `maxComponentLength` - limits the length of each scope component Example configuration: @@ -716,6 +719,7 @@ Example configuration: metrics.reporter.stsd.class: org.apache.flink.metrics.statsd.StatsDReporter metrics.reporter.stsd.host: localhost metrics.reporter.stsd.port: 8125 +metrics.reporter.stsd.maxComponentLength: 80 {% endhighlight %} diff --git a/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/ScheduledDropwizardReporter.java b/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/ScheduledDropwizardReporter.java index e3184dafc0fce..67f500fffef2f 100644 --- a/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/ScheduledDropwizardReporter.java +++ b/flink-metrics/flink-metrics-dropwizard/src/main/java/org/apache/flink/dropwizard/ScheduledDropwizardReporter.java @@ -61,6 +61,7 @@ public abstract class ScheduledDropwizardReporter implements MetricReporter, Sch public static final String ARG_PREFIX = "prefix"; public static final String ARG_CONVERSION_RATE = "rateConversion"; public static final String ARG_CONVERSION_DURATION = "durationConversion"; + public static final String ARG_MAX_COMPONENT_LENGTH = "maxComponentLength"; // ------------------------------------------------------------------------ @@ -73,6 +74,8 @@ public abstract class ScheduledDropwizardReporter implements MetricReporter, Sch private final Map histograms = new HashMap<>(); private final Map meters = new HashMap<>(); + private int maxComponentLength = 80; + // ------------------------------------------------------------------------ protected ScheduledDropwizardReporter() { @@ -109,6 +112,7 @@ Map getHistograms() { @Override public void open(MetricConfig config) { + this.maxComponentLength = config.getInteger(ARG_MAX_COMPONENT_LENGTH, 80); this.reporter = getReporter(config); } @@ -184,7 +188,11 @@ public void notifyOfRemovedMetric(Metric metric, String metricName, MetricGroup @Override public String filterCharacters(String metricName) { char[] chars = null; - final int strLen = metricName.length(); + int strLen = metricName.length(); + if (strLen > maxComponentLength) { + log.warn("The metric name component {} exceeded the {} characters length limit and was truncated.", metricName, maxComponentLength); + strLen = maxComponentLength; + } int pos = 0; for (int i = 0; i < strLen; i++) { diff --git a/flink-metrics/flink-metrics-dropwizard/src/test/java/org/apache/flink/dropwizard/ScheduledDropwizardReporterTest.java b/flink-metrics/flink-metrics-dropwizard/src/test/java/org/apache/flink/dropwizard/ScheduledDropwizardReporterTest.java index b69b8d8e3fa51..3ce536c7a9a45 100644 --- a/flink-metrics/flink-metrics-dropwizard/src/test/java/org/apache/flink/dropwizard/ScheduledDropwizardReporterTest.java +++ b/flink-metrics/flink-metrics-dropwizard/src/test/java/org/apache/flink/dropwizard/ScheduledDropwizardReporterTest.java @@ -56,6 +56,23 @@ */ public class ScheduledDropwizardReporterTest { + @Test + public void testNameTruncating() { + ScheduledDropwizardReporter reporter = new ScheduledDropwizardReporter() { + @Override + public ScheduledReporter getReporter(MetricConfig config) { + return null; + } + }; + + MetricConfig config = new MetricConfig(); + config.setProperty(ScheduledDropwizardReporter.ARG_MAX_COMPONENT_LENGTH, "10"); + + reporter.open(config); + + assertEquals("0123456789", reporter.filterCharacters("0123456789DEADBEEF")); + } + @Test public void testInvalidCharacterReplacement() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { ScheduledDropwizardReporter reporter = new ScheduledDropwizardReporter() { diff --git a/flink-metrics/flink-metrics-statsd/src/main/java/org/apache/flink/metrics/statsd/StatsDReporter.java b/flink-metrics/flink-metrics-statsd/src/main/java/org/apache/flink/metrics/statsd/StatsDReporter.java index 527f9c1076152..da113f42e7913 100644 --- a/flink-metrics/flink-metrics-statsd/src/main/java/org/apache/flink/metrics/statsd/StatsDReporter.java +++ b/flink-metrics/flink-metrics-statsd/src/main/java/org/apache/flink/metrics/statsd/StatsDReporter.java @@ -55,12 +55,15 @@ public class StatsDReporter extends AbstractReporter implements Scheduled { public static final String ARG_HOST = "host"; public static final String ARG_PORT = "port"; + public static final String ARG_MAX_COMPONENT_LENGTH = "maxComponentLength"; private boolean closed = false; private DatagramSocket socket; private InetSocketAddress address; + private int maxComponentLength = 80; + @Override public void open(MetricConfig config) { String host = config.getString(ARG_HOST, null); @@ -77,7 +80,9 @@ public void open(MetricConfig config) { } catch (SocketException e) { throw new RuntimeException("Could not create datagram socket. ", e); } - log.info("Configured StatsDReporter with {host:{}, port:{}}", host, port); + + maxComponentLength = config.getInteger(ARG_MAX_COMPONENT_LENGTH, 80); + log.info("Configured StatsDReporter with {host:{}, port:{}, maxComponentLength:{}}", host, port, maxComponentLength); } @Override @@ -193,7 +198,11 @@ private void send(final String name, final String value) { @Override public String filterCharacters(String input) { char[] chars = null; - final int strLen = input.length(); + int strLen = input.length(); + if (strLen > maxComponentLength) { + log.warn("The metric name component {} exceeded the {} characters length limit and was truncated.", input, maxComponentLength); + strLen = maxComponentLength; + } int pos = 0; for (int i = 0; i < strLen; i++) { diff --git a/flink-metrics/flink-metrics-statsd/src/test/java/org/apache/flink/metrics/statsd/StatsDReporterTest.java b/flink-metrics/flink-metrics-statsd/src/test/java/org/apache/flink/metrics/statsd/StatsDReporterTest.java index c9f5af07a72a1..cf26f55ab5f2f 100644 --- a/flink-metrics/flink-metrics-statsd/src/test/java/org/apache/flink/metrics/statsd/StatsDReporterTest.java +++ b/flink-metrics/flink-metrics-statsd/src/test/java/org/apache/flink/metrics/statsd/StatsDReporterTest.java @@ -60,6 +60,20 @@ */ public class StatsDReporterTest extends TestLogger { + @Test + public void testNameTruncating() { + StatsDReporter reporter = new StatsDReporter(); + + MetricConfig config = new MetricConfig(); + config.setProperty(StatsDReporter.ARG_HOST, "localhost"); + config.setProperty(StatsDReporter.ARG_PORT, "12345"); + config.setProperty(StatsDReporter.ARG_MAX_COMPONENT_LENGTH, "10"); + + reporter.open(config); + + assertEquals("0123456789", reporter.filterCharacters("0123456789DEADBEEF")); + } + @Test public void testReplaceInvalidChars() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { StatsDReporter reporter = new StatsDReporter(); From ae13a5e4179cc416273ee091633e0eeaf52a848a Mon Sep 17 00:00:00 2001 From: zentol Date: Mon, 10 Jul 2017 14:59:42 +0200 Subject: [PATCH 2/2] [hotfix] [metrics] Remove invalid exception declarations --- .../ScheduledDropwizardReporterTest.java | 19 +++++++++++-------- .../metrics/statsd/StatsDReporterTest.java | 19 +++++++++++-------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/flink-metrics/flink-metrics-dropwizard/src/test/java/org/apache/flink/dropwizard/ScheduledDropwizardReporterTest.java b/flink-metrics/flink-metrics-dropwizard/src/test/java/org/apache/flink/dropwizard/ScheduledDropwizardReporterTest.java index 3ce536c7a9a45..2a2d41b944b59 100644 --- a/flink-metrics/flink-metrics-dropwizard/src/test/java/org/apache/flink/dropwizard/ScheduledDropwizardReporterTest.java +++ b/flink-metrics/flink-metrics-dropwizard/src/test/java/org/apache/flink/dropwizard/ScheduledDropwizardReporterTest.java @@ -44,7 +44,6 @@ import com.codahale.metrics.ScheduledReporter; import org.junit.Test; -import java.lang.reflect.InvocationTargetException; import java.util.List; import java.util.Map; @@ -58,23 +57,27 @@ public class ScheduledDropwizardReporterTest { @Test public void testNameTruncating() { - ScheduledDropwizardReporter reporter = new ScheduledDropwizardReporter() { + final MetricConfig config = new MetricConfig(); + config.setProperty(ScheduledDropwizardReporter.ARG_MAX_COMPONENT_LENGTH, "10"); + + final ScheduledDropwizardReporter reporter = new ScheduledDropwizardReporter() { @Override public ScheduledReporter getReporter(MetricConfig config) { return null; } }; - MetricConfig config = new MetricConfig(); - config.setProperty(ScheduledDropwizardReporter.ARG_MAX_COMPONENT_LENGTH, "10"); - - reporter.open(config); + try { + reporter.open(config); - assertEquals("0123456789", reporter.filterCharacters("0123456789DEADBEEF")); + assertEquals("0123456789", reporter.filterCharacters("0123456789DEADBEEF")); + } finally { + reporter.close(); + } } @Test - public void testInvalidCharacterReplacement() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + public void testInvalidCharacterReplacement() { ScheduledDropwizardReporter reporter = new ScheduledDropwizardReporter() { @Override public ScheduledReporter getReporter(MetricConfig config) { diff --git a/flink-metrics/flink-metrics-statsd/src/test/java/org/apache/flink/metrics/statsd/StatsDReporterTest.java b/flink-metrics/flink-metrics-statsd/src/test/java/org/apache/flink/metrics/statsd/StatsDReporterTest.java index cf26f55ab5f2f..b0dcc6084269c 100644 --- a/flink-metrics/flink-metrics-statsd/src/test/java/org/apache/flink/metrics/statsd/StatsDReporterTest.java +++ b/flink-metrics/flink-metrics-statsd/src/test/java/org/apache/flink/metrics/statsd/StatsDReporterTest.java @@ -62,20 +62,23 @@ public class StatsDReporterTest extends TestLogger { @Test public void testNameTruncating() { - StatsDReporter reporter = new StatsDReporter(); - - MetricConfig config = new MetricConfig(); + final MetricConfig config = new MetricConfig(); config.setProperty(StatsDReporter.ARG_HOST, "localhost"); config.setProperty(StatsDReporter.ARG_PORT, "12345"); config.setProperty(StatsDReporter.ARG_MAX_COMPONENT_LENGTH, "10"); - - reporter.open(config); - - assertEquals("0123456789", reporter.filterCharacters("0123456789DEADBEEF")); + + final StatsDReporter reporter = new StatsDReporter(); + try { + reporter.open(config); + + assertEquals("0123456789", reporter.filterCharacters("0123456789DEADBEEF")); + } finally { + reporter.close(); + } } @Test - public void testReplaceInvalidChars() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + public void testReplaceInvalidChars() { StatsDReporter reporter = new StatsDReporter(); assertEquals("", reporter.filterCharacters(""));