Skip to content
This repository has been archived by the owner on Feb 20, 2023. It is now read-only.

Commit

Permalink
Convert LogReporter to ScheduledMetrics2Reporter
Browse files Browse the repository at this point in the history
  • Loading branch information
Felix Barnsteiner committed Mar 12, 2016
1 parent b9c038b commit cecc084
Show file tree
Hide file tree
Showing 22 changed files with 477 additions and 415 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,13 @@
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.stagemonitor.core.metrics.MetricsReporterTestHelper.map;
import static org.stagemonitor.core.metrics.MetricsReporterTestHelper.timer;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
Expand All @@ -24,6 +25,7 @@
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
Expand All @@ -41,7 +43,6 @@
import org.stagemonitor.core.CorePlugin;
import org.stagemonitor.core.MeasurementSession;
import org.stagemonitor.core.configuration.Configuration;
import org.stagemonitor.core.metrics.MetricsReporterTestHelper;
import org.stagemonitor.core.metrics.metrics2.Metric2Registry;
import org.stagemonitor.core.util.JsonUtils;

Expand Down Expand Up @@ -227,15 +228,17 @@ private void checkMetrics() {
}

private void checkMetrics(long timer1Mean, long timer2Mean, long timer3Mean) {
final SortedMap<String, Timer> timers = new TreeMap<String, Timer>();
timers.put("test.timer1", timer(TimeUnit.MILLISECONDS.toNanos(timer1Mean)));
timers.put("test.timer2", timer(TimeUnit.MILLISECONDS.toNanos(timer2Mean)));
timers.put("test.timer3", timer(TimeUnit.MILLISECONDS.toNanos(timer3Mean)));
timers.put("test.some.other.timer", timer(TimeUnit.MILLISECONDS.toNanos(999)));
thresholdMonitoringReporter.report(
MetricsReporterTestHelper.map(),
MetricsReporterTestHelper.map(),
MetricsReporterTestHelper.map(),
MetricsReporterTestHelper.map(),
map("test.timer1", timer(TimeUnit.MILLISECONDS.toNanos(timer1Mean)))
.add("test.timer2", timer(TimeUnit.MILLISECONDS.toNanos(timer2Mean)))
.add("test.timer3", timer(TimeUnit.MILLISECONDS.toNanos(timer3Mean)))
.add("test.some.other.timer", timer(TimeUnit.MILLISECONDS.toNanos(999)))
new TreeMap<String, Gauge>(),
new TreeMap<String, Counter>(),
new TreeMap<String, Histogram>(),
new TreeMap<String, Meter>(),
timers
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,22 +4,22 @@

import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.Locale;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;

import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricFilter;
import com.codahale.metrics.Timer;
import org.apache.commons.io.FileUtils;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeBuilder;
import org.slf4j.LoggerFactory;
import org.stagemonitor.core.Stagemonitor;
import org.stagemonitor.core.metrics.SortedTableLogReporter;
import org.stagemonitor.core.metrics.metrics2.MetricName;

public class InstrumentationPerformanceTest {

Expand Down Expand Up @@ -47,32 +47,26 @@ private static void startElasticsearch() {
nodeBuilder.settings()
.put("name", "junit-es-node")
.put("node.http.enabled", "false")
.put("path.logs", "build/elasticsearch/logs")
.put("path.data", "build/elasticsearch/data")
.put("path.home", "build/elasticsearch")
.put("index.store.fs.memory.enabled", "true")
.put("index.gateway.type", "none")
.put("gateway.type", "none")
.put("index.store.type", "memory")
.put("index.number_of_shards", "1")
.put("index.number_of_replicas", "0")
.put("discovery.zen.ping.multicast.enabled", "false");
.put("index.number_of_replicas", "0");

node = nodeBuilder.node();
node.client().admin().cluster().prepareHealth().setWaitForGreenStatus().get();
}

public static void printResults() throws Exception {
SortedTableLogReporter reporter = SortedTableLogReporter
.forRegistry(Stagemonitor.getMetric2Registry().getMetricRegistry())
.forRegistry(Stagemonitor.getMetric2Registry())
.log(LoggerFactory.getLogger(InstrumentationPerformanceTest.class))
.convertRatesTo(TimeUnit.SECONDS)
.convertDurationsTo(TimeUnit.MILLISECONDS)
.filter(MetricFilter.ALL)
.formattedFor(Locale.US)
.build();
reporter.report(new TreeMap<String, Gauge>(), new TreeMap<String, Counter>(),
new TreeMap<String, Histogram>(), new TreeMap<String, Meter>(),
Stagemonitor.getMetric2Registry().getMetricRegistry().getTimers());
reporter.reportMetrics(new HashMap<MetricName, Gauge>(), new HashMap<MetricName, Counter>(),
new HashMap<MetricName, Histogram>(), new HashMap<MetricName, Meter>(),
Stagemonitor.getMetric2Registry().getTimers());

Stagemonitor.reset();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,33 +15,34 @@
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;

import com.codahale.metrics.Clock;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.JmxReporter;
import com.codahale.metrics.MetricFilter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.ScheduledReporter;
import com.codahale.metrics.graphite.Graphite;
import com.codahale.metrics.graphite.GraphiteReporter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.stagemonitor.core.configuration.Configuration;
import org.stagemonitor.core.configuration.ConfigurationOption;
import org.stagemonitor.core.configuration.converter.ListValueConverter;
import org.stagemonitor.core.configuration.converter.SetValueConverter;
import org.stagemonitor.core.elasticsearch.ElasticsearchClient;
import org.stagemonitor.core.elasticsearch.IndexSelector;
import org.stagemonitor.core.grafana.GrafanaClient;
import org.stagemonitor.core.metrics.AndMetricFilter;
import org.stagemonitor.core.metrics.MetricNameFilter;
import org.stagemonitor.core.metrics.MetricsAggregationReporter;
import org.stagemonitor.core.metrics.MetricsWithCountFilter;
import org.stagemonitor.core.metrics.RegexMetricFilter;
import org.stagemonitor.core.metrics.SortedTableLogReporter;
import org.stagemonitor.core.metrics.metrics2.AndMetric2Filter;
import org.stagemonitor.core.metrics.metrics2.ElasticsearchReporter;
import org.stagemonitor.core.metrics.metrics2.InfluxDbReporter;
import org.stagemonitor.core.metrics.metrics2.Metric2Filter;
import org.stagemonitor.core.metrics.metrics2.Metric2Registry;
import org.stagemonitor.core.metrics.metrics2.MetricName;
import org.stagemonitor.core.metrics.metrics2.MetricNameValueConverter;
import org.stagemonitor.core.metrics.metrics2.ScheduledMetrics2Reporter;
import org.stagemonitor.core.util.HttpClient;
import org.stagemonitor.core.util.IOUtils;
import org.stagemonitor.core.util.StringUtils;
Expand Down Expand Up @@ -262,12 +263,13 @@ public class CorePlugin extends StagemonitorPlugin {
.defaultValue(true)
.configurationCategory(CORE_PLUGIN_NAME)
.build();
private final ConfigurationOption<Collection<Pattern>> excludedMetrics = ConfigurationOption.regexListOption()
private final ConfigurationOption<Collection<MetricName>> excludedMetrics = ConfigurationOption
.builder(new SetValueConverter<MetricName>(new MetricNameValueConverter()), Collection.class)
.key("stagemonitor.metrics.excluded.pattern")
.dynamic(false)
.label("Excluded metrics (regex)")
.label("Excluded metric names")
.description("A comma separated list of metric names that should not be collected.")
.defaultValue(Collections.<Pattern>emptyList())
.defaultValue(Collections.<MetricName>emptyList())
.configurationCategory(CORE_PLUGIN_NAME)
.build();
private final ConfigurationOption<Collection<String>> disabledPlugins = ConfigurationOption.stringsOption()
Expand Down Expand Up @@ -484,52 +486,48 @@ public Integer getValue() {
}

void registerReporters(Metric2Registry metric2Registry, Configuration configuration, MeasurementSession measurementSession) {
Collection<Pattern> excludedMetricsPatterns = getExcludedMetricsPatterns();
MetricFilter regexFilter = MetricFilter.ALL;
Metric2Filter regexFilter = Metric2Filter.ALL;
Collection<MetricName> excludedMetricsPatterns = getExcludedMetricsPatterns();
if (!excludedMetricsPatterns.isEmpty()) {
regexFilter = RegexMetricFilter.excludePatterns(excludedMetricsPatterns);
regexFilter = MetricNameFilter.excludePatterns(excludedMetricsPatterns);
}

MetricFilter allFilters = new AndMetricFilter(regexFilter, new MetricsWithCountFilter());
Metric2Filter allFilters = new AndMetric2Filter(regexFilter, new MetricsWithCountFilter());
MetricRegistry metricRegistry = metric2Registry.getMetricRegistry();

reportToGraphite(metricRegistry, getGraphiteReportingInterval(),
measurementSession, allFilters);
reportToGraphite(metricRegistry, getGraphiteReportingInterval(), measurementSession);
reportToInfluxDb(metric2Registry, reportingIntervalInfluxDb.getValue(),
measurementSession);
reportToElasticsearch(metric2Registry, reportingIntervalElasticsearch.getValue(),
measurementSession, configuration.getConfig(CorePlugin.class));

List<ScheduledReporter> onShutdownReporters = new LinkedList<ScheduledReporter>();
reportToConsole(metricRegistry, getConsoleReportingInterval(), allFilters, onShutdownReporters);
registerAggregationReporter(metricRegistry, allFilters, onShutdownReporters, getAggregationReportingInterval());
List<ScheduledMetrics2Reporter> onShutdownReporters = new LinkedList<ScheduledMetrics2Reporter>();
onShutdownReporters.add(reportToConsole(metric2Registry, getConsoleReportingInterval(), allFilters));
registerAggregationReporter(metric2Registry, onShutdownReporters, getAggregationReportingInterval());
if (configuration.getConfig(CorePlugin.class).isReportToJMX()) {
// Because JMX reporter is on registration and not periodic only the
// regex filter is applicable here (not filtering metrics by count)
reportToJMX(metricRegistry, regexFilter);
reportToJMX(metricRegistry);
}
}

private void registerAggregationReporter(MetricRegistry metricRegistry, MetricFilter allFilters,
List<ScheduledReporter> onShutdownReporters, long reportingInterval) {
private void registerAggregationReporter(Metric2Registry metricRegistry,
List<ScheduledMetrics2Reporter> onShutdownReporters, long reportingInterval) {
if (reportingInterval > 0) {
aggregationReporter = new MetricsAggregationReporter(metricRegistry, allFilters, onShutdownReporters);
aggregationReporter = MetricsAggregationReporter.forRegistry(metricRegistry).onShutdownReporters(onShutdownReporters).build();
aggregationReporter.start(reportingInterval, TimeUnit.SECONDS);
aggregationReporter.report();
reporters.add(aggregationReporter);
}
}

private void reportToGraphite(MetricRegistry metricRegistry, long reportingInterval,
MeasurementSession measurementSession,
MetricFilter filter) {
private void reportToGraphite(MetricRegistry metricRegistry, long reportingInterval, MeasurementSession measurementSession) {
String graphiteHostName = getGraphiteHostName();
if (isReportToGraphite()) {
final GraphiteReporter graphiteReporter = GraphiteReporter.forRegistry(metricRegistry)
.prefixedWith(getGraphitePrefix(measurementSession))
.convertRatesTo(TimeUnit.SECONDS)
.convertDurationsTo(TimeUnit.MILLISECONDS)
.filter(filter)
.build(new Graphite(new InetSocketAddress(graphiteHostName, getGraphitePort())));

graphiteReporter.start(reportingInterval, TimeUnit.SECONDS);
Expand Down Expand Up @@ -581,24 +579,21 @@ private String getGraphitePrefix(MeasurementSession measurementSession) {
sanitizeGraphiteMetricSegment(measurementSession.getHostName()));
}

private void reportToConsole(MetricRegistry metricRegistry, long reportingInterval, MetricFilter filter,
List<ScheduledReporter> onShutdownReporters) {
final SortedTableLogReporter reporter = SortedTableLogReporter.forRegistry(metricRegistry)
private SortedTableLogReporter reportToConsole(Metric2Registry metric2Registry, long reportingInterval, Metric2Filter filter) {
final SortedTableLogReporter reporter = SortedTableLogReporter.forRegistry(metric2Registry)
.convertRatesTo(TimeUnit.SECONDS)
.convertDurationsTo(TimeUnit.MILLISECONDS)
.filter(filter)
.build();
onShutdownReporters.add(reporter);
if (reportingInterval > 0) {
reporter.start(reportingInterval, TimeUnit.SECONDS);
reporters.add(reporter);
}
return reporter;
}

private void reportToJMX(MetricRegistry metricRegistry, MetricFilter filter) {
final JmxReporter reporter = JmxReporter.forRegistry(metricRegistry)
.filter(filter)
.build();
private void reportToJMX(MetricRegistry metricRegistry) {
final JmxReporter reporter = JmxReporter.forRegistry(metricRegistry).build();
reporter.start();
reporters.add(reporter);
}
Expand Down Expand Up @@ -741,7 +736,7 @@ public boolean isDeactivateStagemonitorIfEsConfigSourceIsDown() {
return deactivateStagemonitorIfEsConfigSourceIsDown.getValue();
}

public Collection<Pattern> getExcludedMetricsPatterns() {
public Collection<MetricName> getExcludedMetricsPatterns() {
return excludedMetrics.getValue();
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package org.stagemonitor.core.metrics;

import java.util.Arrays;
import java.util.Collection;

import com.codahale.metrics.Metric;
import org.stagemonitor.core.metrics.metrics2.Metric2Filter;
import org.stagemonitor.core.metrics.metrics2.MetricName;

public class MetricNameFilter implements Metric2Filter {

/**
* A {@link Metric2Filter} that excludes any results
* that match any of the provided patterns.
* @param patterns
* @return
*/
public static MetricNameFilter excludePatterns(MetricName... patterns) {
return new MetricNameFilter(Arrays.asList(patterns), false);
}
/**
* A {@link Metric2Filter} that excludes any results
* that match any of the provided patterns.
* @param patterns
* @return
*/
public static MetricNameFilter excludePatterns(Collection<MetricName> patterns) {
return new MetricNameFilter(patterns, false);
}

/**
* A {@link Metric2Filter} that matches (includes) any results
* that match any of the provided patterns.
* @param patterns
* @return
*/
public static MetricNameFilter includePatterns(MetricName... patterns) {
return new MetricNameFilter(Arrays.asList(patterns), true);
}

private final Collection<MetricName> patterns;
private final boolean matchResult;

/**
* Equivalent to {@link #includePatterns(MetricName...)}
* @param patterns
*/
public MetricNameFilter(Collection<MetricName> patterns) {
this(patterns, true);
}

private MetricNameFilter(Collection<MetricName> patterns, boolean matchResult) {
this.patterns = patterns;
this.matchResult = matchResult;
}

@Override
public boolean matches(MetricName name, Metric metric) {
for (final MetricName pattern : patterns) {
if (name.matches(pattern)) {
return matchResult;
}
}
return !matchResult;
}
}
Loading

0 comments on commit cecc084

Please sign in to comment.