Skip to content
Permalink
Browse files
AMBARI-23567 : Add ability to skip in memory cluster aggregation on c… 
…ertain metrics.
  • Loading branch information
Aravindan Vijayan committed Apr 16, 2018
1 parent 530a5a6 commit 3f2fe295bab66f83d36bfeb2ade7f10aba1863dc
Showing 4 changed files with 28 additions and 18 deletions.
15 ...e/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
View file
@@ -47,6 +47,7 @@
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_PRECISION_TABLE_DURABILITY;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_AGGREGATOR_SINK_CLASS;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getJavaRegexFromSqlRegex;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.ALTER_METRICS_METADATA_TABLE;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CONTAINER_METRICS_TABLE_NAME;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CREATE_CONTAINER_METRICS_TABLE_SQL;
@@ -1275,20 +1276,6 @@ private Collection<List<Function>> findMetricFunctions(Multimap<String, List<Fun
return null;
}

public String getJavaRegexFromSqlRegex(String sqlRegex) {
String javaRegEx;
if (sqlRegex.contains("*") || sqlRegex.contains("__%")) {
//Special case handling for metric name with * and __%.
//For example, dfs.NNTopUserOpCounts.windowMs=300000.op=*.user=%.count
// or dfs.NNTopUserOpCounts.windowMs=300000.op=__%.user=%.count
String metricNameWithEscSeq = sqlRegex.replace("*", "\\*").replace("__%", "..%");
javaRegEx = metricNameWithEscSeq.replace("%", ".*");
} else {
javaRegEx = sqlRegex.replace("%", ".*");
}
return javaRegEx;
}

public void saveHostAggregateRecords(Map<TimelineMetric, MetricHostAggregate> hostAggregateMap,
String phoenixTableName) throws SQLException {

11 ...op/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricsIgniteCache.java
View file
@@ -24,8 +24,10 @@
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_COLLECTOR_IGNITE_BACKUPS;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_COLLECTOR_IGNITE_NODES;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_EVENT_METRIC_PATTERNS;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_AGGREGATION_SQL_FILTERS;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_SERVICE_HTTP_POLICY;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getJavaRegexFromSqlRegex;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getRoundedCheckPointTimeMillis;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getTimeSlices;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.sliceFromTimelineMetric;
@@ -36,11 +38,15 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.cache.Cache;
import javax.cache.expiry.CreatedExpiryPolicy;
@@ -114,7 +120,10 @@ public TimelineMetricsIgniteCache() throws MalformedURLException, URISyntaxExcep
String filteredMetricPatterns = metricConf.get(TIMELINE_METRIC_AGGREGATION_SQL_FILTERS);
if (!StringUtils.isEmpty(filteredMetricPatterns)) {
LOG.info("Skipping aggregation for metric patterns : " + filteredMetricPatterns);
skipAggrPatternStrings.addAll(Arrays.asList(filteredMetricPatterns.split(",")));
for (String patternString : filteredMetricPatterns.split(",")) {
String javaPatternString = getJavaRegexFromSqlRegex(patternString);
skipAggrPatternStrings.add(javaPatternString);
}
}

if (metricConf.get(TIMELINE_METRICS_COLLECTOR_IGNITE_NODES) != null) {
14 ...p/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AggregatorUtils.java
View file
@@ -251,4 +251,18 @@ public static long getRoundedAggregateTimeMillis(long aggregatorPeriod) {
long currentTime = System.currentTimeMillis();
return currentTime - (currentTime % aggregatorPeriod);
}

public static String getJavaRegexFromSqlRegex(String sqlRegex) {
String javaRegEx;
if (sqlRegex.contains("*") || sqlRegex.contains("__%")) {
//Special case handling for metric name with * and __%.
//For example, dfs.NNTopUserOpCounts.windowMs=300000.op=*.user=%.count
// or dfs.NNTopUserOpCounts.windowMs=300000.op=__%.user=%.count
String metricNameWithEscSeq = sqlRegex.replace("*", "\\*").replace("__%", "..%");
javaRegEx = metricNameWithEscSeq.replace("%", ".*");
} else {
javaRegEx = sqlRegex.replace("%", ".*");
}
return javaRegEx;
}
}
6 ...ionhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java
View file
@@ -22,6 +22,7 @@
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_EVENT_METRIC_PATTERNS;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_AGGREGATION_SQL_FILTERS;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getJavaRegexFromSqlRegex;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.getTimeSlices;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.AggregatorUtils.sliceFromTimelineMetric;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_METRIC_SQL;
@@ -68,7 +69,6 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre
protected final boolean interpolationEnabled;
private TimelineMetricMetadataManager metadataManagerInstance;
private String skipAggrPatternStrings;
private String skipInterpolationMetricPatternStrings;
private Set<Pattern> skipInterpolationMetricPatterns = new HashSet<>();
private final static String liveHostsMetricName = "live_hosts";

@@ -95,11 +95,11 @@ public TimelineMetricClusterAggregatorSecond(AGGREGATOR_NAME aggregatorName,
this.serverTimeShiftAdjustment = Long.parseLong(metricsConf.get(SERVER_SIDE_TIMESIFT_ADJUSTMENT, "90000"));
this.interpolationEnabled = Boolean.parseBoolean(metricsConf.get(TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED, "true"));
this.skipAggrPatternStrings = metricsConf.get(TIMELINE_METRIC_AGGREGATION_SQL_FILTERS);
this.skipInterpolationMetricPatternStrings = metricsConf.get(TIMELINE_METRICS_EVENT_METRIC_PATTERNS, "");
String skipInterpolationMetricPatternStrings = metricsConf.get(TIMELINE_METRICS_EVENT_METRIC_PATTERNS, "");

if (StringUtils.isNotEmpty(skipInterpolationMetricPatternStrings)) {
for (String patternString : skipInterpolationMetricPatternStrings.split(",")) {
String javaPatternString = hBaseAccessor.getJavaRegexFromSqlRegex(patternString);
String javaPatternString = getJavaRegexFromSqlRegex(patternString);
LOG.info("SQL pattern " + patternString + " converted to Java pattern : " + javaPatternString);
skipInterpolationMetricPatterns.add(Pattern.compile(javaPatternString));
}

0 comments on commit 3f2fe29

Please sign in to comment.