This repository has been archived by the owner. It is now read-only.
Permalink
Show file tree
Hide file tree
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
AMBARI-23804 : Refine AMS HBase region splitting calculation based on…
… UUID work.
- Loading branch information
Aravindan Vijayan
committed
May 10, 2018
1 parent
056f699
commit a878b0488139b20609c43d6f575ff8546da6ae14
Showing
21 changed files
with
815 additions
and
316 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| @@ -0,0 +1,239 @@ | ||
| /** | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.ambari.metrics.core.timeline; | ||
|
|
||
| import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric; | ||
| import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager; | ||
| import org.apache.commons.lang.StringUtils; | ||
| import org.apache.commons.logging.Log; | ||
| import org.apache.commons.logging.LogFactory; | ||
| import org.apache.hadoop.conf.Configuration; | ||
|
|
||
| import java.io.BufferedReader; | ||
| import java.io.InputStream; | ||
| import java.io.InputStreamReader; | ||
| import java.util.ArrayList; | ||
| import java.util.Arrays; | ||
| import java.util.HashSet; | ||
| import java.util.List; | ||
| import java.util.Set; | ||
|
|
||
| import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_INITIAL_CONFIGURED_MASTER_COMPONENTS; | ||
| import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_INITIAL_CONFIGURED_SLAVE_COMPONENTS; | ||
|
|
||
| public class TimelineMetricSplitPointComputer { | ||
|
|
||
| private static final Log LOG = LogFactory.getLog(TimelineMetricSplitPointComputer.class); | ||
| private Set<String> masterComponents = new HashSet<>(); | ||
| private Set<String> slaveComponents = new HashSet<>(); | ||
|
|
||
| private static final int MINIMUM_PRECISION_TABLE_REGIONS = 4; | ||
| private static final int MINIMUM_AGGREGATE_TABLE_REGIONS = 2; | ||
| private static final int OTHER_TABLE_STATIC_REGIONS = 8; | ||
| private static final int SLAVE_EQUIDISTANT_POINTS = 50; | ||
| private static final int MASTER_EQUIDISTANT_POINTS = 5; | ||
|
|
||
| private List<byte[]> precisionSplitPoints = new ArrayList<>(); | ||
| private List<byte[]> aggregateSplitPoints = new ArrayList<>(); | ||
|
|
||
| public TimelineMetricSplitPointComputer(Configuration metricsConf, | ||
| Configuration hbaseConf, | ||
| TimelineMetricMetadataManager timelineMetricMetadataManager) { | ||
|
|
||
| String componentsString = metricsConf.get(TIMELINE_METRIC_INITIAL_CONFIGURED_MASTER_COMPONENTS, ""); | ||
| if (StringUtils.isNotEmpty(componentsString)) { | ||
| masterComponents.addAll(Arrays.asList(componentsString.split(","))); | ||
| } | ||
|
|
||
| componentsString = metricsConf.get(TIMELINE_METRIC_INITIAL_CONFIGURED_SLAVE_COMPONENTS, ""); | ||
| if (StringUtils.isNotEmpty(componentsString)) { | ||
| slaveComponents.addAll(Arrays.asList(componentsString.split(","))); | ||
| } | ||
|
|
||
| double hbaseTotalHeapsize = metricsConf.getDouble("hbase_total_heapsize", 1024*1024*1024); | ||
| double hbaseMemstoreUpperLimit = hbaseConf.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.5); | ||
| double hbaseMemstoreFlushSize = hbaseConf.getDouble("hbase.hregion.memstore.flush.size", 134217728); | ||
|
|
||
| computeSplitPoints(hbaseTotalHeapsize, hbaseMemstoreUpperLimit, hbaseMemstoreFlushSize, timelineMetricMetadataManager); | ||
| } | ||
|
|
||
|
|
||
| private void computeSplitPoints(double hbaseTotalHeapsize, | ||
| double hbaseMemstoreUpperLimit, | ||
| double hbaseMemstoreFlushSize, | ||
| TimelineMetricMetadataManager timelineMetricMetadataManager) { | ||
|
|
||
| double memstoreMaxMemory = hbaseMemstoreUpperLimit * hbaseTotalHeapsize; | ||
| int maxInMemoryRegions = (int) ((memstoreMaxMemory / hbaseMemstoreFlushSize) - OTHER_TABLE_STATIC_REGIONS); | ||
|
|
||
| int targetPrecisionTableRegionCount = MINIMUM_PRECISION_TABLE_REGIONS; | ||
| int targetAggregateTableRegionCount = MINIMUM_AGGREGATE_TABLE_REGIONS; | ||
|
|
||
| if (maxInMemoryRegions > 2) { | ||
| targetPrecisionTableRegionCount = Math.max(4, (int)(0.70 * maxInMemoryRegions)); | ||
| targetAggregateTableRegionCount = Math.max(2, (int)(0.15 * maxInMemoryRegions)); | ||
| } | ||
|
|
||
| List<MetricApp> metricList = new ArrayList<>(); | ||
|
|
||
| for (String component : masterComponents) { | ||
| metricList.addAll(getSortedMetricListForSplitPoint(component, false)); | ||
| } | ||
|
|
||
| for (String component : slaveComponents) { | ||
| metricList.addAll(getSortedMetricListForSplitPoint(component, true)); | ||
| } | ||
|
|
||
| int totalMetricLength = metricList.size(); | ||
|
|
||
| if (targetPrecisionTableRegionCount > 1) { | ||
| int idx = (int) Math.ceil(totalMetricLength / targetPrecisionTableRegionCount); | ||
| int index = idx; | ||
| for (int i = 0; i < targetPrecisionTableRegionCount; i++) { | ||
| if (index < totalMetricLength - 1) { | ||
| MetricApp metricAppService = metricList.get(index); | ||
| byte[] uuid = timelineMetricMetadataManager.getUuid( | ||
| new TimelineClusterMetric(metricAppService.metricName, metricAppService.appId, null, -1), | ||
| true); | ||
| precisionSplitPoints.add(uuid); | ||
| index += idx; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| if (targetAggregateTableRegionCount > 1) { | ||
| int idx = (int) Math.ceil(totalMetricLength / targetAggregateTableRegionCount); | ||
| int index = idx; | ||
| for (int i = 0; i < targetAggregateTableRegionCount; i++) { | ||
| if (index < totalMetricLength - 1) { | ||
| MetricApp metricAppService = metricList.get(index); | ||
| byte[] uuid = timelineMetricMetadataManager.getUuid( | ||
| new TimelineClusterMetric(metricAppService.metricName, metricAppService.appId, null, -1), | ||
| true); | ||
| aggregateSplitPoints.add(uuid); | ||
| index += idx; | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private List<MetricApp> getSortedMetricListForSplitPoint(String component, boolean isSlave) { | ||
|
|
||
| String appId = getAppId(component); | ||
| List<MetricApp> metricList = new ArrayList<>(); | ||
|
|
||
| ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); | ||
| if (classLoader == null) { | ||
| classLoader = getClass().getClassLoader(); | ||
| } | ||
|
|
||
| String strLine; | ||
| BufferedReader bufferedReader; | ||
|
|
||
| try (InputStream inputStream = classLoader.getResourceAsStream("metrics_def/" + appId.toUpperCase() + ".dat")) { | ||
|
|
||
| if (inputStream != null) { | ||
| bufferedReader = new BufferedReader(new InputStreamReader(inputStream)); | ||
| LOG.info("Found split point candidate metrics for : " + appId); | ||
|
|
||
| while ((strLine = bufferedReader.readLine()) != null) { | ||
| metricList.add(new MetricApp(strLine.trim(), appId)); | ||
| } | ||
| } else { | ||
| LOG.info("Split point candidate metrics not found for : " + appId); | ||
| } | ||
| } catch (Exception e) { | ||
| LOG.info("Error reading split point candidate metrics for component : " + component); | ||
| LOG.error(e); | ||
| } | ||
|
|
||
| if (isSlave) { | ||
| return getEquidistantMetrics(metricList, SLAVE_EQUIDISTANT_POINTS); | ||
| } else { | ||
| return getEquidistantMetrics(metricList, MASTER_EQUIDISTANT_POINTS); | ||
| } | ||
| } | ||
|
|
||
| private List<MetricApp> getEquidistantMetrics(List<MetricApp> metrics, int distance) { | ||
| List<MetricApp> selectedMetricApps = new ArrayList<>(); | ||
|
|
||
| int idx = metrics.size() / distance; | ||
| if (idx == 0) { | ||
| return metrics; | ||
| } | ||
|
|
||
| int index = idx; | ||
| for (int i = 0; i < distance; i++) { | ||
| selectedMetricApps.add(metrics.get(index - 1)); | ||
| index += idx; | ||
| } | ||
| return selectedMetricApps; | ||
| } | ||
|
|
||
|
|
||
| public List<byte[]> getPrecisionSplitPoints() { | ||
| return precisionSplitPoints; | ||
| } | ||
|
|
||
| public List<byte[]> getClusterAggregateSplitPoints() { | ||
| return aggregateSplitPoints; | ||
| } | ||
|
|
||
| public List<byte[]> getHostAggregateSplitPoints() { | ||
| return aggregateSplitPoints; | ||
| } | ||
|
|
||
| private String getAppId(String component) { | ||
|
|
||
| if (component.equalsIgnoreCase("METRICS_COLLECTOR")) { | ||
| return "ams-hbase"; | ||
| } | ||
|
|
||
| if (component.equalsIgnoreCase("METRICS_MONITOR")) { | ||
| return "HOST"; | ||
| } | ||
| return component; | ||
| } | ||
| } | ||
|
|
||
| class MetricApp implements Comparable{ | ||
| String metricName; | ||
| String appId; | ||
|
|
||
| MetricApp(String metricName, String appId) { | ||
| this.metricName = metricName; | ||
| if (appId.startsWith("hbase")) { | ||
| this.appId = "hbase"; | ||
| } else { | ||
| this.appId = appId; | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public int compareTo(Object o) { | ||
| MetricApp that = (MetricApp)o; | ||
|
|
||
| int metricCompare = metricName.compareTo(that.metricName); | ||
| if (metricCompare != 0) { | ||
| return metricCompare; | ||
| } | ||
|
|
||
| return appId.compareTo(that.appId); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.