Skip to content

Commit

Permalink
HBASE-16956 Refactor FavoredNodePlan to use regionNames as keys (Thir…
Browse files Browse the repository at this point in the history 
…uvel Thirumoolan)
  • Loading branch information
Devaraj Das committed Nov 17, 2016
1 parent 4c7eac8 commit 5753d18
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 52 deletions.
28 changes: 17 additions & 11 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java
View file
Expand Up @@ -409,7 +409,7 @@ private void genAssignmentPlan(TableName tableName,
favoredServers.add(ServerName.valueOf(s.getHostname(), s.getPort(), favoredServers.add(ServerName.valueOf(s.getHostname(), s.getPort(),
ServerName.NON_STARTCODE)); ServerName.NON_STARTCODE));
// Update the assignment plan // Update the assignment plan
plan.updateAssignmentPlan(regions.get(i), favoredServers); plan.updateFavoredNodesMap(regions.get(i), favoredServers);
} }
LOG.info("Generated the assignment plan for " + numRegions + LOG.info("Generated the assignment plan for " + numRegions +
" regions from table " + tableName + " with " + " regions from table " + tableName + " with " +
Expand Down Expand Up @@ -444,7 +444,7 @@ private void genAssignmentPlan(TableName tableName,
favoredServers.add(ServerName.valueOf(s.getHostname(), s.getPort(), favoredServers.add(ServerName.valueOf(s.getHostname(), s.getPort(),
ServerName.NON_STARTCODE)); ServerName.NON_STARTCODE));
// Update the assignment plan // Update the assignment plan
plan.updateAssignmentPlan(regions.get(i), favoredServers); plan.updateFavoredNodesMap(regions.get(i), favoredServers);
} }
LOG.info("Generated the assignment plan for " + numRegions + LOG.info("Generated the assignment plan for " + numRegions +
" regions from table " + tableName + " with " + " regions from table " + tableName + " with " +
Expand Down Expand Up @@ -614,13 +614,13 @@ public static void printAssignmentPlan(FavoredNodesPlan plan) {
if (plan == null) return; if (plan == null) return;
LOG.info("========== Start to print the assignment plan ================"); LOG.info("========== Start to print the assignment plan ================");
// sort the map based on region info // sort the map based on region info
Map<HRegionInfo, List<ServerName>> assignmentMap = Map<String, List<ServerName>> assignmentMap =
new TreeMap<HRegionInfo, List<ServerName>>(plan.getAssignmentMap()); new TreeMap<String, List<ServerName>>(plan.getAssignmentMap());


for (Map.Entry<HRegionInfo, List<ServerName>> entry : assignmentMap.entrySet()) { for (Map.Entry<String, List<ServerName>> entry : assignmentMap.entrySet()) {


String serverList = FavoredNodeAssignmentHelper.getFavoredNodesAsString(entry.getValue()); String serverList = FavoredNodeAssignmentHelper.getFavoredNodesAsString(entry.getValue());
String regionName = entry.getKey().getRegionNameAsString(); String regionName = entry.getKey();
LOG.info("Region: " + regionName ); LOG.info("Region: " + regionName );
LOG.info("Its favored nodes: " + serverList); LOG.info("Its favored nodes: " + serverList);
} }
Expand All @@ -636,9 +636,15 @@ public void updateAssignmentPlanToMeta(FavoredNodesPlan plan)
throws IOException { throws IOException {
try { try {
LOG.info("Start to update the hbase:meta with the new assignment plan"); LOG.info("Start to update the hbase:meta with the new assignment plan");
Map<HRegionInfo, List<ServerName>> assignmentMap = Map<String, List<ServerName>> assignmentMap = plan.getAssignmentMap();
plan.getAssignmentMap(); Map<HRegionInfo, List<ServerName>> planToUpdate = new HashMap<>(assignmentMap.size());
FavoredNodeAssignmentHelper.updateMetaWithFavoredNodesInfo(assignmentMap, conf); Map<String, HRegionInfo> regionToRegionInfoMap =
getRegionAssignmentSnapshot().getRegionNameToRegionInfoMap();
for (Map.Entry<String, List<ServerName>> entry : assignmentMap.entrySet()) {
planToUpdate.put(regionToRegionInfoMap.get(entry.getKey()), entry.getValue());
}

FavoredNodeAssignmentHelper.updateMetaWithFavoredNodesInfo(planToUpdate, conf);
LOG.info("Updated the hbase:meta with the new assignment plan"); LOG.info("Updated the hbase:meta with the new assignment plan");
} catch (Exception e) { } catch (Exception e) {
LOG.error("Failed to update hbase:meta with the new assignment" + LOG.error("Failed to update hbase:meta with the new assignment" +
Expand Down Expand Up @@ -680,7 +686,7 @@ private void updateAssignmentPlanToRegionServers(FavoredNodesPlan plan)
singleServerPlan = new FavoredNodesPlan(); singleServerPlan = new FavoredNodesPlan();
} }
// Update the single server update // Update the single server update
singleServerPlan.updateAssignmentPlan(region, favoredServerList); singleServerPlan.updateFavoredNodesMap(region, favoredServerList);
regionUpdateInfos.add( regionUpdateInfos.add(
new Pair<HRegionInfo, List<ServerName>>(region, favoredServerList)); new Pair<HRegionInfo, List<ServerName>>(region, favoredServerList));
} }
Expand Down Expand Up @@ -1104,7 +1110,7 @@ public static void main(String args[]) throws IOException {
LOG.error("Cannot parse the invalid favored nodes because " + e); LOG.error("Cannot parse the invalid favored nodes because " + e);
} }
FavoredNodesPlan newPlan = new FavoredNodesPlan(); FavoredNodesPlan newPlan = new FavoredNodesPlan();
newPlan.updateAssignmentPlan(regionInfo, favoredNodes); newPlan.updateFavoredNodesMap(regionInfo, favoredNodes);
rp.updateAssignmentPlan(newPlan); rp.updateAssignmentPlan(newPlan);
} }
} else { } else {
Expand Down
54 changes: 17 additions & 37 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredNodesPlan.java
View file
Expand Up @@ -22,55 +22,51 @@
import java.util.Map; import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;


import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.ServerName;


/** /**
* This class contains the mapping information between each region and * This class contains the mapping information between each region name and
* its favored region server list. Used by {@link FavoredNodeLoadBalancer} set * its favored region server list. Used by {@link FavoredNodeLoadBalancer} set
* of classes and from unit tests (hence the class is public) * of classes and from unit tests (hence the class is public)
* *
* All the access to this class is thread-safe. * All the access to this class is thread-safe.
*/ */
@InterfaceAudience.Private @InterfaceAudience.Private
public class FavoredNodesPlan { public class FavoredNodesPlan {
private static final Log LOG = LogFactory.getLog(
FavoredNodesPlan.class.getName());


/** the map between each region and its favored region server list */ /** the map between each region name and its favored region server list */
private Map<HRegionInfo, List<ServerName>> favoredNodesMap; private Map<String, List<ServerName>> favoredNodesMap;


public static enum Position { public static enum Position {
PRIMARY, PRIMARY,
SECONDARY, SECONDARY,
TERTIARY; TERTIARY
}; }


public FavoredNodesPlan() { public FavoredNodesPlan() {
favoredNodesMap = new ConcurrentHashMap<HRegionInfo, List<ServerName>>(); favoredNodesMap = new ConcurrentHashMap<String, List<ServerName>>();
} }


/** /**
* Add an assignment to the plan * Update an assignment to the plan
* @param region * @param region
* @param servers * @param servers
*/ */
public synchronized void updateFavoredNodesMap(HRegionInfo region, public void updateFavoredNodesMap(HRegionInfo region, List<ServerName> servers) {
List<ServerName> servers) { if (region == null || servers == null || servers.size() == 0) {
if (region == null || servers == null || servers.size() ==0)
return; return;
this.favoredNodesMap.put(region, servers); }
this.favoredNodesMap.put(region.getRegionNameAsString(), servers);
} }


/** /**
* @param region * @param region
* @return the list of favored region server for this region based on the plan * @return the list of favored region server for this region based on the plan
*/ */
public synchronized List<ServerName> getFavoredNodes(HRegionInfo region) { public List<ServerName> getFavoredNodes(HRegionInfo region) {
return favoredNodesMap.get(region); return favoredNodesMap.get(region.getRegionNameAsString());
} }


/** /**
Expand All @@ -97,23 +93,8 @@ public static Position getFavoredServerPosition(
/** /**
* @return the mapping between each region to its favored region server list * @return the mapping between each region to its favored region server list
*/ */
public synchronized Map<HRegionInfo, List<ServerName>> getAssignmentMap() { public Map<String, List<ServerName>> getAssignmentMap() {
return this.favoredNodesMap; return favoredNodesMap;
}

/**
* Add an assignment to the plan
* @param region
* @param servers
*/
public synchronized void updateAssignmentPlan(HRegionInfo region,
List<ServerName> servers) {
if (region == null || servers == null || servers.size() ==0)
return;
this.favoredNodesMap.put(region, servers);
LOG.info("Update the assignment plan for region " +
region.getRegionNameAsString() + " ; favored nodes " +
FavoredNodeAssignmentHelper.getFavoredNodesAsString(servers));
} }


@Override @Override
Expand All @@ -128,15 +109,14 @@ public boolean equals(Object o) {
return false; return false;
} }
// To compare the map from objec o is identical to current assignment map. // To compare the map from objec o is identical to current assignment map.
Map<HRegionInfo, List<ServerName>> comparedMap= Map<String, List<ServerName>> comparedMap = ((FavoredNodesPlan)o).getAssignmentMap();
((FavoredNodesPlan)o).getAssignmentMap();


// compare the size // compare the size
if (comparedMap.size() != this.favoredNodesMap.size()) if (comparedMap.size() != this.favoredNodesMap.size())
return false; return false;


// compare each element in the assignment map // compare each element in the assignment map
for (Map.Entry<HRegionInfo, List<ServerName>> entry : for (Map.Entry<String, List<ServerName>> entry :
comparedMap.entrySet()) { comparedMap.entrySet()) {
List<ServerName> serverList = this.favoredNodesMap.get(entry.getKey()); List<ServerName> serverList = this.favoredNodesMap.get(entry.getKey());
if (serverList == null && entry.getValue() != null) { if (serverList == null && entry.getValue() != null) {
Expand Down
9 changes: 5 additions & 4 deletions hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
View file
Expand Up @@ -299,12 +299,13 @@ public void testRandomizedMatrix() {
* @return * @return
*/ */
private FavoredNodesPlan shuffleAssignmentPlan(FavoredNodesPlan plan, private FavoredNodesPlan shuffleAssignmentPlan(FavoredNodesPlan plan,
FavoredNodesPlan.Position p1, FavoredNodesPlan.Position p2) { FavoredNodesPlan.Position p1, FavoredNodesPlan.Position p2) throws IOException {
FavoredNodesPlan shuffledPlan = new FavoredNodesPlan(); FavoredNodesPlan shuffledPlan = new FavoredNodesPlan();


for (Map.Entry<HRegionInfo, List<ServerName>> entry : Map<String, HRegionInfo> regionToHRegion =
rp.getRegionAssignmentSnapshot().getRegionNameToRegionInfoMap();
for (Map.Entry<String, List<ServerName>> entry :
plan.getAssignmentMap().entrySet()) { plan.getAssignmentMap().entrySet()) {
HRegionInfo region = entry.getKey();


// copy the server list from the original plan // copy the server list from the original plan
List<ServerName> shuffledServerList = new ArrayList<ServerName>(); List<ServerName> shuffledServerList = new ArrayList<ServerName>();
Expand All @@ -315,7 +316,7 @@ private FavoredNodesPlan shuffleAssignmentPlan(FavoredNodesPlan plan,
shuffledServerList.set(p2.ordinal(), entry.getValue().get(p1.ordinal())); shuffledServerList.set(p2.ordinal(), entry.getValue().get(p1.ordinal()));


// update the plan // update the plan
shuffledPlan.updateAssignmentPlan(region, shuffledServerList); shuffledPlan.updateFavoredNodesMap(regionToHRegion.get(entry.getKey()), shuffledServerList);
} }
return shuffledPlan; return shuffledPlan;
} }
Expand Down

0 comments on commit 5753d18

Please sign in to comment.