Permalink
Browse files

Report free resources instead of used.

Summary:
Modify ClusterNodeInfo to specify free resources instead of used.

This helps simplify the reporting of free disk space when there are
multiple disks. We just report the largest free space on any one disk.

Test Plan:
unit tests.

Revert Plan:

Reviewers: dms, aching, pyang

Reviewed By: dms
  • Loading branch information...
1 parent 24bbb49 commit 903c399a05078c572a15e5ecb3bd84caa924de4f rvadali committed with Alex Feinberg Jun 23, 2012
@@ -41,7 +41,7 @@ struct ClusterNodeInfo {
1: required string name,
2: required InetAddress address,
3: required ComputeSpecs total,
- 4: optional ComputeSpecs used,
+ 4: optional ComputeSpecs free,
5: optional map<ResourceType, string> resourceInfos
}
Oops, something went wrong.
@@ -37,7 +37,7 @@
public boolean deleted = false;
public final Node hostNode;
private ClusterNodeInfo clusterNodeInfo;
- private volatile ComputeSpecs usedSpecs;
+ private volatile ComputeSpecs freeSpecs;
private Map<ResourceType, Integer> resourceTypeToMaxCpu =
new EnumMap<ResourceType, Integer>(ResourceType.class);
private Map<ResourceType, Stats> resourceTypeToStatsMap =
@@ -169,7 +169,7 @@ public ClusterNode(
Map<Integer, Map<ResourceType, Integer>> cpuToResourcePartitioning) {
clusterNodeInfo.address.host = clusterNodeInfo.address.host.intern();
this.clusterNodeInfo = clusterNodeInfo;
- this.usedSpecs = clusterNodeInfo.getUsed();
+ this.freeSpecs = clusterNodeInfo.getFree();
lastHeartbeatTime = ClusterManager.clock.getTime();
this.hostNode = node;
initResourceTypeToCpu(cpuToResourcePartitioning);
@@ -214,14 +214,14 @@ public boolean checkForGrant(
int cpuAlloced = resourceTypeToStatsMap.get(req.type).allocatedCpu;
Integer cpuMax = resourceTypeToMaxCpu.get(req.type);
- boolean enoughCpu = cpuMax.intValue() >= req.specs.numCpus + cpuAlloced;
+ boolean enoughCpu = cpuMax.intValue() >= req.getSpecs().numCpus + cpuAlloced;
boolean enoughMem = resourceLimit.hasEnoughResource(this);
return enoughCpu && enoughMem;
}
public void heartbeat(ClusterNodeInfo newClusterNodeInfo) {
lastHeartbeatTime = ClusterManager.clock.getTime();
- usedSpecs = newClusterNodeInfo.getUsed();
+ freeSpecs = newClusterNodeInfo.getFree();
}
public String getName() {
@@ -236,8 +236,8 @@ public InetAddress getAddress() {
return clusterNodeInfo.address;
}
- public ComputeSpecs getUsed() {
- return usedSpecs;
+ public ComputeSpecs getFree() {
+ return freeSpecs;
}
public ComputeSpecs getTotal() {
@@ -76,14 +76,12 @@ public boolean hasEnoughResource(ClusterNode node) {
* @return true if the node has enough memory, false otherwise
*/
public boolean hasEnoughMemory(ClusterNode node) {
- int used = node.getUsed().memoryMB;
int total = node.getTotal().memoryMB;
- int free = total - used;
+ int free = node.getFree().memoryMB;
if (free < nodeReservedMemoryMB) {
if (LOG.isDebugEnabled()) {
LOG.debug(node.getHost() + " not enough memory." +
" totalMB:" + total +
- " used:" + used +
" free:" + free +
" limit:" + nodeReservedMemoryMB);
}
@@ -98,14 +96,12 @@ public boolean hasEnoughMemory(ClusterNode node) {
* @return true if the node has enough space, false otherwise
*/
private boolean hasEnoughDiskSpace(ClusterNode node) {
- int used = node.getUsed().diskGB;
int total = node.getTotal().diskGB;
- int free = total - used;
+ int free = node.getFree().diskGB;
if (free < nodeReservedDiskGB) {
if (LOG.isDebugEnabled()) {
LOG.debug(node.getHost() + " not enough disk space." +
" totalMB:" + total +
- " used:" + used +
" free:" + free +
" limit:" + nodeReservedDiskGB);
}
@@ -253,17 +253,18 @@ private void heartbeatToClusterManager() throws IOException {
if (cpuUsage == ResourceCalculatorPlugin.UNAVAILABLE) {
cpuUsage = 0;
}
- ComputeSpecs used = new ComputeSpecs((short)(numCpu * cpuUsage / 100D));
- used.setNetworkMBps((short)10);
+ ComputeSpecs free = new ComputeSpecs((short)(numCpu * cpuUsage / 100D));
+ // TODO find free network.
+ free.setNetworkMBps((short)100);
int availableMemoryMB =
(int)(resourceCalculatorPlugin.
getAvailablePhysicalMemorySize() / 1024D / 1024);
- used.setMemoryMB(totalMemoryMB - availableMemoryMB);
- used.setDiskGB((int)(getDiskSpace(true) / 1024D / 1024 / 1024));
+ free.setMemoryMB(availableMemoryMB);
+ free.setDiskGB((int)(getDiskSpace(true) / 1024D / 1024 / 1024));
// TT puts it's MR specific host:port tuple here
ClusterNodeInfo node = new ClusterNodeInfo
(this.getName(), clusterManagerCallbackServerAddr, total);
- node.setUsed(used);
+ node.setFree(free);
node.setResourceInfos(resourceInfos);
LOG.debug("ClusterManager heartbeat: " + node.toString());
@@ -61,7 +61,7 @@ public static void main(String[] args) throws Exception {
new InetAddress(TstUtils.getNodeHost(i),
TstUtils.getNodePort(i)),
TstUtils.std_spec);
- nodes[i].setUsed(TstUtils.free_spec);
+ nodes[i].setFree(TstUtils.std_spec);
nodes[i].setResourceInfos(resourceInfos);
}
@@ -62,7 +62,7 @@ protected void setUp() throws IOException {
new InetAddress(TstUtils.getNodeHost(i),
TstUtils.getNodePort(i)),
TstUtils.std_spec);
- nodes[i].setUsed(TstUtils.free_spec);
+ nodes[i].setFree(TstUtils.std_spec);
nodes[i].setResourceInfos(resourceInfos);
}
@@ -57,7 +57,7 @@ protected void setUp() throws IOException {
new InetAddress(TstUtils.getNodeHost(i),
TstUtils.getNodePort(i)),
TstUtils.std_spec);
- nodes[i].setUsed(TstUtils.free_spec);
+ nodes[i].setFree(TstUtils.std_spec);
nodes[i].setResourceInfos(resourceInfos);
}
@@ -60,7 +60,7 @@ protected void setUp() throws IOException {
new InetAddress(TstUtils.getNodeHost(i),
TstUtils.getNodePort(i)),
TstUtils.std_spec);
- nodes[i].setUsed(TstUtils.free_spec);
+ nodes[i].setFree(TstUtils.std_spec);
nodes[i].setResourceInfos(resourceInfos);
}
@@ -52,7 +52,7 @@ protected void setUp(int reservedMemoryMB, int reservedDiskGB)
new InetAddress(TstUtils.getNodeHost(i),
TstUtils.getNodePort(i)),
TstUtils.std_spec);
- nodes[i].setUsed(TstUtils.free_spec);
+ nodes[i].setFree(TstUtils.std_spec);
nodes[i].setResourceInfos(resourceInfos);
}
sessionInfo = new SessionInfo(
@@ -135,7 +135,7 @@ public void testResourceUpdate() throws Exception {
addSomeNodes(1);
ClusterNodeInfo newInfo = new ClusterNodeInfo(nodes[0]);
// Fully used.
- newInfo.setUsed(TstUtils.std_spec);
+ newInfo.setFree(TstUtils.nothing_free_spec);
cm.nodeHeartbeat(newInfo);
String handle = TstUtils.startSession(cm, sessionInfo);
@@ -152,7 +152,7 @@ public void testResourceUpdate() throws Exception {
// Node is free
newInfo = new ClusterNodeInfo(newInfo);
- newInfo.setUsed(TstUtils.free_spec);
+ newInfo.setFree(TstUtils.std_spec);
cm.nodeHeartbeat(newInfo);
TstUtils.reliableSleep(500);
Oops, something went wrong.

0 comments on commit 903c399

Please sign in to comment.