New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Better task balancing #1482

Merged
merged 73 commits into from Jun 8, 2017
Commits
Jump to file or symbol
Failed to load files and symbols.
+39 鈭43
Diff settings

Always

Just for now

Viewing a subset of changes. View all

switch to enum for resource usages

  • Loading branch information...
darcatron committed Apr 20, 2017
commit d7867e95e9ec89d38450deed0f3e6f408f041398
@@ -8,8 +8,10 @@
public class SingularitySlaveUsage {
public static final String CPU_USED = "cpusUsed";
public static final String MEMORY_BYTES_USED = "memoryRssBytes";
public enum ResourceUsageType {
CPU_USED, MEMORY_BYTES_USED
}
public static final long BYTES_PER_MEGABYTE = 1024L * 1024L;

This comment has been minimized.

@ssalinas

ssalinas Apr 20, 2017

Member

was about to comment that there must be some type of easy class/enum for this like there is with TimeUnit, but apparently there isn't... weird...

@ssalinas

ssalinas Apr 20, 2017

Member

was about to comment that there must be some type of easy class/enum for this like there is with TimeUnit, but apparently there isn't... weird...

This comment has been minimized.

@darcatron

darcatron Apr 20, 2017

Contributor

Yeah, I was sad to see there wasn't a lib method for this too 馃槩

@darcatron

darcatron Apr 20, 2017

Contributor

Yeah, I was sad to see there wasn't a lib method for this too 馃槩

private final long memoryBytesUsed;
@@ -18,7 +20,7 @@
private final double cpusUsed;
private final Optional<Long> memoryMbTotal;
private final Optional<Double> cpuTotal;
private final Map<RequestType, Map<String, Number>> usagePerRequestType;
private final Map<RequestType, Map<ResourceUsageType, Number>> usagePerRequestType;
@JsonCreator
public SingularitySlaveUsage(@JsonProperty("memoryBytesUsed") long memoryBytesUsed,
@@ -27,7 +29,7 @@ public SingularitySlaveUsage(@JsonProperty("memoryBytesUsed") long memoryBytesUs
@JsonProperty("numTasks") int numTasks,
@JsonProperty("memoryMbTotal") Optional<Long> memoryMbTotal,
@JsonProperty("cpuTotal") Optional<Double> cpuTotal,
@JsonProperty("usagePerRequestType") Map<RequestType, Map<String, Number>> usagePerRequestType) {
@JsonProperty("usagePerRequestType") Map<RequestType, Map<ResourceUsageType, Number>> usagePerRequestType) {
this.memoryBytesUsed = memoryBytesUsed;
this.timestamp = timestamp;
this.cpusUsed = cpusUsed;
@@ -65,18 +67,10 @@ public int getNumTasks() {
return cpuTotal;
}
public Map<RequestType, Map<String, Number>> getUsagePerRequestType() {
public Map<RequestType, Map<ResourceUsageType, Number>> getUsagePerRequestType() {
return usagePerRequestType;
}
public double getCpusUsedForRequestType(RequestType type) {
return usagePerRequestType.get(type).get(CPU_USED).doubleValue();
}
public long getMemBytesUsedForRequestType(RequestType type) {
return usagePerRequestType.get(type).get(MEMORY_BYTES_USED).longValue();
}
@Override
public String toString() {
return "SingularitySlaveUsage [memoryBytesUsed=" + memoryBytesUsed + ", numTasks=" + numTasks + ", timestamp=" + timestamp + ", cpusUsed=" + cpusUsed + "]";
@@ -23,7 +23,7 @@
import com.hubspot.mesos.Resources;
import com.hubspot.singularity.RequestType;
import com.hubspot.singularity.SingularityPendingTaskId;
import com.hubspot.singularity.SingularitySlaveUsage;
import com.hubspot.singularity.SingularitySlaveUsage.ResourceUsageType;
import com.hubspot.singularity.SingularitySlaveUsageWithId;
import com.hubspot.singularity.SingularityTask;
import com.hubspot.singularity.SingularityTaskRequest;
@@ -266,11 +266,11 @@ private double score(SingularityOfferHolder offerHolder, SingularitySchedulerSta
}
SingularitySlaveUsageWithId slaveUsage = maybeSlaveUsage.get();
Map<RequestType, Map<String, Number>> usagesPerRequestType = slaveUsage.getUsagePerRequestType();
Map<String, Number> usagePerResource = usagesPerRequestType.get(taskRequest.getRequest().getRequestType());
Map<RequestType, Map<ResourceUsageType, Number>> usagesPerRequestType = slaveUsage.getUsagePerRequestType();
Map<ResourceUsageType, Number> usagePerResource = usagesPerRequestType.get(taskRequest.getRequest().getRequestType());
score += configuration.getRequestTypeCpuWeightForOffer() * (1 - usagePerResource.get(SingularitySlaveUsage.CPU_USED).doubleValue() / slaveUsage.getCpuTotal().get());
score += configuration.getRequestTypeMemWeightForOffer() * (1 - ((double) usagePerResource.get(SingularitySlaveUsage.MEMORY_BYTES_USED).longValue() / slaveUsage.getMemoryBytesTotal().get()));
score += configuration.getRequestTypeCpuWeightForOffer() * (1 - usagePerResource.get(ResourceUsageType.CPU_USED).doubleValue() / slaveUsage.getCpuTotal().get());
score += configuration.getRequestTypeMemWeightForOffer() * (1 - ((double) usagePerResource.get(ResourceUsageType.MEMORY_BYTES_USED).longValue() / slaveUsage.getMemoryBytesTotal().get()));
score += configuration.getFreeCpuWeightForOffer() * (MesosUtils.getNumCpus(offer) / slaveUsage.getCpuTotal().get());
score += configuration.getFreeMemWeightForOffer() * (MesosUtils.getMemory(offer) / slaveUsage.getMemoryMbTotal().get());
@@ -16,6 +16,7 @@
import com.hubspot.singularity.RequestType;
import com.hubspot.singularity.SingularitySlave;
import com.hubspot.singularity.SingularitySlaveUsage;
import com.hubspot.singularity.SingularitySlaveUsage.ResourceUsageType;
import com.hubspot.singularity.SingularityTaskCurrentUsage;
import com.hubspot.singularity.SingularityTaskId;
import com.hubspot.singularity.SingularityTaskUsage;
@@ -55,7 +56,7 @@
@Override
public void runActionOnPoll() {
final long now = System.currentTimeMillis();
Map<RequestType, Map<String, Number>> usagesPerRequestType = new HashMap<>();
Map<RequestType, Map<ResourceUsageType, Number>> usagesPerRequestType = new HashMap<>();
for (SingularitySlave slave : usageHelper.getSlavesToTrackUsageFor()) {
Optional<Long> memoryMbTotal = Optional.empty();
@@ -129,23 +130,23 @@ private RequestType getRequestType(MesosTaskMonitorObject task) {
return requestManager.getRequest(SingularityTaskId.valueOf(task.getSource()).getRequestId()).get().getRequest().getRequestType();
}
private void updateUsagesPerRequestType(Map<RequestType, Map<String, Number>> usagePerRequestType, RequestType type, long memBytesUsed, double cpuUsed) {
private void updateUsagesPerRequestType(Map<RequestType, Map<ResourceUsageType, Number>> usagePerRequestType, RequestType type, long memBytesUsed, double cpuUsed) {
if (usagePerRequestType.containsKey(type)) {
long oldMemUsed = 0L;
double oldCpuUsed = 0;
if (usagePerRequestType.get(type).containsKey(SingularitySlaveUsage.MEMORY_BYTES_USED)) {
oldMemUsed = usagePerRequestType.get(type).get(SingularitySlaveUsage.MEMORY_BYTES_USED).longValue();
if (usagePerRequestType.get(type).containsKey(ResourceUsageType.MEMORY_BYTES_USED)) {
oldMemUsed = usagePerRequestType.get(type).get(ResourceUsageType.MEMORY_BYTES_USED).longValue();
}
if (usagePerRequestType.get(type).containsKey(SingularitySlaveUsage.CPU_USED)) {
oldCpuUsed = usagePerRequestType.get(type).get(SingularitySlaveUsage.CPU_USED).doubleValue();
if (usagePerRequestType.get(type).containsKey(ResourceUsageType.CPU_USED)) {
oldCpuUsed = usagePerRequestType.get(type).get(ResourceUsageType.CPU_USED).doubleValue();
}
usagePerRequestType.get(type).put(SingularitySlaveUsage.MEMORY_BYTES_USED, oldMemUsed + memBytesUsed);
usagePerRequestType.get(type).put(SingularitySlaveUsage.CPU_USED, oldCpuUsed + cpuUsed);
usagePerRequestType.get(type).put(ResourceUsageType.MEMORY_BYTES_USED, oldMemUsed + memBytesUsed);
usagePerRequestType.get(type).put(ResourceUsageType.CPU_USED, oldCpuUsed + cpuUsed);
} else {
usagePerRequestType.put(type, ImmutableMap.of(SingularitySlaveUsage.MEMORY_BYTES_USED, memBytesUsed));
usagePerRequestType.put(type, ImmutableMap.of(SingularitySlaveUsage.CPU_USED, cpuUsed));
usagePerRequestType.put(type, ImmutableMap.of(ResourceUsageType.MEMORY_BYTES_USED, memBytesUsed));
usagePerRequestType.put(type, ImmutableMap.of(ResourceUsageType.CPU_USED, cpuUsed));
}
}
}
@@ -27,6 +27,7 @@
import com.hubspot.singularity.SingularityPendingTaskId;
import com.hubspot.singularity.SingularityRequest;
import com.hubspot.singularity.SingularitySlaveUsage;
import com.hubspot.singularity.SingularitySlaveUsage.ResourceUsageType;
import com.hubspot.singularity.SingularitySlaveUsageWithId;
import com.hubspot.singularity.SingularityTaskRequest;
@@ -56,59 +57,59 @@ public void setup() {
@Test
public void itGetsTheCorrectScore() {
String slaveId = "slave";
Map<RequestType, Map<String, Number>> usagePerRequestType = new HashMap<>();
Map<RequestType, Map<ResourceUsageType, Number>> usagePerRequestType = new HashMap<>();
setRequestType(RequestType.SERVICE);
// no usage tracked -> default score
assertScoreIs(0.10, scheduler.score(getOffer(10, 10, slaveId), taskRequest, Optional.empty()));
// new slave (no resources used) -> perfect score
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(SingularitySlaveUsage.CPU_USED, 0, SingularitySlaveUsage.MEMORY_BYTES_USED, 0));
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(ResourceUsageType.CPU_USED, 0, ResourceUsageType.MEMORY_BYTES_USED, 0));
assertScoreIs(1, scheduler.score(getOffer(10, 10, slaveId), taskRequest, Optional.of(getUsage(0, 0, 10, 10, usagePerRequestType, slaveId))));
// cpu used, no mem used --- different request type
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(SingularitySlaveUsage.CPU_USED, 0, SingularitySlaveUsage.MEMORY_BYTES_USED, 0));
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(ResourceUsageType.CPU_USED, 0, ResourceUsageType.MEMORY_BYTES_USED, 0));
assertScoreIs(0.90, scheduler.score(getOffer(5, 10, slaveId), taskRequest, Optional.of(getUsage(0, 5, 10, 10, usagePerRequestType, slaveId))));
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(SingularitySlaveUsage.CPU_USED, 0, SingularitySlaveUsage.MEMORY_BYTES_USED, 0));
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(ResourceUsageType.CPU_USED, 0, ResourceUsageType.MEMORY_BYTES_USED, 0));
assertScoreIs(0.84, scheduler.score(getOffer(2, 10, slaveId), taskRequest, Optional.of(getUsage(0, 8, 10, 10, usagePerRequestType, slaveId))));
// cpu used, no mem used --- same request type
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(SingularitySlaveUsage.CPU_USED, 5, SingularitySlaveUsage.MEMORY_BYTES_USED, 0));
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(ResourceUsageType.CPU_USED, 5, ResourceUsageType.MEMORY_BYTES_USED, 0));
assertScoreIs(0.80, scheduler.score(getOffer(5, 10, slaveId), taskRequest, Optional.of(getUsage(0, 5, 10, 10, usagePerRequestType, slaveId))));
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(SingularitySlaveUsage.CPU_USED, 8, SingularitySlaveUsage.MEMORY_BYTES_USED, 0));
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(ResourceUsageType.CPU_USED, 8, ResourceUsageType.MEMORY_BYTES_USED, 0));
assertScoreIs(0.68, scheduler.score(getOffer(2, 10, slaveId), taskRequest, Optional.of(getUsage(0, 8, 10, 10, usagePerRequestType, slaveId))));
// cpu used, no mem used --- different request type
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(SingularitySlaveUsage.CPU_USED, 0, SingularitySlaveUsage.MEMORY_BYTES_USED, 0));
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(ResourceUsageType.CPU_USED, 0, ResourceUsageType.MEMORY_BYTES_USED, 0));
assertScoreIs(0.85, scheduler.score(getOffer(10, 5, slaveId), taskRequest, Optional.of(getUsage(mbToBytes(5), 0, 10, 10, usagePerRequestType, slaveId))));
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(SingularitySlaveUsage.CPU_USED, 0, SingularitySlaveUsage.MEMORY_BYTES_USED, 0));
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(ResourceUsageType.CPU_USED, 0, ResourceUsageType.MEMORY_BYTES_USED, 0));
assertScoreIs(0.76, scheduler.score(getOffer(10, 2, slaveId), taskRequest, Optional.of(getUsage(mbToBytes(8), 0, 10, 10, usagePerRequestType, slaveId))));
// no cpu used, mem used --- same request type
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(SingularitySlaveUsage.CPU_USED, 0, SingularitySlaveUsage.MEMORY_BYTES_USED, mbToBytes(5)));
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(ResourceUsageType.CPU_USED, 0, ResourceUsageType.MEMORY_BYTES_USED, mbToBytes(5)));
assertScoreIs(0.70, scheduler.score(getOffer(10, 5, slaveId), taskRequest, Optional.of(getUsage(mbToBytes(5), 0, 10, 10, usagePerRequestType, slaveId))));
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(SingularitySlaveUsage.CPU_USED, 0, SingularitySlaveUsage.MEMORY_BYTES_USED, mbToBytes(8)));
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(ResourceUsageType.CPU_USED, 0, ResourceUsageType.MEMORY_BYTES_USED, mbToBytes(8)));
assertScoreIs(0.52, scheduler.score(getOffer(10, 2, slaveId), taskRequest, Optional.of(getUsage(mbToBytes(8), 0, 10, 10, usagePerRequestType, slaveId))));
// cpu used, mem used --- different request type
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(SingularitySlaveUsage.CPU_USED, 0, SingularitySlaveUsage.MEMORY_BYTES_USED, 0));
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(ResourceUsageType.CPU_USED, 0, ResourceUsageType.MEMORY_BYTES_USED, 0));
assertScoreIs(0.75, scheduler.score(getOffer(5, 5, slaveId), taskRequest, Optional.of(getUsage(mbToBytes(5), 5, 10, 10, usagePerRequestType, slaveId))));
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(SingularitySlaveUsage.CPU_USED, 0, SingularitySlaveUsage.MEMORY_BYTES_USED, 0));
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(ResourceUsageType.CPU_USED, 0, ResourceUsageType.MEMORY_BYTES_USED, 0));
assertScoreIs(0.60, scheduler.score(getOffer(2, 2, slaveId), taskRequest, Optional.of(getUsage(mbToBytes(8), 8, 10, 10, usagePerRequestType, slaveId))));
// cpu used, mem used --- same request type
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(SingularitySlaveUsage.CPU_USED, 5, SingularitySlaveUsage.MEMORY_BYTES_USED, mbToBytes(5)));
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(ResourceUsageType.CPU_USED, 5, ResourceUsageType.MEMORY_BYTES_USED, mbToBytes(5)));
assertScoreIs(0.50, scheduler.score(getOffer(5, 5, slaveId), taskRequest, Optional.of(getUsage(mbToBytes(5), 5, 10, 10, usagePerRequestType, slaveId))));
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(SingularitySlaveUsage.CPU_USED, 8, SingularitySlaveUsage.MEMORY_BYTES_USED, mbToBytes(8)));
usagePerRequestType.put(RequestType.SERVICE, ImmutableMap.of(ResourceUsageType.CPU_USED, 8, ResourceUsageType.MEMORY_BYTES_USED, mbToBytes(8)));
assertScoreIs(0.20, scheduler.score(getOffer(2, 2, slaveId), taskRequest, Optional.of(getUsage(mbToBytes(8), 8, 10, 10, usagePerRequestType, slaveId))));
}
@@ -159,7 +160,7 @@ private long mbToBytes(long memMb) {
return memMb * 1024L * 1024L;
}
private SingularitySlaveUsageWithId getUsage(long memBytes, double cpus, long memMbTotal, double cpusTotal, Map<RequestType, Map<String, Number>> usagePerRequestType, String slaveId) {
private SingularitySlaveUsageWithId getUsage(long memBytes, double cpus, long memMbTotal, double cpusTotal, Map<RequestType, Map<ResourceUsageType, Number>> usagePerRequestType, String slaveId) {
return new SingularitySlaveUsageWithId(new SingularitySlaveUsage(memBytes, 0L, cpus, 1, Optional.of(memMbTotal), Optional.of(cpusTotal), usagePerRequestType), slaveId);
}
ProTip! Use n and p to navigate between commits in a pull request.