New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Better task balancing #1482

Merged
merged 73 commits into from Jun 8, 2017
Commits
Jump to file or symbol
Failed to load files and symbols.
+116 鈭24
Diff settings

Always

Just for now

Viewing a subset of changes. View all

make offer scoring configurable

  • Loading branch information...
darcatron committed Apr 20, 2017
commit b6c010876189db585160b79d5fb14b4016b66109
@@ -171,13 +171,13 @@ These settings should live under the "mesos" field inside the root configuration
#### Resource Limits ####
| Parameter | Default | Description | Type |
|-----------|---------|-------------|------|
| defaultCpus | 1 | Number of CPUs to request for a task if none are specified | int |
| defaultCpus | 1 | Number of CPUs to request for a task if none are specified | int |
| defaultMemory | 64 | MB of memory to request for a task if none is specified | int |
| maxNumInstancesPerRequest | 25 | Max instances (tasks) to allow for a request (requests using over this will return a 400) | int |
| maxNumCpusPerInstance | 50 | Max number of CPUs allowed on a given task | int |
| maxNumCpusPerRequest | 900 | Max number of CPUs allowed for a given request (cpus per task * task instance) | int |
| maxMemoryMbPerInstance | 24000 | Max MB of memory allowed on a given task | int |
| maxMemoryMbPerRequest | 450000 | Max MB of memory allowed for a given request (memoryMb per task * task instances) | int |
| maxNumCpusPerInstance | 50 | Max number of CPUs allowed on a given task | int |
| maxNumCpusPerRequest | 900 | Max number of CPUs allowed for a given request (cpus per task * task instance) | int |
| maxMemoryMbPerInstance | 24000 | Max MB of memory allowed on a given task | int |
| maxMemoryMbPerRequest | 450000 | Max MB of memory allowed for a given request (memoryMb per task * task instances) | int |
#### Racks ####
| Parameter | Default | Description | Type |
@@ -189,7 +189,19 @@ These settings should live under the "mesos" field inside the root configuration
| Parameter | Default | Description | Type |
|-----------|---------|-------------|------|
| slaveHttpPort | 5051 | The port to talk to slaves on | int |
| slaveHttpsPort | absent | The HTTPS port to talk to slaves on | Integer (Optional) |
| slaveHttpsPort | absent | The HTTPS port to talk to slaves on | Integer (Optional) |
#### Offers ####
| Parameter | Default | Description | Type |
|-----------|---------|-------------|------|
| minOfferScore | 0.80 | The starting minimum score a task will accept for a mesos offer. The best possible offer score is 1.00 | double |
| maxOfferAttemptsPerTask | 20 | The max number of offers a task will reject | int |
| maxMillisPastDuePerTask | 600000 | The max milliseconds a task can be past due when scoring an offer | long |
| requestTypeCpuWeightForOffer | 0.20 | The weight the request type's utilization carries when scoring an offer | double |
| requestTypeMemWeightForOffer | 0.30 | The weight the request type's utilization carries when scoring an offer | double |
| freeCpuWeightForOffer | 0.20 | The weight the slave's cpu utilization carries when scoring an offer | double |
| freeMemWeightForOffer | 0.30 | The weight the slave's memory utilization carries when scoring an offer | double |
| defaultOfferScoreForMissingUsage | 0.10 | The default offer score used for offers without utilization metrics | double |
## Database ##
@@ -188,6 +188,22 @@
private int maxTasksPerOfferPerRequest = 0;
private double minOfferScore = 0.80;
private int maxOfferAttemptsPerTask = 20;
private long maxMillisPastDuePerTask = TimeUnit.MILLISECONDS.convert(10, TimeUnit.MINUTES);
private double requestTypeCpuWeightForOffer = 0.20;
private double requestTypeMemWeightForOffer = 0.30;
private double freeCpuWeightForOffer = 0.20;
private double freeMemWeightForOffer = 0.30;
private double defaultOfferScoreForMissingUsage = 0.10;
private int maxRequestIdSize = 100;
private int maxUserIdSize = 100;
@@ -618,6 +634,38 @@ public int getMaxTasksPerOfferPerRequest() {
return maxTasksPerOfferPerRequest;
}
public double getMinOfferScore() {
return minOfferScore;
}
public int getMaxOfferAttemptsPerTask() {
return maxOfferAttemptsPerTask;
}
public long getMaxMillisPastDuePerTask() {
return maxMillisPastDuePerTask;
}
public double getRequestTypeCpuWeightForOffer() {
return requestTypeCpuWeightForOffer;
}
public double getFreeCpuWeightForOffer() {
return freeCpuWeightForOffer;
}
public double getDefaultOfferScoreForMissingUsage() {
return defaultOfferScoreForMissingUsage;
}
public double getFreeMemWeightForOffer() {
return freeMemWeightForOffer;
}
public double getRequestTypeMemWeightForOffer() {
return requestTypeMemWeightForOffer;
}
public MesosConfiguration getMesosConfiguration() {
return mesosConfiguration;
}
@@ -978,6 +1026,46 @@ public void setMaxTasksPerOfferPerRequest(int maxTasksPerOfferPerRequest) {
this.maxTasksPerOfferPerRequest = maxTasksPerOfferPerRequest;
}
public SingularityConfiguration setMinOfferScore(double minOfferScore) {
this.minOfferScore = minOfferScore;
return this;
}
public SingularityConfiguration setMaxOfferAttemptsPerTask(int maxOfferAttemptsPerTask) {
this.maxOfferAttemptsPerTask = maxOfferAttemptsPerTask;
return this;
}
public SingularityConfiguration setMaxMillisPastDuePerTask(long maxMillisPastDuePerTask) {
this.maxMillisPastDuePerTask = maxMillisPastDuePerTask;
return this;
}
public SingularityConfiguration setRequestTypeCpuWeightForOffer(double requestTypeCpuWeightForOffer) {
this.requestTypeCpuWeightForOffer = requestTypeCpuWeightForOffer;
return this;
}
public SingularityConfiguration setRequestTypeMemWeightForOffer(double requestTypeMemWeightForOffer) {
this.requestTypeMemWeightForOffer = requestTypeMemWeightForOffer;
return this;
}
public SingularityConfiguration setFreeCpuWeightForOffer(double freeCpuWeightForOffer) {
this.freeCpuWeightForOffer = freeCpuWeightForOffer;
return this;
}
public SingularityConfiguration setFreeMemWeightForOffer(double freeMemWeightForOffer) {
this.freeMemWeightForOffer = freeMemWeightForOffer;
return this;
}
public SingularityConfiguration setDefaultOfferScoreForMissingUsage(double defaultOfferScoreForMissingUsage) {
this.defaultOfferScoreForMissingUsage = defaultOfferScoreForMissingUsage;
return this;
}
public void setMesosConfiguration(MesosConfiguration mesosConfiguration) {
this.mesosConfiguration = mesosConfiguration;
}
@@ -6,7 +6,6 @@
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.inject.Singleton;
@@ -128,7 +127,7 @@ public SingularityMesosOfferScheduler(MesosConfiguration mesosConfiguration,
Map<SingularityOfferHolder, Double> scorePerOffer = new HashMap<>();
double minScore = minScore(taskRequestHolder.getTaskRequest(), offerMatchAttemptsPerTask, System.currentTimeMillis());
LOG.info("Minimum score for task {} is {}", taskRequestHolder.getTaskRequest().getPendingTask().getPendingTaskId().getId(), minScore);
LOG.trace("Minimum score for task {} is {}", taskRequestHolder.getTaskRequest().getPendingTask().getPendingTaskId().getId(), minScore);
for (SingularityOfferHolder offerHolder : offerHolders) {
@@ -259,39 +258,32 @@ private double score(SingularityOfferHolder offerHolder, SingularitySchedulerSta
@VisibleForTesting
double score(Offer offer, SingularityTaskRequest taskRequest, Optional<SingularitySlaveUsageWithId> maybeSlaveUsage) {

This comment has been minimized.

@ssalinas

ssalinas Apr 20, 2017

Member

Let's go over this one in-person, think we are getting close, just easier to chat than typing a novel in github ;)

@ssalinas

ssalinas Apr 20, 2017

Member

Let's go over this one in-person, think we are getting close, just easier to chat than typing a novel in github ;)

double requestTypeCpuWeight = 0.20;
double requestTypeMemWeight = 0.30;
double freeCpuWeight = 0.20;
double freeMemWeight = 0.30;
double score = 0;
double defaultScoreForMissingUsage = 0.10;
if (!maybeSlaveUsage.isPresent() || !maybeSlaveUsage.get().getCpuTotal().isPresent() || !maybeSlaveUsage.get().getMemoryMbTotal().isPresent()) {
LOG.info("Slave {} has no total usage data. Will default to {}", offer.getSlaveId().getValue(), defaultScoreForMissingUsage);
return defaultScoreForMissingUsage;
LOG.info("Slave {} has no total usage data. Will default to {}", offer.getSlaveId().getValue(), configuration.getDefaultOfferScoreForMissingUsage());
return configuration.getDefaultOfferScoreForMissingUsage();
}
SingularitySlaveUsageWithId slaveUsage = maybeSlaveUsage.get();
Map<RequestType, Map<String, Number>> usagesPerRequestType = slaveUsage.getUsagePerRequestType();
Map<String, Number> usagePerResource = usagesPerRequestType.get(taskRequest.getRequest().getRequestType());
score += requestTypeCpuWeight * (1 - usagePerResource.get(SingularitySlaveUsage.CPU_USED).doubleValue() / slaveUsage.getCpuTotal().get());
score += requestTypeMemWeight * (1 - ((double) usagePerResource.get(SingularitySlaveUsage.MEMORY_BYTES_USED).longValue() / slaveUsage.getMemoryBytesTotal().get()));
score += configuration.getRequestTypeCpuWeightForOffer() * (1 - usagePerResource.get(SingularitySlaveUsage.CPU_USED).doubleValue() / slaveUsage.getCpuTotal().get());
score += configuration.getRequestTypeMemWeightForOffer() * (1 - ((double) usagePerResource.get(SingularitySlaveUsage.MEMORY_BYTES_USED).longValue() / slaveUsage.getMemoryBytesTotal().get()));
score += freeCpuWeight * (MesosUtils.getNumCpus(offer) / slaveUsage.getCpuTotal().get());
score += freeMemWeight * (MesosUtils.getMemory(offer) / slaveUsage.getMemoryMbTotal().get());
score += configuration.getFreeCpuWeightForOffer() * (MesosUtils.getNumCpus(offer) / slaveUsage.getCpuTotal().get());
score += configuration.getFreeMemWeightForOffer() * (MesosUtils.getMemory(offer) / slaveUsage.getMemoryMbTotal().get());
return score;
}
@VisibleForTesting
double minScore(SingularityTaskRequest taskRequest, Map<String, Integer> offerMatchAttemptsPerTask, long now) {
double minScore = 0.80;
int maxOfferAttempts = 20;
long maxMillisPastDue = TimeUnit.MILLISECONDS.convert(10, TimeUnit.MINUTES);
double minScore = configuration.getMinOfferScore();
minScore -= offerMatchAttemptsPerTask.getOrDefault(taskRequest.getPendingTask().getPendingTaskId().getId(), 0) / (double) maxOfferAttempts;
minScore -= millisPastDue(taskRequest, now) / (double) maxMillisPastDue;
minScore -= offerMatchAttemptsPerTask.getOrDefault(taskRequest.getPendingTask().getPendingTaskId().getId(), 0) / (double) configuration.getMaxOfferAttemptsPerTask();
minScore -= millisPastDue(taskRequest, now) / (double) configuration.getMaxMillisPastDuePerTask();
return Math.max(minScore, 0);
}
ProTip! Use n and p to navigate between commits in a pull request.