New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Better task balancing #1482

Merged
merged 73 commits into from Jun 8, 2017
Commits
Jump to file or symbol
Failed to load files and symbols.
+8 鈭41
Diff settings

Always

Just for now

Viewing a subset of changes. View all

don't save history usage summary data

  • Loading branch information...
darcatron committed May 23, 2017
commit 1f6d650a34895ca4dbb0c6de2648a7a89f138ba3
@@ -197,7 +197,6 @@ These settings should live under the "mesos" field inside the root configuration
#### Offers ####
| Parameter | Default | Description | Type |
|-----------|---------|-------------|------|
| minOfferScore | 0.70 | The starting minimum score a task will accept for a mesos offer. The best possible offer score is 1.00 | double |
| maxMillisPastDuePerTask | 300000 (5 min) | The max milliseconds a task can be past due when scoring an offer | long |
| longRunningUsedCpuWeightForOffer | 0.30 | The weight long running tasks' cpu utilization carries when scoring an offer (should add up to 1 with longRunningUsedMemWeightForOffer) | double |
| longRunningUsedMemWeightForOffer | 0.70 | The weight long running tasks' memory utilization carries when scoring an offer (should add up to 1 with longRunningUsedCpuWeightForOffer) | double |
@@ -11,6 +11,7 @@
import org.apache.curator.utils.ZKPaths;
import com.codahale.metrics.MetricRegistry;
import com.google.common.base.Optional;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.hubspot.singularity.SingularityClusterUtilization;
@@ -104,10 +105,6 @@ private String getCurrentSlaveUsagePath(String slaveId) {
return ZKPaths.makePath(getSlaveUsagePath(slaveId), CURRENT_USAGE_NODE_KEY);
}
private String getSpecificClusterUtilizationPath(long timestamp) {
return ZKPaths.makePath(USAGE_SUMMARY_PATH, Long.toString(timestamp));
}
private String getCurrentTaskUsagePath(String taskId) {
return ZKPaths.makePath(getTaskUsagePath(taskId), CURRENT_USAGE_NODE_KEY);
}
@@ -120,10 +117,6 @@ public SingularityDeleteResult deleteTaskUsage(String taskId) {
return delete(getTaskUsagePath(taskId));
}
public SingularityDeleteResult deleteSpecificClusterUtilization(long timestamp) {
return delete(getSpecificClusterUtilizationPath(timestamp));
}
public SingularityDeleteResult deleteSpecificSlaveUsage(String slaveId, long timestamp) {
return delete(getSpecificSlaveUsagePath(slaveId, timestamp));
}
@@ -140,8 +133,8 @@ public SingularityCreateResult saveSpecificTaskUsage(String taskId, SingularityT
return save(getSpecificTaskUsagePath(taskId, usage.getTimestamp()), usage, taskUsageTranscoder);
}
public SingularityCreateResult saveSpecificClusterUtilization(SingularityClusterUtilization utilization) {
return save(getSpecificClusterUtilizationPath(utilization.getTimestamp()) , utilization, clusterUtilizationTranscoder);
public SingularityCreateResult saveClusterUtilization(SingularityClusterUtilization utilization) {
return save(USAGE_SUMMARY_PATH, utilization, clusterUtilizationTranscoder);
}
public SingularityCreateResult saveSpecificSlaveUsageAndSetCurrent(String slaveId, SingularitySlaveUsage usage) {
@@ -179,20 +172,8 @@ public int compare(SingularityTaskUsage o1, SingularityTaskUsage o2) {
return children;
}
private static final Comparator<SingularityClusterUtilization> CLUSTER_UTILIZATION_COMPARATOR_TIMESTAMP_ASC = new Comparator<SingularityClusterUtilization>() {
@Override
public int compare(SingularityClusterUtilization o1, SingularityClusterUtilization o2) {
return Long.compare(o1.getTimestamp(), o2.getTimestamp());
}
};
public List<SingularityClusterUtilization> getClusterUtilization() {
List<SingularityClusterUtilization> children = getAsyncChildren(USAGE_SUMMARY_PATH, clusterUtilizationTranscoder);
children.sort(CLUSTER_UTILIZATION_COMPARATOR_TIMESTAMP_ASC);
return children;
public Optional<SingularityClusterUtilization> getClusterUtilization() {
return getData(USAGE_SUMMARY_PATH, clusterUtilizationTranscoder);
}
public List<SingularitySlaveUsageWithId> getCurrentSlaveUsages(List<String> slaveIds) {
@@ -135,7 +135,7 @@ public SingularityMesosOfferScheduler(MesosConfiguration mesosConfiguration,
SingularityTaskRequestHolder taskRequestHolder = iterator.next();
Map<SingularityOfferHolder, Double> scorePerOffer = new HashMap<>();
double minScore = minScore(taskRequestHolder.getTaskRequest(), offerMatchAttemptsPerTask, getLatestClusterUtilization(), System.currentTimeMillis());
double minScore = minScore(taskRequestHolder.getTaskRequest(), offerMatchAttemptsPerTask, usageManager.getClusterUtilization(), System.currentTimeMillis());
LOG.trace("Minimum score {} for task {}", minScore, taskRequestHolder.getTaskRequest().getPendingTask().getPendingTaskId().getId());
@@ -192,11 +192,6 @@ public SingularityMesosOfferScheduler(MesosConfiguration mesosConfiguration,
return offerHolders;
}
private Optional<SingularityClusterUtilization> getLatestClusterUtilization() {
List<SingularityClusterUtilization> clusterUtilizations = usageManager.getClusterUtilization();
return clusterUtilizations.isEmpty() ? Optional.absent() : Optional.of(clusterUtilizations.get(clusterUtilizations.size() - 1));
}
private double getNormalizedWeight(ResourceUsageType type) {
double freeCpuWeight = configuration.getFreeCpuWeightForOffer();
double freeMemWeight = configuration.getFreeMemWeightForOffer();
@@ -159,7 +159,7 @@ public void runActionOnPoll() {
exceptionNotifier.notify(message, e);
}
saveClusterUtilization(totalMemBytesUsed, totalMemBytesAvailable, totalCpuUsed, totalCpuAvailable, now);
usageManager.saveClusterUtilization(new SingularityClusterUtilization(totalMemBytesUsed, totalMemBytesAvailable, totalCpuUsed, totalCpuAvailable, now));
}
}
@@ -190,12 +190,4 @@ private void updateLongRunningTasksUsage(Map<ResourceUsageType, Number> longRunn
longRunningTasksUsage.compute(ResourceUsageType.MEMORY_BYTES_USED, (k, v) -> (v == null) ? memBytesUsed : v.longValue() + memBytesUsed);
longRunningTasksUsage.compute(ResourceUsageType.CPU_USED, (k, v) -> (v == null) ? cpuUsed : v.doubleValue() + cpuUsed);
}
private void saveClusterUtilization(long totalMemBytesUsed, long totalMemBytesAvailable, double totalCpuUsed, double totalCpuAvailable, long now) {
List<SingularityClusterUtilization> utilizations = usageManager.getClusterUtilization();
if (utilizations.size() + 1 > configuration.getNumUsageToKeep()) {
usageManager.deleteSpecificClusterUtilization(utilizations.get(0).getTimestamp());
}
usageManager.saveSpecificClusterUtilization(new SingularityClusterUtilization(totalMemBytesUsed, totalMemBytesAvailable, totalCpuUsed, totalCpuAvailable, now));
}
}
@@ -289,7 +289,7 @@ public void itCorrectlyScoresLongNonLongRunningTasks() {
}
@Test
public void itGetsTheCorrectMinScore() {
public void itGetsTheCorrectMinScore() {
long now = System.currentTimeMillis();
String taskId = "taskId";
setNextRunAt(now);
ProTip! Use n and p to navigate between commits in a pull request.