Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Track average scheduling delay when accepting tasks. #1583

Merged
merged 4 commits into from
Jul 14, 2017
Merged

Track average scheduling delay when accepting tasks. #1583

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4be6599
Represent the average scheduling delay in per-(request,deploy) statis…
baconmania Jul 5, 2017
3c738b8
Update average scheduling delay when accepting a task for a Mesos lau…
baconmania Jul 5, 2017
a452fde
Track this new metric after a task completes, to avoid having to main…
baconmania Jul 5, 2017
64ca644
Math.
baconmania Jul 6, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeployStatistics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,13 @@ public class SingularityDeployStatistics {
private final Optional<ExtendedTaskState> lastTaskState;

private final Optional<Long> averageRuntimeMillis;
private final Optional<Long> averageSchedulingDelayMillis;

@JsonCreator
public SingularityDeployStatistics(@JsonProperty("requestId") String requestId, @JsonProperty("deployId") String deployId, @JsonProperty("numSuccess") int numSuccess, @JsonProperty("numFailures") int numFailures,
@JsonProperty("numSequentialRetries") int numSequentialRetries, @JsonProperty("lastFinishAt") Optional<Long> lastFinishAt, @JsonProperty("lastTaskState") Optional<ExtendedTaskState> lastTaskState,
@JsonProperty("instanceSequentialFailureTimestamps") ListMultimap<Integer, Long> instanceSequentialFailureTimestamps, @JsonProperty("numTasks") int numTasks, @JsonProperty("averageRuntimeMillis") Optional<Long> averageRuntimeMillis) {
@JsonProperty("instanceSequentialFailureTimestamps") ListMultimap<Integer, Long> instanceSequentialFailureTimestamps, @JsonProperty("numTasks") int numTasks,
@JsonProperty("averageRuntimeMillis") Optional<Long> averageRuntimeMillis, @JsonProperty("averageSchedulingDelayMillis") Optional<Long> averageSchedulingDelayMillis) {
this.requestId = requestId;
this.deployId = deployId;
this.numSuccess = numSuccess;
Expand All @@ -39,6 +41,7 @@ public SingularityDeployStatistics(@JsonProperty("requestId") String requestId,
this.numSequentialRetries = numSequentialRetries;
this.numTasks = numTasks;
this.averageRuntimeMillis = averageRuntimeMillis;
this.averageSchedulingDelayMillis = averageSchedulingDelayMillis;
this.instanceSequentialFailureTimestamps = instanceSequentialFailureTimestamps == null ? ImmutableListMultimap.<Integer, Long> of() : ImmutableListMultimap.copyOf(instanceSequentialFailureTimestamps);
}

Expand All @@ -51,6 +54,7 @@ public SingularityDeployStatisticsBuilder toBuilder() {
.setNumSuccess(numSuccess)
.setNumTasks(numTasks)
.setAverageRuntimeMillis(averageRuntimeMillis)
.setAverageSchedulingDelayMillis(averageSchedulingDelayMillis)
.setInstanceSequentialFailureTimestamps(ArrayListMultimap.create(instanceSequentialFailureTimestamps));
}

Expand All @@ -62,6 +66,10 @@ public Optional<Long> getAverageRuntimeMillis() {
return averageRuntimeMillis;
}

public Optional<Long> getAverageSchedulingDelayMillis() {
return averageSchedulingDelayMillis;
}

public String getRequestId() {
return requestId;
}
Expand Down Expand Up @@ -107,6 +115,7 @@ public String toString() {
", lastFinishAt=" + lastFinishAt +
", lastTaskState=" + lastTaskState +
", averageRuntimeMillis=" + averageRuntimeMillis +
", averageSchedulingDelayMillis=" + averageSchedulingDelayMillis +
'}';
}
}
13 changes: 12 additions & 1 deletion ...ularityBase/src/main/java/com/hubspot/singularity/SingularityDeployStatisticsBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ public class SingularityDeployStatisticsBuilder {
private Optional<ExtendedTaskState> lastTaskState;

private Optional<Long> averageRuntimeMillis;
private Optional<Long> averageSchedulingDelayMillis; // Delta between between when each task was supposed to run vs. actually being submitted to Mesos

public SingularityDeployStatisticsBuilder(String requestId, String deployId) {
this.requestId = requestId;
Expand All @@ -29,10 +30,11 @@ public SingularityDeployStatisticsBuilder(String requestId, String deployId) {
this.lastFinishAt = Optional.absent();
this.lastTaskState = Optional.absent();
this.averageRuntimeMillis = Optional.absent();
this.averageSchedulingDelayMillis = Optional.absent();
}

public SingularityDeployStatistics build() {
return new SingularityDeployStatistics(requestId, deployId, numSuccess, numFailures, numSequentialRetries, lastFinishAt, lastTaskState, instanceSequentialFailureTimestamps, numTasks, averageRuntimeMillis);
return new SingularityDeployStatistics(requestId, deployId, numSuccess, numFailures, numSequentialRetries, lastFinishAt, lastTaskState, instanceSequentialFailureTimestamps, numTasks, averageRuntimeMillis, averageSchedulingDelayMillis);
}

public ListMultimap<Integer, Long> getInstanceSequentialFailureTimestamps() {
Expand Down Expand Up @@ -111,6 +113,15 @@ public SingularityDeployStatisticsBuilder setAverageRuntimeMillis(Optional<Long>
return this;
}

public Optional<Long> getAverageSchedulingDelayMillis() {
return averageSchedulingDelayMillis;
}

public SingularityDeployStatisticsBuilder setAverageSchedulingDelayMillis(Optional<Long> averageSchedulingDelayMillis) {
this.averageSchedulingDelayMillis = averageSchedulingDelayMillis;
return this;
}

public String getDeployId() {
return deployId;
}
Expand Down
28 changes: 28 additions & 0 deletions ...tyService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosOfferScheduler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import com.hubspot.mesos.Resources;
import com.hubspot.singularity.RequestType;
import com.hubspot.singularity.SingularityDeployStatistics;
import com.hubspot.singularity.SingularityDeployStatisticsBuilder;
import com.hubspot.singularity.SingularityPendingTaskId;
import com.hubspot.singularity.SingularitySlaveUsage.ResourceUsageType;
import com.hubspot.singularity.SingularitySlaveUsageWithId;
Expand Down Expand Up @@ -373,9 +374,36 @@ private SingularityTask acceptTask(SingularityOfferHolder offerHolder, Singulari
addRequestToMapByOfferHost(tasksPerOfferPerRequest, offerHolder.getHostname(), taskRequest.getRequest().getId());
stateCache.getScheduledTasks().remove(taskRequest.getPendingTask());

updateDeployStatistics(taskRequestHolder);

return task;
}

private void updateDeployStatistics(SingularityTaskRequestHolder taskRequestHolder) {
Optional<SingularityDeployStatistics> statistics = deployManager.getDeployStatistics(
taskRequestHolder.getTaskRequest().getRequest().getId(),
taskRequestHolder.getTaskRequest().getDeploy().getId()
);

if (statistics.isPresent()) {
SingularityDeployStatisticsBuilder bldr = statistics.get().toBuilder();

long dueTime = taskRequestHolder.getTaskRequest().getPendingTask().getPendingTaskId().getNextRunAt();
long now = System.currentTimeMillis();

if (bldr.getAverageSchedulingDelayMillis().isPresent()) {
long newAverageSchedulingDelayMillis = (bldr.getAverageSchedulingDelayMillis().get() * bldr.getNumTasks() + (now - dueTime)) / (bldr.getNumTasks() + 1);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this could get funky along with tracking the avgRunTime. We would be incrementing the numTasks when the tasks were finished, but updating this based on when they were started. Might want to track a separate numTasks for this

bldr.setAverageSchedulingDelayMillis(Optional.of(newAverageSchedulingDelayMillis));
} else {
bldr.setAverageSchedulingDelayMillis(Optional.of(now - dueTime));
}

final SingularityDeployStatistics newStatistics = bldr.build();

deployManager.saveDeployStatistics(newStatistics);
}
}

private void addRequestToMapByOfferHost(Map<String, Map<String, Integer>> tasksPerOfferHostPerRequest, String hostname, String requestId) {
if (tasksPerOfferHostPerRequest.containsKey(hostname)) {
if (tasksPerOfferHostPerRequest.get(hostname).containsKey(requestId)) {
Expand Down