From 2287497fe717ed2f65daed58b73021c8df465eae Mon Sep 17 00:00:00 2001 From: Stephen Salinas Date: Tue, 24 Apr 2018 08:57:21 -0400 Subject: [PATCH 1/4] Add ability to set attributes that mark slave for only preemtible tasks --- .../config/SingularityConfiguration.java | 20 +++++++++++++++++++ .../mesos/SingularityMesosOfferScheduler.java | 16 +++++++++++++-- .../mesos/SingularitySlaveAndRackManager.java | 14 ++++++++++--- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java index 4f6e61e2ac..d497a39fda 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java @@ -390,6 +390,10 @@ public class SingularityConfiguration extends Configuration { // If cpuHardLimit is specified and a task is requesting a base cpu of > cpuHardLimit, that task's new hard limit is requested cpus * cpuHardLimitScaleFactor private double cpuHardLimitScaleFactor = 1.25; + private Map preemptableTasksOnlyMachineAttributes = Collections.emptyMap(); + + private long preemptableTaskMaxExpectedRuntimeMs = 900000; // 15 minutes + public long getAskDriverToKillTasksAgainAfterMillis() { return askDriverToKillTasksAgainAfterMillis; } @@ -1670,4 +1674,20 @@ public SingularityConfiguration setCpuHardLimitScaleFactor(double cpuHardLimitSc this.cpuHardLimitScaleFactor = cpuHardLimitScaleFactor; return this; } + + public Map getPreemptableTasksOnlyMachineAttributes() { + return preemptableTasksOnlyMachineAttributes; + } + + public void setPreemptableTasksOnlyMachineAttributes(Map preemptableTasksOnlyMachineAttributes) { + this.preemptableTasksOnlyMachineAttributes = preemptableTasksOnlyMachineAttributes; + } + + public long getPreemptableTaskMaxExpectedRuntimeMs() { + return preemptableTaskMaxExpectedRuntimeMs; + } + + public void setPreemptableTaskMaxExpectedRuntimeMs(long preemptableTaskMaxExpectedRuntimeMs) { + this.preemptableTaskMaxExpectedRuntimeMs = preemptableTaskMaxExpectedRuntimeMs; + } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosOfferScheduler.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosOfferScheduler.java index 22c983b468..8a284adcd8 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosOfferScheduler.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosOfferScheduler.java @@ -374,8 +374,7 @@ private double score(SingularityOfferHolder offerHolder, Map ta if (!matchesResources) { return 0; } - - final SlaveMatchState slaveMatchState = slaveAndRackManager.doesOfferMatch(offerHolder, taskRequest, activeTaskIdsForRequest); + final SlaveMatchState slaveMatchState = slaveAndRackManager.doesOfferMatch(offerHolder, taskRequest, activeTaskIdsForRequest, isPreemtableTask(taskRequest)); if (slaveMatchState.isMatchAllowed()) { return score(offerHolder.getHostname(), taskRequest, maybeSlaveUsage); @@ -387,6 +386,19 @@ private double score(SingularityOfferHolder offerHolder, Map ta return 0; } + private boolean isPreemtableTask(SingularityTaskRequest taskRequest) { + // A long running task can be replaced + killed easily + if (taskRequest.getRequest().getRequestType().isLongRunning()) { + return true; + } + + // A short, non-long-running task + Optional deployStatistics = deployManager.getDeployStatistics(taskRequest.getRequest().getId(), taskRequest.getDeploy().getId()); + return deployStatistics.isPresent() + && deployStatistics.get().getAverageRuntimeMillis().isPresent() + && deployStatistics.get().getAverageRuntimeMillis().get() < configuration.getPreemptableTaskMaxExpectedRuntimeMs(); + } + @VisibleForTesting double score(String hostname, SingularityTaskRequest taskRequest, Optional maybeSlaveUsage) { if (!maybeSlaveUsage.isPresent() || maybeSlaveUsage.get().isMissingUsageData()) { diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularitySlaveAndRackManager.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularitySlaveAndRackManager.java index 6b78b9257e..7678b57566 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularitySlaveAndRackManager.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularitySlaveAndRackManager.java @@ -81,7 +81,7 @@ public class SingularitySlaveAndRackManager { this.leaderCache = leaderCache; } - SlaveMatchState doesOfferMatch(SingularityOfferHolder offerHolder, SingularityTaskRequest taskRequest, List activeTaskIdsForRequest) { + SlaveMatchState doesOfferMatch(SingularityOfferHolder offerHolder, SingularityTaskRequest taskRequest, List activeTaskIdsForRequest, boolean isPreemtibleTask) { final String host = offerHolder.getHostname(); final String rackId = offerHolder.getRackId(); final String slaveId = offerHolder.getSlaveId(); @@ -113,7 +113,7 @@ SlaveMatchState doesOfferMatch(SingularityOfferHolder offerHolder, SingularityTa } } - if (!isSlaveAttributesMatch(offerHolder, taskRequest)) { + if (!isSlaveAttributesMatch(offerHolder, taskRequest, isPreemtibleTask)) { return SlaveMatchState.SLAVE_ATTRIBUTES_DO_NOT_MATCH; } @@ -241,7 +241,7 @@ SlaveMatchState doesOfferMatch(SingularityOfferHolder offerHolder, SingularityTa return SlaveMatchState.OK; } - private boolean isSlaveAttributesMatch(SingularityOfferHolder offer, SingularityTaskRequest taskRequest) { + private boolean isSlaveAttributesMatch(SingularityOfferHolder offer, SingularityTaskRequest taskRequest, boolean isPreemtibleTask) { if (offer.hasReservedSlaveAttributes()) { Map reservedSlaveAttributes = offer.getReservedSlaveAttributes(); @@ -260,6 +260,14 @@ private boolean isSlaveAttributesMatch(SingularityOfferHolder offer, Singularity } } + if (!configuration.getPreemptableTasksOnlyMachineAttributes().isEmpty()) { + if (slaveAndRackHelper.hasRequiredAttributes(offer.getTextAttributes(), configuration.getPreemptableTasksOnlyMachineAttributes()) + && !isPreemtibleTask) { + LOG.debug("Host {} is reserved for preemptible tasks", offer.getHostname()); + return false; + } + } + if (taskRequest.getRequest().getRequiredSlaveAttributes().isPresent() && !slaveAndRackHelper.hasRequiredAttributes(offer.getTextAttributes(), taskRequest.getRequest().getRequiredSlaveAttributes().get())) { LOG.trace("Task requires slave with attributes {}, (slave attributes are {})", taskRequest.getRequest().getRequiredSlaveAttributes().get(), offer.getTextAttributes()); From 152c52c56bcc7da5febb3a1b5c79e70a08882e74 Mon Sep 17 00:00:00 2001 From: Stephen Salinas Date: Tue, 24 Apr 2018 08:58:57 -0400 Subject: [PATCH 2/4] typos --- .../config/SingularityConfiguration.java | 20 +++++++++---------- .../mesos/SingularityMesosOfferScheduler.java | 2 +- .../mesos/SingularitySlaveAndRackManager.java | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java index d497a39fda..b1f4025e0b 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java @@ -390,9 +390,9 @@ public class SingularityConfiguration extends Configuration { // If cpuHardLimit is specified and a task is requesting a base cpu of > cpuHardLimit, that task's new hard limit is requested cpus * cpuHardLimitScaleFactor private double cpuHardLimitScaleFactor = 1.25; - private Map preemptableTasksOnlyMachineAttributes = Collections.emptyMap(); + private Map preemptibleTasksOnlyMachineAttributes = Collections.emptyMap(); - private long preemptableTaskMaxExpectedRuntimeMs = 900000; // 15 minutes + private long preemptibleTaskMaxExpectedRuntimeMs = 900000; // 15 minutes public long getAskDriverToKillTasksAgainAfterMillis() { return askDriverToKillTasksAgainAfterMillis; @@ -1675,19 +1675,19 @@ public SingularityConfiguration setCpuHardLimitScaleFactor(double cpuHardLimitSc return this; } - public Map getPreemptableTasksOnlyMachineAttributes() { - return preemptableTasksOnlyMachineAttributes; + public Map getPreemptibleTasksOnlyMachineAttributes() { + return preemptibleTasksOnlyMachineAttributes; } - public void setPreemptableTasksOnlyMachineAttributes(Map preemptableTasksOnlyMachineAttributes) { - this.preemptableTasksOnlyMachineAttributes = preemptableTasksOnlyMachineAttributes; + public void setPreemptibleTasksOnlyMachineAttributes(Map preemptibleTasksOnlyMachineAttributes) { + this.preemptibleTasksOnlyMachineAttributes = preemptibleTasksOnlyMachineAttributes; } - public long getPreemptableTaskMaxExpectedRuntimeMs() { - return preemptableTaskMaxExpectedRuntimeMs; + public long getPreemptibleTaskMaxExpectedRuntimeMs() { + return preemptibleTaskMaxExpectedRuntimeMs; } - public void setPreemptableTaskMaxExpectedRuntimeMs(long preemptableTaskMaxExpectedRuntimeMs) { - this.preemptableTaskMaxExpectedRuntimeMs = preemptableTaskMaxExpectedRuntimeMs; + public void setPreemptibleTaskMaxExpectedRuntimeMs(long preemptibleTaskMaxExpectedRuntimeMs) { + this.preemptibleTaskMaxExpectedRuntimeMs = preemptibleTaskMaxExpectedRuntimeMs; } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosOfferScheduler.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosOfferScheduler.java index 8a284adcd8..d1d03791c4 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosOfferScheduler.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosOfferScheduler.java @@ -396,7 +396,7 @@ private boolean isPreemtableTask(SingularityTaskRequest taskRequest) { Optional deployStatistics = deployManager.getDeployStatistics(taskRequest.getRequest().getId(), taskRequest.getDeploy().getId()); return deployStatistics.isPresent() && deployStatistics.get().getAverageRuntimeMillis().isPresent() - && deployStatistics.get().getAverageRuntimeMillis().get() < configuration.getPreemptableTaskMaxExpectedRuntimeMs(); + && deployStatistics.get().getAverageRuntimeMillis().get() < configuration.getPreemptibleTaskMaxExpectedRuntimeMs(); } @VisibleForTesting diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularitySlaveAndRackManager.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularitySlaveAndRackManager.java index 7678b57566..f1dcb8dcec 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularitySlaveAndRackManager.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularitySlaveAndRackManager.java @@ -260,8 +260,8 @@ private boolean isSlaveAttributesMatch(SingularityOfferHolder offer, Singularity } } - if (!configuration.getPreemptableTasksOnlyMachineAttributes().isEmpty()) { - if (slaveAndRackHelper.hasRequiredAttributes(offer.getTextAttributes(), configuration.getPreemptableTasksOnlyMachineAttributes()) + if (!configuration.getPreemptibleTasksOnlyMachineAttributes().isEmpty()) { + if (slaveAndRackHelper.hasRequiredAttributes(offer.getTextAttributes(), configuration.getPreemptibleTasksOnlyMachineAttributes()) && !isPreemtibleTask) { LOG.debug("Host {} is reserved for preemptible tasks", offer.getHostname()); return false; From 0fc33ec46e9b91a1be5f7d91a4ea16139f060e8a Mon Sep 17 00:00:00 2001 From: Stephen Salinas Date: Tue, 24 Apr 2018 09:57:19 -0400 Subject: [PATCH 3/4] typo --- .../singularity/mesos/SingularityMesosOfferScheduler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosOfferScheduler.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosOfferScheduler.java index d1d03791c4..9b74cd0acd 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosOfferScheduler.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosOfferScheduler.java @@ -374,7 +374,7 @@ private double score(SingularityOfferHolder offerHolder, Map ta if (!matchesResources) { return 0; } - final SlaveMatchState slaveMatchState = slaveAndRackManager.doesOfferMatch(offerHolder, taskRequest, activeTaskIdsForRequest, isPreemtableTask(taskRequest)); + final SlaveMatchState slaveMatchState = slaveAndRackManager.doesOfferMatch(offerHolder, taskRequest, activeTaskIdsForRequest, isPreemtibleTask(taskRequest)); if (slaveMatchState.isMatchAllowed()) { return score(offerHolder.getHostname(), taskRequest, maybeSlaveUsage); @@ -386,7 +386,7 @@ private double score(SingularityOfferHolder offerHolder, Map ta return 0; } - private boolean isPreemtableTask(SingularityTaskRequest taskRequest) { + private boolean isPreemtibleTask(SingularityTaskRequest taskRequest) { // A long running task can be replaced + killed easily if (taskRequest.getRequest().getRequestType().isLongRunning()) { return true; From d9f645f57d4d45678306d62e688dd693045284bd Mon Sep 17 00:00:00 2001 From: Stephen Salinas Date: Tue, 24 Apr 2018 09:59:23 -0400 Subject: [PATCH 4/4] learn to english --- .../singularity/mesos/SingularityMesosOfferScheduler.java | 4 ++-- .../singularity/mesos/SingularitySlaveAndRackManager.java | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosOfferScheduler.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosOfferScheduler.java index 9b74cd0acd..cee89be183 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosOfferScheduler.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosOfferScheduler.java @@ -374,7 +374,7 @@ private double score(SingularityOfferHolder offerHolder, Map ta if (!matchesResources) { return 0; } - final SlaveMatchState slaveMatchState = slaveAndRackManager.doesOfferMatch(offerHolder, taskRequest, activeTaskIdsForRequest, isPreemtibleTask(taskRequest)); + final SlaveMatchState slaveMatchState = slaveAndRackManager.doesOfferMatch(offerHolder, taskRequest, activeTaskIdsForRequest, isPreemptibleTask(taskRequest)); if (slaveMatchState.isMatchAllowed()) { return score(offerHolder.getHostname(), taskRequest, maybeSlaveUsage); @@ -386,7 +386,7 @@ private double score(SingularityOfferHolder offerHolder, Map ta return 0; } - private boolean isPreemtibleTask(SingularityTaskRequest taskRequest) { + private boolean isPreemptibleTask(SingularityTaskRequest taskRequest) { // A long running task can be replaced + killed easily if (taskRequest.getRequest().getRequestType().isLongRunning()) { return true; diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularitySlaveAndRackManager.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularitySlaveAndRackManager.java index f1dcb8dcec..b3d5ea1190 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularitySlaveAndRackManager.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularitySlaveAndRackManager.java @@ -81,7 +81,7 @@ public class SingularitySlaveAndRackManager { this.leaderCache = leaderCache; } - SlaveMatchState doesOfferMatch(SingularityOfferHolder offerHolder, SingularityTaskRequest taskRequest, List activeTaskIdsForRequest, boolean isPreemtibleTask) { + SlaveMatchState doesOfferMatch(SingularityOfferHolder offerHolder, SingularityTaskRequest taskRequest, List activeTaskIdsForRequest, boolean isPreemptibleTask) { final String host = offerHolder.getHostname(); final String rackId = offerHolder.getRackId(); final String slaveId = offerHolder.getSlaveId(); @@ -113,7 +113,7 @@ SlaveMatchState doesOfferMatch(SingularityOfferHolder offerHolder, SingularityTa } } - if (!isSlaveAttributesMatch(offerHolder, taskRequest, isPreemtibleTask)) { + if (!isSlaveAttributesMatch(offerHolder, taskRequest, isPreemptibleTask)) { return SlaveMatchState.SLAVE_ATTRIBUTES_DO_NOT_MATCH; } @@ -241,7 +241,7 @@ SlaveMatchState doesOfferMatch(SingularityOfferHolder offerHolder, SingularityTa return SlaveMatchState.OK; } - private boolean isSlaveAttributesMatch(SingularityOfferHolder offer, SingularityTaskRequest taskRequest, boolean isPreemtibleTask) { + private boolean isSlaveAttributesMatch(SingularityOfferHolder offer, SingularityTaskRequest taskRequest, boolean isPreemptibleTask) { if (offer.hasReservedSlaveAttributes()) { Map reservedSlaveAttributes = offer.getReservedSlaveAttributes(); @@ -262,7 +262,7 @@ private boolean isSlaveAttributesMatch(SingularityOfferHolder offer, Singularity if (!configuration.getPreemptibleTasksOnlyMachineAttributes().isEmpty()) { if (slaveAndRackHelper.hasRequiredAttributes(offer.getTextAttributes(), configuration.getPreemptibleTasksOnlyMachineAttributes()) - && !isPreemtibleTask) { + && !isPreemptibleTask) { LOG.debug("Host {} is reserved for preemptible tasks", offer.getHostname()); return false; }