From 827d11ca56341507f79f688c4c81b1f3ccfded93 Mon Sep 17 00:00:00 2001 From: Stephen Salinas Date: Fri, 16 Feb 2018 19:22:55 -0500 Subject: [PATCH 1/5] add trace logging to offer cache --- .../com/hubspot/singularity/mesos/SingularityOfferCache.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityOfferCache.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityOfferCache.java index 1755ef8f66..fbcde9430e 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityOfferCache.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityOfferCache.java @@ -75,11 +75,13 @@ public void onRemoval(RemovalNotification notification) { @Override public void rescindOffer(OfferID offerId) { + LOG.trace("Offer {} rescinded, removing from cache", offerId.getValue()); offerCache.invalidate(offerId.getValue()); } @Override public void useOffer(CachedOffer cachedOffer) { + LOG.trace("Using cached offer {}", cachedOffer.getOfferId()); offerCache.invalidate(cachedOffer.offerId); } @@ -94,6 +96,7 @@ public List checkoutOffers() { List offers = new ArrayList<>((int) offerCache.size()); for (CachedOffer cachedOffer : offerCache.asMap().values()) { + LOG.trace("Checking out offer {}", cachedOffer.getOfferId()); cachedOffer.checkOut(); offers.add(cachedOffer); } @@ -116,8 +119,10 @@ public List peekOffers() { public void returnOffer(CachedOffer cachedOffer) { synchronized (offerCache) { if (cachedOffer.offerState == OfferState.EXPIRED) { + LOG.trace("Declining returned offer {}", cachedOffer.getOfferId()); declineOffer(cachedOffer); } else { + LOG.trace("Returning offer {} to cache", cachedOffer.getOfferId()); cachedOffer.checkIn(); } } From 3aea2ddd63fc8cd1f500c0dd8853a053f3e865eb Mon Sep 17 00:00:00 2001 From: Stephen Salinas Date: Fri, 16 Feb 2018 19:24:09 -0500 Subject: [PATCH 2/5] more trace logging --- .../hubspot/singularity/mesos/SingularityMesosSchedulerImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerImpl.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerImpl.java index 6eeec20cb3..abcffbb3f9 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerImpl.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerImpl.java @@ -209,6 +209,7 @@ public void resourceOffers(List offers) { for (SingularityOfferHolder offerHolder : offerHolders) { if (!offerHolder.getAcceptedTasks().isEmpty()) { List leftoverOffers = offerHolder.launchTasksAndGetUnusedOffers(mesosSchedulerClient); + LOG.debug("Leftover offers: {}", leftoverOffers.stream().map(Offer::getId)); leftoverOffers.forEach((o) -> { if (cachedOffers.containsKey(o.getId().getValue())) { From d74c93123a05642b71c5cf172d88071ac3272da9 Mon Sep 17 00:00:00 2001 From: Stephen Salinas Date: Fri, 16 Feb 2018 19:37:48 -0500 Subject: [PATCH 3/5] additional logging --- .../singularity/mesos/SingularityMesosSchedulerImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerImpl.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerImpl.java index abcffbb3f9..554c9bb065 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerImpl.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerImpl.java @@ -209,7 +209,7 @@ public void resourceOffers(List offers) { for (SingularityOfferHolder offerHolder : offerHolders) { if (!offerHolder.getAcceptedTasks().isEmpty()) { List leftoverOffers = offerHolder.launchTasksAndGetUnusedOffers(mesosSchedulerClient); - LOG.debug("Leftover offers: {}", leftoverOffers.stream().map(Offer::getId)); + LOG.debug("Leftover offers: {}", leftoverOffers.stream().map(Offer::getId).collect(Collectors.toList())); leftoverOffers.forEach((o) -> { if (cachedOffers.containsKey(o.getId().getValue())) { @@ -221,6 +221,7 @@ public void resourceOffers(List offers) { List offersAcceptedFromSlave = offerHolder.getOffers(); offersAcceptedFromSlave.removeAll(leftoverOffers); + LOG.trace("Accepted offers {}", offersAcceptedFromSlave.stream().map(Offer::getId).collect(Collectors.toList())); offersAcceptedFromSlave.stream() .filter((offer) -> cachedOffers.containsKey(offer.getId().getValue())) .map((o) -> cachedOffers.get(o.getId().getValue())) From 501822924a9194bc28d4628bec9855ed0c8876f8 Mon Sep 17 00:00:00 2001 From: Stephen Salinas Date: Fri, 16 Feb 2018 20:56:38 -0500 Subject: [PATCH 4/5] Use original cached offer object, not a copy --- .../mesos/SingularityMesosSchedulerImpl.java | 14 +++++++++----- .../singularity/mesos/SingularityOfferCache.java | 3 --- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerImpl.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerImpl.java index 554c9bb065..b6286fa5cb 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerImpl.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosSchedulerImpl.java @@ -4,6 +4,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; @@ -11,7 +12,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Function; import java.util.stream.Collectors; import javax.inject.Singleton; @@ -186,8 +186,14 @@ public void resourceOffers(List offers) { } List offersToCheck = new ArrayList<>(offers); - Map cachedOffers = offerCache.checkoutOffers().stream().collect(Collectors.toMap(CachedOffer::getOfferId, Function.identity())); - offersToCheck.addAll(cachedOffers.values().stream().map(CachedOffer::getOffer).collect(Collectors.toList())); + + List cachedOfferList = offerCache.checkoutOffers(); + Map cachedOffers = new HashMap<>(); + for (CachedOffer cachedOffer : cachedOfferList) { + cachedOffers.put(cachedOffer.getOfferId(), cachedOffer); + offersToCheck.add(cachedOffer.getOffer()); + } + offers.parallelStream().forEach((offer) -> { String rolesInfo = MesosUtils.getRoles(offer).toString(); LOG.debug("Received offer ID {} with roles {} from {} ({}) for {} cpu(s), {} memory, {} ports, and {} disk", offer.getId().getValue(), rolesInfo, offer.getHostname(), offer.getAgentId().getValue(), MesosUtils.getNumCpus(offer), MesosUtils.getMemory(offer), @@ -209,7 +215,6 @@ public void resourceOffers(List offers) { for (SingularityOfferHolder offerHolder : offerHolders) { if (!offerHolder.getAcceptedTasks().isEmpty()) { List leftoverOffers = offerHolder.launchTasksAndGetUnusedOffers(mesosSchedulerClient); - LOG.debug("Leftover offers: {}", leftoverOffers.stream().map(Offer::getId).collect(Collectors.toList())); leftoverOffers.forEach((o) -> { if (cachedOffers.containsKey(o.getId().getValue())) { @@ -221,7 +226,6 @@ public void resourceOffers(List offers) { List offersAcceptedFromSlave = offerHolder.getOffers(); offersAcceptedFromSlave.removeAll(leftoverOffers); - LOG.trace("Accepted offers {}", offersAcceptedFromSlave.stream().map(Offer::getId).collect(Collectors.toList())); offersAcceptedFromSlave.stream() .filter((offer) -> cachedOffers.containsKey(offer.getId().getValue())) .map((o) -> cachedOffers.get(o.getId().getValue())) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityOfferCache.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityOfferCache.java index fbcde9430e..a4ceb64c88 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityOfferCache.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityOfferCache.java @@ -75,13 +75,11 @@ public void onRemoval(RemovalNotification notification) { @Override public void rescindOffer(OfferID offerId) { - LOG.trace("Offer {} rescinded, removing from cache", offerId.getValue()); offerCache.invalidate(offerId.getValue()); } @Override public void useOffer(CachedOffer cachedOffer) { - LOG.trace("Using cached offer {}", cachedOffer.getOfferId()); offerCache.invalidate(cachedOffer.offerId); } @@ -96,7 +94,6 @@ public List checkoutOffers() { List offers = new ArrayList<>((int) offerCache.size()); for (CachedOffer cachedOffer : offerCache.asMap().values()) { - LOG.trace("Checking out offer {}", cachedOffer.getOfferId()); cachedOffer.checkOut(); offers.add(cachedOffer); } From 4b24710c170e8b92c68d12dc2d5174316ed71654 Mon Sep 17 00:00:00 2001 From: Stephen Salinas Date: Fri, 16 Feb 2018 20:58:06 -0500 Subject: [PATCH 5/5] remove extra trace logging --- .../com/hubspot/singularity/mesos/SingularityOfferCache.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityOfferCache.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityOfferCache.java index a4ceb64c88..1755ef8f66 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityOfferCache.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityOfferCache.java @@ -116,10 +116,8 @@ public List peekOffers() { public void returnOffer(CachedOffer cachedOffer) { synchronized (offerCache) { if (cachedOffer.offerState == OfferState.EXPIRED) { - LOG.trace("Declining returned offer {}", cachedOffer.getOfferId()); declineOffer(cachedOffer); } else { - LOG.trace("Returning offer {} to cache", cachedOffer.getOfferId()); cachedOffer.checkIn(); } }