diff --git a/src/com/facebook/buck/event/listener/AbstractConsoleEventBusListener.java b/src/com/facebook/buck/event/listener/AbstractConsoleEventBusListener.java index 9f1f49c56df..e2b7fdfe402 100644 --- a/src/com/facebook/buck/event/listener/AbstractConsoleEventBusListener.java +++ b/src/com/facebook/buck/event/listener/AbstractConsoleEventBusListener.java @@ -21,10 +21,6 @@ import com.facebook.buck.core.model.BuildId; import com.facebook.buck.core.model.UnflavoredBuildTarget; import com.facebook.buck.core.util.log.Logger; -import com.facebook.buck.distributed.DistBuildStatus; -import com.facebook.buck.distributed.DistBuildStatusEvent; -import com.facebook.buck.distributed.build_client.DistBuildRemoteProgressEvent; -import com.facebook.buck.distributed.thrift.CoordinatorBuildProgress; import com.facebook.buck.event.ActionGraphEvent; import com.facebook.buck.event.BuckEvent; import com.facebook.buck.event.BuckEventBus; @@ -40,6 +36,7 @@ import com.facebook.buck.event.listener.stats.cache.RemoteArtifactUploadStats; import com.facebook.buck.event.listener.stats.cache.RemoteDownloadStats; import com.facebook.buck.event.listener.stats.parse.ParseStatsTracker; +import com.facebook.buck.event.listener.stats.stampede.DistBuildStatsTracker; import com.facebook.buck.event.listener.util.EventInterval; import com.facebook.buck.event.listener.util.ProgressEstimator; import com.facebook.buck.test.TestRuleEvent; @@ -81,7 +78,6 @@ import java.util.logging.Level; import java.util.stream.Stream; import javax.annotation.Nullable; -import javax.annotation.concurrent.GuardedBy; import org.stringtemplate.v4.ST; /** @@ -145,17 +141,10 @@ public abstract class AbstractConsoleEventBusListener implements BuckEventListen protected final NetworkStatsTracker networkStatsTracker; protected final ParseStatsTracker parseStats; - - protected volatile int distBuildTotalRulesCount = 0; - protected volatile int distBuildFinishedRulesCount = 0; + protected final DistBuildStatsTracker distStatsTracker; protected BuildRuleThreadTracker buildRuleThreadTracker; - protected final Object distBuildStatusLock = new Object(); - - @GuardedBy("distBuildStatusLock") - protected Optional distBuildStatus = Optional.empty(); - /** Commands that should print out the build details, if provided */ protected final ImmutableSet buildDetailsCommands = ImmutableSet.of("build", "test", "install"); @@ -173,6 +162,7 @@ public AbstractConsoleEventBusListener( this.console = console; this.parseStats = new ParseStatsTracker(); this.networkStatsTracker = new NetworkStatsTracker(); + this.distStatsTracker = new DistBuildStatsTracker(); this.clock = clock; this.locale = locale; this.ansi = console.getAnsi(); @@ -206,6 +196,7 @@ public void register(BuckEventBus buildEventBus) { buildEventBus.register(this); buildEventBus.register(parseStats); buildEventBus.register(networkStatsTracker); + buildEventBus.register(distStatsTracker); } public static String getBuildDetailsLine(BuildId buildId, String buildDetailsTemplate) { @@ -238,15 +229,6 @@ protected String formatElapsedTime(long elapsedTimeMs) { return minutes == 0 ? String.valueOf(seconds) : String.format("%2$dm %1$s", seconds, minutes); } - protected Optional getApproximateDistBuildProgress() { - if (distBuildTotalRulesCount == 0) { - return Optional.of(0.0); - } - - double buildRatio = (double) distBuildFinishedRulesCount / distBuildTotalRulesCount; - return Optional.of(Math.floor(100 * buildRatio) / 100.0); - } - /** Local build progress. */ protected Optional getApproximateLocalBuildProgress() { if (progressEstimator.isPresent()) { @@ -258,7 +240,7 @@ protected Optional getApproximateLocalBuildProgress() { protected Optional getApproximateBuildProgress() { if (distBuildStarted != null && distBuildFinished == null) { - return getApproximateDistBuildProgress(); + return distStatsTracker.getApproximateProgress(); } else { return getApproximateLocalBuildProgress(); } @@ -880,22 +862,6 @@ public void installFinished(InstallEvent.Finished finished) { installFinished = finished; } - @Subscribe - public void onDistBuildStatusEvent(DistBuildStatusEvent event) { - synchronized (distBuildStatusLock) { - distBuildStatus = Optional.of(event.getStatus()); - } - } - - /** Update distributed build progress. */ - @Subscribe - public void onDistBuildProgressEvent(DistBuildRemoteProgressEvent event) { - CoordinatorBuildProgress buildProgress = event.getBuildProgress(); - distBuildTotalRulesCount = - buildProgress.getTotalRulesCount() - buildProgress.getSkippedRulesCount(); - distBuildFinishedRulesCount = buildProgress.getBuiltRulesCount(); - } - @Subscribe public void commandFinished(CommandEvent.Finished event) { commandFinished = event; diff --git a/src/com/facebook/buck/event/listener/BUCK b/src/com/facebook/buck/event/listener/BUCK index 793999d9659..970e24b8d95 100644 --- a/src/com/facebook/buck/event/listener/BUCK +++ b/src/com/facebook/buck/event/listener/BUCK @@ -39,6 +39,7 @@ java_immutables_library( "//src/com/facebook/buck/event/listener/interfaces:interfaces", "//src/com/facebook/buck/event/listener/stats/cache:cache", "//src/com/facebook/buck/event/listener/stats/parse:parse", + "//src/com/facebook/buck/event/listener/stats/stampede:stampede", "//src/com/facebook/buck/event/listener/util:util", "//src/com/facebook/buck/httpserver:httpserver", "//src/com/facebook/buck/io:io", diff --git a/src/com/facebook/buck/event/listener/SimpleConsoleEventBusListener.java b/src/com/facebook/buck/event/listener/SimpleConsoleEventBusListener.java index bdabce4135f..e7e33ca1e5a 100644 --- a/src/com/facebook/buck/event/listener/SimpleConsoleEventBusListener.java +++ b/src/com/facebook/buck/event/listener/SimpleConsoleEventBusListener.java @@ -26,16 +26,15 @@ import com.facebook.buck.core.test.event.TestStatusMessageEvent; import com.facebook.buck.distributed.DistBuildCreatedEvent; import com.facebook.buck.distributed.DistBuildRunEvent; -import com.facebook.buck.distributed.DistBuildStatusEvent; import com.facebook.buck.distributed.build_client.StampedeConsoleEvent; import com.facebook.buck.distributed.thrift.BuildSlaveInfo; -import com.facebook.buck.distributed.thrift.BuildSlaveRunId; import com.facebook.buck.distributed.thrift.BuildStatus; import com.facebook.buck.event.ActionGraphEvent; import com.facebook.buck.event.CommandEvent; import com.facebook.buck.event.ConsoleEvent; import com.facebook.buck.event.InstallEvent; import com.facebook.buck.event.listener.interfaces.AdditionalConsoleLineProvider; +import com.facebook.buck.event.listener.stats.stampede.DistBuildStatsTracker; import com.facebook.buck.event.listener.util.EventInterval; import com.facebook.buck.test.TestStatusMessage; import com.facebook.buck.test.config.TestResultSummaryVerbosity; @@ -47,11 +46,8 @@ import com.google.common.eventbus.Subscribe; import java.nio.file.Path; import java.util.Collection; -import java.util.LinkedHashMap; import java.util.Locale; -import java.util.Map; import java.util.Optional; -import javax.annotation.concurrent.GuardedBy; /** * Implementation of {@code AbstractConsoleEventBusListener} for terminals that don't support ansi. @@ -66,11 +62,6 @@ public class SimpleConsoleEventBusListener extends AbstractConsoleEventBusListen private final boolean hideSucceededRules; public final ImmutableList buildFinishedLineProvider; - @GuardedBy("distBuildSlaveTracker") - private final Map distBuildSlaveTracker; - - private volatile Optional stampedeBuildStatus = Optional.empty(); - public SimpleConsoleEventBusListener( RenderingConsole console, Clock clock, @@ -109,8 +100,6 @@ public SimpleConsoleEventBusListener( locale, Optional.of(testLogPath)); - this.distBuildSlaveTracker = new LinkedHashMap<>(); - if (printBuildId) { printLines(ImmutableList.of(getBuildLogLine(buildId))); } @@ -121,6 +110,31 @@ public SimpleConsoleEventBusListener( this.parseStats.registerListener(this::parseFinished); this.networkStatsTracker.registerListener(this::onCacheUploadsFinished); + this.distStatsTracker.registerListener( + new DistBuildStatsTracker.Listener() { + @Override + public void onWorkerJoined(BuildSlaveInfo slaveInfo) { + printLine( + "STAMPEDE WORKER [%s][%s] JOINED BUILD WITH STATUS [%s]", + slaveInfo.getHostname(), + slaveInfo.getBuildSlaveRunId().getId(), + slaveInfo.getStatus().name()); + } + + @Override + public void onWorkerStatusChanged(BuildSlaveInfo slaveInfo) { + printLine( + "STAMPEDE WORKER [%s][%s] CHANGED STATUS TO [%s]", + slaveInfo.getHostname(), + slaveInfo.getBuildSlaveRunId().getId(), + slaveInfo.getStatus().name()); + } + + @Override + public void onDistBuildStateChanged(BuildStatus distBuildState) { + printLine("STAMPEDE JOB STATUS CHANGED TO [%s]", distBuildState); + } + }); } private void parseFinished() { @@ -243,48 +257,6 @@ public void logStampedeConsoleEvent(StampedeConsoleEvent event) { logEvent(event.getConsoleEvent()); } - @Override - @Subscribe - public void onDistBuildStatusEvent(DistBuildStatusEvent event) { - super.onDistBuildStatusEvent(event); - - ImmutableList.Builder builder = ImmutableList.builder(); - - BuildStatus newJobStatus = event.getJob().getStatus(); - if (!stampedeBuildStatus.isPresent() || !stampedeBuildStatus.get().equals(newJobStatus)) { - builder.add(String.format("STAMPEDE JOB STATUS CHANGED TO [%s]", newJobStatus)); - } - stampedeBuildStatus = Optional.of(newJobStatus); - - synchronized (distBuildSlaveTracker) { - // Don't track the status of failed or lost minions - for (BuildSlaveInfo slaveInfo : event.getJob().getBuildSlaves()) { - if (!distBuildSlaveTracker.containsKey(slaveInfo.getBuildSlaveRunId())) { - builder.add( - String.format( - "STAMPEDE WORKER [%s][%s] JOINED BUILD WITH STATUS [%s]", - slaveInfo.getHostname(), - slaveInfo.getBuildSlaveRunId().getId(), - slaveInfo.getStatus().name())); - } else { - BuildStatus existingStatus = distBuildSlaveTracker.get(slaveInfo.getBuildSlaveRunId()); - if (!existingStatus.equals(slaveInfo.getStatus())) { - builder.add( - String.format( - "STAMPEDE WORKER [%s][%s] CHANGED STATUS TO [%s]", - slaveInfo.getHostname(), - slaveInfo.getBuildSlaveRunId().getId(), - slaveInfo.getStatus().name())); - } - } - - distBuildSlaveTracker.put(slaveInfo.getBuildSlaveRunId(), slaveInfo.getStatus()); - } - } - - printLines(builder); - } - @Override public void printSevereWarningDirectly(String line) { logEvent(ConsoleEvent.severe(line)); diff --git a/src/com/facebook/buck/event/listener/SuperConsoleEventBusListener.java b/src/com/facebook/buck/event/listener/SuperConsoleEventBusListener.java index 3e0d3619e7a..84219f60d98 100644 --- a/src/com/facebook/buck/event/listener/SuperConsoleEventBusListener.java +++ b/src/com/facebook/buck/event/listener/SuperConsoleEventBusListener.java @@ -23,14 +23,8 @@ import com.facebook.buck.core.test.event.TestSummaryEvent; import com.facebook.buck.core.util.log.Logger; import com.facebook.buck.distributed.DistBuildCreatedEvent; -import com.facebook.buck.distributed.DistBuildStatusEvent; -import com.facebook.buck.distributed.StampedeLocalBuildStatusEvent; import com.facebook.buck.distributed.build_client.DistBuildSuperConsoleEvent; import com.facebook.buck.distributed.build_client.StampedeConsoleEvent; -import com.facebook.buck.distributed.thrift.BuildSlaveInfo; -import com.facebook.buck.distributed.thrift.BuildSlaveRunId; -import com.facebook.buck.distributed.thrift.BuildSlaveStatus; -import com.facebook.buck.distributed.thrift.BuildStatus; import com.facebook.buck.event.ActionGraphEvent; import com.facebook.buck.event.ArtifactCompressionEvent; import com.facebook.buck.event.CommandEvent.Finished; @@ -44,6 +38,8 @@ import com.facebook.buck.event.WatchmanStatusEvent; import com.facebook.buck.event.listener.interfaces.AdditionalConsoleLineProvider; import com.facebook.buck.event.listener.stats.cache.CacheRateStatsKeeper; +import com.facebook.buck.event.listener.stats.cache.CacheRateStatsKeeper.CacheRateStatsUpdateEvent; +import com.facebook.buck.event.listener.stats.stampede.DistBuildTrackedStatus; import com.facebook.buck.event.listener.util.EventInterval; import com.facebook.buck.step.StepEvent; import com.facebook.buck.test.TestResultSummary; @@ -68,7 +64,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -84,7 +79,6 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.logging.Level; -import javax.annotation.concurrent.GuardedBy; /** Console that provides rich, updating ansi output about the current build. */ public class SuperConsoleEventBusListener extends AbstractConsoleEventBusListener { @@ -99,11 +93,13 @@ public class SuperConsoleEventBusListener extends AbstractConsoleEventBusListene private final Locale locale; private final Function formatTimeFunction; + + private final ConcurrentMap> threadsToRunningStep; + private final ConcurrentMap> threadsToRunningTestSummaryEvent; private final ConcurrentMap> threadsToRunningTestStatusMessageEvent; - private final ConcurrentMap> threadsToRunningStep; private final TestResultFormatter testFormatter; @@ -135,18 +131,11 @@ public class SuperConsoleEventBusListener extends AbstractConsoleEventBusListene // way the user can know that their changes, if they made any, were not picked up from Watchman. private boolean isZeroFileChanges = false; - private final Object distBuildSlaveTrackerLock = new Object(); - private long minimumDurationMillisecondsToShowParse; private long minimumDurationMillisecondsToShowActionGraph; private long minimumDurationMillisecondsToShowWatchman; private boolean hideEmptyDownload; - @GuardedBy("distBuildSlaveTrackerLock") - private final Map distBuildSlaveTracker; - - private volatile StampedeLocalBuildStatusEvent stampedeLocalBuildStatus = - new StampedeLocalBuildStatusEvent("init"); private volatile Optional stampedeSuperConsoleEvent = Optional.empty(); private Optional stampedeIdLogLine = Optional.empty(); @@ -253,9 +242,6 @@ public SuperConsoleEventBusListener( DateFormat dateFormat = new SimpleDateFormat("[yyyy-MM-dd HH:mm:ss.SSS]", this.locale); dateFormat.setTimeZone(timeZone); - // Using LinkedHashMap because we want a predictable iteration order. - this.distBuildSlaveTracker = new LinkedHashMap<>(); - int outputMaxColumns = 80; if (config.getThreadLineOutputMaxColumns().isPresent()) { outputMaxColumns = config.getThreadLineOutputMaxColumns().getAsInt(); @@ -356,7 +342,7 @@ public ImmutableList createRenderLinesAtTime(long currentTimeMillis) { String localBuildLinePrefix = "Building"; if (stampedeSuperConsoleEvent.isPresent()) { - localBuildLinePrefix = stampedeLocalBuildStatus.getLocalBuildLinePrefix(); + localBuildLinePrefix = distStatsTracker.getLocalBuildLinePrefix(); stampedeIdLogLine.ifPresent(lines::add); stampedeSuperConsoleEvent @@ -371,17 +357,16 @@ public ImmutableList createRenderLinesAtTime(long currentTimeMillis) { 0, this.distBuildStarted, this.distBuildFinished, - getApproximateDistBuildProgress(), + distStatsTracker.getApproximateProgress(), Optional.empty(), lines); if (distBuildMs == UNFINISHED_EVENT_PAIR) { MultiStateRenderer renderer; - synchronized (distBuildSlaveTrackerLock) { - renderer = - new DistBuildSlaveStateRenderer( - ansi, currentTimeMillis, ImmutableList.copyOf(distBuildSlaveTracker.values())); - } + + renderer = + new DistBuildSlaveStateRenderer( + ansi, currentTimeMillis, distStatsTracker.getSlaveStatuses()); renderLines(renderer, lines, maxThreadLines, true); } @@ -529,57 +514,48 @@ private void getTotalTimeLine(ImmutableList.Builder lines) { private Optional getOptionalDistBuildLineSuffix() { List columns = new ArrayList<>(); + DistBuildTrackedStatus distBuildStatus = distStatsTracker.getTrackedStatus(); - synchronized (distBuildStatusLock) { - if (!distBuildStatus.isPresent()) { - columns.add("remote status: init"); - } else { - distBuildStatus - .get() - .getStatus() - .ifPresent(status -> columns.add("remote status: " + status.toLowerCase())); - - int totalUploadErrorsCount = 0; - ImmutableList.Builder slaveCacheStats = - new ImmutableList.Builder<>(); - - for (BuildSlaveStatus slaveStatus : distBuildStatus.get().getSlaveStatuses()) { - totalUploadErrorsCount += slaveStatus.getHttpArtifactUploadsFailureCount(); - - if (slaveStatus.isSetCacheRateStats()) { - slaveCacheStats.add( - CacheRateStatsKeeper.getCacheRateStatsUpdateEventFromSerializedStats( - slaveStatus.getCacheRateStats())); - } - } + if (!distBuildStatus.hasRemoteStatus()) { + columns.add("remote status: init"); + } else { + distBuildStatus + .getStatus() + .ifPresent(status -> columns.add("remote status: " + status.toLowerCase())); - if (distBuildTotalRulesCount > 0) { - columns.add( - String.format("%d/%d jobs", distBuildFinishedRulesCount, distBuildTotalRulesCount)); - } + int totalUploadErrorsCount = 0; + for (CacheRateStatsUpdateEvent slaveCacheStat : distBuildStatus.getSlaveCacheStats()) { + totalUploadErrorsCount += slaveCacheStat.getCacheErrorCount(); + } - CacheRateStatsKeeper.CacheRateStatsUpdateEvent aggregatedCacheStats = - CacheRateStatsKeeper.getAggregatedCacheRateStats(slaveCacheStats.build()); + if (distBuildStatus.getTotalRulesCount() > 0) { + columns.add( + String.format( + "%d/%d jobs", + distBuildStatus.getFinishedRulesCount(), distBuildStatus.getTotalRulesCount())); + } - if (aggregatedCacheStats.getTotalRulesCount() != 0) { - columns.add(String.format("%.1f%% cache miss", aggregatedCacheStats.getCacheMissRate())); + CacheRateStatsKeeper.CacheRateStatsUpdateEvent aggregatedCacheStats = + CacheRateStatsKeeper.getAggregatedCacheRateStats(distBuildStatus.getSlaveCacheStats()); - if (aggregatedCacheStats.getCacheErrorCount() != 0) { - columns.add( - String.format( - "%d [%.1f%%] cache errors", - aggregatedCacheStats.getCacheErrorCount(), - aggregatedCacheStats.getCacheErrorRate())); - } - } + if (aggregatedCacheStats.getTotalRulesCount() != 0) { + columns.add(String.format("%.1f%% cache miss", aggregatedCacheStats.getCacheMissRate())); - if (totalUploadErrorsCount > 0) { - columns.add(String.format("%d upload errors", totalUploadErrorsCount)); + if (aggregatedCacheStats.getCacheErrorCount() != 0) { + columns.add( + String.format( + "%d [%.1f%%] cache errors", + aggregatedCacheStats.getCacheErrorCount(), + aggregatedCacheStats.getCacheErrorRate())); } } + + if (totalUploadErrorsCount > 0) { + columns.add(String.format("%d upload errors", totalUploadErrorsCount)); + } } - String localStatus = String.format("local status: %s", stampedeLocalBuildStatus.getStatus()); + String localStatus = String.format("local status: %s", distBuildStatus.getLocalStatus()); String remoteStatusAndSummary = String.join(", ", columns); if (remoteStatusAndSummary.length() == 0) { return Optional.of(localStatus); @@ -708,30 +684,6 @@ public void ruleKeyCalculationFinished(RuleKeyCalculationEvent.Finished finished runningStepFinished(finished.getThreadId()); } - @Override - @Subscribe - public void onDistBuildStatusEvent(DistBuildStatusEvent event) { - super.onDistBuildStatusEvent(event); - synchronized (distBuildSlaveTrackerLock) { - for (BuildSlaveStatus status : event.getStatus().getSlaveStatuses()) { - distBuildSlaveTracker.put(status.buildSlaveRunId, status); - } - - // Don't track the status of failed or lost minions - for (BuildSlaveInfo slaveInfo : event.getJob().getBuildSlaves()) { - if (slaveInfo.getStatus().equals(BuildStatus.FAILED) - || slaveInfo.getStatus().equals(BuildStatus.LOST)) { - distBuildSlaveTracker.remove(slaveInfo.buildSlaveRunId); - } - } - } - } - - @Subscribe - public void onStampedeLocalBuildStatusEvent(StampedeLocalBuildStatusEvent event) { - this.stampedeLocalBuildStatus = event; - } - @Subscribe public void onDistBuildCreatedEvent(DistBuildCreatedEvent event) { stampedeIdLogLine = Optional.of(event.getConsoleLogLine()); diff --git a/src/com/facebook/buck/event/listener/stats/stampede/AbstractDistBuildTrackedStatus.java b/src/com/facebook/buck/event/listener/stats/stampede/AbstractDistBuildTrackedStatus.java new file mode 100644 index 00000000000..ff2b9ba42d4 --- /dev/null +++ b/src/com/facebook/buck/event/listener/stats/stampede/AbstractDistBuildTrackedStatus.java @@ -0,0 +1,39 @@ +/* + * Copyright 2019-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.facebook.buck.event.listener.stats.stampede; + +import com.facebook.buck.core.util.immutables.BuckStyleTuple; +import com.facebook.buck.event.listener.stats.cache.CacheRateStatsKeeper.CacheRateStatsUpdateEvent; +import com.google.common.collect.ImmutableList; +import java.util.Optional; +import org.immutables.value.Value; + +/** Holds data about the distributed build status. */ +@Value.Immutable +@BuckStyleTuple +interface AbstractDistBuildTrackedStatus { + boolean hasRemoteStatus(); + + Optional getStatus(); + + String getLocalStatus(); + + int getTotalRulesCount(); + + int getFinishedRulesCount(); + + ImmutableList getSlaveCacheStats(); +} diff --git a/src/com/facebook/buck/event/listener/stats/stampede/BUCK b/src/com/facebook/buck/event/listener/stats/stampede/BUCK new file mode 100644 index 00000000000..12174dc1e3b --- /dev/null +++ b/src/com/facebook/buck/event/listener/stats/stampede/BUCK @@ -0,0 +1,23 @@ +load( + "//tools/build_rules:java_rules.bzl", + "java_immutables_library", +) + +java_immutables_library( + name = "stampede", + srcs = glob(["*.java"]), + visibility = [ + "PUBLIC", + ], + deps = [ + "//src/com/facebook/buck/core/build/event:event", + "//src/com/facebook/buck/distributed:common", + "//src/com/facebook/buck/distributed/build_client:build_client", + "//src/com/facebook/buck/distributed/build_slave:build_slave", + "//src/com/facebook/buck/event:interfaces", + "//src/com/facebook/buck/event/external:external_lib", + "//src/com/facebook/buck/event/listener/stats/cache:cache", + "//src/com/facebook/buck/event/listener/util:util", + "//third-party/java/guava:guava", + ], +) diff --git a/src/com/facebook/buck/event/listener/stats/stampede/DistBuildStatsTracker.java b/src/com/facebook/buck/event/listener/stats/stampede/DistBuildStatsTracker.java new file mode 100644 index 00000000000..c3248d57ff8 --- /dev/null +++ b/src/com/facebook/buck/event/listener/stats/stampede/DistBuildStatsTracker.java @@ -0,0 +1,177 @@ +/* + * Copyright 2019-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.facebook.buck.event.listener.stats.stampede; + +import com.facebook.buck.distributed.DistBuildStatus; +import com.facebook.buck.distributed.DistBuildStatusEvent; +import com.facebook.buck.distributed.StampedeLocalBuildStatusEvent; +import com.facebook.buck.distributed.build_client.DistBuildRemoteProgressEvent; +import com.facebook.buck.distributed.thrift.BuildSlaveInfo; +import com.facebook.buck.distributed.thrift.BuildSlaveRunId; +import com.facebook.buck.distributed.thrift.BuildSlaveStatus; +import com.facebook.buck.distributed.thrift.BuildStatus; +import com.facebook.buck.distributed.thrift.CoordinatorBuildProgress; +import com.facebook.buck.event.listener.stats.cache.CacheRateStatsKeeper; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.eventbus.Subscribe; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentLinkedQueue; +import javax.annotation.concurrent.GuardedBy; + +/** Tracks distbuild related events and maintains stats about the build. */ +public class DistBuildStatsTracker { + + /** The Listener gets callbacks for interesting events. */ + public interface Listener { + void onWorkerJoined(BuildSlaveInfo slaveInfo); + + void onWorkerStatusChanged(BuildSlaveInfo slaveInfo); + + void onDistBuildStateChanged(BuildStatus distBuildState); + } + + @VisibleForTesting volatile int distBuildTotalRulesCount = 0; + @VisibleForTesting volatile int distBuildFinishedRulesCount = 0; + + private final Object distBuildStatusLock = new Object(); + + @GuardedBy("distBuildStatusLock") + private Optional distBuildStatus = Optional.empty(); + + private BuildStatus distBuildState = BuildStatus.UNKNOWN; + + // TODO(cjhopman): This is so strange that we track two different "Status" objects. + // Using LinkedHashMap because we want a predictable iteration order. + @GuardedBy("distBuildStatusLock") + private final Map distBuildSlaveTracker = new LinkedHashMap<>(); + + @GuardedBy("distBuildStatusLock") + private final Map distBuildSlaveStatusTracker = + new LinkedHashMap<>(); + + private final ConcurrentLinkedQueue listeners = new ConcurrentLinkedQueue<>(); + + private volatile StampedeLocalBuildStatusEvent stampedeLocalBuildStatus = + new StampedeLocalBuildStatusEvent("init"); + + /** Registers a listener. */ + public void registerListener(Listener listener) { + listeners.add(listener); + } + + /** Gets the current build slave statuses. */ + public ImmutableList getSlaveStatuses() { + synchronized (distBuildStatusLock) { + return ImmutableList.copyOf(distBuildSlaveStatusTracker.values()); + } + } + + /** Gets the prefix for the local build line. */ + public String getLocalBuildLinePrefix() { + return stampedeLocalBuildStatus.getLocalBuildLinePrefix(); + } + + /** Gets the current approximate progress of the distributed build. */ + public Optional getApproximateProgress() { + synchronized (distBuildStatusLock) { + if (distBuildTotalRulesCount == 0) { + return Optional.of(0.0); + } + + double buildRatio = (double) distBuildFinishedRulesCount / distBuildTotalRulesCount; + return Optional.of(Math.floor(100 * buildRatio) / 100.0); + } + } + + /** Gets the current tracked distributed build status. */ + public DistBuildTrackedStatus getTrackedStatus() { + synchronized (distBuildStatusLock) { + ImmutableList.Builder slaveCacheStatsBuilder = + ImmutableList.builder(); + + if (distBuildStatus.isPresent()) { + for (BuildSlaveStatus slaveStatus : distBuildStatus.get().getSlaveStatuses()) { + + if (slaveStatus.isSetCacheRateStats()) { + slaveCacheStatsBuilder.add( + CacheRateStatsKeeper.getCacheRateStatsUpdateEventFromSerializedStats( + slaveStatus.getCacheRateStats())); + } + } + } + return DistBuildTrackedStatus.of( + distBuildStatus.isPresent(), + distBuildStatus.flatMap(DistBuildStatus::getStatus), + stampedeLocalBuildStatus.getStatus(), + distBuildTotalRulesCount, + distBuildFinishedRulesCount, + slaveCacheStatsBuilder.build()); + } + } + + @Subscribe + private void onStampedeLocalBuildStatusEvent(StampedeLocalBuildStatusEvent event) { + this.stampedeLocalBuildStatus = event; + } + + /** Update distributed build progress. */ + @Subscribe + private void onDistBuildProgressEvent(DistBuildRemoteProgressEvent event) { + CoordinatorBuildProgress buildProgress = event.getBuildProgress(); + distBuildTotalRulesCount = + buildProgress.getTotalRulesCount() - buildProgress.getSkippedRulesCount(); + distBuildFinishedRulesCount = buildProgress.getBuiltRulesCount(); + } + + @Subscribe + private void onDistBuildStatusEvent(DistBuildStatusEvent event) { + synchronized (distBuildStatusLock) { + distBuildStatus = Optional.of(event.getStatus()); + + BuildStatus previousState = this.distBuildState; + this.distBuildState = event.getJob().getStatus(); + if (previousState != distBuildState) { + listeners.forEach(listener -> listener.onDistBuildStateChanged(distBuildState)); + } + + for (BuildSlaveStatus slaveStatus : event.getStatus().getSlaveStatuses()) { + // Don't track the status of failed or lost minions + distBuildSlaveStatusTracker.put(slaveStatus.getBuildSlaveRunId(), slaveStatus); + } + + for (BuildSlaveInfo slaveInfo : event.getJob().getBuildSlaves()) { + if (!distBuildSlaveTracker.containsKey(slaveInfo.getBuildSlaveRunId())) { + listeners.forEach(listener -> listener.onWorkerJoined(slaveInfo)); + } else { + BuildStatus existingStatus = distBuildSlaveTracker.get(slaveInfo.getBuildSlaveRunId()); + if (!existingStatus.equals(slaveInfo.getStatus())) { + listeners.forEach(listener -> listener.onWorkerStatusChanged(slaveInfo)); + } + } + distBuildSlaveTracker.put(slaveInfo.getBuildSlaveRunId(), slaveInfo.getStatus()); + if (slaveInfo.getStatus().equals(BuildStatus.FAILED) + || slaveInfo.getStatus().equals(BuildStatus.LOST)) { + synchronized (distBuildSlaveStatusTracker) { + distBuildSlaveStatusTracker.remove(slaveInfo.buildSlaveRunId); + } + } + } + } + } +} diff --git a/test/com/facebook/buck/event/listener/AbstractConsoleEventBusListenerTest.java b/test/com/facebook/buck/event/listener/AbstractConsoleEventBusListenerTest.java index 4e7e29e358d..7725d124913 100644 --- a/test/com/facebook/buck/event/listener/AbstractConsoleEventBusListenerTest.java +++ b/test/com/facebook/buck/event/listener/AbstractConsoleEventBusListenerTest.java @@ -28,7 +28,6 @@ import java.util.Collection; import java.util.List; import java.util.Locale; -import java.util.Optional; import java.util.OptionalLong; import org.junit.Test; @@ -50,23 +49,6 @@ public void printSevereWarningDirectly(String line) {} }; } - @Test - public void testApproximateDistBuildProgressDoesNotLosePrecision() { - AbstractConsoleEventBusListener listener = createAbstractConsoleInstance(); - - listener.distBuildTotalRulesCount = 0; - listener.distBuildFinishedRulesCount = 0; - assertEquals(Optional.of(0.0), listener.getApproximateDistBuildProgress()); - - listener.distBuildTotalRulesCount = 100; - listener.distBuildFinishedRulesCount = 50; - assertEquals(Optional.of(0.5), listener.getApproximateDistBuildProgress()); - - listener.distBuildTotalRulesCount = 17; - listener.distBuildFinishedRulesCount = 4; - assertEquals(Optional.of(0.23), listener.getApproximateDistBuildProgress()); - } - @Test public void testGetEventsBetween() { EventInterval zeroToOneHundred = EventInterval.proxy(0, 100); diff --git a/test/com/facebook/buck/event/listener/BUCK b/test/com/facebook/buck/event/listener/BUCK index ed6a7ff7dd6..1f14aff4435 100644 --- a/test/com/facebook/buck/event/listener/BUCK +++ b/test/com/facebook/buck/event/listener/BUCK @@ -149,6 +149,7 @@ standard_java_test( "//src/com/facebook/buck/event/listener:listener", "//src/com/facebook/buck/event/listener/interfaces:interfaces", "//src/com/facebook/buck/event/listener/stats/cache:cache", + "//src/com/facebook/buck/event/listener/stats/stampede:stampede", "//src/com/facebook/buck/event/listener/util:util", "//src/com/facebook/buck/httpserver:httpserver", "//src/com/facebook/buck/io:executable-finder", diff --git a/test/com/facebook/buck/event/listener/stats/stampede/BUCK b/test/com/facebook/buck/event/listener/stats/stampede/BUCK new file mode 100644 index 00000000000..9837a1f3284 --- /dev/null +++ b/test/com/facebook/buck/event/listener/stats/stampede/BUCK @@ -0,0 +1,17 @@ +load( + "//tools/build_rules:java_rules.bzl", + "standard_java_test", +) + +standard_java_test( + name = "listener", + deps = [ + "//src-gen:thrift", + "//src/com/facebook/buck/core/build/event:event", + "//src/com/facebook/buck/distributed:common", + "//src/com/facebook/buck/distributed/build_client:build_client", + "//src/com/facebook/buck/event/listener/stats/stampede:stampede", + "//test/com/facebook/buck/event:testutil", + "//third-party/java/junit:junit", + ], +) diff --git a/test/com/facebook/buck/event/listener/stats/stampede/DistBuildStatsTrackerTest.java b/test/com/facebook/buck/event/listener/stats/stampede/DistBuildStatsTrackerTest.java new file mode 100644 index 00000000000..4477db1df7d --- /dev/null +++ b/test/com/facebook/buck/event/listener/stats/stampede/DistBuildStatsTrackerTest.java @@ -0,0 +1,52 @@ +/* + * Copyright 2019-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.facebook.buck.event.listener.stats.stampede; + +import static org.junit.Assert.assertEquals; + +import com.facebook.buck.distributed.build_client.DistBuildRemoteProgressEvent; +import com.facebook.buck.distributed.thrift.CoordinatorBuildProgress; +import com.facebook.buck.event.BuckEvent; +import com.facebook.buck.event.BuckEventBus; +import com.facebook.buck.event.BuckEventBusForTests; +import java.util.Optional; +import org.junit.Test; + +public class DistBuildStatsTrackerTest { + @Test + public void testApproximateDistBuildProgressDoesNotLosePrecision() { + DistBuildStatsTracker tracker = new DistBuildStatsTracker(); + BuckEventBus eventBus = BuckEventBusForTests.newInstance(); + eventBus.register(tracker); + + eventBus.post(progressEvent(100, 0, 0)); + assertEquals(Optional.of(0.0), tracker.getApproximateProgress()); + + eventBus.post(progressEvent(100, 2, 0)); + assertEquals(Optional.of(0.02), tracker.getApproximateProgress()); + + eventBus.post(progressEvent(100, 4, 83)); + assertEquals(Optional.of(0.23), tracker.getApproximateProgress()); + } + + private BuckEvent progressEvent(int totalRules, int builtRules, int skippedRules) { + CoordinatorBuildProgress coordinatorProgress = new CoordinatorBuildProgress(); + coordinatorProgress.setTotalRulesCount(totalRules); + coordinatorProgress.setBuiltRulesCount(builtRules); + coordinatorProgress.setSkippedRulesCount(skippedRules); + return new DistBuildRemoteProgressEvent(coordinatorProgress); + } +}