Skip to content
This repository has been archived by the owner on Nov 10, 2023. It is now read-only.

Commit

Permalink
Extract DistBuildStatsTracker
Browse files Browse the repository at this point in the history 
Summary:
This extracts the tracking of dist build events to a separate stats tracker
object. The console eventbus listeners get the current state of
the distributed build from this stats tracker. This better
separates concerns.

Reviewed By: ruibm

fbshipit-source-id: 081fc2ff6f
  • Loading branch information
@cjhopman @facebook-github-bot
cjhopman authored and facebook-github-bot committed Feb 16, 2019
1 parent d3b281d commit 074240a
Show file tree
Hide file tree
Showing 11 changed files with 384 additions and 202 deletions.
44 changes: 5 additions & 39 deletions src/com/facebook/buck/event/listener/AbstractConsoleEventBusListener.java
View file
Expand Up @@ -21,10 +21,6 @@
import com.facebook.buck.core.model.BuildId;
import com.facebook.buck.core.model.UnflavoredBuildTarget;
import com.facebook.buck.core.util.log.Logger;
import com.facebook.buck.distributed.DistBuildStatus;
import com.facebook.buck.distributed.DistBuildStatusEvent;
import com.facebook.buck.distributed.build_client.DistBuildRemoteProgressEvent;
import com.facebook.buck.distributed.thrift.CoordinatorBuildProgress;
import com.facebook.buck.event.ActionGraphEvent;
import com.facebook.buck.event.BuckEvent;
import com.facebook.buck.event.BuckEventBus;
Expand All @@ -40,6 +36,7 @@
import com.facebook.buck.event.listener.stats.cache.RemoteArtifactUploadStats;
import com.facebook.buck.event.listener.stats.cache.RemoteDownloadStats;
import com.facebook.buck.event.listener.stats.parse.ParseStatsTracker;
import com.facebook.buck.event.listener.stats.stampede.DistBuildStatsTracker;
import com.facebook.buck.event.listener.util.EventInterval;
import com.facebook.buck.event.listener.util.ProgressEstimator;
import com.facebook.buck.test.TestRuleEvent;
Expand Down Expand Up @@ -81,7 +78,6 @@
import java.util.logging.Level;
import java.util.stream.Stream;
import javax.annotation.Nullable;
import javax.annotation.concurrent.GuardedBy;
import org.stringtemplate.v4.ST;

/**
Expand Down Expand Up @@ -145,17 +141,10 @@ public abstract class AbstractConsoleEventBusListener implements BuckEventListen

protected final NetworkStatsTracker networkStatsTracker;
protected final ParseStatsTracker parseStats;

protected volatile int distBuildTotalRulesCount = 0;
protected volatile int distBuildFinishedRulesCount = 0;
protected final DistBuildStatsTracker distStatsTracker;

protected BuildRuleThreadTracker buildRuleThreadTracker;

protected final Object distBuildStatusLock = new Object();

@GuardedBy("distBuildStatusLock")
protected Optional<DistBuildStatus> distBuildStatus = Optional.empty();

/** Commands that should print out the build details, if provided */
protected final ImmutableSet<String> buildDetailsCommands =
ImmutableSet.of("build", "test", "install");
Expand All @@ -173,6 +162,7 @@ public AbstractConsoleEventBusListener(
this.console = console;
this.parseStats = new ParseStatsTracker();
this.networkStatsTracker = new NetworkStatsTracker();
this.distStatsTracker = new DistBuildStatsTracker();
this.clock = clock;
this.locale = locale;
this.ansi = console.getAnsi();
Expand Down Expand Up @@ -206,6 +196,7 @@ public void register(BuckEventBus buildEventBus) {
buildEventBus.register(this);
buildEventBus.register(parseStats);
buildEventBus.register(networkStatsTracker);
buildEventBus.register(distStatsTracker);
}

public static String getBuildDetailsLine(BuildId buildId, String buildDetailsTemplate) {
Expand Down Expand Up @@ -238,15 +229,6 @@ protected String formatElapsedTime(long elapsedTimeMs) {
return minutes == 0 ? String.valueOf(seconds) : String.format("%2$dm %1$s", seconds, minutes);
}

protected Optional<Double> getApproximateDistBuildProgress() {
if (distBuildTotalRulesCount == 0) {
return Optional.of(0.0);
}

double buildRatio = (double) distBuildFinishedRulesCount / distBuildTotalRulesCount;
return Optional.of(Math.floor(100 * buildRatio) / 100.0);
}

/** Local build progress. */
protected Optional<Double> getApproximateLocalBuildProgress() {
if (progressEstimator.isPresent()) {
Expand All @@ -258,7 +240,7 @@ protected Optional<Double> getApproximateLocalBuildProgress() {

protected Optional<Double> getApproximateBuildProgress() {
if (distBuildStarted != null && distBuildFinished == null) {
return getApproximateDistBuildProgress();
return distStatsTracker.getApproximateProgress();
} else {
return getApproximateLocalBuildProgress();
}
Expand Down Expand Up @@ -880,22 +862,6 @@ public void installFinished(InstallEvent.Finished finished) {
installFinished = finished;
}

@Subscribe
public void onDistBuildStatusEvent(DistBuildStatusEvent event) {
synchronized (distBuildStatusLock) {
distBuildStatus = Optional.of(event.getStatus());
}
}

/** Update distributed build progress. */
@Subscribe
public void onDistBuildProgressEvent(DistBuildRemoteProgressEvent event) {
CoordinatorBuildProgress buildProgress = event.getBuildProgress();
distBuildTotalRulesCount =
buildProgress.getTotalRulesCount() - buildProgress.getSkippedRulesCount();
distBuildFinishedRulesCount = buildProgress.getBuiltRulesCount();
}

@Subscribe
public void commandFinished(CommandEvent.Finished event) {
commandFinished = event;
Expand Down
1 change: 1 addition & 0 deletions src/com/facebook/buck/event/listener/BUCK
View file
Expand Up @@ -39,6 +39,7 @@ java_immutables_library(
"//src/com/facebook/buck/event/listener/interfaces:interfaces",
"//src/com/facebook/buck/event/listener/stats/cache:cache",
"//src/com/facebook/buck/event/listener/stats/parse:parse",
"//src/com/facebook/buck/event/listener/stats/stampede:stampede",
"//src/com/facebook/buck/event/listener/util:util",
"//src/com/facebook/buck/httpserver:httpserver",
"//src/com/facebook/buck/io:io",
Expand Down
80 changes: 26 additions & 54 deletions src/com/facebook/buck/event/listener/SimpleConsoleEventBusListener.java
View file
Expand Up @@ -26,16 +26,15 @@
import com.facebook.buck.core.test.event.TestStatusMessageEvent;
import com.facebook.buck.distributed.DistBuildCreatedEvent;
import com.facebook.buck.distributed.DistBuildRunEvent;
import com.facebook.buck.distributed.DistBuildStatusEvent;
import com.facebook.buck.distributed.build_client.StampedeConsoleEvent;
import com.facebook.buck.distributed.thrift.BuildSlaveInfo;
import com.facebook.buck.distributed.thrift.BuildSlaveRunId;
import com.facebook.buck.distributed.thrift.BuildStatus;
import com.facebook.buck.event.ActionGraphEvent;
import com.facebook.buck.event.CommandEvent;
import com.facebook.buck.event.ConsoleEvent;
import com.facebook.buck.event.InstallEvent;
import com.facebook.buck.event.listener.interfaces.AdditionalConsoleLineProvider;
import com.facebook.buck.event.listener.stats.stampede.DistBuildStatsTracker;
import com.facebook.buck.event.listener.util.EventInterval;
import com.facebook.buck.test.TestStatusMessage;
import com.facebook.buck.test.config.TestResultSummaryVerbosity;
Expand All @@ -47,11 +46,8 @@
import com.google.common.eventbus.Subscribe;
import java.nio.file.Path;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import javax.annotation.concurrent.GuardedBy;

/**
* Implementation of {@code AbstractConsoleEventBusListener} for terminals that don't support ansi.
Expand All @@ -66,11 +62,6 @@ public class SimpleConsoleEventBusListener extends AbstractConsoleEventBusListen
private final boolean hideSucceededRules;
public final ImmutableList<AdditionalConsoleLineProvider> buildFinishedLineProvider;

@GuardedBy("distBuildSlaveTracker")
private final Map<BuildSlaveRunId, BuildStatus> distBuildSlaveTracker;

private volatile Optional<BuildStatus> stampedeBuildStatus = Optional.empty();

public SimpleConsoleEventBusListener(
RenderingConsole console,
Clock clock,
Expand Down Expand Up @@ -109,8 +100,6 @@ public SimpleConsoleEventBusListener(
locale,
Optional.of(testLogPath));

this.distBuildSlaveTracker = new LinkedHashMap<>();

if (printBuildId) {
printLines(ImmutableList.of(getBuildLogLine(buildId)));
}
Expand All @@ -121,6 +110,31 @@ public SimpleConsoleEventBusListener(

this.parseStats.registerListener(this::parseFinished);
this.networkStatsTracker.registerListener(this::onCacheUploadsFinished);
this.distStatsTracker.registerListener(
new DistBuildStatsTracker.Listener() {
@Override
public void onWorkerJoined(BuildSlaveInfo slaveInfo) {
printLine(
"STAMPEDE WORKER [%s][%s] JOINED BUILD WITH STATUS [%s]",
slaveInfo.getHostname(),
slaveInfo.getBuildSlaveRunId().getId(),
slaveInfo.getStatus().name());
}

@Override
public void onWorkerStatusChanged(BuildSlaveInfo slaveInfo) {
printLine(
"STAMPEDE WORKER [%s][%s] CHANGED STATUS TO [%s]",
slaveInfo.getHostname(),
slaveInfo.getBuildSlaveRunId().getId(),
slaveInfo.getStatus().name());
}

@Override
public void onDistBuildStateChanged(BuildStatus distBuildState) {
printLine("STAMPEDE JOB STATUS CHANGED TO [%s]", distBuildState);
}
});
}

private void parseFinished() {
Expand Down Expand Up @@ -243,48 +257,6 @@ public void logStampedeConsoleEvent(StampedeConsoleEvent event) {
logEvent(event.getConsoleEvent());
}

@Override
@Subscribe
public void onDistBuildStatusEvent(DistBuildStatusEvent event) {
super.onDistBuildStatusEvent(event);

ImmutableList.Builder<String> builder = ImmutableList.builder();

BuildStatus newJobStatus = event.getJob().getStatus();
if (!stampedeBuildStatus.isPresent() || !stampedeBuildStatus.get().equals(newJobStatus)) {
builder.add(String.format("STAMPEDE JOB STATUS CHANGED TO [%s]", newJobStatus));
}
stampedeBuildStatus = Optional.of(newJobStatus);

synchronized (distBuildSlaveTracker) {
// Don't track the status of failed or lost minions
for (BuildSlaveInfo slaveInfo : event.getJob().getBuildSlaves()) {
if (!distBuildSlaveTracker.containsKey(slaveInfo.getBuildSlaveRunId())) {
builder.add(
String.format(
"STAMPEDE WORKER [%s][%s] JOINED BUILD WITH STATUS [%s]",
slaveInfo.getHostname(),
slaveInfo.getBuildSlaveRunId().getId(),
slaveInfo.getStatus().name()));
} else {
BuildStatus existingStatus = distBuildSlaveTracker.get(slaveInfo.getBuildSlaveRunId());
if (!existingStatus.equals(slaveInfo.getStatus())) {
builder.add(
String.format(
"STAMPEDE WORKER [%s][%s] CHANGED STATUS TO [%s]",
slaveInfo.getHostname(),
slaveInfo.getBuildSlaveRunId().getId(),
slaveInfo.getStatus().name()));
}
}

distBuildSlaveTracker.put(slaveInfo.getBuildSlaveRunId(), slaveInfo.getStatus());
}
}

printLines(builder);
}

@Override
public void printSevereWarningDirectly(String line) {
logEvent(ConsoleEvent.severe(line));
Expand Down

0 comments on commit 074240a

Please sign in to comment.