Skip to content

Commit

Permalink
Encode remaining SimpleSpawnResult failures with FailureDetails
Browse files Browse the repository at this point in the history 
RELNOTES: None.
PiperOrigin-RevId: 318514673
  • Loading branch information
@anakanemison
anakanemison authored and Copybara-Service committed Jun 26, 2020
1 parent 577d907 commit 6fba77d
Show file tree
Hide file tree
Showing 8 changed files with 108 additions and 17 deletions.
1 change: 1 addition & 0 deletions src/main/java/com/google/devtools/build/lib/exec/BUILD
View file
Expand Up @@ -216,6 +216,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/actions",
"//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code",
"//src/main/java/com/google/devtools/build/lib/util:exit_code",
"//src/main/protobuf:failure_details_java_proto",
"//third_party:guava",
],
)
Expand Down
10 changes: 9 additions & 1 deletion src/main/java/com/google/devtools/build/lib/exec/SpawnExecException.java
View file
Expand Up @@ -22,6 +22,9 @@
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.SpawnResult;
import com.google.devtools.build.lib.actions.SpawnResult.Status;
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.devtools.build.lib.server.FailureDetails.Spawn;
import com.google.devtools.build.lib.server.FailureDetails.Spawn.Code;
import com.google.devtools.build.lib.util.DetailedExitCode;
import com.google.devtools.build.lib.util.ExitCode;

Expand Down Expand Up @@ -77,7 +80,12 @@ private DetailedExitCode getDetailedExitCode() {
ExitCode exitCode =
result.status().isConsideredUserError() ? ExitCode.BUILD_FAILURE : ExitCode.REMOTE_ERROR;
if (result.failureDetail() == null) {
return DetailedExitCode.justExitCode(exitCode);
return DetailedExitCode.of(
exitCode,
FailureDetail.newBuilder()
.setMessage("spawn failed")
.setSpawn(Spawn.newBuilder().setCode(Code.UNSPECIFIED_EXECUTION_FAILURE))
.build());
}
return DetailedExitCode.of(exitCode, result.failureDetail());
}
Expand Down
25 changes: 25 additions & 0 deletions src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
View file
Expand Up @@ -74,6 +74,8 @@
import com.google.devtools.build.lib.remote.util.TracingMetadataUtils;
import com.google.devtools.build.lib.remote.util.Utils;
import com.google.devtools.build.lib.remote.util.Utils.InMemoryOutput;
import com.google.devtools.build.lib.server.FailureDetails;
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.devtools.build.lib.util.ExitCode;
import com.google.devtools.build.lib.util.io.FileOutErr;
import com.google.devtools.build.lib.vfs.Path;
Expand Down Expand Up @@ -602,16 +604,31 @@ private SpawnResult handleError(
.setRunnerName(getName())
.setStatus(Status.TIMEOUT)
.setExitCode(SpawnResult.POSIX_TIMEOUT_EXIT_CODE)
.setFailureDetail(
FailureDetail.newBuilder()
.setMessage("remote spawn timed out")
.setSpawn(
FailureDetails.Spawn.newBuilder()
.setCode(FailureDetails.Spawn.Code.TIMEOUT))
.build())
.build();
}
}
final Status status;
FailureDetails.Spawn.Code detailedCode;
boolean catastrophe;
if (RemoteRetrierUtils.causedByStatus(exception, Code.UNAVAILABLE)) {
status = Status.EXECUTION_FAILED_CATASTROPHICALLY;
detailedCode = FailureDetails.Spawn.Code.EXECUTION_FAILED;
catastrophe = true;
} else if (remoteCacheFailed) {
status = Status.REMOTE_CACHE_FAILED;
detailedCode = FailureDetails.Spawn.Code.REMOTE_CACHE_FAILED;
catastrophe = false;
} else {
status = Status.EXECUTION_FAILED;
detailedCode = FailureDetails.Spawn.Code.EXECUTION_FAILED;
catastrophe = false;
}

final String errorMessage;
Expand All @@ -627,6 +644,14 @@ private SpawnResult handleError(
.setStatus(status)
.setExitCode(ExitCode.REMOTE_ERROR.getNumericExitCode())
.setFailureMessage(errorMessage)
.setFailureDetail(
FailureDetail.newBuilder()
.setMessage("remote spawn failed: " + errorMessage)
.setSpawn(
FailureDetails.Spawn.newBuilder()
.setCode(detailedCode)
.setCatastrophic(catastrophe))
.build())
.build();
}

Expand Down
1 change: 1 addition & 0 deletions src/main/java/com/google/devtools/build/lib/remote/util/BUILD
View file
Expand Up @@ -22,6 +22,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/remote/options",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
"//src/main/protobuf:failure_details_java_proto",
"//third_party:guava",
"//third_party:jsr305",
"//third_party/grpc:grpc-jar",
Expand Down
10 changes: 10 additions & 0 deletions src/main/java/com/google/devtools/build/lib/remote/util/Utils.java
View file
Expand Up @@ -29,6 +29,9 @@
import com.google.devtools.build.lib.remote.common.CacheNotFoundException;
import com.google.devtools.build.lib.remote.common.RemoteCacheClient.ActionKey;
import com.google.devtools.build.lib.remote.options.RemoteOutputsMode;
import com.google.devtools.build.lib.server.FailureDetails;
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.devtools.build.lib.server.FailureDetails.Spawn.Code;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
Expand Down Expand Up @@ -98,6 +101,13 @@ public static SpawnResult createSpawnResult(
.setCacheHit(cacheHit)
.setSpawnMetrics(spawnMetrics)
.setRemote(true);
if (exitCode != 0) {
builder.setFailureDetail(
FailureDetail.newBuilder()
.setMessage("remote spawn failed")
.setSpawn(FailureDetails.Spawn.newBuilder().setCode(Code.NON_ZERO_EXIT))
.build());
}
if (inMemoryOutput != null) {
builder.setInMemoryOutput(inMemoryOutput.getOutput(), inMemoryOutput.getContents());
}
Expand Down
54 changes: 43 additions & 11 deletions src/main/java/com/google/devtools/build/lib/sandbox/AbstractSandboxSpawnRunner.java
View file
Expand Up @@ -36,6 +36,7 @@
import com.google.devtools.build.lib.profiler.Profiler;
import com.google.devtools.build.lib.profiler.SilentCloseable;
import com.google.devtools.build.lib.runtime.CommandEnvironment;
import com.google.devtools.build.lib.server.FailureDetails;
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.devtools.build.lib.server.FailureDetails.Sandbox;
import com.google.devtools.build.lib.server.FailureDetails.Sandbox.Code;
Expand Down Expand Up @@ -193,11 +194,13 @@ private final SpawnResult run(
.getErrorStream()
.write(("Action failed to execute: java.io.IOException: " + msg + "\n").getBytes(UTF_8));
outErr.getErrorStream().flush();
String message = makeFailureMessage(originalSpawn, sandbox);
return new SpawnResult.Builder()
.setRunnerName(getName())
.setStatus(Status.EXECUTION_FAILED)
.setExitCode(LOCAL_EXEC_ERROR)
.setFailureMessage(makeFailureMessage(originalSpawn, sandbox))
.setFailureMessage(message)
.setFailureDetail(createFailureDetail(message, Code.EXECUTION_FAILED))
.build();
}

Expand All @@ -206,23 +209,52 @@ private final SpawnResult run(
boolean wasTimeout =
(useSubprocessTimeout && terminationStatus.timedOut())
|| (!useSubprocessTimeout && wasTimeout(timeout, wallTime));
int exitCode =
wasTimeout ? SpawnResult.POSIX_TIMEOUT_EXIT_CODE : terminationStatus.getRawExitCode();
Status status =
wasTimeout
? Status.TIMEOUT
: (exitCode == 0) ? Status.SUCCESS : Status.NON_ZERO_EXIT;

int exitCode;
Status status;
String failureMessage;
FailureDetail failureDetail;
if (wasTimeout) {
exitCode = SpawnResult.POSIX_TIMEOUT_EXIT_CODE;
status = Status.TIMEOUT;
failureMessage = makeFailureMessage(originalSpawn, sandbox);
failureDetail =
FailureDetail.newBuilder()
.setMessage(failureMessage)
.setSpawn(
FailureDetails.Spawn.newBuilder().setCode(FailureDetails.Spawn.Code.TIMEOUT))
.build();
} else {
exitCode = terminationStatus.getRawExitCode();
if (exitCode == 0) {
status = Status.SUCCESS;
failureMessage = "";
failureDetail = null;
} else {
status = Status.NON_ZERO_EXIT;
failureMessage = makeFailureMessage(originalSpawn, sandbox);
failureDetail =
FailureDetail.newBuilder()
.setMessage(failureMessage)
.setSpawn(
FailureDetails.Spawn.newBuilder()
.setCode(FailureDetails.Spawn.Code.NON_ZERO_EXIT)
.setSpawnExitCode(exitCode))
.build();
}
}

SpawnResult.Builder spawnResultBuilder =
new SpawnResult.Builder()
.setRunnerName(getName())
.setStatus(status)
.setExitCode(exitCode)
.setWallTime(wallTime)
.setFailureMessage(
status != Status.SUCCESS || exitCode != 0
? makeFailureMessage(originalSpawn, sandbox)
: "");
.setFailureMessage(failureMessage);

if (failureDetail != null) {
spawnResultBuilder.setFailureDetail(failureDetail);
}

Path statisticsPath = sandbox.getStatisticsPath();
if (statisticsPath != null) {
Expand Down
20 changes: 15 additions & 5 deletions src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnRunner.java
View file
Expand Up @@ -33,6 +33,7 @@
import com.google.devtools.build.lib.actions.SpawnExecutedEvent;
import com.google.devtools.build.lib.actions.SpawnMetrics;
import com.google.devtools.build.lib.actions.SpawnResult;
import com.google.devtools.build.lib.actions.SpawnResult.Status;
import com.google.devtools.build.lib.actions.Spawns;
import com.google.devtools.build.lib.actions.UserExecException;
import com.google.devtools.build.lib.events.Event;
Expand Down Expand Up @@ -209,19 +210,28 @@ private SpawnResult actuallyExec(Spawn spawn, SpawnExecutionContext context)
response.getOutputBytes().writeTo(outErr.getErrorStream());

int exitCode = response.getExitCode();
SpawnResult result =
SpawnResult.Builder builder =
new SpawnResult.Builder()
.setRunnerName(getName())
.setExitCode(exitCode)
.setStatus(
exitCode == 0 ? SpawnResult.Status.SUCCESS : SpawnResult.Status.NON_ZERO_EXIT)
.setStatus(exitCode == 0 ? Status.SUCCESS : Status.NON_ZERO_EXIT)
.setWallTime(wallTime)
.setSpawnMetrics(
SpawnMetrics.Builder.forWorkerExec()
.setTotalTime(wallTime)
.setExecutionWallTime(wallTime)
.build())
.build();
.build());
if (exitCode != 0) {
builder.setFailureDetail(
FailureDetail.newBuilder()
.setMessage("worker spawn failed")
.setSpawn(
FailureDetails.Spawn.newBuilder()
.setCode(FailureDetails.Spawn.Code.NON_ZERO_EXIT)
.setSpawnExitCode(exitCode))
.build());
}
SpawnResult result = builder.build();
reporter.post(new SpawnExecutedEvent(spawn, result, startTime));
return result;
}
Expand Down
4 changes: 4 additions & 0 deletions src/main/protobuf/failure_details.proto
View file
Expand Up @@ -205,6 +205,9 @@ message Spawn {
INVALID_TIMEOUT = 9 [(metadata) = { exit_code: 1 }];
INVALID_REMOTE_EXECUTION_PROPERTIES = 10 [(metadata) = { exit_code: 1 }];
NO_USABLE_STRATEGY_FOUND = 11 [(metadata) = { exit_code: 1 }];
// TODO(b/138456686): this code should be deprecated when SpawnResult is
// refactored to prohibit undetailed failures
UNSPECIFIED_EXECUTION_FAILURE = 12 [(metadata) = { exit_code: 1 }];
}
Code code = 1;

Expand Down Expand Up @@ -653,6 +656,7 @@ message Sandbox {
MOUNT_SOURCE_DOES_NOT_EXIST = 7 [(metadata) = { exit_code: 1 }];
MOUNT_SOURCE_TARGET_TYPE_MISMATCH = 8 [(metadata) = { exit_code: 1 }];
MOUNT_TARGET_DOES_NOT_EXIST = 9 [(metadata) = { exit_code: 1 }];
EXECUTION_FAILED = 10 [(metadata) = { exit_code: 1 }];
}

Code code = 1;
Expand Down

0 comments on commit 6fba77d

Please sign in to comment.