diff --git a/src/main/java/com/google/devtools/build/lib/exec/BUILD b/src/main/java/com/google/devtools/build/lib/exec/BUILD index 9251478f386c06..eabf79feafb84c 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/BUILD +++ b/src/main/java/com/google/devtools/build/lib/exec/BUILD @@ -216,6 +216,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/actions", "//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code", "//src/main/java/com/google/devtools/build/lib/util:exit_code", + "//src/main/protobuf:failure_details_java_proto", "//third_party:guava", ], ) diff --git a/src/main/java/com/google/devtools/build/lib/exec/SpawnExecException.java b/src/main/java/com/google/devtools/build/lib/exec/SpawnExecException.java index 7c7763678e15ca..ae9b7a9b5b9a32 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/SpawnExecException.java +++ b/src/main/java/com/google/devtools/build/lib/exec/SpawnExecException.java @@ -22,6 +22,9 @@ import com.google.devtools.build.lib.actions.ExecException; import com.google.devtools.build.lib.actions.SpawnResult; import com.google.devtools.build.lib.actions.SpawnResult.Status; +import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; +import com.google.devtools.build.lib.server.FailureDetails.Spawn; +import com.google.devtools.build.lib.server.FailureDetails.Spawn.Code; import com.google.devtools.build.lib.util.DetailedExitCode; import com.google.devtools.build.lib.util.ExitCode; @@ -77,7 +80,12 @@ private DetailedExitCode getDetailedExitCode() { ExitCode exitCode = result.status().isConsideredUserError() ? ExitCode.BUILD_FAILURE : ExitCode.REMOTE_ERROR; if (result.failureDetail() == null) { - return DetailedExitCode.justExitCode(exitCode); + return DetailedExitCode.of( + exitCode, + FailureDetail.newBuilder() + .setMessage("spawn failed") + .setSpawn(Spawn.newBuilder().setCode(Code.UNSPECIFIED_EXECUTION_FAILURE)) + .build()); } return DetailedExitCode.of(exitCode, result.failureDetail()); } diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java index 7966e08649a9e9..649e24aca0a807 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java @@ -74,6 +74,8 @@ import com.google.devtools.build.lib.remote.util.TracingMetadataUtils; import com.google.devtools.build.lib.remote.util.Utils; import com.google.devtools.build.lib.remote.util.Utils.InMemoryOutput; +import com.google.devtools.build.lib.server.FailureDetails; +import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; import com.google.devtools.build.lib.util.ExitCode; import com.google.devtools.build.lib.util.io.FileOutErr; import com.google.devtools.build.lib.vfs.Path; @@ -602,16 +604,31 @@ private SpawnResult handleError( .setRunnerName(getName()) .setStatus(Status.TIMEOUT) .setExitCode(SpawnResult.POSIX_TIMEOUT_EXIT_CODE) + .setFailureDetail( + FailureDetail.newBuilder() + .setMessage("remote spawn timed out") + .setSpawn( + FailureDetails.Spawn.newBuilder() + .setCode(FailureDetails.Spawn.Code.TIMEOUT)) + .build()) .build(); } } final Status status; + FailureDetails.Spawn.Code detailedCode; + boolean catastrophe; if (RemoteRetrierUtils.causedByStatus(exception, Code.UNAVAILABLE)) { status = Status.EXECUTION_FAILED_CATASTROPHICALLY; + detailedCode = FailureDetails.Spawn.Code.EXECUTION_FAILED; + catastrophe = true; } else if (remoteCacheFailed) { status = Status.REMOTE_CACHE_FAILED; + detailedCode = FailureDetails.Spawn.Code.REMOTE_CACHE_FAILED; + catastrophe = false; } else { status = Status.EXECUTION_FAILED; + detailedCode = FailureDetails.Spawn.Code.EXECUTION_FAILED; + catastrophe = false; } final String errorMessage; @@ -627,6 +644,14 @@ private SpawnResult handleError( .setStatus(status) .setExitCode(ExitCode.REMOTE_ERROR.getNumericExitCode()) .setFailureMessage(errorMessage) + .setFailureDetail( + FailureDetail.newBuilder() + .setMessage("remote spawn failed: " + errorMessage) + .setSpawn( + FailureDetails.Spawn.newBuilder() + .setCode(detailedCode) + .setCatastrophic(catastrophe)) + .build()) .build(); } diff --git a/src/main/java/com/google/devtools/build/lib/remote/util/BUILD b/src/main/java/com/google/devtools/build/lib/remote/util/BUILD index 7612bda8a185ba..e727163c8ed55b 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/util/BUILD +++ b/src/main/java/com/google/devtools/build/lib/remote/util/BUILD @@ -22,6 +22,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/remote/options", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", + "//src/main/protobuf:failure_details_java_proto", "//third_party:guava", "//third_party:jsr305", "//third_party/grpc:grpc-jar", diff --git a/src/main/java/com/google/devtools/build/lib/remote/util/Utils.java b/src/main/java/com/google/devtools/build/lib/remote/util/Utils.java index c6b6357cb47550..71847e7655b4d7 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/util/Utils.java +++ b/src/main/java/com/google/devtools/build/lib/remote/util/Utils.java @@ -29,6 +29,9 @@ import com.google.devtools.build.lib.remote.common.CacheNotFoundException; import com.google.devtools.build.lib.remote.common.RemoteCacheClient.ActionKey; import com.google.devtools.build.lib.remote.options.RemoteOutputsMode; +import com.google.devtools.build.lib.server.FailureDetails; +import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; +import com.google.devtools.build.lib.server.FailureDetails.Spawn.Code; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; @@ -98,6 +101,13 @@ public static SpawnResult createSpawnResult( .setCacheHit(cacheHit) .setSpawnMetrics(spawnMetrics) .setRemote(true); + if (exitCode != 0) { + builder.setFailureDetail( + FailureDetail.newBuilder() + .setMessage("remote spawn failed") + .setSpawn(FailureDetails.Spawn.newBuilder().setCode(Code.NON_ZERO_EXIT)) + .build()); + } if (inMemoryOutput != null) { builder.setInMemoryOutput(inMemoryOutput.getOutput(), inMemoryOutput.getContents()); } diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/AbstractSandboxSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/sandbox/AbstractSandboxSpawnRunner.java index 886f82ce8bc1b1..8807993e986d30 100644 --- a/src/main/java/com/google/devtools/build/lib/sandbox/AbstractSandboxSpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/sandbox/AbstractSandboxSpawnRunner.java @@ -36,6 +36,7 @@ import com.google.devtools.build.lib.profiler.Profiler; import com.google.devtools.build.lib.profiler.SilentCloseable; import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.server.FailureDetails; import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; import com.google.devtools.build.lib.server.FailureDetails.Sandbox; import com.google.devtools.build.lib.server.FailureDetails.Sandbox.Code; @@ -193,11 +194,13 @@ private final SpawnResult run( .getErrorStream() .write(("Action failed to execute: java.io.IOException: " + msg + "\n").getBytes(UTF_8)); outErr.getErrorStream().flush(); + String message = makeFailureMessage(originalSpawn, sandbox); return new SpawnResult.Builder() .setRunnerName(getName()) .setStatus(Status.EXECUTION_FAILED) .setExitCode(LOCAL_EXEC_ERROR) - .setFailureMessage(makeFailureMessage(originalSpawn, sandbox)) + .setFailureMessage(message) + .setFailureDetail(createFailureDetail(message, Code.EXECUTION_FAILED)) .build(); } @@ -206,12 +209,40 @@ private final SpawnResult run( boolean wasTimeout = (useSubprocessTimeout && terminationStatus.timedOut()) || (!useSubprocessTimeout && wasTimeout(timeout, wallTime)); - int exitCode = - wasTimeout ? SpawnResult.POSIX_TIMEOUT_EXIT_CODE : terminationStatus.getRawExitCode(); - Status status = - wasTimeout - ? Status.TIMEOUT - : (exitCode == 0) ? Status.SUCCESS : Status.NON_ZERO_EXIT; + + int exitCode; + Status status; + String failureMessage; + FailureDetail failureDetail; + if (wasTimeout) { + exitCode = SpawnResult.POSIX_TIMEOUT_EXIT_CODE; + status = Status.TIMEOUT; + failureMessage = makeFailureMessage(originalSpawn, sandbox); + failureDetail = + FailureDetail.newBuilder() + .setMessage(failureMessage) + .setSpawn( + FailureDetails.Spawn.newBuilder().setCode(FailureDetails.Spawn.Code.TIMEOUT)) + .build(); + } else { + exitCode = terminationStatus.getRawExitCode(); + if (exitCode == 0) { + status = Status.SUCCESS; + failureMessage = ""; + failureDetail = null; + } else { + status = Status.NON_ZERO_EXIT; + failureMessage = makeFailureMessage(originalSpawn, sandbox); + failureDetail = + FailureDetail.newBuilder() + .setMessage(failureMessage) + .setSpawn( + FailureDetails.Spawn.newBuilder() + .setCode(FailureDetails.Spawn.Code.NON_ZERO_EXIT) + .setSpawnExitCode(exitCode)) + .build(); + } + } SpawnResult.Builder spawnResultBuilder = new SpawnResult.Builder() @@ -219,10 +250,11 @@ private final SpawnResult run( .setStatus(status) .setExitCode(exitCode) .setWallTime(wallTime) - .setFailureMessage( - status != Status.SUCCESS || exitCode != 0 - ? makeFailureMessage(originalSpawn, sandbox) - : ""); + .setFailureMessage(failureMessage); + + if (failureDetail != null) { + spawnResultBuilder.setFailureDetail(failureDetail); + } Path statisticsPath = sandbox.getStatisticsPath(); if (statisticsPath != null) { diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnRunner.java index 898a6ac0f71c92..bdb9b1d76c1b15 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnRunner.java @@ -33,6 +33,7 @@ import com.google.devtools.build.lib.actions.SpawnExecutedEvent; import com.google.devtools.build.lib.actions.SpawnMetrics; import com.google.devtools.build.lib.actions.SpawnResult; +import com.google.devtools.build.lib.actions.SpawnResult.Status; import com.google.devtools.build.lib.actions.Spawns; import com.google.devtools.build.lib.actions.UserExecException; import com.google.devtools.build.lib.events.Event; @@ -209,19 +210,28 @@ private SpawnResult actuallyExec(Spawn spawn, SpawnExecutionContext context) response.getOutputBytes().writeTo(outErr.getErrorStream()); int exitCode = response.getExitCode(); - SpawnResult result = + SpawnResult.Builder builder = new SpawnResult.Builder() .setRunnerName(getName()) .setExitCode(exitCode) - .setStatus( - exitCode == 0 ? SpawnResult.Status.SUCCESS : SpawnResult.Status.NON_ZERO_EXIT) + .setStatus(exitCode == 0 ? Status.SUCCESS : Status.NON_ZERO_EXIT) .setWallTime(wallTime) .setSpawnMetrics( SpawnMetrics.Builder.forWorkerExec() .setTotalTime(wallTime) .setExecutionWallTime(wallTime) - .build()) - .build(); + .build()); + if (exitCode != 0) { + builder.setFailureDetail( + FailureDetail.newBuilder() + .setMessage("worker spawn failed") + .setSpawn( + FailureDetails.Spawn.newBuilder() + .setCode(FailureDetails.Spawn.Code.NON_ZERO_EXIT) + .setSpawnExitCode(exitCode)) + .build()); + } + SpawnResult result = builder.build(); reporter.post(new SpawnExecutedEvent(spawn, result, startTime)); return result; } diff --git a/src/main/protobuf/failure_details.proto b/src/main/protobuf/failure_details.proto index 7beafc6d3f200b..d125f3919dd39e 100644 --- a/src/main/protobuf/failure_details.proto +++ b/src/main/protobuf/failure_details.proto @@ -205,6 +205,9 @@ message Spawn { INVALID_TIMEOUT = 9 [(metadata) = { exit_code: 1 }]; INVALID_REMOTE_EXECUTION_PROPERTIES = 10 [(metadata) = { exit_code: 1 }]; NO_USABLE_STRATEGY_FOUND = 11 [(metadata) = { exit_code: 1 }]; + // TODO(b/138456686): this code should be deprecated when SpawnResult is + // refactored to prohibit undetailed failures + UNSPECIFIED_EXECUTION_FAILURE = 12 [(metadata) = { exit_code: 1 }]; } Code code = 1; @@ -653,6 +656,7 @@ message Sandbox { MOUNT_SOURCE_DOES_NOT_EXIST = 7 [(metadata) = { exit_code: 1 }]; MOUNT_SOURCE_TARGET_TYPE_MISMATCH = 8 [(metadata) = { exit_code: 1 }]; MOUNT_TARGET_DOES_NOT_EXIST = 9 [(metadata) = { exit_code: 1 }]; + EXECUTION_FAILED = 10 [(metadata) = { exit_code: 1 }]; } Code code = 1;