Skip to content

Commit

Permalink
[credentialhelper] Ignore failing to provide request
Browse files Browse the repository at this point in the history 
This fixes a race condition where writing to stdin of the subprocess fails because the subprocess terminated quickly (e.g., because it did not need the request to provide credentials, or some other error occurred in the helper before it reads from stdin).

```
Starting local Bazel server and connecting to it...
INFO: Writing tracer profile to '/private/var/tmp/_bazel_yannic/bb6c3b02ccc98aaca966df445b5024eb/command.profile.gz'
INFO: Invocation ID: 86e227bc-187e-4d28-8f04-40b1ce9a36a4
ERROR: java.io.IOException: Stream closed
	at java.base/java.lang.ProcessBuilder$NullOutputStream.write(Unknown Source)
	at java.base/java.io.OutputStream.write(Unknown Source)
	at java.base/java.io.BufferedOutputStream.flushBuffer(Unknown Source)
	at java.base/java.io.BufferedOutputStream.flush(Unknown Source)
	at java.base/java.io.FilterOutputStream.close(Unknown Source)
	at java.base/sun.nio.cs.StreamEncoder.implClose(Unknown Source)
	at java.base/sun.nio.cs.StreamEncoder.close(Unknown Source)
	at java.base/java.io.OutputStreamWriter.close(Unknown Source)
	at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelper.getCredentials(CredentialHelper.java:82)
	at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getCredentialsFromHelper(CredentialHelperCredentials.java:108)
	at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2413)
	at java.base/java.util.concurrent.ConcurrentHashMap.compute(Unknown Source)
	at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2411)
	at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2394)
	at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108)
	at com.github.benmanes.caffeine.cache.LocalManualCache.get(LocalManualCache.java:62)
	at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getRequestMetadata(CredentialHelperCredentials.java:80)
	at com.google.auth.Credentials.blockingGetToCallback(Credentials.java:112)
	at com.google.auth.Credentials$1.run(Credentials.java:98)
	at io.grpc.stub.ClientCalls$ThreadlessExecutor.runQuietly(ClientCalls.java:771)
	at io.grpc.stub.ClientCalls$ThreadlessExecutor.waitAndDrain(ClientCalls.java:754)
	at io.grpc.stub.ClientCalls.blockingUnaryCall(ClientCalls.java:157)
	at build.bazel.remote.execution.v2.CapabilitiesGrpc$CapabilitiesBlockingStub.getCapabilities(CapabilitiesGrpc.java:218)
	at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$0(RemoteServerCapabilities.java:84)
	at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannelBlocking$2(ReferenceCountedChannel.java:85)
	at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannel$4(ReferenceCountedChannel.java:108)
	at io.reactivex.rxjava3.internal.operators.single.SingleUsing.subscribeActual(SingleUsing.java:59)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85)
	at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback$FlatMapSingleObserver.onSuccess(SingleFlatMap.java:112)
	at io.reactivex.rxjava3.internal.operators.single.SingleMap$MapSingleObserver.onSuccess(SingleMap.java:65)
	at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose$DoOnDisposeObserver.onSuccess(SingleDoOnDispose.java:84)
	at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError$DoOnError.onSuccess(SingleDoOnError.java:52)
	at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle$SingleElementObserver.onComplete(ObservableSingleSingle.java:110)
	at io.reactivex.rxjava3.internal.observers.DeferredScalarDisposable.complete(DeferredScalarDisposable.java:85)
	at io.reactivex.rxjava3.subjects.AsyncSubject.subscribeActual(AsyncSubject.java:233)
	at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176)
	at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle.subscribeActual(ObservableSingleSingle.java:36)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError.subscribeActual(SingleDoOnError.java:35)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose.subscribeActual(SingleDoOnDispose.java:38)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleMap.subscribeActual(SingleMap.java:35)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85)
	at io.reactivex.rxjava3.internal.operators.single.SingleCreate$Emitter.onSuccess(SingleCreate.java:68)
	at com.google.devtools.build.lib.remote.grpc.TokenBucket$1.onNext(TokenBucket.java:79)
	at io.reactivex.rxjava3.internal.util.NotificationLite.accept(NotificationLite.java:247)
	at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.test(BehaviorSubject.java:507)
	at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.emitFirst(BehaviorSubject.java:468)
	at io.reactivex.rxjava3.subjects.BehaviorSubject.subscribeActual(BehaviorSubject.java:224)
	at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176)
	at com.google.devtools.build.lib.remote.grpc.TokenBucket.lambda$acquireToken$0(TokenBucket.java:64)
	at io.reactivex.rxjava3.internal.operators.single.SingleCreate.subscribeActual(SingleCreate.java:40)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleDefer.subscribeActual(SingleDefer.java:43)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.core.Single.blockingGet(Single.java:3644)
	at com.google.devtools.build.lib.remote.ReferenceCountedChannel.withChannelBlocking(ReferenceCountedChannel.java:85)
	at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$1(RemoteServerCapabilities.java:83)
	at com.google.devtools.build.lib.remote.Retrier.execute(Retrier.java:244)
	at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:125)
	at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:114)
	at com.google.devtools.build.lib.remote.RemoteServerCapabilities.get(RemoteServerCapabilities.java:81)
	at com.google.devtools.build.lib.remote.RemoteModule.getAndVerifyServerCapabilities(RemoteModule.java:209)
	at com.google.devtools.build.lib.remote.RemoteModule.beforeCommand(RemoteModule.java:542)
	at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.execExclusively(BlazeCommandDispatcher.java:379)
	at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.exec(BlazeCommandDispatcher.java:233)
	at com.google.devtools.build.lib.server.GrpcServerImpl.executeCommand(GrpcServerImpl.java:550)
	at com.google.devtools.build.lib.server.GrpcServerImpl.lambda$run$1(GrpcServerImpl.java:614)
	at io.grpc.Context$1.run(Context.java:566)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
	at java.base/java.lang.Thread.run(Unknown Source)

ERROR: Failed to query remote execution capabilities: Stream closed
java.io.IOException: Stream closed
	at java.base/java.lang.ProcessBuilder$NullOutputStream.write(Unknown Source)
	at java.base/java.io.OutputStream.write(Unknown Source)
	at java.base/java.io.BufferedOutputStream.flushBuffer(Unknown Source)
	at java.base/java.io.BufferedOutputStream.flush(Unknown Source)
	at java.base/java.io.FilterOutputStream.close(Unknown Source)
	at java.base/sun.nio.cs.StreamEncoder.implClose(Unknown Source)
	at java.base/sun.nio.cs.StreamEncoder.close(Unknown Source)
	at java.base/java.io.OutputStreamWriter.close(Unknown Source)
	at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelper.getCredentials(CredentialHelper.java:82)
	at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getCredentialsFromHelper(CredentialHelperCredentials.java:108)
	at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2413)
	at java.base/java.util.concurrent.ConcurrentHashMap.compute(Unknown Source)
	at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2411)
	at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2394)
	at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:108)
	at com.github.benmanes.caffeine.cache.LocalManualCache.get(LocalManualCache.java:62)
	at com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials.getRequestMetadata(CredentialHelperCredentials.java:80)
	at com.google.auth.Credentials.blockingGetToCallback(Credentials.java:112)
	at com.google.auth.Credentials$1.run(Credentials.java:98)
	at io.grpc.stub.ClientCalls$ThreadlessExecutor.runQuietly(ClientCalls.java:771)
	at io.grpc.stub.ClientCalls$ThreadlessExecutor.waitAndDrain(ClientCalls.java:754)
	at io.grpc.stub.ClientCalls.blockingUnaryCall(ClientCalls.java:157)
	at build.bazel.remote.execution.v2.CapabilitiesGrpc$CapabilitiesBlockingStub.getCapabilities(CapabilitiesGrpc.java:218)
	at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$0(RemoteServerCapabilities.java:84)
	at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannelBlocking$2(ReferenceCountedChannel.java:85)
	at com.google.devtools.build.lib.remote.ReferenceCountedChannel.lambda$withChannel$4(ReferenceCountedChannel.java:108)
	at io.reactivex.rxjava3.internal.operators.single.SingleUsing.subscribeActual(SingleUsing.java:59)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85)
	at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback$FlatMapSingleObserver.onSuccess(SingleFlatMap.java:112)
	at io.reactivex.rxjava3.internal.operators.single.SingleMap$MapSingleObserver.onSuccess(SingleMap.java:65)
	at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose$DoOnDisposeObserver.onSuccess(SingleDoOnDispose.java:84)
	at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError$DoOnError.onSuccess(SingleDoOnError.java:52)
	at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle$SingleElementObserver.onComplete(ObservableSingleSingle.java:110)
	at io.reactivex.rxjava3.internal.observers.DeferredScalarDisposable.complete(DeferredScalarDisposable.java:85)
	at io.reactivex.rxjava3.subjects.AsyncSubject.subscribeActual(AsyncSubject.java:233)
	at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176)
	at io.reactivex.rxjava3.internal.operators.observable.ObservableSingleSingle.subscribeActual(ObservableSingleSingle.java:36)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleDoOnError.subscribeActual(SingleDoOnError.java:35)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleDoOnDispose.subscribeActual(SingleDoOnDispose.java:38)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleMap.subscribeActual(SingleMap.java:35)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap$SingleFlatMapCallback.onSuccess(SingleFlatMap.java:85)
	at io.reactivex.rxjava3.internal.operators.single.SingleCreate$Emitter.onSuccess(SingleCreate.java:68)
	at com.google.devtools.build.lib.remote.grpc.TokenBucket$1.onNext(TokenBucket.java:79)
	at io.reactivex.rxjava3.internal.util.NotificationLite.accept(NotificationLite.java:247)
	at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.test(BehaviorSubject.java:507)
	at io.reactivex.rxjava3.subjects.BehaviorSubject$BehaviorDisposable.emitFirst(BehaviorSubject.java:468)
	at io.reactivex.rxjava3.subjects.BehaviorSubject.subscribeActual(BehaviorSubject.java:224)
	at io.reactivex.rxjava3.core.Observable.subscribe(Observable.java:13176)
	at com.google.devtools.build.lib.remote.grpc.TokenBucket.lambda$acquireToken$0(TokenBucket.java:64)
	at io.reactivex.rxjava3.internal.operators.single.SingleCreate.subscribeActual(SingleCreate.java:40)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleDefer.subscribeActual(SingleDefer.java:43)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.internal.operators.single.SingleFlatMap.subscribeActual(SingleFlatMap.java:37)
	at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
	at io.reactivex.rxjava3.core.Single.blockingGet(Single.java:3644)
	at com.google.devtools.build.lib.remote.ReferenceCountedChannel.withChannelBlocking(ReferenceCountedChannel.java:85)
	at com.google.devtools.build.lib.remote.RemoteServerCapabilities.lambda$get$1(RemoteServerCapabilities.java:83)
	at com.google.devtools.build.lib.remote.Retrier.execute(Retrier.java:244)
	at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:125)
	at com.google.devtools.build.lib.remote.RemoteRetrier.execute(RemoteRetrier.java:114)
	at com.google.devtools.build.lib.remote.RemoteServerCapabilities.get(RemoteServerCapabilities.java:81)
	at com.google.devtools.build.lib.remote.RemoteModule.getAndVerifyServerCapabilities(RemoteModule.java:209)
	at com.google.devtools.build.lib.remote.RemoteModule.beforeCommand(RemoteModule.java:542)
	at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.execExclusively(BlazeCommandDispatcher.java:379)
	at com.google.devtools.build.lib.runtime.BlazeCommandDispatcher.exec(BlazeCommandDispatcher.java:233)
	at com.google.devtools.build.lib.server.GrpcServerImpl.executeCommand(GrpcServerImpl.java:550)
	at com.google.devtools.build.lib.server.GrpcServerImpl.lambda$run$1(GrpcServerImpl.java:614)
	at io.grpc.Context$1.run(Context.java:566)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
	at java.base/java.lang.Thread.run(Unknown Source)

INFO: Streaming build results to: https://opal.cluster.engflow.com/invocation/86e227bc-187e-4d28-8f04-40b1ce9a36a4
```

Closes bazelbuild#17666.

PiperOrigin-RevId: 514377908
Change-Id: I772a5e462146e3e9699b45090120634a057a5532
  • Loading branch information
@Yannic
Yannic authored and Copybara-Service committed Mar 6, 2023
1 parent f4c5a95 commit 7c235ff
Showing 1 changed file with 8 additions and 0 deletions.
8 changes: 8 additions & 0 deletions ...main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialHelper.java
View file
Expand Up @@ -80,6 +80,14 @@ public GetCredentialsResponse getCredentials(CredentialHelperEnvironment environ
Reader stderr = new InputStreamReader(process.getErrorStream(), UTF_8)) {
try (Writer stdin = new OutputStreamWriter(process.getOutputStream(), UTF_8)) {
GSON.toJson(GetCredentialsRequest.newBuilder().setUri(uri).build(), stdin);
} catch (IOException e) {
// This can happen if the helper prints a static set of credentials without reading from
// stdin (e.g., with a simple shell script running `echo "{...}"`). If the process is
// already finished even though we failed to write to its stdin, ignore the error and
// assume the process did not need the request payload.
if (!process.finished()) {
throw e;
}
}

try {
Expand Down

0 comments on commit 7c235ff

Please sign in to comment.