Skip to content

Commit

Permalink
Add netrc support to --bes_backend
Browse files Browse the repository at this point in the history
  • Loading branch information
Yannic committed Jul 20, 2022
1 parent b6f580a commit e7e8562
Show file tree
Hide file tree
Showing 12 changed files with 235 additions and 192 deletions.
2 changes: 2 additions & 0 deletions src/main/java/com/google/devtools/build/lib/authandtls/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ java_library(
srcs = glob(["*.java"]),
deps = [
"//src/main/java/com/google/devtools/build/lib/concurrent",
"//src/main/java/com/google/devtools/build/lib/events",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/common/options",
"//third_party:auth",
"//third_party:auto_value",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.Path;
import io.grpc.CallCredentials;
import io.grpc.ClientInterceptor;
import io.grpc.ManagedChannel;
Expand All @@ -41,6 +43,8 @@
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
Expand Down Expand Up @@ -191,7 +195,20 @@ private static NettyChannelBuilder newNettyChannelBuilder(String targetUrl, Stri
* @throws IOException in case the call credentials can't be constructed.
*/
@Nullable
public static CallCredentials newCallCredentials(AuthAndTLSOptions options) throws IOException {
public static CallCredentials newCallCredentials(
Map<String, String> clientEnv,
FileSystem fileSystem,
AuthAndTLSOptions options) throws IOException {
Credentials creds = newCredentials(clientEnv, fileSystem, options);
if (creds != null) {
return MoreCallCredentials.from(creds);
}
return null;
}

@Nullable
@VisibleForTesting
public static CallCredentials newCallCredentialsForTesting(AuthAndTLSOptions options) throws IOException {
Credentials creds = newCredentials(options);
if (creds != null) {
return MoreCallCredentials.from(creds);
Expand All @@ -217,12 +234,46 @@ public static CallCredentialsProvider newCallCredentialsProvider(@Nullable Crede
*/
@Nullable
public static Credentials newCredentials(@Nullable AuthAndTLSOptions options) throws IOException {
Optional<Credentials> credentials = newGoogleCredentials(options);

return credentials.orElse(null);
}

/**
* Create a new {@link Credentials} with following order:
*
* <ol>
* <li>If authentication enabled by flags, use it to create credentials
* <li>Use .netrc to provide credentials if exists
* <li>Otherwise, return {@code null}
* </ol>
*
* @throws IOException in case the credentials can't be constructed.
*/
@VisibleForTesting
public static Credentials newCredentials(
Map<String, String> clientEnv,
FileSystem fileSystem,
AuthAndTLSOptions authAndTlsOptions)
throws IOException {
Optional<Credentials> credentials = newGoogleCredentials(authAndTlsOptions);

if (credentials.isEmpty()) {
// Fallback to .netrc if it exists.
credentials = newCredentialsFromNetrc(clientEnv, fileSystem);
}

return credentials.orElse(null);
}

private static Optional<Credentials> newGoogleCredentials(
@Nullable AuthAndTLSOptions options) throws IOException {
if (options == null) {
return null;
return Optional.empty();
} else if (options.googleCredentials != null) {
// Credentials from file
try (InputStream authFile = new FileInputStream(options.googleCredentials)) {
return newCredentials(authFile, options.googleAuthScopes);
return Optional.of(newGoogleCredentials(authFile, options.googleAuthScopes));
} catch (FileNotFoundException e) {
String message =
String.format(
Expand All @@ -231,10 +282,10 @@ public static Credentials newCredentials(@Nullable AuthAndTLSOptions options) th
throw new IOException(message, e);
}
} else if (options.useGoogleDefaultCredentials) {
return newCredentials(
null /* Google Application Default Credentials */, options.googleAuthScopes);
return Optional.of(newGoogleCredentials(
null /* Google Application Default Credentials */, options.googleAuthScopes));
}
return null;
return Optional.empty();
}

/**
Expand All @@ -243,7 +294,7 @@ public static Credentials newCredentials(@Nullable AuthAndTLSOptions options) th
* @throws IOException in case the credentials can't be constructed.
*/
@VisibleForTesting
public static Credentials newCredentials(
public static Credentials newGoogleCredentials(
@Nullable InputStream credentialsFile, List<String> authScopes) throws IOException {
try {
GoogleCredentials creds =
Expand All @@ -259,4 +310,43 @@ public static Credentials newCredentials(
throw new IOException(message, e);
}
}

/**
* Create a new {@link Credentials} object by parsing the .netrc file with following order to
* search it:
*
* <ol>
* <li>If environment variable $NETRC exists, use it as the path to the .netrc file
* <li>Fallback to $HOME/.netrc
* </ol>
*
* @return the {@link Credentials} object or {@code null} if there is no .netrc file.
* @throws IOException in case the credentials can't be constructed.
*/
@VisibleForTesting
static Optional<Credentials> newCredentialsFromNetrc(Map<String, String> clientEnv, FileSystem fileSystem)
throws IOException {
Optional<String> netrcFileString =
Optional.ofNullable(clientEnv.get("NETRC"))
.or(
() ->
Optional.ofNullable(clientEnv.get("HOME"))
.map(home -> home + "/.netrc"));
if (netrcFileString.isEmpty()) {
return Optional.empty();
}

Path netrcFile = fileSystem.getPath(netrcFileString.get());
if (!netrcFile.exists()) {
return Optional.empty();
}

try {
Netrc netrc = NetrcParser.parseAndClose(netrcFile.getInputStream());
return Optional.of(new NetrcCredentials(netrc));
} catch (IOException e) {
throw new IOException(
"Failed to parse " + netrcFile.getPathString() + ": " + e.getMessage(), e);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
import com.google.devtools.build.lib.buildeventservice.client.BuildEventServiceClient;
import com.google.devtools.build.lib.buildeventservice.client.BuildEventServiceGrpcClient;
import com.google.devtools.build.lib.runtime.CommandEnvironment;
import io.grpc.ClientInterceptor;
import io.grpc.ManagedChannel;
import io.grpc.Metadata;
Expand Down Expand Up @@ -70,15 +71,16 @@ protected Class<BuildEventServiceOptions> optionsClass() {

@Override
protected BuildEventServiceClient getBesClient(
BuildEventServiceOptions besOptions, AuthAndTLSOptions authAndTLSOptions) throws IOException {
CommandEnvironment env, BuildEventServiceOptions besOptions, AuthAndTLSOptions authAndTLSOptions) throws IOException {
BackendConfig newConfig = BackendConfig.create(besOptions, authAndTLSOptions);
if (client == null || !Objects.equals(config, newConfig)) {
clearBesClient();
config = newConfig;
client =
new BuildEventServiceGrpcClient(
newGrpcChannel(config),
GoogleAuthUtils.newCallCredentials(config.authAndTLSOptions()),
GoogleAuthUtils.newCallCredentials(
env.getClientEnv(), env.getRuntime().getFileSystem(), config.authAndTLSOptions()),
makeGrpcInterceptor(config));
}
return client;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -699,7 +699,7 @@ private BuildEventServiceTransport createBesTransport(

final BuildEventServiceClient besClient;
try {
besClient = getBesClient(besOptions, authTlsOptions);
besClient = getBesClient(cmdEnv, besOptions, authTlsOptions);
} catch (IOException | OptionsParsingException e) {
reportError(
reporter,
Expand Down Expand Up @@ -845,7 +845,7 @@ private static AbruptExitException createAbruptExitException(
protected abstract Class<OptionsT> optionsClass();

protected abstract BuildEventServiceClient getBesClient(
OptionsT besOptions, AuthAndTLSOptions authAndTLSOptions)
CommandEnvironment env, OptionsT besOptions, AuthAndTLSOptions authAndTLSOptions)
throws IOException, OptionsParsingException;

protected abstract void clearBesClient();
Expand Down
119 changes: 27 additions & 92 deletions src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,6 @@
import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions.UnresolvedScopedCredentialHelper;
import com.google.devtools.build.lib.authandtls.CallCredentialsProvider;
import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
import com.google.devtools.build.lib.authandtls.Netrc;
import com.google.devtools.build.lib.authandtls.NetrcCredentials;
import com.google.devtools.build.lib.authandtls.NetrcParser;
import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperEnvironment;
import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperProvider;
import com.google.devtools.build.lib.bazel.repository.downloader.Downloader;
Expand Down Expand Up @@ -1047,95 +1044,6 @@ RemoteActionContextProvider getActionContextProvider() {
return actionContextProvider;
}

/**
* Create a new {@link Credentials} object by parsing the .netrc file with following order to
* search it:
*
* <ol>
* <li>If environment variable $NETRC exists, use it as the path to the .netrc file
* <li>Fallback to $HOME/.netrc
* </ol>
*
* @return the {@link Credentials} object or {@code null} if there is no .netrc file.
* @throws IOException in case the credentials can't be constructed.
*/
@Nullable
@VisibleForTesting
static Credentials newCredentialsFromNetrc(Map<String, String> clientEnv, FileSystem fileSystem)
throws IOException {
String netrcFileString =
Optional.ofNullable(clientEnv.get("NETRC"))
.orElseGet(
() ->
Optional.ofNullable(clientEnv.get("HOME"))
.map(home -> home + "/.netrc")
.orElse(null));
if (netrcFileString == null) {
return null;
}

Path netrcFile = fileSystem.getPath(netrcFileString);
if (netrcFile.exists()) {
try {
Netrc netrc = NetrcParser.parseAndClose(netrcFile.getInputStream());
return new NetrcCredentials(netrc);
} catch (IOException e) {
throw new IOException(
"Failed to parse " + netrcFile.getPathString() + ": " + e.getMessage(), e);
}
} else {
return null;
}
}

/**
* Create a new {@link Credentials} with following order:
*
* <ol>
* <li>If authentication enabled by flags, use it to create credentials
* <li>Use .netrc to provide credentials if exists
* <li>Otherwise, return {@code null}
* </ol>
*
* @throws IOException in case the credentials can't be constructed.
*/
@VisibleForTesting
static Credentials newCredentials(
Map<String, String> clientEnv,
FileSystem fileSystem,
Reporter reporter,
AuthAndTLSOptions authAndTlsOptions,
RemoteOptions remoteOptions)
throws IOException {
Credentials creds = GoogleAuthUtils.newCredentials(authAndTlsOptions);

// Fallback to .netrc if it exists
if (creds == null) {
try {
creds = newCredentialsFromNetrc(clientEnv, fileSystem);
} catch (IOException e) {
reporter.handle(Event.warn(e.getMessage()));
}

try {
if (creds != null
&& remoteOptions.remoteCache != null
&& Ascii.toLowerCase(remoteOptions.remoteCache).startsWith("http://")
&& !creds.getRequestMetadata(new URI(remoteOptions.remoteCache)).isEmpty()) {
reporter.handle(
Event.warn(
"Username and password from .netrc is transmitted in plaintext to "
+ remoteOptions.remoteCache
+ ". Please consider using an HTTPS endpoint."));
}
} catch (URISyntaxException e) {
throw new IOException(e.getMessage(), e);
}
}

return creds;
}

@VisibleForTesting
static CredentialHelperProvider newCredentialHelperProvider(
CredentialHelperEnvironment environment,
Expand All @@ -1159,6 +1067,33 @@ static CredentialHelperProvider newCredentialHelperProvider(
return builder.build();
}

static Credentials newCredentials(
Map<String, String> clientEnv,
FileSystem fileSystem,
Reporter reporter,
AuthAndTLSOptions authAndTlsOptions,
RemoteOptions remoteOptions) throws IOException {
Credentials credentials = GoogleAuthUtils.newCredentials(clientEnv, fileSystem, authAndTlsOptions);

try {
if (credentials != null
&& remoteOptions.remoteCache != null
&& Ascii.toLowerCase(remoteOptions.remoteCache).startsWith("http://")
&& !credentials.getRequestMetadata(new URI(remoteOptions.remoteCache)).isEmpty()) {
// TODO(yannic): Make this a error aborting the build.
reporter.handle(
Event.warn(
"Credentials are transmitted in plaintext to "
+ remoteOptions.remoteCache
+ ". Please consider using an HTTPS endpoint."));
}
} catch (URISyntaxException e) {
throw new IOException(e.getMessage(), e);
}

return credentials;
}

@VisibleForTesting
@AutoValue
abstract static class ScopedCredentialHelper {
Expand Down
4 changes: 4 additions & 0 deletions src/test/java/com/google/devtools/build/lib/authandtls/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,11 @@ java_library(
),
deps = [
"//src/main/java/com/google/devtools/build/lib/authandtls",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs",
"//src/main/java/com/google/devtools/common/options",
"//src/test/java/com/google/devtools/build/lib/testutil",
"//third_party:auth_checked_in",
"//third_party:guava",
"//third_party:junit4",
"//third_party:truth",
Expand Down

0 comments on commit e7e8562

Please sign in to comment.