diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index b883cbd0091f79..8f489b3dbc09a7 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -325,7 +325,10 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { try { UrlRewriter rewriter = UrlRewriter.getDownloaderUrlRewriter( - repoOptions == null ? null : repoOptions.downloaderConfig, env.getReporter()); + repoOptions == null ? null : repoOptions.downloaderConfig, + env.getReporter(), + env.getClientEnv(), + env.getRuntime().getFileSystem()); downloadManager.setUrlRewriter(rewriter); } catch (UrlRewriterParseException e) { // It's important that the build stops ASAP, because this config file may be required for diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD index 2c5d148128da8d..8b2a5db0cbf93a 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD @@ -13,6 +13,7 @@ java_library( srcs = glob(["*.java"]), deps = [ "//src/main/java/com/google/devtools/build/lib/analysis:blaze_version_info", + "//src/main/java/com/google/devtools/build/lib/authandtls", "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache", "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache:events", "//src/main/java/com/google/devtools/build/lib/buildeventstream", @@ -20,9 +21,12 @@ java_library( "//src/main/java/com/google/devtools/build/lib/concurrent", "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/util", + "//src/main/java/com/google/devtools/build/lib/util:os", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//src/main/java/net/starlark/java/syntax", + "//third_party:auth", + "//third_party:auto_value", "//third_party:guava", "//third_party:jsr305", ], diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java index 00aaee646e7eac..1548ea7b0bc1f7 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java @@ -15,6 +15,7 @@ package com.google.devtools.build.lib.bazel.repository.downloader; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; import com.google.common.base.MoreObjects; import com.google.common.base.Optional; @@ -108,12 +109,14 @@ public Path download( throw new InterruptedException(); } - List rewrittenUrls = originalUrls; + ImmutableList rewrittenUrls = ImmutableList.copyOf(originalUrls); Map> rewrittenAuthHeaders = authHeaders; if (rewriter != null) { - rewrittenUrls = rewriter.amend(originalUrls); - rewrittenAuthHeaders = rewriter.updateAuthHeaders(rewrittenUrls, authHeaders); + ImmutableList rewrittenUrlMappings = rewriter.amend(originalUrls); + rewrittenUrls = + rewrittenUrlMappings.stream().map(url -> url.url()).collect(toImmutableList()); + rewrittenAuthHeaders = rewriter.updateAuthHeaders(rewrittenUrlMappings, authHeaders); } URL mainUrl; // The "main" URL for this request diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/UrlRewriter.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/UrlRewriter.java index f02a2dc67514f7..d8cce6295dd120 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/UrlRewriter.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/UrlRewriter.java @@ -16,6 +16,8 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static java.nio.charset.StandardCharsets.ISO_8859_1; +import com.google.auth.Credentials; +import com.google.auto.value.AutoValue; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Ascii; import com.google.common.base.Preconditions; @@ -23,8 +25,14 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.authandtls.Netrc; +import com.google.devtools.build.lib.authandtls.NetrcCredentials; +import com.google.devtools.build.lib.authandtls.NetrcParser; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.Path; import java.io.BufferedReader; import java.io.IOException; import java.io.Reader; @@ -38,14 +46,17 @@ import java.nio.file.Paths; import java.util.Base64; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.function.Consumer; import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.annotation.Nullable; +import net.starlark.java.syntax.Location; /** * Helper class for taking URLs and converting them according to an optional config specified by @@ -59,13 +70,19 @@ public class UrlRewriter { private static final ImmutableSet REWRITABLE_SCHEMES = ImmutableSet.of("http", "https"); private final UrlRewriterConfig config; - private final Function> rewriter; + private final Function> rewriter; + @Nullable private final Credentials netrcCreds; @VisibleForTesting - UrlRewriter(Consumer log, String filePathForErrorReporting, Reader reader) + UrlRewriter( + Consumer log, + String filePathForErrorReporting, + Reader reader, + @Nullable Credentials netrcCreds) throws UrlRewriterParseException { Preconditions.checkNotNull(reader, "UrlRewriterConfig source must be set"); this.config = new UrlRewriterConfig(filePathForErrorReporting, reader); + this.netrcCreds = netrcCreds; this.rewriter = this::rewrite; } @@ -75,17 +92,30 @@ public class UrlRewriter { * * @param configPath Path to the config file to use. May be null. * @param reporter Used for logging when URLs are rewritten. + * @param clientEnv a map of the current Bazel command's environment + * @param fileSystem the Blaze file system */ - public static UrlRewriter getDownloaderUrlRewriter(String configPath, Reporter reporter) + public static UrlRewriter getDownloaderUrlRewriter( + String configPath, + Reporter reporter, + ImmutableMap clientEnv, + FileSystem fileSystem) throws UrlRewriterParseException { Consumer log = str -> reporter.handle(Event.info(str)); + // "empty" UrlRewriter shouldn't alter auth headers if (Strings.isNullOrEmpty(configPath)) { - return new UrlRewriter(log, "", new StringReader("")); + return new UrlRewriter(log, "", new StringReader(""), null); } + Credentials creds = null; + try { + creds = newCredentialsFromNetrc(clientEnv, fileSystem); + } catch (UrlRewriterParseException e) { + // If the credentials extraction failed, we're letting bazel try without credentials. + } try (BufferedReader reader = Files.newBufferedReader(Paths.get(configPath))) { - return new UrlRewriter(log, configPath, reader); + return new UrlRewriter(log, configPath, reader, creds); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -93,71 +123,93 @@ public static UrlRewriter getDownloaderUrlRewriter(String configPath, Reporter r /** * Rewrites {@code urls} using the configuration provided to {@link - * #getDownloaderUrlRewriter(String, Reporter)}. The returned list of URLs may be empty if the - * configuration used blocks all the input URLs. + * #getDownloaderUrlRewriter(String, Reporter, ImmutableMap, FileSystem)}. The returned list of + * URLs may be empty if the configuration used blocks all the input URLs. * * @param urls The input list of {@link URL}s. May be empty. * @return The amended lists of URLs. */ - public List amend(List urls) { + public ImmutableList amend(List urls) { Objects.requireNonNull(urls, "URLS to check must be set but may be empty"); - ImmutableList rewritten = - urls.stream().map(rewriter).flatMap(Collection::stream).collect(toImmutableList()); - - return rewritten; + return urls.stream().map(rewriter).flatMap(Collection::stream).collect(toImmutableList()); } /** - * Updates {@code authHeaders} using the userInfo available in the provided {@code urls}. + * Updates {@code authHeaders} using the userInfo available in the provided {@code urls}. Note + * that if the same url is present in both {@code authHeaders} and download config then it + * will be overridden with the value from download config. * * @param urls The input list of {@link URL}s. May be empty. * @param authHeaders A map of the URLs and their corresponding auth tokens. * @return A map of the updated authentication headers. */ public Map> updateAuthHeaders( - List urls, Map> authHeaders) { - ImmutableMap.Builder> authHeadersBuilder = - ImmutableMap.>builder().putAll(authHeaders); + List urls, Map> authHeaders) { + Map> updatedAuthHeaders = new HashMap<>(authHeaders); - for (URL url : urls) { - String userInfo = url.getUserInfo(); + for (RewrittenURL url : urls) { + // if URL was not re-written by UrlRewriter in first place, we should not attach auth headers + // to it + if (!url.rewritten()) { + continue; + } + + String userInfo = url.url().getUserInfo(); if (userInfo != null) { try { String token = "Basic " + Base64.getEncoder().encodeToString(userInfo.getBytes(ISO_8859_1)); - authHeadersBuilder.put(url.toURI(), ImmutableMap.of("Authorization", token)); + updatedAuthHeaders.put(url.url().toURI(), ImmutableMap.of("Authorization", token)); } catch (URISyntaxException e) { // If the credentials extraction failed, we're letting bazel try without credentials. } + } else if (this.netrcCreds != null) { + try { + Map> urlAuthHeaders = + this.netrcCreds.getRequestMetadata(url.url().toURI()); + if (urlAuthHeaders == null || urlAuthHeaders.isEmpty()) { + continue; + } + // there could be multiple Auth headers, take the first one + Map.Entry> firstAuthHeader = + urlAuthHeaders.entrySet().stream().findFirst().get(); + if (firstAuthHeader.getValue() != null && !firstAuthHeader.getValue().isEmpty()) { + updatedAuthHeaders.put( + url.url().toURI(), + ImmutableMap.of(firstAuthHeader.getKey(), firstAuthHeader.getValue().get(0))); + } + } catch (URISyntaxException | IOException e) { + // If the credentials extraction failed, we're letting bazel try without credentials. + } } } - return authHeadersBuilder.build(); + return ImmutableMap.copyOf(updatedAuthHeaders); } - private ImmutableList rewrite(URL url) { + private ImmutableList rewrite(URL url) { Preconditions.checkNotNull(url); // Cowardly refuse to rewrite non-HTTP(S) urls if (REWRITABLE_SCHEMES.stream() .noneMatch(scheme -> Ascii.equalsIgnoreCase(scheme, url.getProtocol()))) { - return ImmutableList.of(url); + return ImmutableList.of(RewrittenURL.create(url, false)); } - List rewrittenUrls = applyRewriteRules(url); + ImmutableList rewrittenUrls = applyRewriteRules(url); - ImmutableList.Builder toReturn = ImmutableList.builder(); + ImmutableList.Builder toReturn = ImmutableList.builder(); // Now iterate over the URLs - for (URL consider : rewrittenUrls) { + for (RewrittenURL consider : rewrittenUrls) { // If there's an allow entry, add it to the set to return and continue - if (isAllowMatched(consider)) { + if (isAllowMatched(consider.url())) { toReturn.add(consider); continue; } // If there's no block that matches the domain, add it to the set to return and continue - if (!isBlockMatched(consider)) { + if (!isBlockMatched(consider.url())) { toReturn.add(consider); } } @@ -192,7 +244,7 @@ private static boolean isMatchingHostName(URL url, String host) { return host.equals(url.getHost()) || url.getHost().endsWith("." + host); } - private ImmutableList applyRewriteRules(URL url) { + private ImmutableList applyRewriteRules(URL url) { String withoutScheme = url.toString().substring(url.getProtocol().length() + 3); ImmutableSet.Builder rewrittenUrls = ImmutableSet.builder(); @@ -210,11 +262,12 @@ private ImmutableList applyRewriteRules(URL url) { } if (!matchMade) { - return ImmutableList.of(url); + return ImmutableList.of(RewrittenURL.create(url, false)); } return rewrittenUrls.build().stream() .map(urlString -> prefixWithProtocol(urlString, url.getProtocol())) + .map(plainUrl -> RewrittenURL.create(plainUrl, true)) .collect(toImmutableList()); } @@ -232,8 +285,64 @@ private static URL prefixWithProtocol(String url, String protocol) { } } + /** + * Create a new {@link Credentials} object by parsing the .netrc file with following order to + * search it: + * + *
    + *
  1. If environment variable $NETRC exists, use it as the path to the .netrc file + *
  2. Fallback to $HOME/.netrc or $USERPROFILE/.netrc + *
+ * + * @return the {@link Credentials} object or {@code null} if there is no .netrc file. + * @throws UrlRewriterParseException in case the credentials can't be constructed. + */ + // TODO : consider re-using RemoteModule.newCredentialsFromNetrc + @VisibleForTesting + static Credentials newCredentialsFromNetrc(Map clientEnv, FileSystem fileSystem) + throws UrlRewriterParseException { + final Optional homeDir; + if (OS.getCurrent() == OS.WINDOWS) { + homeDir = Optional.ofNullable(clientEnv.get("USERPROFILE")); + } else { + homeDir = Optional.ofNullable(clientEnv.get("HOME")); + } + String netrcFileString = + Optional.ofNullable(clientEnv.get("NETRC")) + .orElseGet(() -> homeDir.map(home -> home + "/.netrc").orElse(null)); + if (netrcFileString == null) { + return null; + } + Location location = Location.fromFileLineColumn(netrcFileString, 0, 0); + + Path netrcFile = fileSystem.getPath(netrcFileString); + if (netrcFile.exists()) { + try { + Netrc netrc = NetrcParser.parseAndClose(netrcFile.getInputStream()); + return new NetrcCredentials(netrc); + } catch (IOException e) { + throw new UrlRewriterParseException( + "Failed to parse " + netrcFile.getPathString() + ": " + e.getMessage(), location); + } + } else { + return null; + } + } + @Nullable public String getAllBlockedMessage() { return config.getAllBlockedMessage(); } + + /** Holds the URL along with meta-info, such as whether URL was re-written or not. */ + @AutoValue + public abstract static class RewrittenURL { + static RewrittenURL create(URL url, boolean rewritten) { + return new AutoValue_UrlRewriter_RewrittenURL(url, rewritten); + } + + abstract URL url(); + + abstract boolean rewritten(); + } } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD index e156c86e07247c..05a7e941a5e203 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD @@ -16,13 +16,16 @@ java_library( name = "DownloaderTestSuite_lib", srcs = glob(["*.java"]), deps = [ + "//src/main/java/com/google/devtools/build/lib/authandtls", "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache", "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/util", "//src/main/java/com/google/devtools/build/lib/vfs", + "//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs", "//src/main/java/net/starlark/java/syntax", "//src/test/java/com/google/devtools/build/lib/testutil", + "//third_party:auth", "//third_party:guava", "//third_party:jsr305", "//third_party:junit4", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/UrlRewriterTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/UrlRewriterTest.java index 15148ec345ee43..422b6c5bf9cc6f 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/UrlRewriterTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/UrlRewriterTest.java @@ -13,12 +13,22 @@ // limitations under the License. package com.google.devtools.build.lib.bazel.repository.downloader; +import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.truth.Truth.assertThat; import static java.nio.charset.StandardCharsets.ISO_8859_1; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.fail; +import com.google.auth.Credentials; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.authandtls.BasicHttpAuthenticationEncoder; +import com.google.devtools.build.lib.testutil.Scratch; +import com.google.devtools.build.lib.vfs.DigestHashFunction; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; +import java.io.IOException; import java.io.StringReader; import java.net.URI; import java.net.URL; @@ -36,10 +46,11 @@ public class UrlRewriterTest { @Test public void byDefaultTheUrlRewriterDoesNothing() throws Exception { - UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader("")); + UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(""), null); List urls = ImmutableList.of(new URL("http://example.com")); - List amended = munger.amend(urls); + ImmutableList amended = + munger.amend(urls).stream().map(url -> url.url()).collect(toImmutableList()); assertThat(amended).isEqualTo(urls); } @@ -47,14 +58,15 @@ public void byDefaultTheUrlRewriterDoesNothing() throws Exception { @Test public void shouldBeAbleToBlockParticularHostsRegardlessOfScheme() throws Exception { String config = "block example.com"; - UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config)); + UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config), null); List urls = ImmutableList.of( new URL("http://example.com"), new URL("https://example.com"), new URL("http://localhost")); - List amended = munger.amend(urls); + ImmutableList amended = + munger.amend(urls).stream().map(url -> url.url()).collect(toImmutableList()); assertThat(amended).containsExactly(new URL("http://localhost")); } @@ -62,10 +74,11 @@ public void shouldBeAbleToBlockParticularHostsRegardlessOfScheme() throws Except @Test public void shouldAllowAUrlToBeRewritten() throws Exception { String config = "rewrite example.com/foo/(.*) mycorp.com/$1/foo"; - UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config)); + UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config), null); List urls = ImmutableList.of(new URL("https://example.com/foo/bar")); - List amended = munger.amend(urls); + ImmutableList amended = + munger.amend(urls).stream().map(url -> url.url()).collect(toImmutableList()); assertThat(amended).containsExactly(new URL("https://mycorp.com/bar/foo")); } @@ -75,10 +88,11 @@ public void rewritesCanExpandToMoreThanOneUrl() throws Exception { String config = "rewrite example.com/foo/(.*) mycorp.com/$1/somewhere\n" + "rewrite example.com/foo/(.*) mycorp.com/$1/elsewhere"; - UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config)); + UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config), null); List urls = ImmutableList.of(new URL("https://example.com/foo/bar")); - List amended = munger.amend(urls); + ImmutableList amended = + munger.amend(urls).stream().map(url -> url.url()).collect(toImmutableList()); // There's no guarantee about the ordering of the rewrites assertThat(amended).contains(new URL("https://mycorp.com/bar/somewhere")); @@ -89,14 +103,15 @@ public void rewritesCanExpandToMoreThanOneUrl() throws Exception { public void shouldBlockAllUrlsOtherThanSpecificOnes() throws Exception { String config = "" + "block *\n" + "allow example.com"; - UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config)); + UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config), null); List urls = ImmutableList.of( new URL("https://foo.com"), new URL("https://example.com/foo/bar"), new URL("https://subdomain.example.com/qux")); - List amended = munger.amend(urls); + ImmutableList amended = + munger.amend(urls).stream().map(url -> url.url()).collect(toImmutableList()); assertThat(amended) .containsExactly( @@ -112,10 +127,11 @@ public void commentsArePrecededByTheHashCharacter() throws Exception { + "# But allow example.com\n" + "allow example.com"; - UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config)); + UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config), null); List urls = ImmutableList.of(new URL("https://foo.com"), new URL("https://example.com")); - List amended = munger.amend(urls); + ImmutableList amended = + munger.amend(urls).stream().map(url -> url.url()).collect(toImmutableList()); assertThat(amended).containsExactly(new URL("https://example.com")); } @@ -124,9 +140,12 @@ public void commentsArePrecededByTheHashCharacter() throws Exception { public void allowListAppliesToSubdomainsToo() throws Exception { String config = "" + "block *\n" + "allow example.com"; - UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config)); + UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config), null); - List amended = munger.amend(ImmutableList.of(new URL("https://subdomain.example.com"))); + ImmutableList amended = + munger.amend(ImmutableList.of(new URL("https://subdomain.example.com"))).stream() + .map(url -> url.url()) + .collect(toImmutableList()); assertThat(amended).containsExactly(new URL("https://subdomain.example.com")); } @@ -135,9 +154,12 @@ public void allowListAppliesToSubdomainsToo() throws Exception { public void blockListAppliesToSubdomainsToo() throws Exception { String config = "block example.com"; - UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config)); + UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config), null); - List amended = munger.amend(ImmutableList.of(new URL("https://subdomain.example.com"))); + ImmutableList amended = + munger.amend(ImmutableList.of(new URL("https://subdomain.example.com"))).stream() + .map(url -> url.url()) + .collect(toImmutableList()); assertThat(amended).isEmpty(); } @@ -146,9 +168,12 @@ public void blockListAppliesToSubdomainsToo() throws Exception { public void emptyLinesAreFine() throws Exception { String config = "" + "\n" + " \n" + "block *\n" + "\t \n" + "allow example.com"; - UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config)); + UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config), null); - List amended = munger.amend(ImmutableList.of(new URL("https://subdomain.example.com"))); + ImmutableList amended = + munger.amend(ImmutableList.of(new URL("https://subdomain.example.com"))).stream() + .map(url -> url.url()) + .collect(toImmutableList()); assertThat(amended).containsExactly(new URL("https://subdomain.example.com")); } @@ -157,11 +182,16 @@ public void emptyLinesAreFine() throws Exception { public void rewritingUrlsIsAppliedBeforeBlocking() throws Exception { String config = "" + "block bad.com\n" + "rewrite bad.com/foo/(.*) mycorp.com/$1"; - UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config)); + UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config), null); List amended = - munger.amend( - ImmutableList.of(new URL("https://www.bad.com"), new URL("https://bad.com/foo/bar"))); + munger + .amend( + ImmutableList.of( + new URL("https://www.bad.com"), new URL("https://bad.com/foo/bar"))) + .stream() + .map(url -> url.url()) + .collect(toImmutableList()); assertThat(amended).containsExactly(new URL("https://mycorp.com/bar")); } @@ -171,11 +201,16 @@ public void rewritingUrlsIsAppliedBeforeAllowing() throws Exception { String config = "" + "block *\n" + "allow mycorp.com\n" + "rewrite bad.com/foo/(.*) mycorp.com/$1"; - UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config)); + UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config), null); List amended = - munger.amend( - ImmutableList.of(new URL("https://www.bad.com"), new URL("https://bad.com/foo/bar"))); + munger + .amend( + ImmutableList.of( + new URL("https://www.bad.com"), new URL("https://bad.com/foo/bar"))) + .stream() + .map(url -> url.url()) + .collect(toImmutableList()); assertThat(amended).containsExactly(new URL("https://mycorp.com/bar")); } @@ -228,14 +263,18 @@ public void rewritingUrlsAllowsProtocolRewrite() throws Exception { + "rewrite bad.com/foo/(.*) http://mycorp.com/$1\n" + "rewrite bad.com/bar/(.*) https://othercorp.com/bar/$1\n"; - UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config)); + UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config), null); List amended = - munger.amend( - ImmutableList.of( - new URL("https://www.bad.com"), - new URL("https://bad.com/foo/bar"), - new URL("http://bad.com/bar/xyz"))); + munger + .amend( + ImmutableList.of( + new URL("https://www.bad.com"), + new URL("https://bad.com/foo/bar"), + new URL("http://bad.com/bar/xyz"))) + .stream() + .map(url -> url.url()) + .collect(toImmutableList()); assertThat(amended) .containsExactly( @@ -245,19 +284,133 @@ public void rewritingUrlsAllowsProtocolRewrite() throws Exception { @Test public void rewritingUrlsWithAuthHeaders() throws Exception { String creds = "user:password"; - String config = "rewrite my.example.com/foo/(.*) " + creds + "@mycorp.com/foo/$1\n"; - - UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config)); - - List amended = munger.amend(ImmutableList.of(new URL("https://my.example.com/foo/bar"))); + String firstNetrcCreds = "netrc_user_0:netrc_pw_0"; + String secondNetrcCreds = "netrc_user_1:netrc_pw_1"; + Credentials netrc = + parseNetrc( + "machine mycorp.com login netrc_user_0 password netrc_pw_0\n" + + "machine myothercorp.com login netrc_user_1 password netrc_pw_1\n" + + "machine no-override.com login netrc_user_2 password netrc_pw_2\n"); + String config = + "" + + "rewrite my.example.com/foo/(.*) " + + creds + + "@mycorp.com/foo/$1\n" // this cred should from download config file + + "rewrite my.example.com/from_netrc/(.*) mycorp.com/from_netrc/$1\n" // this cred + // should come + // from netrc + + "rewrite" + + " my.example.com/from_other_netrc_entry/(.*)" + + " myothercorp.com/from_netrc/$1\n" // this cred should come from netrc + + "rewrite my.example.com/no_creds/(.*) myopencorp.com/no_creds/$1\n"; // should be + // re-written, + // but no auth + // headers added + + UrlRewriter munger = new UrlRewriter(str -> {}, "/dev/null", new StringReader(config), netrc); + + ImmutableList amended = + munger.amend( + ImmutableList.of( + new URL("https://my.example.com/foo/bar"), + new URL("https://my.example.com/from_netrc/bar"), + new URL("https://my.example.com/from_other_netrc_entry/bar"), + new URL("https://my.example.com/no_creds/bar"), + new URL("https://should-not-be-overridden.com/"))); Map> updatedAuthHeaders = munger.updateAuthHeaders(amended, ImmutableMap.of()); String expectedToken = "Basic " + Base64.getEncoder().encodeToString(creds.getBytes(ISO_8859_1)); + String expectedFirstNetrcToken = + "Basic " + Base64.getEncoder().encodeToString(firstNetrcCreds.getBytes(ISO_8859_1)); + String expectedSecondNetrcToken = + "Basic " + Base64.getEncoder().encodeToString(secondNetrcCreds.getBytes(ISO_8859_1)); + // only three URLs should have auth headers assertThat(updatedAuthHeaders) .containsExactly( new URI("https://user:password@mycorp.com/foo/bar"), - ImmutableMap.of("Authorization", expectedToken)); + ImmutableMap.of("Authorization", expectedToken), + new URI("https://mycorp.com/from_netrc/bar"), + ImmutableMap.of("Authorization", expectedFirstNetrcToken), + new URI("https://myothercorp.com/from_netrc/bar"), + ImmutableMap.of("Authorization", expectedSecondNetrcToken)); + // yet all four urls should be present + assertThat(amended) + .containsExactly( + UrlRewriter.RewrittenURL.create( + new URL("https://user:password@mycorp.com/foo/bar"), true), + UrlRewriter.RewrittenURL.create(new URL("https://mycorp.com/from_netrc/bar"), true), + UrlRewriter.RewrittenURL.create( + new URL("https://myothercorp.com/from_netrc/bar"), true), + UrlRewriter.RewrittenURL.create(new URL("https://myopencorp.com/no_creds/bar"), true), + UrlRewriter.RewrittenURL.create( + new URL("https://should-not-be-overridden.com/"), false)); + } + + @Test + public void testNetrc_emptyEnv_shouldIgnore() throws Exception { + ImmutableMap clientEnv = ImmutableMap.of(); + FileSystem fileSystem = new InMemoryFileSystem(DigestHashFunction.SHA256); + + Credentials credentials = UrlRewriter.newCredentialsFromNetrc(clientEnv, fileSystem); + + assertThat(credentials).isNull(); + } + + @Test + public void testNetrc_netrcNotExist_shouldIgnore() throws Exception { + String home = "/home/foo"; + ImmutableMap clientEnv = ImmutableMap.of("HOME", home, "USERPROFILE", home); + FileSystem fileSystem = new InMemoryFileSystem(DigestHashFunction.SHA256); + + Credentials credentials = UrlRewriter.newCredentialsFromNetrc(clientEnv, fileSystem); + + assertThat(credentials).isNull(); + } + + @Test + public void testNetrc_netrcExist_shouldUse() throws Exception { + Credentials credentials = parseNetrc("machine foo.example.org login foouser password foopass"); + + assertThat(credentials).isNotNull(); + assertRequestMetadata( + credentials.getRequestMetadata(URI.create("https://foo.example.org")), + "foouser", + "foopass"); + } + + @Test + public void testNetrc_netrcExist_cant_parse() throws Exception { + String home = "/home/foo"; + ImmutableMap clientEnv = ImmutableMap.of("HOME", home, "USERPROFILE", home); + FileSystem fileSystem = new InMemoryFileSystem(DigestHashFunction.SHA256); + Scratch scratch = new Scratch(fileSystem); + scratch.file(home + "/.netrc", "mach foo.example.org log foouser password foopass"); + + try { + UrlRewriter.newCredentialsFromNetrc(clientEnv, fileSystem); + fail(); + } catch (UrlRewriterParseException e) { + assertThat(e.getLocation()).isEqualTo(Location.fromFileLineColumn("/home/foo/.netrc", 0, 0)); + } + } + + private static Credentials parseNetrc(String content) + throws IOException, UrlRewriterParseException { + String home = "/home/foo"; + ImmutableMap clientEnv = ImmutableMap.of("HOME", home, "USERPROFILE", home); + FileSystem fileSystem = new InMemoryFileSystem(DigestHashFunction.SHA256); + Scratch scratch = new Scratch(fileSystem); + scratch.file(home + "/.netrc", content); + + return UrlRewriter.newCredentialsFromNetrc(clientEnv, fileSystem); + } + + private static void assertRequestMetadata( + Map> requestMetadata, String username, String password) { + assertThat(requestMetadata.keySet()).containsExactly("Authorization"); + assertThat(Iterables.getOnlyElement(requestMetadata.values())) + .containsExactly(BasicHttpAuthenticationEncoder.encode(username, password, UTF_8)); } }