diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DelegatingDownloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DelegatingDownloader.java index 249cc22c7c0d19..129a730bcfc8b6 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DelegatingDownloader.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DelegatingDownloader.java @@ -47,6 +47,7 @@ public void setDelegate(@Nullable Downloader delegate) { @Override public void download( List urls, + Map> headers, Credentials credentials, Optional checksum, String canonicalId, @@ -60,6 +61,6 @@ public void download( downloader = delegate; } downloader.download( - urls, credentials, checksum, canonicalId, destination, eventHandler, clientEnv, type); + urls, headers, credentials, checksum, canonicalId, destination, eventHandler, clientEnv, type); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java index ae9fe3543a2806..ac3e49e04476cf 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java @@ -122,6 +122,7 @@ public void setCredentialFactory(CredentialFactory credentialFactory) { */ public Path download( List originalUrls, + Map> headers, Map>> authHeaders, Optional checksum, String canonicalId, @@ -267,6 +268,7 @@ public Path download( try { downloader.download( rewrittenUrls, + headers, credentialFactory.create(rewrittenAuthHeaders), checksum, canonicalId, diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/Downloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/Downloader.java index 1e8fc932b43b08..79a0076a3f56e1 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/Downloader.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/Downloader.java @@ -42,6 +42,7 @@ public interface Downloader { */ void download( List urls, + Map> headers, Credentials credentials, Optional checksum, String canonicalId, diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexer.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexer.java index 1d5ba9b08a435c..72ef98afe59575 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexer.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexer.java @@ -75,7 +75,7 @@ final class HttpConnectorMultiplexer { } public HttpStream connect(URL url, Optional checksum) throws IOException { - return connect(url, checksum, StaticCredentials.EMPTY, Optional.empty()); + return connect(url, checksum, Map.of(), StaticCredentials.EMPTY, Optional.empty()); } /** @@ -96,14 +96,18 @@ public HttpStream connect(URL url, Optional checksum) throws IOExcepti * @throws IllegalArgumentException if {@code urls} is empty or has an unsupported protocol */ public HttpStream connect( - URL url, Optional checksum, Credentials credentials, Optional type) + URL url, Optional checksum, Map> headers, Credentials credentials, Optional type) throws IOException { Preconditions.checkArgument(HttpUtils.isUrlSupportedByDownloader(url)); if (Thread.interrupted()) { throw new InterruptedIOException(); } + Map> baseHeaders = new HashMap<>(headers); + // REQUEST_HEADERS should not be overriable by user provided headers + baseHeaders.putAll(REQUEST_HEADERS); + Function>> headerFunction = - getHeaderFunction(REQUEST_HEADERS, credentials); + getHeaderFunction(baseHeaders, credentials); URLConnection connection = connector.connect(url, headerFunction); return httpStreamFactory.create( connection, diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java index 1cc14c1dd183c8..6464aad4caeffd 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java @@ -15,6 +15,8 @@ package com.google.devtools.build.lib.bazel.repository.downloader; import com.google.auth.Credentials; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.io.ByteStreams; @@ -74,6 +76,7 @@ public void setMaxRetryTimeout(Duration maxRetryTimeout) { @Override public void download( List urls, + Map> headers, Credentials credentials, Optional checksum, String canonicalId, @@ -93,7 +96,7 @@ public void download( for (URL url : urls) { SEMAPHORE.acquire(); - try (HttpStream payload = multiplexer.connect(url, checksum, credentials, type); + try (HttpStream payload = multiplexer.connect(url, checksum, headers, credentials, type); OutputStream out = destination.getOutputStream()) { try { ByteStreams.copy(payload, out); @@ -152,7 +155,7 @@ public byte[] downloadAndReadOneUrl( ByteArrayOutputStream out = new ByteArrayOutputStream(); SEMAPHORE.acquire(); try (HttpStream payload = - multiplexer.connect(url, Optional.empty(), credentials, Optional.empty())) { + multiplexer.connect(url, Optional.empty(), ImmutableMap.of(), credentials, Optional.empty())) { ByteStreams.copy(payload, out); } catch (SocketTimeoutException e) { // SocketTimeoutExceptions are InterruptedIOExceptions; however they do not signify diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkBaseExternalContext.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkBaseExternalContext.java index cd5f6d3a644c76..8badaacea4d380 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkBaseExternalContext.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkBaseExternalContext.java @@ -223,44 +223,18 @@ private static ImmutableMap>> getAuthHeaders( return res; } - private static ImmutableMap>> getHeaders( - Map>> auth) throws RepositoryFunctionException, EvalException { - ImmutableMap.Builder>> headers = new ImmutableMap.Builder<>(); - for (Map.Entry>> entry : auth.entrySet()) { - try { - URL url = new URL(entry.getKey()); - Map> headerMap = entry.getValue(); - headers.put(url.toURI(), headerMap); - } catch (MalformedURLException e) { - throw new RepositoryFunctionException(e, Transience.PERSISTENT); - } catch (URISyntaxException e) { - throw new EvalException(e); - } - } - return headers.buildOrThrow(); - } - - private static ImmutableMap>> getHeaderContents(Dict x, String what) + private static ImmutableMap> getHeaderContents(Dict x, String what) throws EvalException { // Dict.cast returns Dict. @SuppressWarnings({"unchecked", "rawtypes"}) - Map> urlHeaderMapUnchecked = (Map) Dict.cast(x, String.class, Dict.class, what); - - ImmutableMap.Builder>> urlHeadersMap = new ImmutableMap.Builder<>(); - - for (Map.Entry> urlHeaderEntry : urlHeaderMapUnchecked.entrySet()) { - - ImmutableMap.Builder> headers = new ImmutableMap.Builder<>(); - Dict headersUnchecked = Dict.cast(urlHeaderEntry.getValue(), String.class, List.class, "header entry"); - - for (Map.Entry headerEntry : headersUnchecked.entrySet()) { - List headerValue = headerEntry.getValue().stream().map(r -> r.toString()).toList(); - headers.put(headerEntry.getKey(), headerValue); - } + Dict headersUnchecked = (Dict) Dict.cast(x, String.class, List.class, what); + ImmutableMap.Builder> headers = new ImmutableMap.Builder<>(); - urlHeadersMap.put(urlHeaderEntry.getKey(), headers.buildOrThrow()); + for (Map.Entry headerEntry : headersUnchecked.entrySet()) { + List headerValue = headerEntry.getValue().stream().map(r -> r.toString()).toList(); + headers.put(headerEntry.getKey(), headerValue); } - return urlHeadersMap.buildOrThrow(); + return headers.buildOrThrow(); } private static ImmutableList checkAllUrls(Iterable urlList) throws EvalException { @@ -488,7 +462,7 @@ public StructImpl download( Boolean executable, Boolean allowFail, String canonicalId, - Dict authUnchecked, // expected + Dict authUnchecked, // > expected Dict headersUnchecked, // expected String integrity, StarlarkThread thread) @@ -496,12 +470,7 @@ public StructImpl download( ImmutableMap>> authHeaders = getAuthHeaders(getAuthContents(authUnchecked, "auth")); - ImmutableMap>> headers = getHeaders(getHeaderContents(headersUnchecked, "headers")); - - ImmutableMap>> allHeaders = new ImmutableMap.Builder>>() - .putAll(authHeaders) - .putAll(headers) - .buildOrThrow(); + ImmutableMap> headers = getHeaderContents(headersUnchecked, "headers"); ImmutableList urls = getUrls( @@ -536,7 +505,8 @@ public StructImpl download( downloadedPath = downloadManager.download( urls, - allHeaders, + headers, + authHeaders, checksum, canonicalId, Optional.empty(), @@ -696,12 +666,7 @@ public StructImpl downloadAndExtract( ImmutableMap>> authHeaders = getAuthHeaders(getAuthContents(authUnchecked, "auth")); - ImmutableMap>> headers = getHeaders(getHeaderContents(headersUnchecked, "headers")); - - ImmutableMap>> allHeaders = new ImmutableMap.Builder>>() - .putAll(authHeaders) - .putAll(headers) - .buildOrThrow(); + ImmutableMap> headers = getHeaderContents(headersUnchecked, "headers"); ImmutableList urls = getUrls( @@ -750,7 +715,8 @@ public StructImpl downloadAndExtract( downloadedPath = downloadManager.download( urls, - allHeaders, + headers, + authHeaders, checksum, canonicalId, Optional.of(type), diff --git a/src/main/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloader.java b/src/main/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloader.java index 1a201e25cc4bd2..6ce50222e9cb89 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloader.java +++ b/src/main/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloader.java @@ -110,6 +110,7 @@ public void close() { @Override public void download( List urls, + Map> headers, Credentials credentials, Optional checksum, String canonicalId, @@ -154,7 +155,7 @@ public void download( eventHandler.handle( Event.warn("Remote Cache: " + Utils.grpcAwareErrorMessage(e, verboseFailures))); fallbackDownloader.download( - urls, credentials, checksum, canonicalId, destination, eventHandler, clientEnv, type); + urls, headers, credentials, checksum, canonicalId, destination, eventHandler, clientEnv, type); } }