From bfa9c6acc6a98fb1c6f95b33b2d3f7da8765500e Mon Sep 17 00:00:00 2001 From: "bazel.build machine account" Date: Thu, 23 May 2024 18:33:46 -0400 Subject: [PATCH] [7.2.0] Add support for overlays in source.json (#22525) This is a continuation of #22155 that adds the newly added 'remote_files' attribute for http_archive to the bzlmod functionality. The end goal is to then update BCR to this new functionality to overlay files rather than use patch files when providing MODULE/WORKSPACE/BUILD files. https://github.com/bazelbuild/bazel-central-registry/issues/1566 has a good discussion of the rationale. Co-authored-by: Fabian Meumertzheim Closes #22349. PiperOrigin-RevId: 636682112 Change-Id: Ief070985598a7c0f427a98cd3daeb69a0984f7be Commit https://github.com/bazelbuild/bazel/commit/c4167e309a29384ee1cf827b4ecf4ff5b0210fc9 Co-authored-by: Farid Zakaria --- .../bazel/bzlmod/ArchiveRepoSpecBuilder.java | 23 ++++++++++++ .../build/lib/bazel/bzlmod/IndexRegistry.java | 22 ++++++++++++ .../lib/bazel/bzlmod/IndexRegistryTest.java | 35 +++++++++++++++++++ src/test/py/bazel/bzlmod/mod_command_test.py | 34 ++++++++++-------- 4 files changed, 100 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java index 62074385419042..a68a4b45c1370a 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java @@ -15,11 +15,15 @@ package com.google.devtools.build.lib.bazel.bzlmod; +import static com.google.common.collect.ImmutableMap.toImmutableMap; + import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.cmdline.Label; import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.util.List; +import java.util.Map.Entry; import net.starlark.java.eval.StarlarkInt; /** @@ -78,6 +82,19 @@ public ArchiveRepoSpecBuilder setRemotePatches(ImmutableMap remo return this; } + @CanIgnoreReturnValue + public ArchiveRepoSpecBuilder setOverlay(ImmutableMap overlay) { + ImmutableMap> remoteFiles = + overlay.entrySet().stream() + .collect(toImmutableMap(Entry::getKey, e -> e.getValue().urls())); + ImmutableMap remoteFilesIntegrity = + overlay.entrySet().stream() + .collect(toImmutableMap(Entry::getKey, e -> e.getValue().integrity())); + attrBuilder.put("remote_file_urls", remoteFiles); + attrBuilder.put("remote_file_integrity", remoteFilesIntegrity); + return this; + } + @CanIgnoreReturnValue public ArchiveRepoSpecBuilder setRemotePatchStrip(int remotePatchStrip) { attrBuilder.put("remote_patch_strip", StarlarkInt.of(remotePatchStrip)); @@ -99,4 +116,10 @@ public RepoSpec build() { .setAttributes(AttributeValues.create(attrBuilder.buildOrThrow())) .build(); } + + /** + * A simple pojo to track remote files that are offered at multiple urls (mirrors) with a single + * integrity. We split up the file here to simplify the dependency. + */ + public record RemoteFile(String integrity, List urls) {} } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java index 7c01d12651fcca..57881065649dd6 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java @@ -15,6 +15,7 @@ package com.google.devtools.build.lib.bazel.bzlmod; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.base.Preconditions; @@ -222,6 +223,7 @@ private static class ArchiveSourceJson { String integrity; String stripPrefix; Map patches; + Map overlay; int patchStrip; String archiveType; } @@ -412,11 +414,31 @@ private RepoSpec createArchiveRepoSpec( } } + ImmutableMap sourceJsonOverlay = + sourceJson.overlay != null ? ImmutableMap.copyOf(sourceJson.overlay) : ImmutableMap.of(); + ImmutableMap overlay = + sourceJsonOverlay.entrySet().stream() + .collect( + toImmutableMap( + entry -> entry.getKey(), + entry -> + new ArchiveRepoSpecBuilder.RemoteFile( + entry.getValue(), // integrity + // URLs in the registry itself are not mirrored. + ImmutableList.of( + constructUrl( + getUrl(), + "modules", + key.getName(), + key.getVersion().toString(), + entry.getKey()))))); + return new ArchiveRepoSpecBuilder() .setUrls(urls.build()) .setIntegrity(sourceJson.integrity) .setStripPrefix(Strings.nullToEmpty(sourceJson.stripPrefix)) .setRemotePatches(remotePatches.buildOrThrow()) + .setOverlay(overlay) .setRemotePatchStrip(sourceJson.patchStrip) .setArchiveType(sourceJson.archiveType) .build(); diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java index e76f499ff020b1..9e20b46e83c65d 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java @@ -183,6 +183,17 @@ public void testGetArchiveRepoSpec() throws Exception { " },", " \"patch_strip\": 3", "}"); + server.serve( + "/modules/baz/3.0/source.json", + """ + { + "url": "https://example.com/archive.jar?with=query", + "integrity": "sha256-bleh", + "overlay": { + "BUILD.bazel": "sha256-bleh-overlay" + } + } + """); server.start(); Registry registry = @@ -199,6 +210,7 @@ public void testGetArchiveRepoSpec() throws Exception { .setIntegrity("sha256-blah") .setStripPrefix("pref") .setRemotePatches(ImmutableMap.of()) + .setOverlay(ImmutableMap.of()) .setRemotePatchStrip(0) .build()); assertThat(registry.getRepoSpec(createModuleKey("bar", "2.0"), reporter)) @@ -217,6 +229,27 @@ public void testGetArchiveRepoSpec() throws Exception { server.getUrl() + "/modules/bar/2.0/patches/2.fix-that.patch", "sha256-kek")) .setRemotePatchStrip(3) + .setOverlay(ImmutableMap.of()) + .build()); + assertThat(registry.getRepoSpec(createModuleKey("baz", "3.0"), reporter)) + .isEqualTo( + new ArchiveRepoSpecBuilder() + .setUrls( + ImmutableList.of( + "https://mirror.bazel.build/example.com/archive.jar?with=query", + "file:///home/bazel/mymirror/example.com/archive.jar?with=query", + "https://example.com/archive.jar?with=query")) + .setIntegrity("sha256-bleh") + .setStripPrefix("") + .setOverlay( + ImmutableMap.of( + "BUILD.bazel", + new ArchiveRepoSpecBuilder.RemoteFile( + "sha256-bleh-overlay", + // URLs in the registry itself are not mirrored. + ImmutableList.of(server.getUrl() + "/modules/baz/3.0/BUILD.bazel")))) + .setRemotePatches(ImmutableMap.of()) + .setRemotePatchStrip(0) .build()); } @@ -265,6 +298,7 @@ public void testGetRepoInvalidRegistryJsonSpec() throws Exception { .setIntegrity("sha256-blah") .setStripPrefix("pref") .setRemotePatches(ImmutableMap.of()) + .setOverlay(ImmutableMap.of()) .setRemotePatchStrip(0) .build()); } @@ -352,6 +386,7 @@ public void testArchiveWithExplicitType() throws Exception { .setArchiveType("zip") .setRemotePatches(ImmutableMap.of()) .setRemotePatchStrip(0) + .setOverlay(ImmutableMap.of()) .build()); } diff --git a/src/test/py/bazel/bzlmod/mod_command_test.py b/src/test/py/bazel/bzlmod/mod_command_test.py index cc4c844f5a1c62..eac9852c4a378d 100644 --- a/src/test/py/bazel/bzlmod/mod_command_test.py +++ b/src/test/py/bazel/bzlmod/mod_command_test.py @@ -397,13 +397,15 @@ def testShowModuleAndExtensionReposFromBaseModule(self): ) self.assertRegex(stdout.pop(4), r'^ urls = \[".*"\],$') self.assertRegex(stdout.pop(4), r'^ integrity = ".*",$') - stdout.pop(11) - self.assertRegex(stdout.pop(16), r'^ path = ".*",$') - stdout.pop(29) - stdout.pop(39) - self.assertRegex(stdout.pop(44), r'^ urls = \[".*"\],$') - self.assertRegex(stdout.pop(44), r'^ integrity = ".*",$') - stdout.pop(51) + self.assertRegex(stdout.pop(19), r'^ path = ".*",$') + # lines after 'Rule data_repo defined at (most recent call last):' + stdout.pop(32) + stdout.pop(42) + self.assertRegex(stdout.pop(47), r'^ urls = \[".*"\],$') + self.assertRegex(stdout.pop(47), r'^ integrity = ".*",$') + # lines after '# Rule http_archive defined at (most recent call last):' + stdout.pop(13) + stdout.pop(55) self.assertListEqual( stdout, [ @@ -414,19 +416,21 @@ def testShowModuleAndExtensionReposFromBaseModule(self): # pop(4) -- urls=[...] # pop(4) -- integrity=... ' strip_prefix = "",', + ' remote_file_urls = {},', + ' remote_file_integrity = {},', ' remote_patches = {},', ' remote_patch_strip = 0,', ')', '# Rule bar~ instantiated at (most recent call last):', '# in ', '# Rule http_archive defined at (most recent call last):', - # pop(11) + # pop(13) '', '## ext@1.0:', '# ', 'local_repository(', ' name = "ext~",', - # pop(16) -- path=... + # pop(19) -- path=... ')', '# Rule ext~ instantiated at (most recent call last):', '# in ', @@ -440,7 +444,7 @@ def testShowModuleAndExtensionReposFromBaseModule(self): '# Rule ext~~ext~repo3 instantiated at (most recent call last):', '# in ', '# Rule data_repo defined at (most recent call last):', - # pop(29) + # pop(32) '', '## @my_repo4:', '# ', @@ -451,22 +455,24 @@ def testShowModuleAndExtensionReposFromBaseModule(self): '# Rule ext~~ext~repo4 instantiated at (most recent call last):', '# in ', '# Rule data_repo defined at (most recent call last):', - # pop(39) + # pop(42) '', '## bar@2.0:', '# ', 'http_archive(', ' name = "bar~",', - # pop(44) -- urls=[...] - # pop(44) -- integrity=... + # pop(47) -- urls=[...] + # pop(47) -- integrity=... ' strip_prefix = "",', + ' remote_file_urls = {},', + ' remote_file_integrity = {},', ' remote_patches = {},', ' remote_patch_strip = 0,', ')', '# Rule bar~ instantiated at (most recent call last):', '# in ', '# Rule http_archive defined at (most recent call last):', - # pop(51) + # pop(55) '', ], 'wrong output in the show query for module and extension-generated'