Skip to content

Commit

Permalink
Write a repo mapping manifest in the runfiles directory (bazelbuild#1…
Browse files Browse the repository at this point in the history
…6555)

* Write a repo mapping manifest in the runfiles directory

To ensure we can use repo mappings in the runfiles library, this change writes an extra file "my_binary_target.repo_mapping", which contains a bunch of (base_repo_canonical_name, apparent_repo_name, canonical_repo_name) triples. See https://github.com/bazelbuild/proposals/blob/main/designs/2022-07-21-locating-runfiles-with-bzlmod.md for more information.

The extra file is written using a new action "RepoMappingManifestAction", and it's only executed if Bzlmod is enabled. This avoids generating a lot of extra actions that are essentially useless for monorepo setups such as Google's.

Work towards bazelbuild#16124

PiperOrigin-RevId: 483735699
Change-Id: I885b4df093bd2c783c57d19f995f420b9b29b53c

* Remove erroneous check about a repo mapping entry with an empty apparent name

It turns out that not only the main repo can have such an entry; any repo generated by a module extension hosted in the main repo can (and will) also have such an entry, because they can see all repos that the main repo can see, which includes the main repo itself.

PiperOrigin-RevId: 483938753
Change-Id: I6192496f2edf08c9b1723c5b041c0ac44f928b3e
  • Loading branch information
Wyverald committed Oct 27, 2022
1 parent b88779e commit 75b0f3a
Show file tree
Hide file tree
Showing 9 changed files with 612 additions and 23 deletions.
20 changes: 20 additions & 0 deletions src/main/java/com/google/devtools/build/lib/analysis/BUILD
Expand Up @@ -336,6 +336,7 @@ java_library(
":package_specification_provider",
":platform_options",
":provider_collection",
":repo_mapping_manifest_action",
":required_config_fragments_provider",
":resolved_toolchain_context",
":rule_configured_object_value",
Expand Down Expand Up @@ -982,6 +983,25 @@ java_library(
],
)

java_library(
name = "repo_mapping_manifest_action",
srcs = ["RepoMappingManifestAction.java"],
deps = [
":actions/abstract_file_write_action",
":actions/deterministic_writer",
"//src/main/java/com/google/devtools/build/lib/actions",
"//src/main/java/com/google/devtools/build/lib/actions:artifacts",
"//src/main/java/com/google/devtools/build/lib/actions:commandline_item",
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/collect/nestedset",
"//src/main/java/com/google/devtools/build/lib/util",
"//src/main/java/net/starlark/java/eval",
"//third_party:auto_value",
"//third_party:guava",
"//third_party:jsr305",
],
)

java_library(
name = "required_config_fragments_provider",
srcs = ["RequiredConfigFragmentsProvider.java"],
Expand Down
@@ -0,0 +1,122 @@
// Copyright 2022 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.analysis;

import static java.nio.charset.StandardCharsets.ISO_8859_1;
import static java.util.Comparator.comparing;

import com.google.auto.value.AutoValue;
import com.google.common.collect.ImmutableList;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.ActionKeyContext;
import com.google.devtools.build.lib.actions.ActionOwner;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander;
import com.google.devtools.build.lib.actions.CommandLineExpansionException;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction;
import com.google.devtools.build.lib.analysis.actions.DeterministicWriter;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.collect.nestedset.Order;
import com.google.devtools.build.lib.util.Fingerprint;
import java.io.PrintWriter;
import java.util.List;
import java.util.UUID;
import javax.annotation.Nullable;
import net.starlark.java.eval.EvalException;

/** Creates a manifest file describing the repos and mappings relevant for a runfile tree. */
public class RepoMappingManifestAction extends AbstractFileWriteAction {
private static final UUID MY_UUID = UUID.fromString("458e351c-4d30-433d-b927-da6cddd4737f");

private final ImmutableList<Entry> entries;
private final String workspaceName;

/** An entry in the repo mapping manifest file. */
@AutoValue
public abstract static class Entry {
public static Entry of(
RepositoryName sourceRepo, String targetRepoApparentName, RepositoryName targetRepo) {
return new AutoValue_RepoMappingManifestAction_Entry(
sourceRepo, targetRepoApparentName, targetRepo);
}

public abstract RepositoryName sourceRepo();

public abstract String targetRepoApparentName();

public abstract RepositoryName targetRepo();
}

public RepoMappingManifestAction(
ActionOwner owner, Artifact output, List<Entry> entries, String workspaceName) {
super(owner, NestedSetBuilder.emptySet(Order.STABLE_ORDER), output, /*makeExecutable=*/ false);
this.entries =
ImmutableList.sortedCopyOf(
comparing((Entry e) -> e.sourceRepo().getName())
.thenComparing(Entry::targetRepoApparentName),
entries);
this.workspaceName = workspaceName;
}

@Override
public String getMnemonic() {
return "RepoMappingManifest";
}

@Override
protected String getRawProgressMessage() {
return "writing repo mapping manifest for " + getOwner().getLabel();
}

@Override
protected void computeKey(
ActionKeyContext actionKeyContext,
@Nullable ArtifactExpander artifactExpander,
Fingerprint fp)
throws CommandLineExpansionException, EvalException, InterruptedException {
fp.addUUID(MY_UUID);
fp.addString(workspaceName);
for (Entry entry : entries) {
fp.addString(entry.sourceRepo().getName());
fp.addString(entry.targetRepoApparentName());
fp.addString(entry.targetRepo().getName());
}
}

@Override
public DeterministicWriter newDeterministicWriter(ActionExecutionContext ctx)
throws InterruptedException, ExecException {
return out -> {
PrintWriter writer = new PrintWriter(out, /*autoFlush=*/ false, ISO_8859_1);
for (Entry entry : entries) {
if (entry.targetRepoApparentName().isEmpty()) {
// The apparent repo name can only be empty for the main repo. We skip this line as
// Rlocation paths can't reference an empty apparent name anyway.
continue;
}
// The canonical name of the main repo is the empty string, which is not a valid name for a
// directory, so the "workspace name" is used the name of the directory under the runfiles
// tree for it.
String targetRepoDirectoryName =
entry.targetRepo().isMain() ? workspaceName : entry.targetRepo().getName();
writer.format(
"%s,%s,%s\n",
entry.sourceRepo().getName(), entry.targetRepoApparentName(), targetRepoDirectoryName);
}
writer.flush();
};
}
}
Expand Up @@ -14,26 +14,34 @@

package com.google.devtools.build.lib.analysis;

import static com.google.common.collect.ImmutableSet.toImmutableSet;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.devtools.build.lib.actions.ActionEnvironment;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.CommandLine;
import com.google.devtools.build.lib.analysis.RepoMappingManifestAction.Entry;
import com.google.devtools.build.lib.analysis.SourceManifestAction.ManifestType;
import com.google.devtools.build.lib.analysis.actions.ActionConstructionContext;
import com.google.devtools.build.lib.analysis.actions.SymlinkTreeAction;
import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue;
import com.google.devtools.build.lib.analysis.config.RunUnder;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
import com.google.devtools.build.lib.packages.Package;
import com.google.devtools.build.lib.packages.TargetUtils;
import com.google.devtools.build.lib.packages.Type;
import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -76,11 +84,13 @@ public final class RunfilesSupport {
private static final String RUNFILES_DIR_EXT = ".runfiles";
private static final String INPUT_MANIFEST_EXT = ".runfiles_manifest";
private static final String OUTPUT_MANIFEST_BASENAME = "MANIFEST";
private static final String REPO_MAPPING_MANIFEST_EXT = ".repo_mapping";

private final Runfiles runfiles;

private final Artifact runfilesInputManifest;
private final Artifact runfilesManifest;
private final Artifact repoMappingManifest;
private final Artifact runfilesMiddleman;
private final Artifact owningExecutable;
private final boolean buildRunfileLinks;
Expand Down Expand Up @@ -132,15 +142,19 @@ private static RunfilesSupport create(
runfilesInputManifest = null;
runfilesManifest = null;
}
Artifact repoMappingManifest =
createRepoMappingManifestAction(ruleContext, runfiles, owningExecutable);
Artifact runfilesMiddleman =
createRunfilesMiddleman(ruleContext, owningExecutable, runfiles, runfilesManifest);
createRunfilesMiddleman(
ruleContext, owningExecutable, runfiles, runfilesManifest, repoMappingManifest);

boolean runfilesEnabled = ruleContext.getConfiguration().runfilesEnabled();

return new RunfilesSupport(
runfiles,
runfilesInputManifest,
runfilesManifest,
repoMappingManifest,
runfilesMiddleman,
owningExecutable,
buildRunfileLinks,
Expand All @@ -153,6 +167,7 @@ private RunfilesSupport(
Runfiles runfiles,
Artifact runfilesInputManifest,
Artifact runfilesManifest,
Artifact repoMappingManifest,
Artifact runfilesMiddleman,
Artifact owningExecutable,
boolean buildRunfileLinks,
Expand All @@ -162,6 +177,7 @@ private RunfilesSupport(
this.runfiles = runfiles;
this.runfilesInputManifest = runfilesInputManifest;
this.runfilesManifest = runfilesManifest;
this.repoMappingManifest = repoMappingManifest;
this.runfilesMiddleman = runfilesMiddleman;
this.owningExecutable = owningExecutable;
this.buildRunfileLinks = buildRunfileLinks;
Expand Down Expand Up @@ -268,6 +284,17 @@ public Artifact getRunfilesManifest() {
return runfilesManifest;
}

/**
* Returns the foo.repo_mapping file if Bazel is run with transitive package tracking turned on
* (see {@code SkyframeExecutor#getForcedSingleSourceRootIfNoExecrootSymlinkCreation}) and any of
* the transitive packages come from a repository with strict deps (see {@code
* #collectRepoMappings}). Otherwise, returns null.
*/
@Nullable
public Artifact getRepoMappingManifest() {
return repoMappingManifest;
}

/** Returns the root directory of the runfiles symlink farm; otherwise, returns null. */
@Nullable
public Path getRunfilesDirectory() {
Expand Down Expand Up @@ -327,12 +354,16 @@ private static Artifact createRunfilesMiddleman(
ActionConstructionContext context,
Artifact owningExecutable,
Runfiles runfiles,
@Nullable Artifact runfilesManifest) {
@Nullable Artifact runfilesManifest,
Artifact repoMappingManifest) {
NestedSetBuilder<Artifact> deps = NestedSetBuilder.stableOrder();
deps.addTransitive(runfiles.getAllArtifacts());
if (runfilesManifest != null) {
deps.add(runfilesManifest);
}
if (repoMappingManifest != null) {
deps.add(repoMappingManifest);
}
return context
.getAnalysisEnvironment()
.getMiddlemanFactory()
Expand Down Expand Up @@ -495,4 +526,72 @@ public static Path inputManifestPath(Path runfilesDir) {
public static Path outputManifestPath(Path runfilesDir) {
return runfilesDir.getRelative(OUTPUT_MANIFEST_BASENAME);
}

@Nullable
private static Artifact createRepoMappingManifestAction(
RuleContext ruleContext, Runfiles runfiles, Artifact owningExecutable) {
if (!ruleContext
.getAnalysisEnvironment()
.getStarlarkSemantics()
.getBool(BuildLanguageOptions.ENABLE_BZLMOD)) {
// If Bzlmod is not enabled, we don't need a repo mapping manifest.
return null;
}

PathFragment executablePath =
(owningExecutable != null)
? owningExecutable.getOutputDirRelativePath(
ruleContext.getConfiguration().isSiblingRepositoryLayout())
: ruleContext.getPackageDirectory().getRelative(ruleContext.getLabel().getName());
Artifact repoMappingManifest =
ruleContext.getDerivedArtifact(
executablePath.replaceName(executablePath.getBaseName() + REPO_MAPPING_MANIFEST_EXT),
ruleContext.getBinDirectory());
ruleContext
.getAnalysisEnvironment()
.registerAction(
new RepoMappingManifestAction(
ruleContext.getActionOwner(),
repoMappingManifest,
collectRepoMappings(
Preconditions.checkNotNull(
ruleContext.getTransitivePackagesForRunfileRepoMappingManifest()),
runfiles),
ruleContext.getWorkspaceName()));
return repoMappingManifest;
}

/** Returns the list of entries (unsorted) that should appear in the repo mapping manifest. */
private static ImmutableList<Entry> collectRepoMappings(
NestedSet<Package> transitivePackages, Runfiles runfiles) {
// NOTE: It might appear that the flattening of `transitivePackages` is better suited to the
// execution phase rather than here in the analysis phase, but we can't do that since it would
// necessitate storing `transitivePackages` in an action, which breaks skyframe serialization
// since packages cannot be serialized here.

ImmutableSet<RepositoryName> reposContributingRunfiles =
runfiles.getAllArtifacts().toList().stream()
.filter(a -> a.getOwner() != null)
.map(a -> a.getOwner().getRepository())
.collect(toImmutableSet());
Set<RepositoryName> seenRepos = new HashSet<>();
ImmutableList.Builder<Entry> entries = ImmutableList.builder();
for (Package pkg : transitivePackages.toList()) {
if (!seenRepos.add(pkg.getPackageIdentifier().getRepository())) {
// Any package from the same repo would have the same repo mapping.
continue;
}
for (Map.Entry<String, RepositoryName> repoMappingEntry :
pkg.getRepositoryMapping().entries().entrySet()) {
if (reposContributingRunfiles.contains(repoMappingEntry.getValue())) {
entries.add(
Entry.of(
pkg.getPackageIdentifier().getRepository(),
repoMappingEntry.getKey(),
repoMappingEntry.getValue()));
}
}
}
return entries.build();
}
}
Expand Up @@ -37,7 +37,8 @@ public abstract class RepositoryMapping {
// Always fallback to the requested name
public static final RepositoryMapping ALWAYS_FALLBACK = createAllowingFallback(ImmutableMap.of());

abstract ImmutableMap<String, RepositoryName> repositoryMapping();
/** Returns all the entries in this repo mapping. */
public abstract ImmutableMap<String, RepositoryName> entries();

/**
* The owner repo of this repository mapping. It is for providing useful debug information when
Expand Down Expand Up @@ -66,7 +67,7 @@ public static RepositoryMapping createAllowingFallback(
*/
public RepositoryMapping withAdditionalMappings(Map<String, RepositoryName> additionalMappings) {
HashMap<String, RepositoryName> allMappings = new HashMap<>(additionalMappings);
allMappings.putAll(repositoryMapping());
allMappings.putAll(entries());
return new AutoValue_RepositoryMapping(ImmutableMap.copyOf(allMappings), ownerRepo());
}

Expand All @@ -76,15 +77,15 @@ public RepositoryMapping withAdditionalMappings(Map<String, RepositoryName> addi
* repo of the given additional mappings is ignored.
*/
public RepositoryMapping withAdditionalMappings(RepositoryMapping additionalMappings) {
return withAdditionalMappings(additionalMappings.repositoryMapping());
return withAdditionalMappings(additionalMappings.entries());
}

/**
* Returns the canonical repository name associated with the given apparent repo name. The
* provided apparent repo name is assumed to be valid.
*/
public RepositoryName get(String preMappingName) {
RepositoryName canonicalRepoName = repositoryMapping().get(preMappingName);
RepositoryName canonicalRepoName = entries().get(preMappingName);
if (canonicalRepoName != null) {
return canonicalRepoName;
}
Expand All @@ -108,7 +109,7 @@ public boolean usesStrictDeps() {
* Returns the first apparent name in this mapping that maps to the given canonical name, if any.
*/
public Optional<String> getInverse(RepositoryName postMappingName) {
return repositoryMapping().entrySet().stream()
return entries().entrySet().stream()
.filter(e -> e.getValue().equals(postMappingName))
.map(Entry::getKey)
.findFirst();
Expand Down

0 comments on commit 75b0f3a

Please sign in to comment.