From 696ccec6af0e56cd0975d861b32592bd1a28eeec Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Fri, 13 Jan 2023 09:39:01 +0100 Subject: [PATCH] runfiles: Apply repo mapping to Rlocation path When a repo mapping manifest is available in runfiles, it is parsed and used to map apparent repository names to canonical ones in paths passed to Rlocation. The current repository, which is required to know which part of the mapping to apply, is either determined using `CurrentRepository` or can be passed in explicitly. With this commit, runfiles lookups should succeed with Bzlmod without code changes. --- examples/bzlmod/MODULE.bazel | 2 +- .../other_module/other_module/pkg/lib.py | 4 + examples/bzlmod/runfiles/BUILD.bazel | 6 +- examples/bzlmod/runfiles/runfiles_test.py | 17 ++ python/runfiles/runfiles.py | 65 +++- tests/runfiles/runfiles_test.py | 288 ++++++++++++++++++ 6 files changed, 375 insertions(+), 7 deletions(-) diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 89e5356c16..48fb4cb3fc 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -29,7 +29,7 @@ pip.parse( ) use_repo(pip, "pip") -bazel_dep(name = "other_module", version = "") +bazel_dep(name = "other_module", version = "", repo_name = "our_other_module") local_path_override( module_name = "other_module", path = "other_module", diff --git a/examples/bzlmod/other_module/other_module/pkg/lib.py b/examples/bzlmod/other_module/other_module/pkg/lib.py index 7ae76d7f85..48f6b58b36 100644 --- a/examples/bzlmod/other_module/other_module/pkg/lib.py +++ b/examples/bzlmod/other_module/other_module/pkg/lib.py @@ -7,3 +7,7 @@ def GetRunfilePathWithCurrentRepository(): # For a non-main repository, the name of the runfiles directory is equal to # the canonical repository name. return r.Rlocation(own_repo + "/other_module/pkg/data/data.txt") + + +def GetRunfilePathWithRepoMapping(): + return runfiles.Create().Rlocation("other_module/other_module/pkg/data/data.txt") diff --git a/examples/bzlmod/runfiles/BUILD.bazel b/examples/bzlmod/runfiles/BUILD.bazel index ad5fc1f0a6..add56b3bd0 100644 --- a/examples/bzlmod/runfiles/BUILD.bazel +++ b/examples/bzlmod/runfiles/BUILD.bazel @@ -5,14 +5,14 @@ py_test( srcs = ["runfiles_test.py"], data = [ "data/data.txt", - "@other_module//other_module/pkg:data/data.txt", + "@our_other_module//other_module/pkg:data/data.txt", ], env = { "DATA_RLOCATIONPATH": "$(rlocationpath data/data.txt)", - "OTHER_MODULE_DATA_RLOCATIONPATH": "$(rlocationpath @other_module//other_module/pkg:data/data.txt)", + "OTHER_MODULE_DATA_RLOCATIONPATH": "$(rlocationpath @our_other_module//other_module/pkg:data/data.txt)", }, deps = [ - "@other_module//other_module/pkg:lib", + "@our_other_module//other_module/pkg:lib", "@rules_python//python/runfiles", ], ) diff --git a/examples/bzlmod/runfiles/runfiles_test.py b/examples/bzlmod/runfiles/runfiles_test.py index 4f17c2a44c..3c3ae75a2c 100644 --- a/examples/bzlmod/runfiles/runfiles_test.py +++ b/examples/bzlmod/runfiles/runfiles_test.py @@ -11,12 +11,29 @@ class RunfilesTest(unittest.TestCase): def testCurrentRepository(self): self.assertEqual(runfiles.Create().CurrentRepository(), "") + def testRunfilesWithRepoMapping(self): + data_path = runfiles.Create().Rlocation("example_bzlmod/runfiles/data/data.txt") + with open(data_path) as f: + self.assertEqual(f.read().strip(), "Hello, example_bzlmod!") + def testRunfileWithRlocationpath(self): data_rlocationpath = os.getenv("DATA_RLOCATIONPATH") data_path = runfiles.Create().Rlocation(data_rlocationpath) with open(data_path) as f: self.assertEqual(f.read().strip(), "Hello, example_bzlmod!") + def testRunfileInOtherModuleWithOurRepoMapping(self): + data_path = runfiles.Create().Rlocation( + "our_other_module/other_module/pkg/data/data.txt" + ) + with open(data_path) as f: + self.assertEqual(f.read().strip(), "Hello, other_module!") + + def testRunfileInOtherModuleWithItsRepoMapping(self): + data_path = lib.GetRunfilePathWithRepoMapping() + with open(data_path) as f: + self.assertEqual(f.read().strip(), "Hello, other_module!") + def testRunfileInOtherModuleWithCurrentRepository(self): data_path = lib.GetRunfilePathWithCurrentRepository() with open(data_path) as f: diff --git a/python/runfiles/runfiles.py b/python/runfiles/runfiles.py index c55b33c227..c310f06b96 100644 --- a/python/runfiles/runfiles.py +++ b/python/runfiles/runfiles.py @@ -126,9 +126,12 @@ def __init__(self, strategy): # type: (Union[_ManifestBased, _DirectoryBased]) -> None self._strategy = strategy self._python_runfiles_root = _FindPythonRunfilesRoot() + self._repo_mapping = _ParseRepoMapping( + strategy.RlocationChecked("_repo_mapping") + ) - def Rlocation(self, path): - # type: (str) -> Optional[str] + def Rlocation(self, path, source_repo=None): + # type: (str, Optional[str]) -> Optional[str] """Returns the runtime path of a runfile. Runfiles are data-dependencies of Bazel-built binaries and tests. @@ -141,6 +144,13 @@ def Rlocation(self, path): Args: path: string; runfiles-root-relative path of the runfile + source_repo: string; optional; the canonical name of the repository + whose repository mapping should be used to resolve apparent to + canonical repository names in `path`. If `None` (default), the + repository mapping of the repository containing the caller of this + method is used. Explicitly setting this parameter should only be + necessary for libraries that want to wrap the runfiles library. Use + `CurrentRepository` to obtain canonical repository names. Returns: the path to the runfile, which the caller should check for existence, or None if the method doesn't know about this runfile @@ -165,7 +175,31 @@ def Rlocation(self, path): raise ValueError('path is absolute without a drive letter: "%s"' % path) if os.path.isabs(path): return path - return self._strategy.RlocationChecked(path) + + if source_repo is None and self._repo_mapping: + # Look up runfiles using the repository mapping of the caller of the + # current method. If the repo mapping is empty, determining this + # name is not necessary. + source_repo = self.CurrentRepository(frame=2) + + # Split off the first path component, which contains the repository + # name (apparent or canonical). + target_repo, _, remainder = path.partition("/") + if not remainder or (source_repo, target_repo) not in self._repo_mapping: + # One of the following is the case: + # - not using Bzlmod, so the repository mapping is empty and + # apparent and canonical repository names are the same + # - target_repo is already a canonical repository name and does not + # have to be mapped. + # - path did not contain a slash and referred to a root symlink, + # which also should not be mapped. + return self._strategy.RlocationChecked(path) + + # target_repo is an apparent repository name. Look up the corresponding + # canonical repository name with respect to the current repository, + # identified by its canonical name. + target_canonical = self._repo_mapping[(source_repo, target_repo)] + return self._strategy.RlocationChecked(target_canonical + "/" + remainder) def EnvVars(self): # type: () -> Dict[str, str] @@ -254,6 +288,31 @@ def _FindPythonRunfilesRoot(): return root +def _ParseRepoMapping(repo_mapping_path): + # type: (Optional[str]) -> Dict[Tuple[str, str], str] + """Parses the repository mapping manifest.""" + # If the repository mapping file can't be found, that is not an error: We + # might be running without Bzlmod enabled or there may not be any runfiles. + # In this case, just apply an empty repo mapping. + if not repo_mapping_path: + return {} + try: + with open(repo_mapping_path, "r") as f: + content = f.read() + except FileNotFoundError: + return {} + + repo_mapping = {} + for line in content.split("\n"): + if not line: + # Empty line following the last line break + break + current_canonical, target_local, target_canonical = line.split(",") + repo_mapping[(current_canonical, target_local)] = target_canonical + + return repo_mapping + + class _ManifestBased(object): """`Runfiles` strategy that parses a runfiles-manifest to look up runfiles.""" diff --git a/tests/runfiles/runfiles_test.py b/tests/runfiles/runfiles_test.py index c234a7b547..966d012c77 100644 --- a/tests/runfiles/runfiles_test.py +++ b/tests/runfiles/runfiles_test.py @@ -205,6 +205,159 @@ def testManifestBasedRlocation(self): else: self.assertEqual(r.Rlocation("/foo"), "/foo") + def testManifestBasedRlocationWithRepoMappingFromMain(self): + with _MockFile( + contents=[ + ",my_module,_main", + ",my_protobuf,protobuf~3.19.2", + ",my_workspace,_main", + "protobuf~3.19.2,protobuf,protobuf~3.19.2", + ] + ) as rm, _MockFile( + contents=[ + "_repo_mapping " + rm.Path(), + "config.json /etc/config.json", + "protobuf~3.19.2/foo/runfile C:/Actual Path\\protobuf\\runfile", + "_main/bar/runfile /the/path/./to/other//other runfile.txt", + "protobuf~3.19.2/bar/dir E:\\Actual Path\\Directory", + ], + ) as mf: + r = runfiles.CreateManifestBased(mf.Path()) + + self.assertEqual( + r.Rlocation("my_module/bar/runfile", ""), + "/the/path/./to/other//other runfile.txt", + ) + self.assertEqual( + r.Rlocation("my_workspace/bar/runfile", ""), + "/the/path/./to/other//other runfile.txt", + ) + self.assertEqual( + r.Rlocation("my_protobuf/foo/runfile", ""), + "C:/Actual Path\\protobuf\\runfile", + ) + self.assertEqual( + r.Rlocation("my_protobuf/bar/dir", ""), "E:\\Actual Path\\Directory" + ) + self.assertEqual( + r.Rlocation("my_protobuf/bar/dir/file", ""), + "E:\\Actual Path\\Directory/file", + ) + self.assertEqual( + r.Rlocation("my_protobuf/bar/dir/de eply/nes ted/fi~le", ""), + "E:\\Actual Path\\Directory/de eply/nes ted/fi~le", + ) + + self.assertIsNone(r.Rlocation("protobuf/foo/runfile")) + self.assertIsNone(r.Rlocation("protobuf/bar/dir")) + self.assertIsNone(r.Rlocation("protobuf/bar/dir/file")) + self.assertIsNone(r.Rlocation("protobuf/bar/dir/dir/de eply/nes ted/fi~le")) + + self.assertEqual( + r.Rlocation("_main/bar/runfile", ""), + "/the/path/./to/other//other runfile.txt", + ) + self.assertEqual( + r.Rlocation("protobuf~3.19.2/foo/runfile", ""), + "C:/Actual Path\\protobuf\\runfile", + ) + self.assertEqual( + r.Rlocation("protobuf~3.19.2/bar/dir", ""), "E:\\Actual Path\\Directory" + ) + self.assertEqual( + r.Rlocation("protobuf~3.19.2/bar/dir/file", ""), + "E:\\Actual Path\\Directory/file", + ) + self.assertEqual( + r.Rlocation("protobuf~3.19.2/bar/dir/de eply/nes ted/fi~le", ""), + "E:\\Actual Path\\Directory/de eply/nes ted/fi~le", + ) + + self.assertEqual(r.Rlocation("config.json", ""), "/etc/config.json") + self.assertIsNone(r.Rlocation("_main", "")) + self.assertIsNone(r.Rlocation("my_module", "")) + self.assertIsNone(r.Rlocation("protobuf", "")) + + def testManifestBasedRlocationWithRepoMappingFromOtherRepo(self): + with _MockFile( + contents=[ + ",my_module,_main", + ",my_protobuf,protobuf~3.19.2", + ",my_workspace,_main", + "protobuf~3.19.2,protobuf,protobuf~3.19.2", + ] + ) as rm, _MockFile( + contents=[ + "_repo_mapping " + rm.Path(), + "config.json /etc/config.json", + "protobuf~3.19.2/foo/runfile C:/Actual Path\\protobuf\\runfile", + "_main/bar/runfile /the/path/./to/other//other runfile.txt", + "protobuf~3.19.2/bar/dir E:\\Actual Path\\Directory", + ], + ) as mf: + r = runfiles.CreateManifestBased(mf.Path()) + + self.assertEqual( + r.Rlocation("protobuf/foo/runfile", "protobuf~3.19.2"), + "C:/Actual Path\\protobuf\\runfile", + ) + self.assertEqual( + r.Rlocation("protobuf/bar/dir", "protobuf~3.19.2"), + "E:\\Actual Path\\Directory", + ) + self.assertEqual( + r.Rlocation("protobuf/bar/dir/file", "protobuf~3.19.2"), + "E:\\Actual Path\\Directory/file", + ) + self.assertEqual( + r.Rlocation( + "protobuf/bar/dir/de eply/nes ted/fi~le", "protobuf~3.19.2" + ), + "E:\\Actual Path\\Directory/de eply/nes ted/fi~le", + ) + + self.assertIsNone(r.Rlocation("my_module/bar/runfile", "protobuf~3.19.2")) + self.assertIsNone(r.Rlocation("my_protobuf/foo/runfile", "protobuf~3.19.2")) + self.assertIsNone(r.Rlocation("my_protobuf/bar/dir", "protobuf~3.19.2")) + self.assertIsNone( + r.Rlocation("my_protobuf/bar/dir/file", "protobuf~3.19.2") + ) + self.assertIsNone( + r.Rlocation( + "my_protobuf/bar/dir/de eply/nes ted/fi~le", "protobuf~3.19.2" + ) + ) + + self.assertEqual( + r.Rlocation("_main/bar/runfile", "protobuf~3.19.2"), + "/the/path/./to/other//other runfile.txt", + ) + self.assertEqual( + r.Rlocation("protobuf~3.19.2/foo/runfile", "protobuf~3.19.2"), + "C:/Actual Path\\protobuf\\runfile", + ) + self.assertEqual( + r.Rlocation("protobuf~3.19.2/bar/dir", "protobuf~3.19.2"), + "E:\\Actual Path\\Directory", + ) + self.assertEqual( + r.Rlocation("protobuf~3.19.2/bar/dir/file", "protobuf~3.19.2"), + "E:\\Actual Path\\Directory/file", + ) + self.assertEqual( + r.Rlocation( + "protobuf~3.19.2/bar/dir/de eply/nes ted/fi~le", "protobuf~3.19.2" + ), + "E:\\Actual Path\\Directory/de eply/nes ted/fi~le", + ) + + self.assertEqual( + r.Rlocation("config.json", "protobuf~3.19.2"), "/etc/config.json" + ) + self.assertIsNone(r.Rlocation("_main", "protobuf~3.19.2")) + self.assertIsNone(r.Rlocation("my_module", "protobuf~3.19.2")) + self.assertIsNone(r.Rlocation("protobuf", "protobuf~3.19.2")) + def testDirectoryBasedRlocation(self): # The _DirectoryBased strategy simply joins the runfiles directory and the # runfile's path on a "/". This strategy does not perform any normalization, @@ -217,6 +370,141 @@ def testDirectoryBasedRlocation(self): else: self.assertEqual(r.Rlocation("/foo"), "/foo") + def testDirectoryBasedRlocationWithRepoMappingFromMain(self): + with _MockFile( + name="_repo_mapping", + contents=[ + ",my_module,_main", + ",my_protobuf,protobuf~3.19.2", + ",my_workspace,_main", + "protobuf~3.19.2,protobuf,protobuf~3.19.2", + ], + ) as rm: + dir = os.path.dirname(rm.Path()) + r = runfiles.CreateDirectoryBased(dir) + + self.assertEqual( + r.Rlocation("my_module/bar/runfile", ""), dir + "/_main/bar/runfile" + ) + self.assertEqual( + r.Rlocation("my_workspace/bar/runfile", ""), dir + "/_main/bar/runfile" + ) + self.assertEqual( + r.Rlocation("my_protobuf/foo/runfile", ""), + dir + "/protobuf~3.19.2/foo/runfile", + ) + self.assertEqual( + r.Rlocation("my_protobuf/bar/dir", ""), dir + "/protobuf~3.19.2/bar/dir" + ) + self.assertEqual( + r.Rlocation("my_protobuf/bar/dir/file", ""), + dir + "/protobuf~3.19.2/bar/dir/file", + ) + self.assertEqual( + r.Rlocation("my_protobuf/bar/dir/de eply/nes ted/fi~le", ""), + dir + "/protobuf~3.19.2/bar/dir/de eply/nes ted/fi~le", + ) + + self.assertEqual( + r.Rlocation("protobuf/foo/runfile", ""), dir + "/protobuf/foo/runfile" + ) + self.assertEqual( + r.Rlocation("protobuf/bar/dir/dir/de eply/nes ted/fi~le", ""), + dir + "/protobuf/bar/dir/dir/de eply/nes ted/fi~le", + ) + + self.assertEqual( + r.Rlocation("_main/bar/runfile", ""), dir + "/_main/bar/runfile" + ) + self.assertEqual( + r.Rlocation("protobuf~3.19.2/foo/runfile", ""), + dir + "/protobuf~3.19.2/foo/runfile", + ) + self.assertEqual( + r.Rlocation("protobuf~3.19.2/bar/dir", ""), + dir + "/protobuf~3.19.2/bar/dir", + ) + self.assertEqual( + r.Rlocation("protobuf~3.19.2/bar/dir/file", ""), + dir + "/protobuf~3.19.2/bar/dir/file", + ) + self.assertEqual( + r.Rlocation("protobuf~3.19.2/bar/dir/de eply/nes ted/fi~le", ""), + dir + "/protobuf~3.19.2/bar/dir/de eply/nes ted/fi~le", + ) + + self.assertEqual(r.Rlocation("config.json", ""), dir + "/config.json") + + def testDirectoryBasedRlocationWithRepoMappingFromOtherRepo(self): + with _MockFile( + name="_repo_mapping", + contents=[ + ",my_module,_main", + ",my_protobuf,protobuf~3.19.2", + ",my_workspace,_main", + "protobuf~3.19.2,protobuf,protobuf~3.19.2", + ], + ) as rm: + dir = os.path.dirname(rm.Path()) + r = runfiles.CreateDirectoryBased(dir) + + self.assertEqual( + r.Rlocation("protobuf/foo/runfile", "protobuf~3.19.2"), + dir + "/protobuf~3.19.2/foo/runfile", + ) + self.assertEqual( + r.Rlocation("protobuf/bar/dir", "protobuf~3.19.2"), + dir + "/protobuf~3.19.2/bar/dir", + ) + self.assertEqual( + r.Rlocation("protobuf/bar/dir/file", "protobuf~3.19.2"), + dir + "/protobuf~3.19.2/bar/dir/file", + ) + self.assertEqual( + r.Rlocation( + "protobuf/bar/dir/de eply/nes ted/fi~le", "protobuf~3.19.2" + ), + dir + "/protobuf~3.19.2/bar/dir/de eply/nes ted/fi~le", + ) + + self.assertEqual( + r.Rlocation("my_module/bar/runfile", "protobuf~3.19.2"), + dir + "/my_module/bar/runfile", + ) + self.assertEqual( + r.Rlocation( + "my_protobuf/bar/dir/de eply/nes ted/fi~le", "protobuf~3.19.2" + ), + dir + "/my_protobuf/bar/dir/de eply/nes ted/fi~le", + ) + + self.assertEqual( + r.Rlocation("_main/bar/runfile", "protobuf~3.19.2"), + dir + "/_main/bar/runfile", + ) + self.assertEqual( + r.Rlocation("protobuf~3.19.2/foo/runfile", "protobuf~3.19.2"), + dir + "/protobuf~3.19.2/foo/runfile", + ) + self.assertEqual( + r.Rlocation("protobuf~3.19.2/bar/dir", "protobuf~3.19.2"), + dir + "/protobuf~3.19.2/bar/dir", + ) + self.assertEqual( + r.Rlocation("protobuf~3.19.2/bar/dir/file", "protobuf~3.19.2"), + dir + "/protobuf~3.19.2/bar/dir/file", + ) + self.assertEqual( + r.Rlocation( + "protobuf~3.19.2/bar/dir/de eply/nes ted/fi~le", "protobuf~3.19.2" + ), + dir + "/protobuf~3.19.2/bar/dir/de eply/nes ted/fi~le", + ) + + self.assertEqual( + r.Rlocation("config.json", "protobuf~3.19.2"), dir + "/config.json" + ) + def testPathsFromEnvvars(self): # Both envvars have a valid value. mf, dr = runfiles._PathsFrom(