Support and require disambiguated file addresses (#10535)

This adds a new syntax for generated subtargets / file deps to specify which original base target they come from: `a/b/c.txt:original`, with the following characteristics:

* If the original target has a "default name", meaning the `name` field is left off, then there is no need for the disambiguation.
* As before, a base target can live in an ancestor directory, e.g. `a/b/c.txt:../../original`.

Thanks to this new syntax, we no longer look for the Owners of explicit file dependencies. We require the values to be unambiguous, which improves performance and removes an edge case where we would error if there were multiple owners of the file.

We use the engine now to parse an `AddressInput` into an `Address`. This is necessary so that we can properly determine if a path is a file or a directory, as `a/b/c` could technically be a file or a directory.

In followups, we can extend this syntax to file specs. It also will allow us to consistently use generated subtargets when using file specs, which will simplify lots of that code and resolve #10455.

src/python/pants/backend/project_info/dependees_test.py

@@ -28,6 +28,7 @@ def rules(cls):
         return (*super().rules(), *dependee_rules())
 
     def setUp(self) -> None:
+        super().setUp()
         self.add_to_build_file("base", "tgt()")
         self.add_to_build_file("intermediate", "tgt(dependencies=['base'])")
         self.add_to_build_file("leaf", "tgt(dependencies=['intermediate'])")
@@ -57,65 +58,63 @@ def test_no_targets(self) -> None:
         self.assert_dependees(targets=[], output_format=OutputFormat.json, expected=["{}"])
 
     def test_normal(self) -> None:
-        self.assert_dependees(targets=["base:base"], expected=["intermediate:intermediate"])
+        self.assert_dependees(targets=["base"], expected=["intermediate"])
         self.assert_dependees(
-            targets=["base:base"],
+            targets=["base"],
             output_format=OutputFormat.json,
             expected=dedent(
                 """\
                 {
-                    "base:base": [
-                        "intermediate:intermediate"
+                    "base": [
+                        "intermediate"
                     ]
                 }"""
             ).splitlines(),
         )
 
     def test_no_dependees(self) -> None:
-        self.assert_dependees(targets=["leaf:leaf"], expected=[])
+        self.assert_dependees(targets=["leaf"], expected=[])
         self.assert_dependees(
-            targets=["leaf:leaf"],
+            targets=["leaf"],
             output_format=OutputFormat.json,
             expected=dedent(
                 """\
                 {
-                    "leaf:leaf": []
+                    "leaf": []
                 }"""
             ).splitlines(),
         )
 
     def test_closed(self) -> None:
-        self.assert_dependees(targets=["leaf:leaf"], closed=True, expected=["leaf:leaf"])
+        self.assert_dependees(targets=["leaf"], closed=True, expected=["leaf"])
         self.assert_dependees(
-            targets=["leaf:leaf"],
+            targets=["leaf"],
             closed=True,
             output_format=OutputFormat.json,
             expected=dedent(
                 """\
                 {
-                    "leaf:leaf": [
-                        "leaf:leaf"
+                    "leaf": [
+                        "leaf"
                     ]
                 }"""
             ).splitlines(),
         )
 
     def test_transitive(self) -> None:
         self.assert_dependees(
-            targets=["base:base"],
-            transitive=True,
-            expected=["intermediate:intermediate", "leaf:leaf"],
+            targets=["base"], transitive=True, expected=["intermediate", "leaf"],
         )
         self.assert_dependees(
-            targets=["base:base"],
+            targets=["base"],
             transitive=True,
             output_format=OutputFormat.json,
             expected=dedent(
                 """\
                 {
-                    "base:base": [
-                        "intermediate:intermediate",
-                        "leaf:leaf"
+                    "base": [
+                        "intermediate",
+                        "leaf"
                     ]
                 }"""
             ).splitlines(),
@@ -125,24 +124,24 @@ def test_multiple_specified_targets(self) -> None:
         # This tests that --output-format=text will deduplicate and that --output-format=json will
         # preserve which dependee belongs to which specified target.
         self.assert_dependees(
-            targets=["base:base", "intermediate:intermediate"],
+            targets=["base", "intermediate"],
             transitive=True,
             # NB: `intermediate` is not included because it's a root and we have `--no-closed`.
-            expected=["leaf:leaf"],
+            expected=["leaf"],
         )
         self.assert_dependees(
-            targets=["base:base", "intermediate:intermediate"],
+            targets=["base", "intermediate"],
             transitive=True,
             output_format=OutputFormat.json,
             expected=dedent(
                 """\
                 {
-                    "base:base": [
-                        "intermediate:intermediate",
-                        "leaf:leaf"
+                    "base": [
+                        "intermediate",
+                        "leaf"
                     ],
-                    "intermediate:intermediate": [
-                        "leaf:leaf"
+                    "intermediate": [
+                        "leaf"
                     ]
                 }"""
             ).splitlines(),

src/python/pants/backend/project_info/dependencies_test.py

@@ -84,18 +84,13 @@ def test_python_dependencies(self) -> None:
         self.assert_dependencies(
             specs=["some/other/target"],
             dependency_type=DependencyType.SOURCE,
-            expected=["3rdparty/python:req2", "some/target:target"],
+            expected=["3rdparty/python:req2", "some/target"],
         )
         self.assert_dependencies(
             specs=["some/other/target"],
             transitive=True,
             dependency_type=DependencyType.SOURCE,
-            expected=[
-                "3rdparty/python:req2",
-                "some/target:target",
-                "3rdparty/python:req1",
-                "dep/target:target",
-            ],
+            expected=["3rdparty/python:req2", "some/target", "3rdparty/python:req1", "dep/target",],
         )
 
         # `--type=3rdparty`
@@ -116,15 +111,15 @@ def test_python_dependencies(self) -> None:
             specs=["some/other/target"],
             transitive=False,
             dependency_type=DependencyType.SOURCE_AND_THIRD_PARTY,
-            expected=["3rdparty/python:req2", "some/target:target", "req2==1.0.0"],
+            expected=["3rdparty/python:req2", "some/target", "req2==1.0.0"],
         )
         self.assert_dependencies(
             specs=["some/other/target"],
             transitive=True,
             dependency_type=DependencyType.SOURCE_AND_THIRD_PARTY,
             expected=[
-                "some/target:target",
-                "dep/target:target",
+                "some/target",
+                "dep/target",
                 "3rdparty/python:req1",
                 "3rdparty/python:req2",
                 "req1==1.0.0",
@@ -136,20 +131,10 @@ def test_python_dependencies(self) -> None:
         # on it.
         self.assert_dependencies(
             specs=["::"],
-            expected=[
-                "3rdparty/python:req1",
-                "3rdparty/python:req2",
-                "dep/target:target",
-                "some/target:target",
-            ],
+            expected=["3rdparty/python:req1", "3rdparty/python:req2", "dep/target", "some/target",],
         )
         self.assert_dependencies(
             specs=["::"],
             transitive=True,
-            expected=[
-                "3rdparty/python:req1",
-                "3rdparty/python:req2",
-                "dep/target:target",
-                "some/target:target",
-            ],
+            expected=["3rdparty/python:req1", "3rdparty/python:req2", "dep/target", "some/target",],
         )

src/python/pants/backend/python/dependency_inference/module_mapper_test.py

@@ -115,19 +115,15 @@ def test_map_first_party_modules_to_addresses(self) -> None:
         assert result.mapping == FrozenDict(
             {
                 "project.util.dirutil": Address(
-                    "src/python/project/util",
-                    target_name="dirutil.py",
-                    generated_base_target_name="util",
+                    "src/python/project/util", relative_file_path="dirutil.py", target_name="util",
                 ),
                 "project.util.tarutil": Address(
-                    "src/python/project/util",
-                    target_name="tarutil.py",
-                    generated_base_target_name="util",
+                    "src/python/project/util", relative_file_path="tarutil.py", target_name="util",
                 ),
                 "project_test.demo_test": Address(
                     "tests/python/project_test/demo_test",
-                    target_name="__init__.py",
-                    generated_base_target_name="demo_test",
+                    relative_file_path="__init__.py",
+                    target_name="demo_test",
                 ),
             }
         )
@@ -198,16 +194,14 @@ def get_owner(module: str) -> Optional[Address]:
         self.create_file("source_root1/project/file2.py")
         self.add_to_build_file("source_root1/project", "python_library()")
         assert get_owner("project.app") == Address(
-            "source_root1/project", target_name="app.py", generated_base_target_name="project"
+            "source_root1/project", relative_file_path="app.py", target_name="project"
         )
 
         # Check a package path
         self.create_file("source_root2/project/subdir/__init__.py")
         self.add_to_build_file("source_root2/project/subdir", "python_library()")
         assert get_owner("project.subdir") == Address(
-            "source_root2/project/subdir",
-            target_name="__init__.py",
-            generated_base_target_name="subdir",
+            "source_root2/project/subdir", relative_file_path="__init__.py", target_name="subdir",
         )
 
         # Test a module with no owner (stdlib). This also sanity checks that we can handle when
@@ -219,5 +213,5 @@ def get_owner(module: str) -> Optional[Address]:
         self.create_file("script.py")
         self.add_to_build_file("", "python_library(name='script')")
         assert get_owner("script.Demo") == Address(
-            "", target_name="script.py", generated_base_target_name="script"
+            "", relative_file_path="script.py", target_name="script"
         )

src/python/pants/backend/python/dependency_inference/rules_test.py

@@ -103,19 +103,19 @@ def run_dep_inference(address: Address) -> InferredDependencies:
 
         # NB: We do not infer `src/python/app.py`, even though it's used by `src/python/f2.py`,
         # because it is part of the requested address.
-        normal_address = Address("src/python", "python")
+        normal_address = Address("src/python")
         assert run_dep_inference(normal_address) == InferredDependencies(
             [
-                Address("3rdparty/python", "Django"),
-                Address("src/python/util", target_name="dep.py", generated_base_target_name="util"),
+                Address("3rdparty/python", target_name="Django"),
+                Address("src/python/util", relative_file_path="dep.py", target_name="util"),
             ]
         )
 
         generated_subtarget_address = Address(
-            "src/python", target_name="f2.py", generated_base_target_name="python"
+            "src/python", relative_file_path="f2.py", target_name="python"
         )
         assert run_dep_inference(generated_subtarget_address) == InferredDependencies(
-            [Address("src/python", target_name="app.py", generated_base_target_name="python")]
+            [Address("src/python", relative_file_path="app.py", target_name="python")]
         )
 
     def test_infer_python_inits(self) -> None:
@@ -141,8 +141,8 @@ def run_dep_inference(address: Address) -> InferredDependencies:
 
         assert run_dep_inference(Address.parse("src/python/root/mid/leaf")) == InferredDependencies(
             [
-                Address("src/python/root", "__init__.py", generated_base_target_name="root"),
-                Address("src/python/root/mid", "__init__.py", generated_base_target_name="mid"),
+                Address("src/python/root", relative_file_path="__init__.py", target_name="root"),
+                Address("src/python/root/mid", relative_file_path="__init__.py", target_name="mid"),
             ]
         )
 
@@ -170,7 +170,7 @@ def run_dep_inference(address: Address) -> InferredDependencies:
 
         assert run_dep_inference(Address.parse("src/python/root/mid/leaf")) == InferredDependencies(
             [
-                Address("src/python/root", "conftest.py", generated_base_target_name="root"),
-                Address("src/python/root/mid", "conftest.py", generated_base_target_name="mid"),
+                Address("src/python/root", relative_file_path="conftest.py", target_name="root"),
+                Address("src/python/root/mid", relative_file_path="conftest.py", target_name="mid"),
             ]
         )

src/python/pants/backend/python/lint/bandit/rules.py

@@ -123,7 +123,7 @@ async def bandit_lint_partition(
     )
 
     address_references = ", ".join(
-        sorted(field_set.address.reference() for field_set in partition.field_sets)
+        sorted(field_set.address.spec for field_set in partition.field_sets)
     )
     report_path = (
         lint_subsystem.reports_dir / "bandit_report.txt" if lint_subsystem.reports_dir else None

src/python/pants/backend/python/lint/black/rules.py

@@ -125,7 +125,7 @@ async def setup(setup_request: SetupRequest, black: Black) -> Setup:
     )
 
     address_references = ", ".join(
-        sorted(field_set.address.reference() for field_set in setup_request.request.field_sets)
+        sorted(field_set.address.spec for field_set in setup_request.request.field_sets)
     )
 
     process = await Get(

src/python/pants/backend/python/lint/docformatter/rules.py

@@ -104,7 +104,7 @@ async def setup(setup_request: SetupRequest, docformatter: Docformatter) -> Setu
     )
 
     address_references = ", ".join(
-        sorted(field_set.address.reference() for field_set in setup_request.request.field_sets)
+        sorted(field_set.address.spec for field_set in setup_request.request.field_sets)
     )
 
     process = await Get(

src/python/pants/backend/python/lint/flake8/rules.py

@@ -123,7 +123,7 @@ async def flake8_lint_partition(
     )
 
     address_references = ", ".join(
-        sorted(field_set.address.reference() for field_set in partition.field_sets)
+        sorted(field_set.address.spec for field_set in partition.field_sets)
     )
     report_path = (
         lint_subsystem.reports_dir / "flake8_report.txt" if lint_subsystem.reports_dir else None

src/python/pants/backend/python/lint/isort/rules.py

@@ -129,7 +129,7 @@ async def setup(setup_request: SetupRequest, isort: Isort) -> Setup:
     )
 
     address_references = ", ".join(
-        sorted(field_set.address.reference() for field_set in setup_request.request.field_sets)
+        sorted(field_set.address.spec for field_set in setup_request.request.field_sets)
     )
 
     process = await Get(

src/python/pants/backend/python/lint/pylint/rules.py

@@ -222,7 +222,7 @@ async def pylint_lint_partition(partition: PylintPartition, pylint: Pylint) -> L
     )
 
     address_references = ", ".join(
-        sorted(field_set.address.reference() for field_set in partition.field_sets)
+        sorted(field_set.address.spec for field_set in partition.field_sets)
     )
 
     result = await Get(

src/python/pants/backend/python/lint/pylint/rules_integration_test.py

@@ -76,7 +76,9 @@ def make_target_with_origin(
                 """
             ),
         )
-        target = self.request_single_product(WrappedTarget, Address(package, name)).target
+        target = self.request_single_product(
+            WrappedTarget, Address(package, target_name=name)
+        ).target
         if origin is None:
             origin = SingleAddress(directory=package, name=name)
         return TargetWithOrigin(target, origin)
@@ -253,7 +255,10 @@ def test_includes_direct_dependencies(self) -> None:
         )
         target = self.make_target_with_origin(
             source_files=[FileContent(f"{self.package}/target.py", source_content.encode())],
-            dependencies=[Address(self.package, "direct_dep"), Address("", "direct_req")],
+            dependencies=[
+                Address(self.package, target_name="direct_dep"),
+                Address("", target_name="direct_req"),
+            ],
         )
 
         result = self.run_pylint([target])

src/python/pants/backend/python/rules/pytest_runner.py

@@ -250,7 +250,7 @@ async def run_python_test(
             argv=test_setup.args,
             input_digest=test_setup.input_digest,
             output_files=tuple(output_files) if output_files else None,
-            description=f"Run Pytest for {field_set.address.reference()}",
+            description=f"Run Pytest for {field_set.address}",
             timeout_seconds=test_setup.timeout_seconds,
             extra_env=env,
             execution_slot_variable=test_setup.execution_slot_variable,
@@ -284,7 +284,7 @@ async def run_python_test(
         result,
         coverage_data=coverage_data,
         xml_results=xml_results_digest,
-        address_ref=field_set.address.reference(),
+        address_ref=field_set.address.spec,
     )
 
 

src/python/pants/backend/python/rules/pytest_runner_integration_test.py

@@ -158,7 +158,7 @@ def run_pytest(
             args.append(f"--pytest-execution-slot-var={execution_slot_var}")
 
         options_bootstrapper = create_options_bootstrapper(args=args)
-        address = Address(self.package, "target")
+        address = Address(self.package, target_name="target")
         if origin is None:
             origin = SingleAddress(directory=address.spec_path, name=address.target_name)
         tgt = PythonTests({}, address=address)

src/python/pants/backend/python/rules/python_sources_test.py

@@ -197,7 +197,7 @@ def test_python_protobuf(self) -> None:
             ),
         )
         self.add_to_build_file("src/protobuf/dir", "protobuf_library()")
-        targets = [ProtobufLibrary({}, address=Address("src/protobuf/dir", "dir"))]
+        targets = [ProtobufLibrary({}, address=Address("src/protobuf/dir"))]
         backend_args = ["--backend-packages=pants.backend.codegen.protobuf.python"]
 
         stripped_result = self.get_stripped_sources(