From 143288b944bb27a199fd46bf5f062fd371b27cce Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Thu, 1 Oct 2020 18:17:38 -0700 Subject: [PATCH 1/2] Add a test for runtime_package_dependencies # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../python/goals/pytest_runner_integration_test.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/python/pants/backend/python/goals/pytest_runner_integration_test.py b/src/python/pants/backend/python/goals/pytest_runner_integration_test.py index cf31601b8a23..a48b998338a5 100644 --- a/src/python/pants/backend/python/goals/pytest_runner_integration_test.py +++ b/src/python/pants/backend/python/goals/pytest_runner_integration_test.py @@ -446,11 +446,17 @@ def test_runtime_package_dependency(rule_runner: RuleRunner) -> None: rule_runner.create_file( f"{PACKAGE}/test_binary_call.py", dedent( - """\ + f"""\ + import os.path import subprocess def test_embedded_binary(): assert b"Hello, test!" in subprocess.check_output(args=['./bin.pex']) + + # Ensure that we didn't accidentally pull in the binary's sources. This is a + # special type of dependency that should not be included with the rest of the + # normal dependencies. + assert os.path.exists("{BINARY_SOURCE.path}") is False """ ), ) From f8fb91d1cdea6e0ac560f79abe11f99bf90d6b04 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 7 Oct 2020 15:32:37 -0700 Subject: [PATCH 2/2] Add SpecialCasedDependencies [ci skip-rust] [ci skip-build-wheels] --- .../pants/backend/project_info/dependees.py | 3 +- .../backend/project_info/dependees_test.py | 15 +++- .../backend/project_info/dependencies.py | 10 ++- .../backend/project_info/dependencies_test.py | 31 ++++++- .../pants/backend/project_info/filedeps.py | 4 +- .../backend/python/goals/pytest_runner.py | 26 +++--- .../pants/backend/python/target_types.py | 12 ++- src/python/pants/core/target_types.py | 33 +++----- src/python/pants/engine/internals/graph.py | 40 ++++++++- .../pants/engine/internals/graph_test.py | 68 ++++++++++++++- src/python/pants/engine/target.py | 83 ++++++++++--------- src/python/pants/engine/target_test.py | 38 +++------ 12 files changed, 250 insertions(+), 113 deletions(-) diff --git a/src/python/pants/backend/project_info/dependees.py b/src/python/pants/backend/project_info/dependees.py index 54936b860344..c47d102559e3 100644 --- a/src/python/pants/backend/project_info/dependees.py +++ b/src/python/pants/backend/project_info/dependees.py @@ -34,7 +34,8 @@ async def map_addresses_to_dependees() -> AddressToDependees: ) all_targets = {*all_expanded_targets, *all_explicit_targets} dependencies_per_target = await MultiGet( - Get(Addresses, DependenciesRequest(tgt.get(Dependencies))) for tgt in all_targets + Get(Addresses, DependenciesRequest(tgt.get(Dependencies), include_special_cased_deps=True)) + for tgt in all_targets ) address_to_dependees = defaultdict(set) diff --git a/src/python/pants/backend/project_info/dependees_test.py b/src/python/pants/backend/project_info/dependees_test.py index cce2522a6d83..5c13ca19cb86 100644 --- a/src/python/pants/backend/project_info/dependees_test.py +++ b/src/python/pants/backend/project_info/dependees_test.py @@ -7,13 +7,17 @@ from pants.backend.project_info.dependees import DependeesGoal from pants.backend.project_info.dependees import DependeesOutputFormat as OutputFormat from pants.backend.project_info.dependees import rules as dependee_rules -from pants.engine.target import Dependencies, Target +from pants.engine.target import Dependencies, SpecialCasedDependencies, Target from pants.testutil.test_base import TestBase +class SpecialDeps(SpecialCasedDependencies): + alias = "special_deps" + + class MockTarget(Target): alias = "tgt" - core_fields = (Dependencies,) + core_fields = (Dependencies, SpecialDeps) class DependeesTest(TestBase): @@ -139,3 +143,10 @@ def test_multiple_specified_targets(self) -> None: }""" ).splitlines(), ) + + def test_special_cased_dependencies(self) -> None: + self.add_to_build_file("special", "tgt(special_deps=['intermediate'])") + self.assert_dependees(targets=["intermediate"], expected=["leaf", "special"]) + self.assert_dependees( + targets=["base"], transitive=True, expected=["intermediate", "leaf", "special"] + ) diff --git a/src/python/pants/backend/project_info/dependencies.py b/src/python/pants/backend/project_info/dependencies.py index 7509f4504729..4bd641273ae3 100644 --- a/src/python/pants/backend/project_info/dependencies.py +++ b/src/python/pants/backend/project_info/dependencies.py @@ -70,12 +70,18 @@ async def dependencies( console: Console, addresses: Addresses, dependencies_subsystem: DependenciesSubsystem ) -> Dependencies: if dependencies_subsystem.transitive: - transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(addresses)) + transitive_targets = await Get( + TransitiveTargets, TransitiveTargetsRequest(addresses, include_special_cased_deps=True) + ) targets = Targets(transitive_targets.dependencies) else: target_roots = await Get(UnexpandedTargets, Addresses, addresses) dependencies_per_target_root = await MultiGet( - Get(Targets, DependenciesRequest(tgt.get(DependenciesField))) for tgt in target_roots + Get( + Targets, + DependenciesRequest(tgt.get(DependenciesField), include_special_cased_deps=True), + ) + for tgt in target_roots ) targets = Targets(itertools.chain.from_iterable(dependencies_per_target_root)) diff --git a/src/python/pants/backend/project_info/dependencies_test.py b/src/python/pants/backend/project_info/dependencies_test.py index 302a630dd5d1..e4bc06e03de7 100644 --- a/src/python/pants/backend/project_info/dependencies_test.py +++ b/src/python/pants/backend/project_info/dependencies_test.py @@ -9,12 +9,26 @@ from pants.backend.project_info.dependencies import Dependencies, DependencyType, rules from pants.backend.python.target_types import PythonLibrary, PythonRequirementLibrary +from pants.engine.target import SpecialCasedDependencies, Target from pants.testutil.rule_runner import RuleRunner +# We verify that any subclasses of `SpecialCasedDependencies` will show up with the `dependencies` +# goal by creating a mock target. +class SpecialDepsField(SpecialCasedDependencies): + alias = "special_deps" + + +class SpecialDepsTarget(Target): + alias = "special_deps_tgt" + core_fields = (SpecialDepsField,) + + @pytest.fixture def rule_runner() -> RuleRunner: - return RuleRunner(rules=rules(), target_types=[PythonLibrary, PythonRequirementLibrary]) + return RuleRunner( + rules=rules(), target_types=[PythonLibrary, PythonRequirementLibrary, SpecialDepsTarget] + ) def create_python_library( @@ -65,6 +79,21 @@ def test_no_dependencies(rule_runner: RuleRunner) -> None: assert_dependencies(rule_runner, specs=["some/target"], expected=[], transitive=True) +def test_special_cased_dependencies(rule_runner: RuleRunner) -> None: + rule_runner.add_to_build_file( + "", + dedent( + """\ + special_deps_tgt(name='t1') + special_deps_tgt(name='t2', special_deps=[':t1']) + special_deps_tgt(name='t3', special_deps=[':t2']) + """ + ), + ) + assert_dependencies(rule_runner, specs=["//:t3"], expected=["//:t2"]) + assert_dependencies(rule_runner, specs=["//:t3"], expected=["//:t1", "//:t2"], transitive=True) + + def test_python_dependencies(rule_runner: RuleRunner) -> None: create_python_requirement_library(rule_runner, name="req1") create_python_requirement_library(rule_runner, name="req2") diff --git a/src/python/pants/backend/project_info/filedeps.py b/src/python/pants/backend/project_info/filedeps.py index 5eb19747b7f8..b556c2ded0a7 100644 --- a/src/python/pants/backend/project_info/filedeps.py +++ b/src/python/pants/backend/project_info/filedeps.py @@ -84,7 +84,9 @@ async def file_deps( ) -> Filedeps: targets: Iterable[Target] if filedeps_subsystem.transitive: - transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest(addresses)) + transitive_targets = await Get( + TransitiveTargets, TransitiveTargetsRequest(addresses, include_special_cased_deps=True) + ) targets = transitive_targets.closure elif filedeps_subsystem.globs: targets = await Get(UnexpandedTargets, Addresses, addresses) diff --git a/src/python/pants/backend/python/goals/pytest_runner.py b/src/python/pants/backend/python/goals/pytest_runner.py index 4cfee50eff08..bb7515ae65e4 100644 --- a/src/python/pants/backend/python/goals/pytest_runner.py +++ b/src/python/pants/backend/python/goals/pytest_runner.py @@ -157,21 +157,23 @@ async def setup_pytest_for_target( # Create any assets that the test depends on through the `runtime_package_dependencies` field. assets: Tuple[BuiltPackage, ...] = () - if ( - request.field_set.runtime_package_dependencies.value - or request.field_set.runtime_binary_dependencies.value - ): - unparsed_addresses = ( - *(request.field_set.runtime_package_dependencies.value or ()), - *(request.field_set.runtime_binary_dependencies.value or ()), - ) - runtime_package_targets = await Get( - Targets, - UnparsedAddressInputs(unparsed_addresses, owning_address=request.field_set.address), + unparsed_runtime_packages = ( + request.field_set.runtime_package_dependencies.to_unparsed_address_inputs() + ) + unparsed_runtime_binaries = ( + request.field_set.runtime_binary_dependencies.to_unparsed_address_inputs() + ) + if unparsed_runtime_packages.values or unparsed_runtime_binaries.values: + runtime_package_targets, runtime_binary_dependencies = await MultiGet( + Get(Targets, UnparsedAddressInputs, unparsed_runtime_packages), + Get(Targets, UnparsedAddressInputs, unparsed_runtime_binaries), ) field_sets_per_target = await Get( FieldSetsPerTarget, - FieldSetsPerTargetRequest(PackageFieldSet, runtime_package_targets), + FieldSetsPerTargetRequest( + PackageFieldSet, + itertools.chain(runtime_package_targets, runtime_binary_dependencies), + ), ) assets = await MultiGet( Get(BuiltPackage, PackageFieldSet, field_set) diff --git a/src/python/pants/backend/python/target_types.py b/src/python/pants/backend/python/target_types.py index 99334f952aed..ce9d45ce3bf1 100644 --- a/src/python/pants/backend/python/target_types.py +++ b/src/python/pants/backend/python/target_types.py @@ -29,9 +29,9 @@ ProvidesField, ScalarField, Sources, + SpecialCasedDependencies, StringField, StringOrStringSequenceField, - StringSequenceField, Target, WrappedTarget, ) @@ -269,9 +269,7 @@ class PythonTestsDependencies(Dependencies): supports_transitive_excludes = True -# TODO(#10888): Teach project introspection goals that this is a special type of the `Dependencies` -# field. -class PythonRuntimePackageDependencies(StringSequenceField): +class PythonRuntimePackageDependencies(SpecialCasedDependencies): """Addresses to targets that can be built with the `./pants package` goal and whose resulting assets should be included in the test run. @@ -286,14 +284,14 @@ class PythonRuntimePackageDependencies(StringSequenceField): alias = "runtime_package_dependencies" -class PythonRuntimeBinaryDependencies(StringSequenceField): +class PythonRuntimeBinaryDependencies(SpecialCasedDependencies): """Deprecated in favor of the `runtime_build_dependencies` field, which works with more target types like `archive` and `python_awslambda`.""" alias = "runtime_binary_dependencies" @classmethod - def compute_value( + def sanitize_raw_value( cls, raw_value: Optional[Iterable[str]], *, address: Address ) -> Optional[Tuple[str, ...]]: if raw_value is not None: @@ -302,7 +300,7 @@ def compute_value( "field is now deprecated in favor of the more flexible " "`runtime_package_dependencies` field, and it will be removed in 2.1.0.dev0." ) - return super().compute_value(raw_value, address=address) + return super().sanitize_raw_value(raw_value, address=address) class PythonTestsTimeout(IntField): diff --git a/src/python/pants/core/target_types.py b/src/python/pants/core/target_types.py index 669eb4a4d447..e36c59329b88 100644 --- a/src/python/pants/core/target_types.py +++ b/src/python/pants/core/target_types.py @@ -2,7 +2,6 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from dataclasses import dataclass -from typing import Tuple from pants.core.goals.package import BuiltPackage, OutputPathField, PackageFieldSet from pants.core.util_rules.archive import ArchiveFormat, CreateArchive @@ -19,8 +18,8 @@ HydratedSources, HydrateSourcesRequest, Sources, + SpecialCasedDependencies, StringField, - StringSequenceField, Target, Targets, WrappedTarget, @@ -62,9 +61,7 @@ class RelocatedFilesSources(Sources): expected_num_files = 0 -# TODO(#10888): Teach project introspection goals that this is a special type of the `Dependencies` -# field. -class RelocatedFilesOriginalTargets(StringSequenceField): +class RelocatedFilesOriginalTargets(SpecialCasedDependencies): """Addresses to the original `files()` targets that you want to relocate, such as `['//:json_files']`. @@ -74,7 +71,6 @@ class RelocatedFilesOriginalTargets(StringSequenceField): alias = "files_targets" required = True - value: Tuple[str, ...] class RelocatedFilesSrcField(StringField): @@ -161,7 +157,11 @@ async def relocate_files(request: RelocateFilesViaCodegenRequest) -> GeneratedSo AddressInput, AddressInput.parse(v, relative_to=request.protocol_target.address.spec_path), ) - for v in request.protocol_target.get(RelocatedFilesOriginalTargets).value + for v in ( + request.protocol_target.get(RelocatedFilesOriginalTargets) + .to_unparsed_address_inputs() + .values + ) ) original_files_sources = await MultiGet( Get(HydratedSources, HydrateSourcesRequest(wrapped_tgt.target.get(Sources))) @@ -222,9 +222,8 @@ class GenericTarget(Target): # `archive` target # ----------------------------------------------------------------------------------------------- -# TODO(#10888): Teach project introspection goals that this is a special type of the `Dependencies` -# field. -class ArchivePackages(StringSequenceField): + +class ArchivePackages(SpecialCasedDependencies): """Addresses to any targets that can be built with `./pants package`. Pants will build the assets as if you had run `./pants package`. It will include the @@ -238,9 +237,7 @@ class ArchivePackages(StringSequenceField): alias = "packages" -# TODO(#10888): Teach project introspection goals that this is a special type of the `Dependencies` -# field. -class ArchiveFiles(StringSequenceField): +class ArchiveFiles(SpecialCasedDependencies): """Addresses to any `files` or `relocated_files` targets to include in the archive, e.g. `["resources:logo"]`. @@ -293,14 +290,8 @@ async def package_archive_target( field_set: ArchiveFieldSet, global_options: GlobalOptions ) -> BuiltPackage: package_targets, files_targets = await MultiGet( - Get( - Targets, - UnparsedAddressInputs(field_set.packages.value or (), owning_address=field_set.address), - ), - Get( - Targets, - UnparsedAddressInputs(field_set.files.value or (), owning_address=field_set.address), - ), + Get(Targets, UnparsedAddressInputs, field_set.packages.to_unparsed_address_inputs()), + Get(Targets, UnparsedAddressInputs, field_set.files.to_unparsed_address_inputs()), ) package_field_sets_per_target = await Get( diff --git a/src/python/pants/engine/internals/graph.py b/src/python/pants/engine/internals/graph.py index 5c9a609a6475..af77ef5797b5 100644 --- a/src/python/pants/engine/internals/graph.py +++ b/src/python/pants/engine/internals/graph.py @@ -56,6 +56,7 @@ InjectedDependencies, RegisteredTargetTypes, Sources, + SpecialCasedDependencies, Subtargets, Target, TargetRootsToFieldSets, @@ -309,7 +310,14 @@ async def transitive_targets(request: TransitiveTargetsRequest) -> TransitiveTar dependency_mapping: Dict[Address, Tuple[Address, ...]] = {} while queued: direct_dependencies = await MultiGet( - Get(Targets, DependenciesRequest(tgt.get(Dependencies))) for tgt in queued + Get( + Targets, + DependenciesRequest( + tgt.get(Dependencies), + include_special_cased_deps=request.include_special_cased_deps, + ), + ) + for tgt in queued ) dependency_mapping.update( @@ -865,6 +873,35 @@ async def resolve_dependencies( t.address for t in subtargets.subtargets if t.address != request.field.address ) + # If the target has `SpecialCasedDependencies`, such as the `archive` target having + # `files` and `packages` fields, then we possibly include those too. We don't want to always + # include those dependencies because they should often be excluded from the result due to + # being handled elsewhere in the calling code. + special_cased: Tuple[Address, ...] = () + if request.include_special_cased_deps: + wrapped_tgt = await Get(WrappedTarget, Address, request.field.address) + # Unlike normal, we don't use `tgt.get()` because there may be >1 subclass of + # SpecialCasedDependencies. + special_cased_fields = tuple( + field + for field in wrapped_tgt.target.field_values.values() + if isinstance(field, SpecialCasedDependencies) + ) + # We can't use the normal `Get(Addresses, UnparsedAddressInputs)` due to a graph cycle. + special_cased = await MultiGet( + Get( + Address, + AddressInput, + AddressInput.parse( + addr, + relative_to=request.field.address.spec_path, + subproject_roots=global_options.options.subproject_roots, + ), + ) + for special_cased_field in special_cased_fields + for addr in special_cased_field.to_unparsed_address_inputs().values + ) + result = { addr for addr in ( @@ -872,6 +909,7 @@ async def resolve_dependencies( *literal_addresses, *itertools.chain.from_iterable(injected), *itertools.chain.from_iterable(inferred), + *special_cased, ) if addr not in ignored_addresses } diff --git a/src/python/pants/engine/internals/graph_test.py b/src/python/pants/engine/internals/graph_test.py index f2beb66ddaae..e059f0e5ee99 100644 --- a/src/python/pants/engine/internals/graph_test.py +++ b/src/python/pants/engine/internals/graph_test.py @@ -61,6 +61,7 @@ InjectDependenciesRequest, InjectedDependencies, Sources, + SpecialCasedDependencies, Tags, Target, TargetRootsToFieldSets, @@ -84,9 +85,17 @@ class MockDependencies(Dependencies): supports_transitive_excludes = True +class SpecialCasedDeps1(SpecialCasedDependencies): + alias = "special_cased_deps1" + + +class SpecialCasedDeps2(SpecialCasedDependencies): + alias = "special_cased_deps2" + + class MockTarget(Target): alias = "target" - core_fields = (MockDependencies, Sources) + core_fields = (MockDependencies, Sources, SpecialCasedDeps1, SpecialCasedDeps2) @pytest.fixture @@ -197,6 +206,63 @@ def get_target(name: str) -> Target: assert transitive_targets.closure == FrozenOrderedSet([root, intermediate]) +def test_special_cased_dependencies(transitive_targets_rule_runner: RuleRunner) -> None: + """Test that subclasses of `SpecialCasedDependencies` show up if requested, but otherwise are + left off. + + This uses the same test setup as `test_transitive_targets`, but does not use the `dependencies` + field like normal. + """ + transitive_targets_rule_runner.add_to_build_file( + "", + dedent( + """\ + target(name='t1') + target(name='t2', special_cased_deps1=[':t1']) + target(name='d1', special_cased_deps1=[':t1']) + target(name='d2', special_cased_deps2=[':t2']) + target(name='d3') + target(name='root', special_cased_deps1=[':d1', ':d2'], special_cased_deps2=[':d3']) + """ + ), + ) + + def get_target(name: str) -> Target: + return transitive_targets_rule_runner.get_target(Address("", target_name=name)) + + t1 = get_target("t1") + t2 = get_target("t2") + d1 = get_target("d1") + d2 = get_target("d2") + d3 = get_target("d3") + root = get_target("root") + + direct_deps = transitive_targets_rule_runner.request( + Targets, [DependenciesRequest(root[Dependencies])] + ) + assert direct_deps == Targets() + + direct_deps = transitive_targets_rule_runner.request( + Targets, [DependenciesRequest(root[Dependencies], include_special_cased_deps=True)] + ) + assert direct_deps == Targets([d1, d2, d3]) + + transitive_targets = transitive_targets_rule_runner.request( + TransitiveTargets, [TransitiveTargetsRequest([root.address, d2.address])] + ) + assert transitive_targets.roots == (root, d2) + assert transitive_targets.dependencies == FrozenOrderedSet() + assert transitive_targets.closure == FrozenOrderedSet([root, d2]) + + transitive_targets = transitive_targets_rule_runner.request( + TransitiveTargets, + [TransitiveTargetsRequest([root.address, d2.address], include_special_cased_deps=True)], + ) + assert transitive_targets.roots == (root, d2) + assert transitive_targets.dependencies == FrozenOrderedSet([d1, d2, d3, t2, t1]) + assert transitive_targets.closure == FrozenOrderedSet([root, d2, d1, d3, t2, t1]) + + def test_transitive_targets_tolerates_subtarget_cycles( transitive_targets_rule_runner: RuleRunner, ) -> None: diff --git a/src/python/pants/engine/target.py b/src/python/pants/engine/target.py index c92c0edfc06a..f9f4f1b202c3 100644 --- a/src/python/pants/engine/target.py +++ b/src/python/pants/engine/target.py @@ -612,9 +612,13 @@ class TransitiveTargetsRequest: """ roots: Tuple[Address, ...] + include_special_cased_deps: bool - def __init__(self, roots: Iterable[Address]) -> None: + def __init__( + self, roots: Iterable[Address], *, include_special_cased_deps: bool = False + ) -> None: self.roots = tuple(roots) + self.include_special_cased_deps = include_special_cased_deps @frozen_after_init @@ -1216,23 +1220,9 @@ def compute_value( return FrozenDict(result) -# ----------------------------------------------------------------------------------------------- -# Sources and codegen -# ----------------------------------------------------------------------------------------------- - - -class Sources(AsyncField): - """A list of files and globs that belong to this target. - - Paths are relative to the BUILD file's directory. You can ignore files/globs by prefixing them - with `!`. Example: `sources=['example.py', 'test_*.py', '!test_ignore.py']`. - """ - - alias = "sources" +class AsyncStringSequenceField(AsyncField): sanitized_raw_value: Optional[Tuple[str, ...]] default: ClassVar[Optional[Tuple[str, ...]]] = None - expected_file_extensions: ClassVar[Optional[Tuple[str, ...]]] = None - expected_num_files: ClassVar[Optional[Union[int, range]]] = None @classmethod def sanitize_raw_value( @@ -1252,6 +1242,23 @@ def sanitize_raw_value( ) return tuple(sorted(value_or_default)) + +# ----------------------------------------------------------------------------------------------- +# Sources and codegen +# ----------------------------------------------------------------------------------------------- + + +class Sources(AsyncStringSequenceField): + """A list of files and globs that belong to this target. + + Paths are relative to the BUILD file's directory. You can ignore files/globs by prefixing them + with `!`. Example: `sources=['example.py', 'test_*.py', '!test_ignore.py']`. + """ + + alias = "sources" + expected_file_extensions: ClassVar[Optional[Tuple[str, ...]]] = None + expected_num_files: ClassVar[Optional[Union[int, range]]] = None + def validate_resolved_files(self, files: Sequence[str]) -> None: """Perform any additional validation on the resulting source files, e.g. ensuring that certain banned files are not used. @@ -1487,7 +1494,7 @@ class GeneratedSources: # NB: To hydrate the dependencies, use one of: # await Get(Addresses, DependenciesRequest(tgt[Dependencies]) # await Get(Targets, DependenciesRequest(tgt[Dependencies]) -class Dependencies(AsyncField): +class Dependencies(AsyncStringSequenceField): """Addresses to other targets that this target depends on, e.g. ['helloworld/subdir:lib']. Alternatively, you may include file names. Pants will find which target owns that file, and @@ -1501,28 +1508,8 @@ class Dependencies(AsyncField): """ alias = "dependencies" - sanitized_raw_value: Optional[Tuple[str, ...]] - default: ClassVar[Optional[Tuple[str, ...]]] = None supports_transitive_excludes = False - @classmethod - def sanitize_raw_value( - cls, raw_value: Optional[Iterable[str]], *, address: Address - ) -> Optional[Tuple[Address, ...]]: - value_or_default = super().sanitize_raw_value(raw_value, address=address) - if value_or_default is None: - return None - try: - ensure_str_list(value_or_default) - except ValueError: - raise InvalidFieldTypeException( - address, - cls.alias, - value_or_default, - expected_type="an iterable of strings (e.g. a list of strings)", - ) - return tuple(sorted(value_or_default)) - @memoized_property def unevaluated_transitive_excludes(self) -> UnparsedAddressInputs: if not self.supports_transitive_excludes or not self.sanitized_raw_value: @@ -1536,6 +1523,7 @@ def unevaluated_transitive_excludes(self) -> UnparsedAddressInputs: @dataclass(frozen=True) class DependenciesRequest(EngineAwareParameter): field: Dependencies + include_special_cased_deps: bool = False def debug_hint(self) -> str: return self.field.address.spec @@ -1667,6 +1655,27 @@ def __iter__(self) -> Iterator[Address]: return iter(self.dependencies) +class SpecialCasedDependencies(AsyncStringSequenceField): + """Subclass this for fields that act similarly to the `dependencies` field, but are handled + differently than normal dependencies. + + For example, you might have a field for package/binary dependencies, which you will call + the equivalent of `./pants package` on. While you could put these in the normal + `dependencies` field, it is often clearer to the user to call out this magic through a + dedicated field. + + This type will ensure that the dependencies show up in project introspection, + like `dependencies` and `dependees`, but not show up when you call `Get(TransitiveTargets, + TransitiveTargetsRequest)` and `Get(Addresses, DependenciesRequest)`. + + To hydrate this field's dependencies, use `await Get(Addresses, UnparsedAddressInputs, + tgt.get(MyField).to_unparsed_address_inputs()`. + """ + + def to_unparsed_address_inputs(self) -> UnparsedAddressInputs: + return UnparsedAddressInputs(self.sanitized_raw_value or (), owning_address=self.address) + + # ----------------------------------------------------------------------------------------------- # Other common Fields used across most targets # ----------------------------------------------------------------------------------------------- diff --git a/src/python/pants/engine/target_test.py b/src/python/pants/engine/target_test.py index be4ffa6f5d72..de21ec1aa3a6 100644 --- a/src/python/pants/engine/target_test.py +++ b/src/python/pants/engine/target_test.py @@ -14,6 +14,7 @@ from pants.engine.rules import Get, rule from pants.engine.target import ( AsyncField, + AsyncStringSequenceField, BoolField, Dependencies, DictStringToStringField, @@ -718,31 +719,14 @@ class ExampleDefault(DictStringToStringSequenceField): assert ExampleDefault(None, address=addr).value == FrozenDict({"default": ("val",)}) -# ----------------------------------------------------------------------------------------------- -# Test Sources and Dependencies. Also see engine/internals/graph_test.py. -# ----------------------------------------------------------------------------------------------- - - -def test_dependencies_and_sources_fields_raw_value_sanitation() -> None: - """Ensure that both Sources and Dependencies behave like a StringSequenceField does. - - Normally, we would use StringSequenceField. However, these are both AsyncFields, and - StringSequenceField is a PrimitiveField, so we end up replicating that validation logic. - """ - addr = Address.parse(":test") - - def assert_flexible_constructor(raw_value: Iterable[str]) -> None: - assert Sources(raw_value, address=addr).sanitized_raw_value == tuple(raw_value) - assert Dependencies(raw_value, address=addr).sanitized_raw_value == tuple(raw_value) - - for v in [("f1.txt", "f2.txt"), ["f1.txt", "f2.txt"], OrderedSet(["f1.txt", "f2.txt"])]: - assert_flexible_constructor(v) - - def assert_invalid_type(raw_value: Any) -> None: - with pytest.raises(InvalidFieldTypeException): - Sources(raw_value, address=addr) - with pytest.raises(InvalidFieldTypeException): - Dependencies(raw_value, address=addr) +def test_async_string_sequence_field() -> None: + class Example(AsyncStringSequenceField): + alias = "example" - for v in [0, object(), "f1.txt"]: # type: ignore[assignment] - assert_invalid_type(v) + addr = Address("", target_name="example") + assert Example(["hello", "world"], address=addr).sanitized_raw_value == ("hello", "world") + assert Example(None, address=addr).sanitized_raw_value is None + with pytest.raises(InvalidFieldTypeException): + Example("strings are technically iterable...", address=addr) + with pytest.raises(InvalidFieldTypeException): + Example(["hello", 0, "world"], address=addr)