From 4780b24ec176b363683b4c448937f180c0e2e4b7 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 17 Jul 2019 17:37:13 -0700 Subject: [PATCH 1/3] Extract out download_pex_bin.py --- src/python/pants/backend/python/register.py | 4 ++- .../backend/python/rules/download_pex_bin.py | 26 +++++++++++++++++++ .../python/rules/resolve_requirements.py | 18 +++++-------- .../python/rules/test_resolve_requirements.py | 12 +++++---- 4 files changed, 43 insertions(+), 17 deletions(-) create mode 100644 src/python/pants/backend/python/rules/download_pex_bin.py diff --git a/src/python/pants/backend/python/register.py b/src/python/pants/backend/python/register.py index 161dad91086..766e39dfba7 100644 --- a/src/python/pants/backend/python/register.py +++ b/src/python/pants/backend/python/register.py @@ -5,7 +5,8 @@ from pants.backend.python.python_artifact import PythonArtifact from pants.backend.python.python_requirement import PythonRequirement from pants.backend.python.python_requirements import PythonRequirements -from pants.backend.python.rules import inject_init, python_test_runner, resolve_requirements +from pants.backend.python.rules import (download_pex_bin, inject_init, python_test_runner, + resolve_requirements) from pants.backend.python.subsystems.python_native_code import PythonNativeCode from pants.backend.python.subsystems.python_native_code import rules as python_native_code_rules from pants.backend.python.subsystems.subprocess_environment import SubprocessEnvironment @@ -88,6 +89,7 @@ def register_goals(): def rules(): return ( + download_pex_bin.rules() + inject_init.rules() + python_test_runner.rules() + python_native_code_rules() + diff --git a/src/python/pants/backend/python/rules/download_pex_bin.py b/src/python/pants/backend/python/rules/download_pex_bin.py new file mode 100644 index 00000000000..678bc33f898 --- /dev/null +++ b/src/python/pants/backend/python/rules/download_pex_bin.py @@ -0,0 +1,26 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from pants.engine.fs import Digest, Snapshot, UrlToFetch +from pants.engine.rules import rule +from pants.engine.selectors import Get +from pants.util.objects import datatype + + +class DownloadedPexBin(datatype([('executable', str), ('directory_digest', Digest)])): + pass + + +@rule(DownloadedPexBin, []) +def download_pex_bin(): + # TODO: Inject versions and digests here through some option, rather than hard-coding it. + url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.8/pex' + digest = Digest('2ca320aede7e7bbcb907af54c9de832707a1df965fb5a0d560f2df29ba8a2f3d', 1866441) + snapshot = yield Get(Snapshot, UrlToFetch(url, digest)) + yield DownloadedPexBin(executable=snapshot.files[0], directory_digest=snapshot.directory_digest) + + +def rules(): + return [ + download_pex_bin, + ] diff --git a/src/python/pants/backend/python/rules/resolve_requirements.py b/src/python/pants/backend/python/rules/resolve_requirements.py index a6cf67c0993..e44d9584201 100644 --- a/src/python/pants/backend/python/rules/resolve_requirements.py +++ b/src/python/pants/backend/python/rules/resolve_requirements.py @@ -1,9 +1,10 @@ # Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +from pants.backend.python.rules.download_pex_bin import DownloadedPexBin from pants.backend.python.subsystems.python_native_code import PexBuildEnvironment, PythonNativeCode from pants.backend.python.subsystems.python_setup import PythonSetup -from pants.engine.fs import Digest, Snapshot, UrlToFetch +from pants.engine.fs import Digest from pants.engine.isolated_process import ExecuteProcessRequest, ExecuteProcessResult from pants.engine.rules import optionable_rule, rule from pants.engine.selectors import Get @@ -26,16 +27,11 @@ class ResolvedRequirementsPex(datatype([('directory_digest', Digest)])): # TODO: This is non-hermetic because the requirements will be resolved on the fly by # pex, where it should be hermetically provided in some way. -@rule(ResolvedRequirementsPex, [ResolveRequirementsRequest, PythonSetup, PexBuildEnvironment]) -def resolve_requirements(request, python_setup, pex_build_environment): +@rule(ResolvedRequirementsPex, [ResolveRequirementsRequest, DownloadedPexBin, PythonSetup, PexBuildEnvironment]) +def resolve_requirements(request, pex_bin, python_setup, pex_build_environment): """Returns a PEX with the given requirements, optional entry point, and optional interpreter constraints.""" - # TODO: Inject versions and digests here through some option, rather than hard-coding it. - url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.8/pex' - digest = Digest('2ca320aede7e7bbcb907af54c9de832707a1df965fb5a0d560f2df29ba8a2f3d', 1866441) - pex_snapshot = yield Get(Snapshot, UrlToFetch(url, digest)) - interpreter_search_paths = create_path_env_var(python_setup.interpreter_search_paths) env = {"PATH": interpreter_search_paths, **pex_build_environment.invocation_environment_dict} @@ -50,7 +46,7 @@ def resolve_requirements(request, python_setup, pex_build_environment): # necessarily the interpreter that PEX will use to execute the generated .pex file. # TODO(#7735): Set --python-setup-interpreter-search-paths differently for the host and target # platforms, when we introduce platforms in https://github.com/pantsbuild/pants/issues/7735. - argv = ["python", "./{}".format(pex_snapshot.files[0]), "-o", request.output_filename] + argv = ["python", f"./{pex_bin.executable}", "-o", request.output_filename] if request.entry_point is not None: argv.extend(["-e", request.entry_point]) argv.extend(interpreter_constraint_args + list(request.requirements)) @@ -58,8 +54,8 @@ def resolve_requirements(request, python_setup, pex_build_environment): request = ExecuteProcessRequest( argv=tuple(argv), env=env, - input_files=pex_snapshot.directory_digest, - description='Resolve requirements: {}'.format(", ".join(request.requirements)), + input_files=pex_bin.directory_digest, + description=f"Resolve requirements: {', '.join(request.requirements)}", output_files=(request.output_filename,), ) diff --git a/tests/python/pants_test/backend/python/rules/test_resolve_requirements.py b/tests/python/pants_test/backend/python/rules/test_resolve_requirements.py index 2a363c3ce42..3e9451352b5 100644 --- a/tests/python/pants_test/backend/python/rules/test_resolve_requirements.py +++ b/tests/python/pants_test/backend/python/rules/test_resolve_requirements.py @@ -5,6 +5,7 @@ import os.path import zipfile +from pants.backend.python.rules.download_pex_bin import download_pex_bin from pants.backend.python.rules.resolve_requirements import (ResolvedRequirementsPex, ResolveRequirementsRequest, resolve_requirements) @@ -26,10 +27,11 @@ class TestResolveRequirements(TestBase): def rules(cls): return super().rules() + [ resolve_requirements, + download_pex_bin, + create_pex_native_build_environment, RootRule(ResolveRequirementsRequest), RootRule(PythonSetup), RootRule(PythonNativeCode), - create_pex_native_build_environment, ] def setUp(self): @@ -37,7 +39,7 @@ def setUp(self): init_subsystems([PythonSetup, PythonNativeCode]) def create_pex_and_get_pex_info( - self, requirements=None, entry_point=None, interpreter_constraints=None + self, *, requirements=None, entry_point=None, interpreter_constraints=None ): def hashify_optional_collection(iterable): return tuple(sorted(iterable)) if iterable is not None else tuple() @@ -64,19 +66,19 @@ def hashify_optional_collection(iterable): pex_info_content = pex_info.readline().decode() return json.loads(pex_info_content) - def test_resolves_dependencies(self): + def test_resolves_dependencies(self) -> None: requirements = {"six==1.12.0", "jsonschema==2.6.0", "requests==2.22.0"} pex_info = self.create_pex_and_get_pex_info(requirements=requirements) # NB: We do not check for transitive dependencies, which PEX-INFO will include. We only check # that at least the dependencies we requested are included. self.assertTrue(requirements.issubset(pex_info["requirements"])) - def test_entry_point(self): + def test_entry_point(self) -> None: entry_point = "pydoc" pex_info = self.create_pex_and_get_pex_info(entry_point=entry_point) self.assertEqual(pex_info["entry_point"], entry_point) - def test_interpreter_constraints(self): + def test_interpreter_constraints(self) -> None: constraints = {"CPython>=2.7,<3", "CPython>=3.6,<4"} pex_info = self.create_pex_and_get_pex_info(interpreter_constraints=constraints) self.assertEqual(set(pex_info["interpreter_constraints"]), constraints) From c6c06581996e0227ead886cad3fcc876a2308fc6 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 19 Jul 2019 17:47:23 -0700 Subject: [PATCH 2/3] Rename resolve_requirements.py -> create_requirements_pex.py --- src/python/pants/backend/python/register.py | 6 ++-- ...irements.py => create_requirements_pex.py} | 14 ++++----- .../python/rules/python_test_runner.py | 6 ++-- .../pants_test/backend/python/rules/BUILD | 31 ++++++++++--------- ...nts.py => test_create_requirements_pex.py} | 16 +++++----- 5 files changed, 36 insertions(+), 37 deletions(-) rename src/python/pants/backend/python/rules/{resolve_requirements.py => create_requirements_pex.py} (86%) rename tests/python/pants_test/backend/python/rules/{test_resolve_requirements.py => test_create_requirements_pex.py} (87%) diff --git a/src/python/pants/backend/python/register.py b/src/python/pants/backend/python/register.py index 766e39dfba7..91afae90652 100644 --- a/src/python/pants/backend/python/register.py +++ b/src/python/pants/backend/python/register.py @@ -5,8 +5,8 @@ from pants.backend.python.python_artifact import PythonArtifact from pants.backend.python.python_requirement import PythonRequirement from pants.backend.python.python_requirements import PythonRequirements -from pants.backend.python.rules import (download_pex_bin, inject_init, python_test_runner, - resolve_requirements) +from pants.backend.python.rules import (create_requirements_pex, download_pex_bin, inject_init, + python_test_runner) from pants.backend.python.subsystems.python_native_code import PythonNativeCode from pants.backend.python.subsystems.python_native_code import rules as python_native_code_rules from pants.backend.python.subsystems.subprocess_environment import SubprocessEnvironment @@ -93,6 +93,6 @@ def rules(): inject_init.rules() + python_test_runner.rules() + python_native_code_rules() + - resolve_requirements.rules() + + create_requirements_pex.rules() + subprocess_environment_rules() ) diff --git a/src/python/pants/backend/python/rules/resolve_requirements.py b/src/python/pants/backend/python/rules/create_requirements_pex.py similarity index 86% rename from src/python/pants/backend/python/rules/resolve_requirements.py rename to src/python/pants/backend/python/rules/create_requirements_pex.py index e44d9584201..0ae85043cc2 100644 --- a/src/python/pants/backend/python/rules/resolve_requirements.py +++ b/src/python/pants/backend/python/rules/create_requirements_pex.py @@ -12,7 +12,7 @@ from pants.util.strutil import create_path_env_var -class ResolveRequirementsRequest(datatype([ +class RequirementsPexRequest(datatype([ ('output_filename', string_type), ('requirements', hashable_string_list), ('interpreter_constraints', hashable_string_list), @@ -21,14 +21,14 @@ class ResolveRequirementsRequest(datatype([ pass -class ResolvedRequirementsPex(datatype([('directory_digest', Digest)])): +class RequirementsPex(datatype([('directory_digest', Digest)])): pass # TODO: This is non-hermetic because the requirements will be resolved on the fly by # pex, where it should be hermetically provided in some way. -@rule(ResolvedRequirementsPex, [ResolveRequirementsRequest, DownloadedPexBin, PythonSetup, PexBuildEnvironment]) -def resolve_requirements(request, pex_bin, python_setup, pex_build_environment): +@rule(RequirementsPex, [RequirementsPexRequest, DownloadedPexBin, PythonSetup, PexBuildEnvironment]) +def create_requirements_pex(request, pex_bin, python_setup, pex_build_environment): """Returns a PEX with the given requirements, optional entry point, and optional interpreter constraints.""" @@ -60,14 +60,12 @@ def resolve_requirements(request, pex_bin, python_setup, pex_build_environment): ) result = yield Get(ExecuteProcessResult, ExecuteProcessRequest, request) - yield ResolvedRequirementsPex( - directory_digest=result.output_directory_digest, - ) + yield RequirementsPex(directory_digest=result.output_directory_digest) def rules(): return [ - resolve_requirements, + create_requirements_pex, optionable_rule(PythonSetup), optionable_rule(PythonNativeCode), ] diff --git a/src/python/pants/backend/python/rules/python_test_runner.py b/src/python/pants/backend/python/rules/python_test_runner.py index 3fa352a0cd5..8a0ae74a2d2 100644 --- a/src/python/pants/backend/python/rules/python_test_runner.py +++ b/src/python/pants/backend/python/rules/python_test_runner.py @@ -1,9 +1,9 @@ # Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +from pants.backend.python.rules.create_requirements_pex import (RequirementsPex, + RequirementsPexRequest) from pants.backend.python.rules.inject_init import InjectedInitDigest -from pants.backend.python.rules.resolve_requirements import (ResolvedRequirementsPex, - ResolveRequirementsRequest) from pants.backend.python.subsystems.pytest import PyTest from pants.backend.python.subsystems.python_setup import PythonSetup from pants.backend.python.subsystems.subprocess_environment import SubprocessEncodingEnvironment @@ -53,7 +53,7 @@ def run_python_test(test_target, pytest, python_setup, source_root_config, subpr all_target_requirements.append(str(py_req.requirement)) all_requirements = all_target_requirements + list(pytest.get_requirement_strings()) resolved_requirements_pex = yield Get( - ResolvedRequirementsPex, ResolveRequirementsRequest( + RequirementsPex, RequirementsPexRequest( output_filename=output_pytest_requirements_pex_filename, requirements=tuple(sorted(all_requirements)), interpreter_constraints=tuple(sorted(interpreter_constraints)), diff --git a/tests/python/pants_test/backend/python/rules/BUILD b/tests/python/pants_test/backend/python/rules/BUILD index 32cdfe91afd..fbf36da3a00 100644 --- a/tests/python/pants_test/backend/python/rules/BUILD +++ b/tests/python/pants_test/backend/python/rules/BUILD @@ -1,6 +1,22 @@ # Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +python_tests( + name='create_requirements_pex', + source='test_create_requirements_pex.py', + dependencies=[ + 'src/python/pants/backend/python/rules', + 'src/python/pants/backend/python/subsystems', + 'src/python/pants/engine:fs', + 'src/python/pants/engine:rules', + 'src/python/pants/engine:selectors', + 'src/python/pants/util:collections', + 'src/python/pants/util:contextutil', + 'tests/python/pants_test:test_base', + 'tests/python/pants_test/subsystem:subsystem_utils', + ] +) + python_tests( name='inject_init', source='test_inject_init.py', @@ -22,18 +38,3 @@ python_tests( tags={'integration'}, ) -python_tests( - name='resolve_requirements', - source='test_resolve_requirements.py', - dependencies=[ - 'src/python/pants/backend/python/rules', - 'src/python/pants/backend/python/subsystems', - 'src/python/pants/engine:fs', - 'src/python/pants/engine:rules', - 'src/python/pants/engine:selectors', - 'src/python/pants/util:collections', - 'src/python/pants/util:contextutil', - 'tests/python/pants_test:test_base', - 'tests/python/pants_test/subsystem:subsystem_utils', - ] -) diff --git a/tests/python/pants_test/backend/python/rules/test_resolve_requirements.py b/tests/python/pants_test/backend/python/rules/test_create_requirements_pex.py similarity index 87% rename from tests/python/pants_test/backend/python/rules/test_resolve_requirements.py rename to tests/python/pants_test/backend/python/rules/test_create_requirements_pex.py index 3e9451352b5..8af6d32d5e2 100644 --- a/tests/python/pants_test/backend/python/rules/test_resolve_requirements.py +++ b/tests/python/pants_test/backend/python/rules/test_create_requirements_pex.py @@ -5,10 +5,10 @@ import os.path import zipfile +from pants.backend.python.rules.create_requirements_pex import (RequirementsPex, + RequirementsPexRequest, + create_requirements_pex) from pants.backend.python.rules.download_pex_bin import download_pex_bin -from pants.backend.python.rules.resolve_requirements import (ResolvedRequirementsPex, - ResolveRequirementsRequest, - resolve_requirements) from pants.backend.python.subsystems.python_native_code import (PythonNativeCode, create_pex_native_build_environment) from pants.backend.python.subsystems.python_setup import PythonSetup @@ -26,10 +26,10 @@ class TestResolveRequirements(TestBase): @classmethod def rules(cls): return super().rules() + [ - resolve_requirements, - download_pex_bin, + create_requirements_pex, create_pex_native_build_environment, - RootRule(ResolveRequirementsRequest), + download_pex_bin, + RootRule(RequirementsPexRequest), RootRule(PythonSetup), RootRule(PythonNativeCode), ] @@ -44,14 +44,14 @@ def create_pex_and_get_pex_info( def hashify_optional_collection(iterable): return tuple(sorted(iterable)) if iterable is not None else tuple() - request = ResolveRequirementsRequest( + request = RequirementsPexRequest( output_filename="test.pex", requirements=hashify_optional_collection(requirements), interpreter_constraints=hashify_optional_collection(interpreter_constraints), entry_point=entry_point, ) requirements_pex = assert_single_element( - self.scheduler.product_request(ResolvedRequirementsPex, [Params( + self.scheduler.product_request(RequirementsPex, [Params( request, PythonSetup.global_instance(), PythonNativeCode.global_instance() From b18677266a0bb7e1eca20c761d8978de5017ed98 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 19 Jul 2019 18:09:06 -0700 Subject: [PATCH 3/3] * Use long CLI args * Fix shadowing variable name * Update description of ProcessRequest --- .../backend/python/rules/create_requirements_pex.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/python/pants/backend/python/rules/create_requirements_pex.py b/src/python/pants/backend/python/rules/create_requirements_pex.py index 0ae85043cc2..cd22d4ab46f 100644 --- a/src/python/pants/backend/python/rules/create_requirements_pex.py +++ b/src/python/pants/backend/python/rules/create_requirements_pex.py @@ -46,20 +46,20 @@ def create_requirements_pex(request, pex_bin, python_setup, pex_build_environmen # necessarily the interpreter that PEX will use to execute the generated .pex file. # TODO(#7735): Set --python-setup-interpreter-search-paths differently for the host and target # platforms, when we introduce platforms in https://github.com/pantsbuild/pants/issues/7735. - argv = ["python", f"./{pex_bin.executable}", "-o", request.output_filename] + argv = ["python", f"./{pex_bin.executable}", "--output-file", request.output_filename] if request.entry_point is not None: - argv.extend(["-e", request.entry_point]) + argv.extend(["--entry-point", request.entry_point]) argv.extend(interpreter_constraint_args + list(request.requirements)) - request = ExecuteProcessRequest( + execute_process_request = ExecuteProcessRequest( argv=tuple(argv), env=env, input_files=pex_bin.directory_digest, - description=f"Resolve requirements: {', '.join(request.requirements)}", + description=f"Create a requirements PEX: {', '.join(request.requirements)}", output_files=(request.output_filename,), ) - result = yield Get(ExecuteProcessResult, ExecuteProcessRequest, request) + result = yield Get(ExecuteProcessResult, ExecuteProcessRequest, execute_process_request) yield RequirementsPex(directory_digest=result.output_directory_digest)