diff --git a/src/python/pants/backend/awslambda/python/awslambda_python_rules_test.py b/src/python/pants/backend/awslambda/python/awslambda_python_rules_test.py index deb57700575..b3a36957a1e 100644 --- a/src/python/pants/backend/awslambda/python/awslambda_python_rules_test.py +++ b/src/python/pants/backend/awslambda/python/awslambda_python_rules_test.py @@ -13,7 +13,6 @@ from pants.backend.awslambda.python.awslambda_python_rules import rules as awslambda_python_rules from pants.backend.awslambda.python.target_types import PythonAWSLambda from pants.backend.python.target_types import PythonLibrary -from pants.core.util_rules.pants_environment import PantsEnvironment from pants.engine.addresses import Address from pants.engine.fs import DigestContents from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -24,7 +23,7 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *awslambda_python_rules(), - QueryRule(CreatedAWSLambda, (PythonAwsLambdaFieldSet, PantsEnvironment)), + QueryRule(CreatedAWSLambda, (PythonAwsLambdaFieldSet,)), ], target_types=[PythonAWSLambda, PythonLibrary], ) @@ -40,7 +39,7 @@ def create_python_awslambda(rule_runner: RuleRunner, addr: str) -> Tuple[str, by ) target = rule_runner.get_target(Address.parse(addr)) created_awslambda = rule_runner.request( - CreatedAWSLambda, [PythonAwsLambdaFieldSet.create(target), PantsEnvironment()] + CreatedAWSLambda, [PythonAwsLambdaFieldSet.create(target)] ) created_awslambda_digest_contents = rule_runner.request( DigestContents, [created_awslambda.digest] diff --git a/src/python/pants/backend/python/goals/pytest_runner_integration_test.py b/src/python/pants/backend/python/goals/pytest_runner_integration_test.py index ff4287a1b99..6bd3d9a493f 100644 --- a/src/python/pants/backend/python/goals/pytest_runner_integration_test.py +++ b/src/python/pants/backend/python/goals/pytest_runner_integration_test.py @@ -5,7 +5,7 @@ import re from pathlib import PurePath from textwrap import dedent -from typing import List, Optional +from typing import List, Mapping, Optional import pytest @@ -23,7 +23,6 @@ from pants.core.goals import binary from pants.core.goals.test import TestDebugRequest, TestResult, get_filtered_environment from pants.core.util_rules import distdir -from pants.core.util_rules.pants_environment import PantsEnvironment from pants.engine.addresses import Address from pants.engine.fs import DigestContents, FileContent from pants.engine.process import InteractiveRunner @@ -43,8 +42,8 @@ def rule_runner() -> RuleRunner: *binary.rules(), *create_python_binary.rules(), get_filtered_environment, - QueryRule(TestResult, (PythonTestFieldSet, PantsEnvironment)), - QueryRule(TestDebugRequest, (PythonTestFieldSet, PantsEnvironment)), + QueryRule(TestResult, (PythonTestFieldSet,)), + QueryRule(TestDebugRequest, (PythonTestFieldSet,)), ], target_types=[PythonBinary, PythonLibrary, PythonTests, PythonRequirementLibrary], ) @@ -144,7 +143,7 @@ def run_pytest( use_coverage: bool = False, execution_slot_var: Optional[str] = None, extra_env_vars: Optional[str] = None, - pants_environment: PantsEnvironment = PantsEnvironment(), + env: Optional[Mapping[str, str]] = None, ) -> TestResult: args = [ "--backend-packages=pants.backend.python", @@ -163,9 +162,9 @@ def run_pytest( args.append("--test-use-coverage") if execution_slot_var: args.append(f"--pytest-execution-slot-var={execution_slot_var}") - rule_runner.set_options(args) + rule_runner.set_options(args, env=env) - inputs = [PythonTestFieldSet.create(test_target), pants_environment] + inputs = [PythonTestFieldSet.create(test_target)] test_result = rule_runner.request(TestResult, inputs) debug_request = rule_runner.request(TestDebugRequest, inputs) if debug_request.process is not None: @@ -433,9 +432,12 @@ def test_args(): ).encode(), ) tgt = create_test_target(rule_runner, [source]) - mock_env = PantsEnvironment({"OTHER_VAR": "other_value"}) - extra_env_vars = '["SOME_VAR=some_value", "OTHER_VAR"]' - result = run_pytest(rule_runner, tgt, extra_env_vars=extra_env_vars, pants_environment=mock_env) + result = run_pytest( + rule_runner, + tgt, + extra_env_vars='["SOME_VAR=some_value", "OTHER_VAR"]', + env={"OTHER_VAR": "other_value"}, + ) assert result.exit_code == 0 diff --git a/src/python/pants/backend/python/lint/bandit/rules_integration_test.py b/src/python/pants/backend/python/lint/bandit/rules_integration_test.py index 503a0c6bec2..f9b879a1953 100644 --- a/src/python/pants/backend/python/lint/bandit/rules_integration_test.py +++ b/src/python/pants/backend/python/lint/bandit/rules_integration_test.py @@ -9,7 +9,6 @@ from pants.backend.python.lint.bandit.rules import rules as bandit_rules from pants.backend.python.target_types import PythonInterpreterCompatibility, PythonLibrary from pants.core.goals.lint import LintResult, LintResults -from pants.core.util_rules.pants_environment import PantsEnvironment from pants.engine.addresses import Address from pants.engine.fs import DigestContents, FileContent from pants.engine.target import Target @@ -22,7 +21,7 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *bandit_rules(), - QueryRule(LintResults, (BanditRequest, PantsEnvironment)), + QueryRule(LintResults, (BanditRequest,)), ], ) @@ -69,7 +68,7 @@ def run_bandit( rule_runner.set_options(args) results = rule_runner.request( LintResults, - [BanditRequest(BanditFieldSet.create(tgt) for tgt in targets), PantsEnvironment()], + [BanditRequest(BanditFieldSet.create(tgt) for tgt in targets)], ) return results.results diff --git a/src/python/pants/backend/python/lint/black/rules_integration_test.py b/src/python/pants/backend/python/lint/black/rules_integration_test.py index 854eba79eb6..e12288ff1e0 100644 --- a/src/python/pants/backend/python/lint/black/rules_integration_test.py +++ b/src/python/pants/backend/python/lint/black/rules_integration_test.py @@ -11,7 +11,6 @@ from pants.backend.python.target_types import PythonLibrary from pants.core.goals.fmt import FmtResult from pants.core.goals.lint import LintResult, LintResults -from pants.core.util_rules.pants_environment import PantsEnvironment from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest from pants.engine.addresses import Address from pants.engine.fs import CreateDigest, Digest, FileContent @@ -25,9 +24,9 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *black_rules(), - QueryRule(LintResults, (BlackRequest, PantsEnvironment)), - QueryRule(FmtResult, (BlackRequest, PantsEnvironment)), - QueryRule(SourceFiles, (SourceFilesRequest, PantsEnvironment)), + QueryRule(LintResults, (BlackRequest,)), + QueryRule(FmtResult, (BlackRequest,)), + QueryRule(SourceFiles, (SourceFilesRequest,)), ], target_types=[PythonLibrary], ) @@ -74,20 +73,17 @@ def run_black( args.append("--black-skip") rule_runner.set_options(args) field_sets = [BlackFieldSet.create(tgt) for tgt in targets] - pants_env = PantsEnvironment() - lint_results = rule_runner.request(LintResults, [BlackRequest(field_sets), pants_env]) + lint_results = rule_runner.request(LintResults, [BlackRequest(field_sets)]) input_sources = rule_runner.request( SourceFiles, [ SourceFilesRequest(field_set.sources for field_set in field_sets), - pants_env, ], ) fmt_result = rule_runner.request( FmtResult, [ BlackRequest(field_sets, prior_formatter_result=input_sources.snapshot), - pants_env, ], ) return lint_results.results, fmt_result diff --git a/src/python/pants/backend/python/lint/docformatter/rules_integration_test.py b/src/python/pants/backend/python/lint/docformatter/rules_integration_test.py index 166c5100d24..649a7192471 100644 --- a/src/python/pants/backend/python/lint/docformatter/rules_integration_test.py +++ b/src/python/pants/backend/python/lint/docformatter/rules_integration_test.py @@ -10,7 +10,6 @@ from pants.backend.python.target_types import PythonLibrary from pants.core.goals.fmt import FmtResult from pants.core.goals.lint import LintResult, LintResults -from pants.core.util_rules.pants_environment import PantsEnvironment from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest from pants.engine.addresses import Address from pants.engine.fs import CreateDigest, Digest, FileContent @@ -23,9 +22,9 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *docformatter_rules(), - QueryRule(LintResults, (DocformatterRequest, PantsEnvironment)), - QueryRule(FmtResult, (DocformatterRequest, PantsEnvironment)), - QueryRule(SourceFiles, (SourceFilesRequest, PantsEnvironment)), + QueryRule(LintResults, (DocformatterRequest,)), + QueryRule(FmtResult, (DocformatterRequest,)), + QueryRule(SourceFiles, (SourceFilesRequest,)), ] ) @@ -55,20 +54,17 @@ def run_docformatter( args.append("--docformatter-skip") rule_runner.set_options(args) field_sets = [DocformatterFieldSet.create(tgt) for tgt in targets] - pants_env = PantsEnvironment() - lint_results = rule_runner.request(LintResults, [DocformatterRequest(field_sets), pants_env]) + lint_results = rule_runner.request(LintResults, [DocformatterRequest(field_sets)]) input_sources = rule_runner.request( SourceFiles, [ SourceFilesRequest(field_set.sources for field_set in field_sets), - pants_env, ], ) fmt_result = rule_runner.request( FmtResult, [ DocformatterRequest(field_sets, prior_formatter_result=input_sources.snapshot), - pants_env, ], ) return lint_results.results, fmt_result diff --git a/src/python/pants/backend/python/lint/flake8/rules_integration_test.py b/src/python/pants/backend/python/lint/flake8/rules_integration_test.py index bf2af1c7780..b786f01ede2 100644 --- a/src/python/pants/backend/python/lint/flake8/rules_integration_test.py +++ b/src/python/pants/backend/python/lint/flake8/rules_integration_test.py @@ -9,7 +9,6 @@ from pants.backend.python.lint.flake8.rules import rules as flake8_rules from pants.backend.python.target_types import PythonInterpreterCompatibility, PythonLibrary from pants.core.goals.lint import LintResult, LintResults -from pants.core.util_rules.pants_environment import PantsEnvironment from pants.engine.addresses import Address from pants.engine.fs import DigestContents, FileContent from pants.engine.target import Target @@ -22,7 +21,7 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *flake8_rules(), - QueryRule(LintResults, (Flake8Request, PantsEnvironment)), + QueryRule(LintResults, (Flake8Request,)), ] ) @@ -70,7 +69,6 @@ def run_flake8( LintResults, [ Flake8Request(Flake8FieldSet.create(tgt) for tgt in targets), - PantsEnvironment(), ], ) return results.results diff --git a/src/python/pants/backend/python/lint/isort/rules_integration_test.py b/src/python/pants/backend/python/lint/isort/rules_integration_test.py index e9f6e84a7c0..4de7f6db6d3 100644 --- a/src/python/pants/backend/python/lint/isort/rules_integration_test.py +++ b/src/python/pants/backend/python/lint/isort/rules_integration_test.py @@ -10,7 +10,6 @@ from pants.backend.python.target_types import PythonLibrary from pants.core.goals.fmt import FmtResult from pants.core.goals.lint import LintResult, LintResults -from pants.core.util_rules.pants_environment import PantsEnvironment from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest from pants.engine.addresses import Address from pants.engine.fs import CreateDigest, Digest, FileContent @@ -23,9 +22,9 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *isort_rules(), - QueryRule(LintResults, (IsortRequest, PantsEnvironment)), - QueryRule(FmtResult, (IsortRequest, PantsEnvironment)), - QueryRule(SourceFiles, (SourceFilesRequest, PantsEnvironment)), + QueryRule(LintResults, (IsortRequest,)), + QueryRule(FmtResult, (IsortRequest,)), + QueryRule(SourceFiles, (SourceFilesRequest,)), ] ) @@ -69,20 +68,17 @@ def run_isort( args.append("--isort-skip") rule_runner.set_options(args) field_sets = [IsortFieldSet.create(tgt) for tgt in targets] - pants_env = PantsEnvironment() - lint_results = rule_runner.request(LintResults, [IsortRequest(field_sets), pants_env]) + lint_results = rule_runner.request(LintResults, [IsortRequest(field_sets)]) input_sources = rule_runner.request( SourceFiles, [ SourceFilesRequest(field_set.sources for field_set in field_sets), - pants_env, ], ) fmt_result = rule_runner.request( FmtResult, [ IsortRequest(field_sets, prior_formatter_result=input_sources.snapshot), - pants_env, ], ) return lint_results.results, fmt_result diff --git a/src/python/pants/backend/python/lint/pylint/rules_integration_test.py b/src/python/pants/backend/python/lint/pylint/rules_integration_test.py index b8d8058ac94..82db4bb0933 100644 --- a/src/python/pants/backend/python/lint/pylint/rules_integration_test.py +++ b/src/python/pants/backend/python/lint/pylint/rules_integration_test.py @@ -12,7 +12,6 @@ from pants.backend.python.lint.pylint.rules import rules as pylint_rules from pants.backend.python.target_types import PythonLibrary, PythonRequirementLibrary from pants.core.goals.lint import LintResult, LintResults -from pants.core.util_rules.pants_environment import PantsEnvironment from pants.engine.addresses import Address from pants.engine.fs import FileContent from pants.engine.target import Target @@ -25,7 +24,7 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *pylint_rules(), - QueryRule(LintResults, (PylintRequest, PantsEnvironment)), + QueryRule(LintResults, (PylintRequest,)), ], target_types=[PythonLibrary, PythonRequirementLibrary, PylintSourcePlugin], ) @@ -98,7 +97,7 @@ def run_pylint( rule_runner.set_options(args) results = rule_runner.request( LintResults, - [PylintRequest(PylintFieldSet.create(tgt) for tgt in targets), PantsEnvironment()], + [PylintRequest(PylintFieldSet.create(tgt) for tgt in targets)], ) return results.results diff --git a/src/python/pants/backend/python/lint/python_fmt_integration_test.py b/src/python/pants/backend/python/lint/python_fmt_integration_test.py index d9333711ce7..1c5bfc5bbea 100644 --- a/src/python/pants/backend/python/lint/python_fmt_integration_test.py +++ b/src/python/pants/backend/python/lint/python_fmt_integration_test.py @@ -10,7 +10,6 @@ from pants.backend.python.lint.python_fmt import PythonFmtTargets, format_python_target from pants.backend.python.target_types import PythonLibrary from pants.core.goals.fmt import LanguageFmtResults -from pants.core.util_rules.pants_environment import PantsEnvironment from pants.engine.addresses import Address from pants.engine.fs import CreateDigest, Digest, FileContent from pants.engine.target import Targets @@ -24,7 +23,7 @@ def rule_runner() -> RuleRunner: format_python_target, *black_rules(), *isort_rules(), - QueryRule(LanguageFmtResults, (PythonFmtTargets, PantsEnvironment)), + QueryRule(LanguageFmtResults, (PythonFmtTargets,)), ] ) @@ -45,7 +44,7 @@ def run_black_and_isort( *(extra_args or []), ] ) - results = rule_runner.request(LanguageFmtResults, [targets, PantsEnvironment()]) + results = rule_runner.request(LanguageFmtResults, [targets]) return results diff --git a/src/python/pants/backend/python/typecheck/mypy/rules_integration_test.py b/src/python/pants/backend/python/typecheck/mypy/rules_integration_test.py index 10350a4fa25..19c1470d6d3 100644 --- a/src/python/pants/backend/python/typecheck/mypy/rules_integration_test.py +++ b/src/python/pants/backend/python/typecheck/mypy/rules_integration_test.py @@ -13,7 +13,6 @@ from pants.backend.python.typecheck.mypy.rules import MyPyFieldSet, MyPyRequest from pants.backend.python.typecheck.mypy.rules import rules as mypy_rules from pants.core.goals.typecheck import TypecheckResult, TypecheckResults -from pants.core.util_rules.pants_environment import PantsEnvironment from pants.engine.addresses import Address from pants.engine.fs import FileContent from pants.engine.rules import QueryRule @@ -31,7 +30,7 @@ def rule_runner() -> RuleRunner: rules=[ *mypy_rules(), *dependency_inference_rules.rules(), # Used for import inference. - QueryRule(TypecheckResults, (MyPyRequest, PantsEnvironment)), + QueryRule(TypecheckResults, (MyPyRequest,)), ], target_types=[PythonLibrary, PythonRequirementLibrary, MyPySourcePlugin], ) @@ -130,7 +129,7 @@ def run_mypy( rule_runner.set_options(args) result = rule_runner.request( TypecheckResults, - [MyPyRequest(MyPyFieldSet.create(tgt) for tgt in targets), PantsEnvironment()], + [MyPyRequest(MyPyFieldSet.create(tgt) for tgt in targets)], ) return result.results diff --git a/src/python/pants/backend/python/util_rules/extract_pex_test.py b/src/python/pants/backend/python/util_rules/extract_pex_test.py index 90f0a803a9f..67c7cf1fa53 100644 --- a/src/python/pants/backend/python/util_rules/extract_pex_test.py +++ b/src/python/pants/backend/python/util_rules/extract_pex_test.py @@ -13,7 +13,6 @@ PexRequest, PexRequirements, ) -from pants.core.util_rules.pants_environment import PantsEnvironment from pants.engine.fs import EMPTY_DIGEST, Snapshot from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -24,7 +23,7 @@ def rule_runner() -> RuleRunner: rules=[ *extract_pex.rules(), *pex.rules(), - QueryRule(Pex, [PexRequest, PantsEnvironment]), + QueryRule(Pex, [PexRequest]), QueryRule(ExtractedPexDistributions, [Pex]), ] ) @@ -48,7 +47,7 @@ def get_distributions( interpreter_constraints=PexInterpreterConstraints([">=3.6"]), internal_only=True, ) - built_pex = rule_runner.request(Pex, [pex_request, PantsEnvironment()]) + built_pex = rule_runner.request(Pex, [pex_request]) return rule_runner.request(ExtractedPexDistributions, [built_pex]) diff --git a/src/python/pants/backend/python/util_rules/pex_cli_test.py b/src/python/pants/backend/python/util_rules/pex_cli_test.py index 973f1a1bd26..3f8674e2055 100644 --- a/src/python/pants/backend/python/util_rules/pex_cli_test.py +++ b/src/python/pants/backend/python/util_rules/pex_cli_test.py @@ -7,7 +7,6 @@ from pants.backend.python.util_rules import pex_cli from pants.backend.python.util_rules.pex_cli import PexCliProcess -from pants.core.util_rules.pants_environment import PantsEnvironment from pants.engine.fs import DigestContents from pants.engine.process import Process from pants.engine.rules import QueryRule @@ -20,7 +19,7 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *pex_cli.rules(), - QueryRule(Process, (PexCliProcess, PantsEnvironment)), + QueryRule(Process, (PexCliProcess,)), ] ) @@ -32,7 +31,7 @@ def test_custom_ca_certs(rule_runner: RuleRunner) -> None: rule_runner.set_options([f"--ca-certs-path={certs_file}"]) proc = rule_runner.request( Process, - [PexCliProcess(argv=["some", "--args"], description=""), PantsEnvironment()], + [PexCliProcess(argv=["some", "--args"], description="")], ) assert proc.argv[2:6] == ("some", "--args", "--cert", "certsfile") files = rule_runner.request(DigestContents, [proc.input_digest]) diff --git a/src/python/pants/backend/python/util_rules/pex_test.py b/src/python/pants/backend/python/util_rules/pex_test.py index 84e4ed9f054..4a48715e407 100644 --- a/src/python/pants/backend/python/util_rules/pex_test.py +++ b/src/python/pants/backend/python/util_rules/pex_test.py @@ -6,7 +6,7 @@ import textwrap import zipfile from dataclasses import dataclass -from typing import Dict, Iterable, Iterator, List, Optional, Tuple, cast +from typing import Dict, Iterable, Iterator, List, Mapping, Optional, Tuple, cast import pytest from pkg_resources import Requirement @@ -21,7 +21,6 @@ PexRequirements, ) from pants.backend.python.util_rules.pex import rules as pex_rules -from pants.core.util_rules.pants_environment import PantsEnvironment from pants.engine.addresses import Address from pants.engine.fs import CreateDigest, Digest, FileContent from pants.engine.process import Process, ProcessResult @@ -317,8 +316,8 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *pex_rules(), - QueryRule(Pex, (PexRequest, PantsEnvironment)), - QueryRule(Process, (PexProcess, PantsEnvironment)), + QueryRule(Pex, (PexRequest,)), + QueryRule(Process, (PexProcess,)), QueryRule(ProcessResult, (Process,)), ] ) @@ -335,6 +334,7 @@ def create_pex_and_get_all_data( additional_inputs: Optional[Digest] = None, additional_pants_args: Tuple[str, ...] = (), additional_pex_args: Tuple[str, ...] = (), + env: Optional[Mapping[str, str]] = None, internal_only: bool = True, ) -> Dict: request = PexRequest( @@ -348,8 +348,10 @@ def create_pex_and_get_all_data( additional_inputs=additional_inputs, additional_args=additional_pex_args, ) - rule_runner.set_options(["--backend-packages=pants.backend.python", *additional_pants_args]) - pex = rule_runner.request(Pex, [request, PantsEnvironment()]) + rule_runner.set_options( + ["--backend-packages=pants.backend.python", *additional_pants_args], env=env + ) + pex = rule_runner.request(Pex, [request]) rule_runner.scheduler.write_digest(pex.digest) pex_path = os.path.join(rule_runner.build_root, "test.pex") with zipfile.ZipFile(pex_path, "r") as zipfp: @@ -452,6 +454,7 @@ def test_pex_environment(rule_runner: RuleRunner) -> None: "--subprocess-environment-env-vars=LANG", # Value should come from environment. "--subprocess-environment-env-vars=ftp_proxy=dummyproxy", ), + env={"LANG": "es_PY.UTF-8"}, ) process = rule_runner.request( @@ -463,7 +466,6 @@ def test_pex_environment(rule_runner: RuleRunner) -> None: input_digest=pex_output["pex"].digest, description="Run the pex and check its reported environment", ), - PantsEnvironment({"LANG": "es_PY.UTF-8"}), ], ) diff --git a/src/python/pants/bin/local_pants_runner.py b/src/python/pants/bin/local_pants_runner.py index 095f29b01e2..dae8650c365 100644 --- a/src/python/pants/bin/local_pants_runner.py +++ b/src/python/pants/bin/local_pants_runner.py @@ -2,6 +2,7 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import logging +import os from dataclasses import dataclass from typing import Mapping, Optional, Tuple @@ -12,9 +13,10 @@ from pants.base.specs_parser import SpecsParser from pants.base.workunit import WorkUnit from pants.build_graph.build_configuration import BuildConfiguration +from pants.core.util_rules.pants_environment import PantsEnvironment from pants.engine.internals.native import Native from pants.engine.internals.scheduler import ExecutionError -from pants.engine.session import SessionValues +from pants.engine.internals.session import SessionValues from pants.engine.unions import UnionMembership from pants.goal.run_tracker import RunTracker from pants.help.help_info_extracter import HelpInfoExtracter @@ -37,7 +39,6 @@ class LocalPantsRunner: build_root: The build root for this run. options: The parsed options for this run. - options_bootstrapper: The OptionsBootstrapper instance to use. build_config: The parsed build configuration for this run. specs: The specs for this run, i.e. either the address or filesystem specs. graph_session: A LegacyGraphSession instance for graph reuse. @@ -46,7 +47,6 @@ class LocalPantsRunner: build_root: str options: Options - options_bootstrapper: OptionsBootstrapper build_config: BuildConfiguration specs: Specs graph_session: GraphSession @@ -85,7 +85,12 @@ def _init_graph_session( dynamic_ui=dynamic_ui, use_colors=use_colors, should_report_workunits=stream_workunits, - session_values=SessionValues({OptionsBootstrapper: options_bootstrapper}), + session_values=SessionValues( + { + OptionsBootstrapper: options_bootstrapper, + PantsEnvironment: PantsEnvironment(os.environ), + } + ), ) @classmethod @@ -137,7 +142,6 @@ def create( return cls( build_root=build_root, options=options, - options_bootstrapper=options_bootstrapper, build_config=build_config, specs=specs, graph_session=graph_session, @@ -179,7 +183,6 @@ def _run_v2(self) -> ExitCode: def _maybe_run_v2_body(self, goals, poll: bool) -> ExitCode: return self.graph_session.run_goal_rules( - options_bootstrapper=self.options_bootstrapper, union_membership=self.union_membership, goals=goals, specs=self.specs, diff --git a/src/python/pants/core/goals/repl_integration_test.py b/src/python/pants/core/goals/repl_integration_test.py index 7e9de9589a9..2b1f6521f32 100644 --- a/src/python/pants/core/goals/repl_integration_test.py +++ b/src/python/pants/core/goals/repl_integration_test.py @@ -10,7 +10,6 @@ from pants.backend.python.util_rules.pex import PexProcess from pants.core.goals.repl import Repl from pants.core.goals.repl import rules as repl_rules -from pants.core.util_rules.pants_environment import PantsEnvironment from pants.engine.process import Process from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -22,7 +21,7 @@ def rule_runner() -> RuleRunner: *repl_rules(), *python_repl.rules(), *pex_from_targets.rules(), - QueryRule(Process, (PexProcess, PantsEnvironment)), + QueryRule(Process, (PexProcess,)), ], target_types=[PythonLibrary, ProtobufLibrary], ) diff --git a/src/python/pants/core/goals/test.py b/src/python/pants/core/goals/test.py index e1151ba78b2..9d85483c87d 100644 --- a/src/python/pants/core/goals/test.py +++ b/src/python/pants/core/goals/test.py @@ -22,7 +22,7 @@ from pants.engine.fs import Digest, MergeDigests, Snapshot, Workspace from pants.engine.goal import Goal, GoalSubsystem from pants.engine.process import FallibleProcessResult, InteractiveProcess, InteractiveRunner -from pants.engine.rules import Get, MultiGet, _uncacheable_rule, collect_rules, goal_rule, rule +from pants.engine.rules import Get, MultiGet, collect_rules, goal_rule, rule from pants.engine.target import ( FieldSet, Sources, @@ -450,7 +450,7 @@ async def run_tests( return Test(exit_code) -@_uncacheable_rule +@rule def get_filtered_environment( test_subsystem: TestSubsystem, pants_env: PantsEnvironment ) -> TestExtraEnv: diff --git a/src/python/pants/core/register.py b/src/python/pants/core/register.py index 70b395a8ae2..b06a9077e28 100644 --- a/src/python/pants/core/register.py +++ b/src/python/pants/core/register.py @@ -14,6 +14,7 @@ external_tool, filter_empty_sources, pants_bin, + pants_environment, source_files, stripped_source_files, subprocess_environment, @@ -39,6 +40,7 @@ def rules(): *stripped_source_files.rules(), *archive.rules(), *external_tool.rules(), + *pants_environment.rules(), *subprocess_environment.rules(), *source_root.rules(), ] diff --git a/src/python/pants/core/util_rules/pants_environment.py b/src/python/pants/core/util_rules/pants_environment.py index 08e347b4966..691582e7a9a 100644 --- a/src/python/pants/core/util_rules/pants_environment.py +++ b/src/python/pants/core/util_rules/pants_environment.py @@ -2,12 +2,12 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import logging -import os import re from dataclasses import dataclass from typing import Dict, Mapping, Optional, Sequence -from pants.engine.rules import side_effecting +from pants.engine.internals.session import SessionValues +from pants.engine.rules import collect_rules, rule from pants.util.frozendict import FrozenDict from pants.util.meta import frozen_after_init @@ -17,19 +17,11 @@ shorthand_re = re.compile(r"([A-Za-z_]\w*)") -@side_effecting @frozen_after_init @dataclass(unsafe_hash=True) class PantsEnvironment: """PantsEnvironment is a representation of the environment variables the currently-executing - pants process was invoked with. - - Since the exact contents of the environment are very specific to a given invocation of pants, - this type is marked as side-effecting, and can only be requested as an input to an uncacheable - rule. An uncacheable rule should accept this type as an input, and use `get_subset` to filter - the environment rules down to only the ones other rules care about as inputs, in order to avoid - a lot of cache invalidation if the environment changes spuriously. - """ + Pants process was invoked with.""" env: FrozenDict[str, str] @@ -39,10 +31,10 @@ def __init__(self, env: Optional[Mapping[str, str]] = None): Explicitly specify the env argument to create a mock environment for testing. """ - self.env = FrozenDict(env if env else os.environ) + self.env = FrozenDict(env or {}) def get_subset( - self, requested: Sequence[str], allowed: Optional[Sequence[str]] = None + self, requested: Sequence[str], *, allowed: Optional[Sequence[str]] = None ) -> FrozenDict[str, str]: """Extract a subset of named env vars. @@ -53,7 +45,8 @@ def get_subset( In the former case, the value for that name is taken from this env. In the latter case the specified value overrides the value in this env. - If `allowed` is specified, variable names must be in that list, or an error will be raised. + If `allowed` is specified, the requested variable names must be in that list, or an error + will be raised. """ allowed_set = None if allowed is None else set(allowed) env_var_subset: Dict[str, str] = {} @@ -80,3 +73,12 @@ def check_and_set(name: str, value: Optional[str]): ) return FrozenDict(env_var_subset) + + +@rule +async def pants_environment(session_values: SessionValues) -> PantsEnvironment: + return session_values[PantsEnvironment] + + +def rules(): + return collect_rules() diff --git a/src/python/pants/core/util_rules/subprocess_environment.py b/src/python/pants/core/util_rules/subprocess_environment.py index f9e4cbbd42d..3417ed74aee 100644 --- a/src/python/pants/core/util_rules/subprocess_environment.py +++ b/src/python/pants/core/util_rules/subprocess_environment.py @@ -6,7 +6,7 @@ from typing import Tuple from pants.core.util_rules.pants_environment import PantsEnvironment -from pants.engine.rules import _uncacheable_rule, collect_rules +from pants.engine.rules import collect_rules, rule from pants.option.subsystem import Subsystem from pants.util.frozendict import FrozenDict @@ -115,12 +115,14 @@ class SubprocessEnvironmentVars: vars: FrozenDict[str, str] -@_uncacheable_rule +@rule def get_subprocess_environment( subproc_env: SubprocessEnvironment, pants_env: PantsEnvironment ) -> SubprocessEnvironmentVars: return SubprocessEnvironmentVars( - pants_env.get_subset(subproc_env.env_vars_to_pass_to_subprocesses, SETTABLE_ENV_VARS) + pants_env.get_subset( + subproc_env.env_vars_to_pass_to_subprocesses, allowed=SETTABLE_ENV_VARS + ) ) diff --git a/src/python/pants/engine/internals/native.py b/src/python/pants/engine/internals/native.py index 9dbba455f5a..7eb61caaa88 100644 --- a/src/python/pants/engine/internals/native.py +++ b/src/python/pants/engine/internals/native.py @@ -24,8 +24,8 @@ PyTasks, PyTypes, ) +from pants.engine.internals.session import SessionValues from pants.engine.rules import Get -from pants.engine.session import SessionValues from pants.engine.unions import union from pants.util.logging import LogLevel from pants.util.memo import memoized_property diff --git a/src/python/pants/engine/internals/options_parsing.py b/src/python/pants/engine/internals/options_parsing.py index 7a4f8d109df..e8130e7301f 100644 --- a/src/python/pants/engine/internals/options_parsing.py +++ b/src/python/pants/engine/internals/options_parsing.py @@ -5,8 +5,8 @@ from typing import cast from pants.build_graph.build_configuration import BuildConfiguration +from pants.engine.internals.session import SessionValues from pants.engine.rules import collect_rules, rule -from pants.engine.session import SessionValues from pants.init.options_initializer import OptionsInitializer from pants.option.global_options import GlobalOptions from pants.option.options import Options @@ -46,11 +46,7 @@ def options(self) -> Options: def parse_options(build_config: BuildConfiguration, session_values: SessionValues) -> _Options: # TODO: Once the OptionsBootstrapper has been removed from all relevant QueryRules, this lookup # should be extracted into a separate @rule. - maybe_options_bootstrapper = session_values.values.get(OptionsBootstrapper) - if maybe_options_bootstrapper is None: - raise ValueError("Expected an OptionsBootstrapper to be provided via SessionValues.") - options_bootstrapper = cast(OptionsBootstrapper, maybe_options_bootstrapper) - + options_bootstrapper = session_values[OptionsBootstrapper] return _Options(options_bootstrapper, build_config) diff --git a/src/python/pants/engine/internals/scheduler.py b/src/python/pants/engine/internals/scheduler.py index 776402d8e58..ccaeafa709c 100644 --- a/src/python/pants/engine/internals/scheduler.py +++ b/src/python/pants/engine/internals/scheduler.py @@ -32,6 +32,7 @@ from pants.engine.internals.native_engine import PyTypes # type: ignore[import] from pants.engine.internals.nodes import Return, Throw from pants.engine.internals.selectors import Params +from pants.engine.internals.session import SessionValues from pants.engine.platform import Platform from pants.engine.process import ( FallibleProcessResultWithPlatform, @@ -40,7 +41,6 @@ MultiPlatformProcess, ) from pants.engine.rules import Rule, RuleIndex, TaskRule -from pants.engine.session import SessionValues from pants.engine.unions import UnionMembership, union from pants.option.global_options import ExecutionOptions from pants.util.contextutil import temporary_file_path diff --git a/src/python/pants/engine/internals/session.py b/src/python/pants/engine/internals/session.py new file mode 100644 index 00000000000..5751ea31134 --- /dev/null +++ b/src/python/pants/engine/internals/session.py @@ -0,0 +1,23 @@ +# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from typing import Any, Type, TypeVar, cast + +from pants.util.frozendict import FrozenDict + +_T = TypeVar("_T") + + +class SessionValues(FrozenDict[Type, Any]): + """Values set for the Session, and exposed to @rules. + + Generally, each type provided via `SessionValues` should have a simple rule that returns the + type so that users can directly request it in a rule, rather than needing to query + `SessionValues`. + """ + + def __getitem__(self, item: Type[_T]) -> _T: + try: + return cast(_T, super().__getitem__(item)) + except KeyError: + raise KeyError(f"Expected {item.__name__} to be provided via SessionValues.") diff --git a/src/python/pants/engine/session.py b/src/python/pants/engine/session.py deleted file mode 100644 index a572e1710db..00000000000 --- a/src/python/pants/engine/session.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from dataclasses import dataclass -from typing import Any, Mapping, Optional, Type - -from pants.util.frozendict import FrozenDict -from pants.util.meta import frozen_after_init - - -@frozen_after_init -@dataclass -class SessionValues: - """Values set for the Session, and exposed to @rules.""" - - values: FrozenDict[Type, Any] - - def __init__(self, values: Optional[Mapping[Type, Any]] = None) -> None: - self.values = FrozenDict(values or {}) diff --git a/src/python/pants/init/engine_initializer.py b/src/python/pants/init/engine_initializer.py index 76128ea94de..c4931801ea8 100644 --- a/src/python/pants/init/engine_initializer.py +++ b/src/python/pants/init/engine_initializer.py @@ -11,7 +11,6 @@ from pants.base.exiter import PANTS_SUCCEEDED_EXIT_CODE from pants.base.specs import Specs from pants.build_graph.build_configuration import BuildConfiguration -from pants.core.util_rules.pants_environment import PantsEnvironment from pants.engine import desktop, fs, process from pants.engine.console import Console from pants.engine.fs import PathGlobs, Snapshot, Workspace @@ -21,10 +20,10 @@ from pants.engine.internals.parser import Parser from pants.engine.internals.scheduler import Scheduler, SchedulerSession from pants.engine.internals.selectors import Params +from pants.engine.internals.session import SessionValues from pants.engine.platform import create_platform_rules from pants.engine.process import InteractiveRunner from pants.engine.rules import QueryRule, collect_rules, rule -from pants.engine.session import SessionValues from pants.engine.target import RegisteredTargetTypes from pants.engine.unions import UnionMembership from pants.init import specs_calculator @@ -69,13 +68,7 @@ class GraphSession: goal_map: Any # NB: Keep this in sync with the method `run_goal_rules`. - goal_param_types: ClassVar[Tuple[Type, ...]] = ( - Specs, - Console, - InteractiveRunner, - Workspace, - PantsEnvironment, - ) + goal_param_types: ClassVar[Tuple[Type, ...]] = (Specs, Console, InteractiveRunner, Workspace) def goal_consumed_subsystem_scopes(self, goal_name: str) -> Tuple[str, ...]: """Return the scopes of subsystems that could be consumed while running the given goal.""" @@ -98,7 +91,6 @@ def goal_consumed_types(self, goal_product: Type) -> Set[Type]: def run_goal_rules( self, *, - options_bootstrapper: OptionsBootstrapper, union_membership: UnionMembership, goals: Iterable[str], specs: Specs, @@ -115,7 +107,6 @@ def run_goal_rules( workspace = Workspace(self.scheduler_session) interactive_runner = InteractiveRunner(self.scheduler_session) - pants_environment = PantsEnvironment() for goal in goals: goal_product = self.goal_map[goal] @@ -128,14 +119,7 @@ def run_goal_rules( if not is_implemented: continue # NB: Keep this in sync with the property `goal_param_types`. - params = Params( - specs, - options_bootstrapper, - self.console, - workspace, - interactive_runner, - pants_environment, - ) + params = Params(specs, self.console, workspace, interactive_runner) logger.debug(f"requesting {goal_product} to satisfy execution of `{goal}` goal") try: exit_code = self.scheduler_session.run_goal_rule( diff --git a/src/python/pants/testutil/rule_runner.py b/src/python/pants/testutil/rule_runner.py index f7fb2cceb2b..0767ee30be5 100644 --- a/src/python/pants/testutil/rule_runner.py +++ b/src/python/pants/testutil/rule_runner.py @@ -27,6 +27,7 @@ from pants.base.specs_parser import SpecsParser from pants.build_graph.build_configuration import BuildConfiguration from pants.build_graph.build_file_aliases import BuildFileAliases +from pants.core.util_rules import pants_environment from pants.core.util_rules.pants_environment import PantsEnvironment from pants.engine.addresses import Address from pants.engine.console import Console @@ -35,10 +36,10 @@ from pants.engine.internals.native import Native from pants.engine.internals.scheduler import SchedulerSession from pants.engine.internals.selectors import Get, Params +from pants.engine.internals.session import SessionValues from pants.engine.process import InteractiveRunner from pants.engine.rules import QueryRule as QueryRule from pants.engine.rules import Rule -from pants.engine.session import SessionValues from pants.engine.target import Target, WrappedTarget from pants.engine.unions import UnionMembership from pants.init.engine_initializer import EngineInitializer @@ -95,6 +96,7 @@ def __init__( all_rules = ( *(rules or ()), *source_root.rules(), + *pants_environment.rules(), QueryRule(WrappedTarget, (Address,)), ) build_config_builder = BuildConfiguration.Builder() @@ -126,7 +128,9 @@ def __init__( execution_options=ExecutionOptions.from_bootstrap_options(global_options), ).new_session( build_id="buildid_for_test", - session_values=SessionValues({OptionsBootstrapper: options_bootstrapper}), + session_values=SessionValues( + {OptionsBootstrapper: options_bootstrapper, PantsEnvironment: PantsEnvironment()} + ), should_report_workunits=True, ) self.scheduler = graph_session.scheduler_session @@ -179,7 +183,6 @@ def run_goal_rule( console, Workspace(self.scheduler), InteractiveRunner(self.scheduler), - PantsEnvironment(), ), ) @@ -187,7 +190,12 @@ def run_goal_rule( return GoalRuleResult(exit_code, stdout.getvalue(), stderr.getvalue()) def set_options(self, args: Iterable[str], *, env: Optional[Mapping[str, str]] = None) -> None: - """Update the engine session with new options. + """Update the engine session with new options and/or environment variables. + + The environment variables will be used to set the `PantsEnvironment`, which is the + environment variables captured by the parent Pants process. Some rules use this to be able + to read arbitrary env vars. Any options that start with `PANTS_` will also be used to set + options. This will override any previously configured values. """ @@ -195,7 +203,9 @@ def set_options(self, args: Iterable[str], *, env: Optional[Mapping[str, str]] = self.scheduler = self.scheduler.scheduler.new_session( build_id="buildid_for_test", should_report_workunits=True, - session_values=SessionValues({OptionsBootstrapper: options_bootstrapper}), + session_values=SessionValues( + {OptionsBootstrapper: options_bootstrapper, PantsEnvironment: PantsEnvironment(env)} + ), ) def _invalidate_for(self, *relpaths): diff --git a/src/python/pants/testutil/test_base.py b/src/python/pants/testutil/test_base.py index 7f4896a29cc..a246737f427 100644 --- a/src/python/pants/testutil/test_base.py +++ b/src/python/pants/testutil/test_base.py @@ -28,6 +28,7 @@ from pants.base.specs_parser import SpecsParser from pants.build_graph.build_configuration import BuildConfiguration from pants.build_graph.build_file_aliases import BuildFileAliases +from pants.core.util_rules import pants_environment from pants.core.util_rules.pants_environment import PantsEnvironment from pants.engine.addresses import Address from pants.engine.console import Console @@ -36,9 +37,9 @@ from pants.engine.internals.native import Native from pants.engine.internals.scheduler import SchedulerSession from pants.engine.internals.selectors import Params +from pants.engine.internals.session import SessionValues from pants.engine.process import InteractiveRunner from pants.engine.rules import QueryRule -from pants.engine.session import SessionValues from pants.engine.target import Target, WrappedTarget from pants.init.engine_initializer import EngineInitializer from pants.init.util import clean_global_runtime_state @@ -112,7 +113,9 @@ def request(self, output_type: Type["TestBase._O"], inputs: Iterable[Any]) -> "T session = self.scheduler.scheduler.new_session( build_id="buildid_for_test", should_report_workunits=True, - session_values=SessionValues({OptionsBootstrapper: value}), + session_values=SessionValues( + {OptionsBootstrapper: value, PantsEnvironment: PantsEnvironment()} + ), ) result = assert_single_element(session.product_request(output_type, [Params(*inputs)])) @@ -142,7 +145,9 @@ def run_goal_rule( session = self.scheduler.scheduler.new_session( build_id="buildid_for_test", should_report_workunits=True, - session_values=SessionValues({OptionsBootstrapper: options_bootstrapper}), + session_values=SessionValues( + {OptionsBootstrapper: options_bootstrapper, PantsEnvironment: PantsEnvironment(env)} + ), ) exit_code = session.run_goal_rule( @@ -152,7 +157,6 @@ def run_goal_rule( console, Workspace(self.scheduler), InteractiveRunner(self.scheduler), - PantsEnvironment(), ), ) @@ -232,6 +236,7 @@ def alias_groups(cls): def rules(cls): return [ *source_root.rules(), + *pants_environment.rules(), QueryRule(WrappedTarget, (Address,)), ] diff --git a/tests/python/pants_test/init/test_plugin_resolver.py b/tests/python/pants_test/init/test_plugin_resolver.py index b99bb399579..4d54b24e5f9 100644 --- a/tests/python/pants_test/init/test_plugin_resolver.py +++ b/tests/python/pants_test/init/test_plugin_resolver.py @@ -16,7 +16,6 @@ from pants.backend.python.util_rules import pex from pants.backend.python.util_rules.pex import Pex, PexProcess, PexRequest, PexRequirements from pants.core.util_rules import archive, external_tool -from pants.core.util_rules.pants_environment import PantsEnvironment from pants.engine.fs import CreateDigest, Digest, FileContent, MergeDigests, Snapshot from pants.engine.process import Process, ProcessResult from pants.init.plugin_resolver import PluginResolver @@ -44,8 +43,8 @@ def rules(cls): *pex.rules(), *external_tool.rules(), *archive.rules(), - QueryRule(Pex, (PexRequest, PantsEnvironment)), - QueryRule(Process, (PexProcess, PantsEnvironment)), + QueryRule(Pex, (PexRequest,)), + QueryRule(Process, (PexProcess,)), QueryRule(ProcessResult, (Process,)), ) @@ -59,7 +58,6 @@ def _create_pex(self) -> Pex: requirements=PexRequirements(["setuptools==44.0.0", "wheel==0.34.2"]), ), create_options_bootstrapper(args=["--backend-packages=pants.backend.python"]), - PantsEnvironment(), ], )