Merge branch 'master' of github.com:pantsbuild/pants into dogfood-cac…

…hing

src/python/pants/VERSION

@@ -1 +1 @@
-2.3.0.dev0
+2.3.0.dev1

src/python/pants/base/specs.py

@@ -1,6 +1,8 @@
 # Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+from __future__ import annotations
+
 import itertools
 import os
 from abc import ABC, ABCMeta, abstractmethod
@@ -306,3 +308,7 @@ class Specs:
     def provided(self) -> bool:
         """Did the user provide specs?"""
         return bool(self.address_specs) or bool(self.filesystem_specs)
+
+    @classmethod
+    def empty(cls) -> Specs:
+        return Specs(AddressSpecs([], filter_by_global_options=True), FilesystemSpecs([]))

src/python/pants/bin/local_pants_runner.py

@@ -46,7 +46,6 @@
 class LocalPantsRunner:
     """Handles a single pants invocation running in the process-local context.
 
-    build_root: The build root for this run.
     options: The parsed options for this run.
     build_config: The parsed build configuration for this run.
     run_tracker: A tracker for metrics for the run.
@@ -55,8 +54,8 @@ class LocalPantsRunner:
     profile_path: The profile path - if any (from from the `PANTS_PROFILE` env var).
     """
 
-    build_root: str
     options: Options
+    options_bootstrapper: OptionsBootstrapper
     build_config: BuildConfiguration
     run_tracker: RunTracker
     specs: Specs
@@ -125,7 +124,6 @@ def create(
         :param options_bootstrapper: The OptionsBootstrapper instance to reuse.
         :param scheduler: If being called from the daemon, a warmed scheduler to use.
         """
-        build_root = get_buildroot()
         global_bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope()
 
         build_config = BuildConfigInitializer.get(options_bootstrapper)
@@ -165,15 +163,15 @@ def create(
         specs = calculate_specs(
             options_bootstrapper=options_bootstrapper,
             options=options,
-            build_root=build_root,
+            build_root=get_buildroot(),
             session=graph_session.scheduler_session,
         )
 
         profile_path = env.get("PANTS_PROFILE")
 
         return cls(
-            build_root=build_root,
             options=options,
+            options_bootstrapper=options_bootstrapper,
             build_config=build_config,
             run_tracker=run_tracker,
             specs=specs,
@@ -252,6 +250,8 @@ def run(self, start_time: float) -> ExitCode:
             streaming_reporter = StreamingWorkunitHandler(
                 self.graph_session.scheduler_session,
                 run_tracker=self.run_tracker,
+                specs=self.specs,
+                options_bootstrapper=self.options_bootstrapper,
                 callbacks=self._get_workunits_callbacks(),
                 report_interval_seconds=global_options.streaming_workunits_report_interval,
             )

src/python/pants/engine/fs.py

@@ -35,8 +35,8 @@
 class Paths:
     """A Paths object is a collection of sorted file paths and dir paths.
 
-    Paths is like a Snapshot, but has a performance optimization that it does digest the files or
-    save them to the LMDB store.
+    Paths is like a Snapshot, but has the performance optimization that it does not digest the files
+    or save them to the LMDB store.
     """
 
     files: Tuple[str, ...]

src/python/pants/engine/internals/engine_test.py

@@ -10,6 +10,10 @@
 
 import pytest
 
+from pants.backend.python.target_types import PythonLibrary
+from pants.base.build_environment import get_buildroot
+from pants.base.specs import Specs
+from pants.base.specs_parser import SpecsParser
 from pants.engine.engine_aware import EngineAwareReturnType
 from pants.engine.fs import (
     EMPTY_FILE_DIGEST,
@@ -302,6 +306,8 @@ def _fixture_for_rules(
             callbacks=[tracker.add],
             report_interval_seconds=0.01,
             max_workunit_verbosity=max_workunit_verbosity,
+            specs=Specs.empty(),
+            options_bootstrapper=create_options_bootstrapper([]),
         )
         return (scheduler, tracker, handler)
 
@@ -674,6 +680,8 @@ def test_more_complicated_engine_aware(rule_runner: RuleRunner, run_tracker: Run
         callbacks=[tracker.add],
         report_interval_seconds=0.01,
         max_workunit_verbosity=LogLevel.TRACE,
+        specs=Specs.empty(),
+        options_bootstrapper=create_options_bootstrapper([]),
     )
     with handler.session():
         input_1 = CreateDigest(
@@ -733,6 +741,8 @@ def test_process_digests_on_streaming_workunits(
         callbacks=[tracker.add],
         report_interval_seconds=0.01,
         max_workunit_verbosity=LogLevel.INFO,
+        specs=Specs.empty(),
+        options_bootstrapper=create_options_bootstrapper([]),
     )
 
     stdout_process = Process(
@@ -763,6 +773,8 @@ def test_process_digests_on_streaming_workunits(
         callbacks=[tracker.add],
         report_interval_seconds=0.01,
         max_workunit_verbosity=LogLevel.INFO,
+        specs=Specs.empty(),
+        options_bootstrapper=create_options_bootstrapper([]),
     )
 
     stderr_process = Process(
@@ -822,6 +834,65 @@ def callback(**kwargs) -> None:
         callbacks=[callback],
         report_interval_seconds=0.01,
         max_workunit_verbosity=LogLevel.INFO,
+        specs=Specs.empty(),
+        options_bootstrapper=create_options_bootstrapper([]),
+    )
+
+    stdout_process = Process(
+        argv=("/bin/bash", "-c", "/bin/echo 'stdout output'"), description="Stdout process"
+    )
+
+    with handler.session():
+        rule_runner.request(ProcessResult, [stdout_process])
+
+
+def test_streaming_workunits_expanded_specs(run_tracker: RunTracker) -> None:
+
+    rule_runner = RuleRunner(
+        target_types=[PythonLibrary],
+        rules=[
+            QueryRule(ProcessResult, (Process,)),
+        ],
+    )
+
+    rule_runner.set_options(["--backend-packages=pants.backend.python"])
+
+    rule_runner.create_file("src/python/somefiles/BUILD", "python_library()")
+    rule_runner.create_file("src/python/somefiles/a.py", "print('')")
+    rule_runner.create_file("src/python/somefiles/b.py", "print('')")
+
+    rule_runner.create_file("src/python/others/BUILD", "python_library()")
+    rule_runner.create_file("src/python/others/a.py", "print('')")
+    rule_runner.create_file("src/python/others/b.py", "print('')")
+
+    specs = SpecsParser(get_buildroot()).parse_specs(
+        ["src/python/somefiles::", "src/python/others/b.py"]
+    )
+
+    def callback(**kwargs) -> None:
+        context = kwargs["context"]
+        assert isinstance(context, StreamingWorkunitContext)
+
+        expanded = context.get_expanded_specs()
+        files = expanded.files
+
+        assert len(files.keys()) == 2
+        assert files["src/python/others/b.py"] == ["src/python/others/b.py"]
+        assert set(files["src/python/somefiles"]) == {
+            "src/python/somefiles/a.py",
+            "src/python/somefiles/b.py",
+        }
+
+    handler = StreamingWorkunitHandler(
+        scheduler=rule_runner.scheduler,
+        run_tracker=run_tracker,
+        callbacks=[callback],
+        report_interval_seconds=0.01,
+        max_workunit_verbosity=LogLevel.INFO,
+        specs=specs,
+        options_bootstrapper=create_options_bootstrapper(
+            ["--backend-packages=pants.backend.python"]
+        ),
     )
 
     stdout_process = Process(

src/python/pants/engine/internals/scheduler.py

@@ -541,7 +541,7 @@ def _raise_on_error(self, throws: List[Throw]) -> NoReturn:
     def run_goal_rule(
         self,
         product: Type,
-        subject: Union[Any, Params],
+        subject: Params,
         poll: bool = False,
         poll_delay: Optional[float] = None,
     ) -> int:

src/python/pants/engine/streaming_workunit_handler.py

@@ -1,25 +1,40 @@
 # Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+import logging
 import threading
 from contextlib import contextmanager
 from dataclasses import dataclass
-from typing import Any, Callable, Iterable, Iterator, Optional, Sequence, Tuple
+from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Sequence, Tuple
 
 from typing_extensions import Protocol
 
+from pants.base.specs import Specs
+from pants.engine.addresses import Addresses
 from pants.engine.fs import Digest, DigestContents, Snapshot
 from pants.engine.internals.scheduler import SchedulerSession, Workunit
+from pants.engine.internals.selectors import Params
 from pants.engine.rules import Get, MultiGet, QueryRule, collect_rules, rule
+from pants.engine.target import Targets
 from pants.engine.unions import UnionMembership, union
 from pants.goal.run_tracker import RunTracker
+from pants.option.options_bootstrapper import OptionsBootstrapper
 from pants.util.logging import LogLevel
 
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class ExpandedSpecs:
+    files: Dict[str, List[str]]
+
 
 @dataclass(frozen=True)
 class StreamingWorkunitContext:
     _scheduler: SchedulerSession
     _run_tracker: RunTracker
+    _specs: Specs
+    _options_bootstrapper: OptionsBootstrapper
 
     @property
     def run_tracker(self) -> RunTracker:
@@ -51,6 +66,25 @@ def get_observation_histograms(self):
         """
         return self._scheduler.get_observation_histograms()
 
+    def get_expanded_specs(self) -> ExpandedSpecs:
+        """Return a dict containing the canonicalized addresses of the specs for this run, and what
+        files they expand to."""
+
+        (unexpanded_addresses,) = self._scheduler.product_request(
+            Addresses, [Params(self._specs, self._options_bootstrapper)]
+        )
+
+        expanded_targets = self._scheduler.product_request(
+            Targets, [Params(Addresses([addr])) for addr in unexpanded_addresses]
+        )
+        files = {}
+        for addr, targets in zip(unexpanded_addresses, expanded_targets):
+            files[addr.spec] = [
+                tgt.address.filename if tgt.address.is_file_target else str(tgt.address)
+                for tgt in targets
+            ]
+        return ExpandedSpecs(files=files)
+
 
 class WorkunitsCallback(Protocol):
     def __call__(
@@ -98,6 +132,8 @@ def __init__(
         scheduler: SchedulerSession,
         run_tracker: RunTracker,
         callbacks: Iterable[WorkunitsCallback],
+        options_bootstrapper: OptionsBootstrapper,
+        specs: Specs,
         report_interval_seconds: float,
         max_workunit_verbosity: LogLevel = LogLevel.TRACE,
     ):
@@ -106,7 +142,10 @@ def __init__(
         self.callbacks = callbacks
         self._thread_runner: Optional[_InnerHandler] = None
         self._context = StreamingWorkunitContext(
-            _scheduler=self.scheduler, _run_tracker=run_tracker
+            _scheduler=self.scheduler,
+            _run_tracker=run_tracker,
+            _specs=specs,
+            _options_bootstrapper=options_bootstrapper,
         )
         # TODO(10092) The max verbosity should be a per-client setting, rather than a global setting.
         self.max_workunit_verbosity = max_workunit_verbosity
@@ -192,4 +231,9 @@ async def construct_workunits_callback_factories(
 
 
 def rules():
-    return [QueryRule(WorkunitsCallbackFactories, (UnionMembership,)), *collect_rules()]
+    return [
+        QueryRule(WorkunitsCallbackFactories, (UnionMembership,)),
+        QueryRule(Targets, (Addresses,)),
+        QueryRule(Addresses, (Specs, OptionsBootstrapper)),
+        *collect_rules(),
+    ]

src/python/pants/init/specs_calculator.py

@@ -2,9 +2,9 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 import logging
-from typing import Optional, cast
+from typing import cast
 
-from pants.base.build_environment import get_buildroot, get_git
+from pants.base.build_environment import get_git
 from pants.base.specs import AddressLiteralSpec, AddressSpecs, FilesystemSpecs, Specs
 from pants.base.specs_parser import SpecsParser
 from pants.engine.addresses import AddressInput
@@ -27,10 +27,9 @@ def calculate_specs(
     options: Options,
     session: SchedulerSession,
     *,
-    build_root: Optional[str] = None,
+    build_root: str,
 ) -> Specs:
     """Determine the specs for a given Pants run."""
-    build_root = build_root or get_buildroot()
     specs = SpecsParser(build_root).parse_specs(options.specs)
     changed_options = ChangedOptions.from_options(options.for_scope("changed"))
 

src/python/pants/notes/2.3.x.md

@@ -1,3 +1,39 @@
+# 2.3.x Stable Releases
+
+See https://www.pantsbuild.org/v2.3/docs/release-notes-2-3 for an overview of the changes in this release series.
+
+## 2.3.0.dev1 (Jan 21, 2021)
+
+### New Features
+
+* Support Pex CLI `--unzip` and `--include-tools`. ([#11483](https://github.com/pantsbuild/pants/pull/11483))
+
+### User API Changes
+
+* Remove deprecated `--pants-bootstrapdir` and `--pants-configdir` options ([#11491](https://github.com/pantsbuild/pants/pull/11491))
+
+### Plugin API Changes
+
+* Remove `RunInfo` class ([#11456](https://github.com/pantsbuild/pants/pull/11456))
+
+### Bug fixes
+
+* Fix use of remote store when in eager cache mode ([#11468](https://github.com/pantsbuild/pants/pull/11468))
+
+* Workaround Rust 1.49 breaking macOS processes with a materialized argv0 ([#11452](https://github.com/pantsbuild/pants/pull/11452))
+
+### Performance
+
+* Use speculation for remote cache reads ([#11429](https://github.com/pantsbuild/pants/pull/11429))
+
+### Documentation
+
+* Change logging level for Scrubbed PYTHONPATH to be debug ([#11480](https://github.com/pantsbuild/pants/pull/11480))
+
+* Add failure logging to the StoreGCService. ([#11460](https://github.com/pantsbuild/pants/pull/11460))
+
+* Document address storage in AsyncFieldMixin. ([#11448](https://github.com/pantsbuild/pants/pull/11448))
+
 ## 2.3.0.dev0 (Jan 10, 2021)
 
 ### User API Changes

src/python/pants/option/global_options.py

@@ -288,24 +288,6 @@ def register_bootstrap_options(cls, register):
             "Useful when printing help messages.",
         )
 
-        register(
-            "--pants-bootstrapdir",
-            advanced=True,
-            metavar="<dir>",
-            default=None,
-            removal_version="2.3.0.dev1",
-            removal_hint="Unused.",
-            help="Unused.",
-        )
-        register(
-            "--pants-configdir",
-            advanced=True,
-            metavar="<dir>",
-            default=None,
-            removal_version="2.3.0.dev1",
-            removal_hint="Unused.",
-            help="Unused.",
-        )
         register(
             "--pants-workdir",
             advanced=True,