Prep support for running Pants with Python 3.10 or 3.11.

package/README.md

@@ -42,7 +42,7 @@ In the course of development you'll probably only be interested in two invocatio
 The primary goal of the package crate as build system is to support development of the `scie-pants`
 binary with an install of Rust as the ~only requirement (CMake is currently needed as well). This
 necessitates dogfooding the same scie mechanism the final `scie-pants` binary uses in production in
-order to bootstrap a [Python tool chain](pbt.lift.json). This is used build the [`tools.pex`](
+order to bootstrap a [Python tool chain](pbt.toml). This is used to build the [`tools.pex`](
 ../tools/README.md) embedded in the final `scie-pants` for use in all the slow-path / high-logic
 steps like Pants configuration, Pants installation and self-update.
 
@@ -53,9 +53,9 @@ src/main.rs):
 1. A [`ptex` binary](https://github.com/a-scie/ptex) is built via `cargo install`. This bootstraps
    the ability to fetch further requirements.
 2. The current production pins of `ptex` and `scie-jump` are fetched and checksum-verified.
-3. A [`pbt`](pbt.lift.json) scie binary is built to facilitate running Python, Pip and Pex tools.
+3. A [`pbt`](pbt.toml) scie binary is built to facilitate running Python, Pip and Pex tools.
 4. The [tools.pex](../tools) is built.
-5. The `scie-pants` scie binary is built.
+5. The [`scie-pants`](scie-pants.toml) scie binary is built.
 6. The `scie-pants` binary is used to run Pants against the Python tools codebase and then to run a
    series of integration tests exercising the ability to install different Pants vintages and
    configure new Pants projects.

package/scie-pants.toml

@@ -28,12 +28,28 @@ release = "20240107"
 lazy = true
 version = "3.9.18"
 
+[[lift.interpreters]]
+id = "cpython310"
+provider = "PythonBuildStandalone"
+release = "20240107"
+lazy = true
+version = "3.10.13"
+
+[[lift.interpreters]]
+id = "cpython311"
+provider = "PythonBuildStandalone"
+release = "20240107"
+lazy = true
+version = "3.11.7"
+
 [[lift.interpreter_groups]]
 id = "cpython"
 selector = "{scie.bindings.configure:PYTHON}"
 members = [
     "cpython38",
     "cpython39",
+    "cpython310",
+    "cpython311",
 ]
 
 [[lift.files]]
@@ -183,6 +199,8 @@ args = [
     "{scie.env.PANTS_TOML}",
     "--github-api-bearer-token",
     "{scie.env.PANTS_BOOTSTRAP_GITHUB_API_BEARER_TOKEN}",
+    "--pants-bootstrap-urls",
+    "{scie.env.PANTS_BOOTSTRAP_URLS}",
     "{scie.bindings}",
 ]
 env.remove_re = [
@@ -209,6 +227,8 @@ args = [
     "install-pants",
     "--pants-version",
     "{scie.bindings.configure:PANTS_VERSION}",
+    "--pants-pex-url",
+    "{scie.bindings.configure:PANTS_PEX_URL}",
     "--find-links",
     "{scie.bindings.configure:FIND_LINKS}",
     "--debug",

package/src/main.rs

@@ -29,6 +29,7 @@ use crate::utils::fs::{base_name, canonicalize, copy, ensure_directory};
 
 const BINARY: &str = "scie-pants";
 
+// The version of a-scie/lift to use by default.
 const SCIENCE_TAG: &str = "v0.3.1";
 
 #[derive(Clone)]
@@ -127,7 +128,7 @@ struct Args {
         long,
         help = format!(
             "Instead of using the released {SCIENCE_TAG} science, package science from the science \
-            project repo at this directory."
+            project repo (github.com/a-scie/lift) at this directory."
         )
     )]
     science: Option<PathBuf>,

package/src/test.rs

@@ -439,7 +439,7 @@ fn test_pants_from_bad_pex_version(scie_pants_scie: &Path) {
 
     let tmpdir = create_tempdir().unwrap();
 
-    let pants_release = "2.18";
+    let pants_release = "2.19";
     let pants_toml_content = format!(
         r#"
         [GLOBAL]
@@ -458,8 +458,11 @@ fn test_pants_from_bad_pex_version(scie_pants_scie: &Path) {
     .unwrap_err();
 
     let error_text = err.to_string();
-    assert!(error_text.contains("Wasn't able to fetch the Pants PEX at"));
-    assert!(error_text.contains("Pants version format not recognized. Please add `.<patch_version>` to the end of the version. For example: `2.18` -> `2.18.0`"));
+    assert!(error_text
+        .contains("Pants version must be a full version, including patch level, got: `2.19`."));
+    assert!(error_text.contains(
+        "Please add `.<patch_version>` to the end of the version. For example: `2.18` -> `2.18.0`."
+    ));
 }
 
 fn test_use_in_repo_with_pants_script(scie_pants_scie: &Path, clone_root: &TempDir) {
@@ -1180,7 +1183,10 @@ fn test_pants_bootstrap_urls(scie_pants_scie: &Path) {
 
     assert_stderr_output(
         &mut command,
-        vec![&format!("Failed to fetch {doesnt_exist_pex_url}")],
+        vec![
+            &format!("Bad URL in PANTS_BOOTSTRAP_URLS for pants.{pants_release}-cp39-"),
+            &format!(".pex: {doesnt_exist_pex_url}"),
+        ],
         ExpectedResult::Failure,
     );
 

tools/README.md

@@ -28,7 +28,7 @@ quickly.
 One is the ability to fetch binaries check-summed in advance just in time and exactly once.
 It does this with a `"ptex"` configuration described [here](
 https://github.com/a-scie/ptex/blob/main/README.md#how-ptex-works) and employed in both the [`pbt`](
-../package/pbt.lift.json) binary and the [`scie-jump`](../package/scie-pants.lift.json) itself. This
+../package/pbt.toml) binary and the [`scie-pants`](../package/scie-pants.toml) itself. This
 allows the `scie-pants` to ship as a small binary that lazily fetches a Python distribution to run
 the tools with / install Pants with. This also gives Python tools code access to a `ptex` binary
 to use when fetching content from the internet. Although the Python code can always reach out to
@@ -39,9 +39,9 @@ The other useful feature of a scie is the ability to run pre-requisite binding c
 steps needed to support user-facing commands. These commands will only ever run once and they can
 record result information in the form of `<key>=<value>` pairs for future invocations to learn the
 one-time binding results. The `"pants"` command in the `scie-pants` binary depends on an `install`
-binding to learn the Path of the Pants venv and that in turn depends on a `configuration` binding 
-to learn the Pants version to install and the find-links repo needed to support that install. In
-turn, both of these bindings depend on Python distributions fetched by the `ptex` fetch mechanism.
+binding to learn the Path of the Pants venv and that in turn depends on a `configuration` binding to
+learn the Pants version to install. In turn, both of these bindings depend on Python distributions
+fetched by the `ptex` fetch mechanism.
 
 All such dependencies are expressed with `{scie.*}` placeholders in the values of command `"exe"`,
 `"args"` and `"env"` values in the scie lift manifest. To follow the flow of a top-level command,
@@ -67,4 +67,4 @@ python {tools.pex} <console script name> <arguments> ...
 Tools may have all of their arguments defined in the lift manifest, or they may take additional
 arguments from the user. If they take additional arguments, it's a good idea to suppress displaying
 help for the arguments defined in the lift manifest since the user can not change those and should
-probably not be concerned with their (hidden) existence.
+probably not be concerned with their (hidden) existence.

tools/src/scie_pants/BUILD

@@ -10,14 +10,18 @@ resource(
     source="BUILD",
 )
 
-resource(
-    name="pants_release_tags",
-    source="pants_release_tags.json",
+resources(
+    name="resources",
+    sources=[
+        # Add new version entries to the top of the list.
+        "pants_python_versions.json",
+        "pants_release_tags.json",
+    ],
 )
 
 python_sources(
     dependencies=[
         ":BUILD",
-        ":pants_release_tags",
+        ":resources",
     ]
 )

tools/src/scie_pants/configure_pants.py

@@ -63,6 +63,11 @@ def main() -> NoReturn:
     parser.add_argument(
         "--github-api-bearer-token", help="The GITHUB_TOKEN to use if running in CI context."
     )
+    parser.add_argument(
+        "--pants-bootstrap-urls",
+        type=str,
+        help="The path to the JSON file containing alternate URLs for downloaded artifacts.",
+    )
     parser.add_argument("base_dir", nargs=1, help="The base directory to create Pants venvs in.")
     options = parser.parse_args()
 
@@ -86,8 +91,8 @@ def main() -> NoReturn:
             pants_version=options.pants_version,
             find_links_dir=find_links_dir,
             github_api_bearer_token=options.github_api_bearer_token,
+            bootstrap_urls_path=options.pants_bootstrap_urls,
         )
-        version = resolve_info.stable_version
     else:
         if pants_config:
             if not prompt_for_pants_version(options.pants_config):
@@ -104,23 +109,22 @@ def main() -> NoReturn:
             pants_config=pants_config,
             find_links_dir=find_links_dir,
             github_api_bearer_token=options.github_api_bearer_token,
+            bootstrap_urls_path=options.pants_bootstrap_urls,
         )
         finalizers.append(configure_version)
-        version = resolve_info.stable_version
-
-    # N.B.: These values must match the lift TOML interpreter ids.
-    python = "cpython38" if version < Version("2.5") else "cpython39"
 
     for finalizer in finalizers:
         finalizer()
 
     with open(env_file, "a") as fp:
+        print(f"PANTS_VERSION={resolve_info.version}", file=fp)
+        print(f"PYTHON={resolve_info.python}", file=fp)
+        if resolve_info.pex_url:
+            print(f"PANTS_PEX_URL={resolve_info.pex_url}", file=fp)
         if resolve_info.find_links:
             print(f"FIND_LINKS={resolve_info.find_links}", file=fp)
         if newly_created_build_root:
             print(f"PANTS_BUILDROOT_OVERRIDE={newly_created_build_root}", file=fp)
-        print(f"PANTS_VERSION={version}", file=fp)
-        print(f"PYTHON={python}", file=fp)
 
     sys.exit(0)
 

tools/src/scie_pants/install_pants.py

@@ -18,13 +18,12 @@
 from packaging.version import Version
 
 from scie_pants.log import debug, fatal, info, init_logging
-from scie_pants.pants_version import PANTS_PEX_GITHUB_RELEASE_VERSION
 from scie_pants.ptex import Ptex
 
 log = logging.getLogger(__name__)
 
 
-def venv_pip_install(venv_dir: Path, *args: str, find_links: str | None) -> None:
+def venv_pip_install(venv_dir: Path, *args: str, find_links: str | None = None) -> None:
     subprocess.run(
         args=[
             str(venv_dir / "bin" / "python"),
@@ -76,19 +75,20 @@ def install_pants_from_pex(
     venv_dir: Path,
     prompt: str,
     version: Version,
+    pex_url: str,
     ptex: Ptex,
     extra_requirements: Iterable[str],
-    find_links: str | None,
     bootstrap_urls_path: str | None,
 ) -> None:
     """Installs Pants into the venv using the platform-specific pre-built PEX."""
-    uname = os.uname()
-    pex_name = f"pants.{version}-cp39-{uname.sysname.lower()}_{uname.machine.lower()}.pex"
-
-    pex_url = f"https://github.com/pantsbuild/pants/releases/download/release_{version}/{pex_name}"
+    pex_name = pex_url.rsplit("/", 1)[-1]
     if bootstrap_urls_path:
         bootstrap_urls = json.loads(Path(bootstrap_urls_path).read_text())
-        urls_info = bootstrap_urls["ptex"]
+        urls_info = bootstrap_urls.get("ptex")
+        if urls_info is None:
+            raise ValueError(
+                f"Missing 'ptex' key in PANTS_BOOTSTRAP_URLS file: {bootstrap_urls_path}"
+            )
         pex_url = urls_info.get(pex_name)
         if pex_url is None:
             raise ValueError(
@@ -105,20 +105,10 @@ def install_pants_from_pex(
         try:
             ptex.fetch_to_fp(pex_url, pants_pex.file)
         except subprocess.CalledProcessError as e:
-            #  if there's only one dot in version, specifically suggest adding the `.patch`)
-            suggestion = (
-                "Pants version format not recognized. Please add `.<patch_version>` to the end of the version. For example: `2.18` -> `2.18.0`.\n\n"
-                if version.base_version.count(".") < 2
-                else ""
-            )
             fatal(
-                f"Wasn't able to fetch the Pants PEX at {pex_url}.\n\n{suggestion}"
-                "Check to see if the URL is reachable (i.e. GitHub isn't down) and if"
-                f" {pex_name} asset exists within the release."
-                " If the asset doesn't exist it may be that this platform isn't yet supported."
-                " If that's the case, please reach out on Slack: https://www.pantsbuild.org/docs/getting-help#slack"
-                " or file an issue on GitHub: https://github.com/pantsbuild/pants/issues/new/choose.\n\n"
-                f"Exception:\n\n{e}"
+                f"Wasn't able to fetch the Pants PEX at {pex_url}.\n"
+                "Check to see if the URL is reachable.\n\n"
+                f"Exception:\n{e}"
             )
         try:
             pants_venv_result = subprocess.run(
@@ -153,9 +143,7 @@ def install_pants_from_pex(
                 print("\n-----", file=fp)
 
     if extra_requirements:
-        venv_pip_install(
-            venv_dir, "--progress-bar", "off", *extra_requirements, find_links=find_links
-        )
+        venv_pip_install(venv_dir, "--progress-bar", "off", *extra_requirements)
 
 
 def chmod_plus_x(path: str) -> None:
@@ -166,12 +154,13 @@ def main() -> NoReturn:
     parser = ArgumentParser()
     get_ptex = Ptex.add_options(parser)
     parser.add_argument(
-        "--pants-version", type=Version, required=True, help="The Pants version to install"
+        "--pants-version", type=Version, required=True, help="The Pants version to install."
     )
+    parser.add_argument("--pants-pex-url", type=str, help="The pants pex release asset url.")
     parser.add_argument(
         "--find-links",
         type=str,
-        help="The find links repo pointing to Pants pre-built wheels for the given Pants version",
+        help="The find links repo pointing to Pants pre-built wheels for the given Pants 1.x version.",
     )
     parser.add_argument(
         "--pants-bootstrap-urls",
@@ -213,14 +202,14 @@ def main() -> NoReturn:
     info(
         f"Installing {' '.join(pants_requirements + extra_requirements)} into a virtual environment at {venv_dir}"
     )
-    if version >= PANTS_PEX_GITHUB_RELEASE_VERSION:
+    if options.pants_pex_url:
         install_pants_from_pex(
             venv_dir=venv_dir,
             prompt=prompt,
             version=version,
+            pex_url=options.pants_pex_url,
             ptex=ptex,
             extra_requirements=extra_requirements,
-            find_links=options.find_links,
             bootstrap_urls_path=options.pants_bootstrap_urls,
         )
     else:
@@ -231,7 +220,7 @@ def main() -> NoReturn:
             find_links=options.find_links,
         )
 
-    info(f"New virtual environment successfully created at {venv_dir}.")
+    info(f"New virtual environment successfully created at {venv_dir}")
 
     pants_server_exe = str(venv_dir / "bin" / "pants")
     # Added in https://github.com/pantsbuild/pants/commit/558d843549204bbe49c351d00cdf23402da262c1

tools/src/scie_pants/pants_python_versions.json

@@ -0,0 +1,4 @@
+[
+  {"pants": "2.5.0.dev0", "python": "cp39"},
+  {"pants": "2.0.0.dev0", "python": "cp38"}
+]