From acae18129e5430c5f7af7ae5ce870081924ab89a Mon Sep 17 00:00:00 2001 From: John Sirois Date: Thu, 25 May 2023 19:42:55 -0700 Subject: [PATCH] Build `scie-pants` using `science`. Instead of using raw `scie-jump` and a hand-crafted JSON lift manifest, use `science` and a TOML lift mabifest to build `scie-pants`. The `science` tool and its more friendly interface are the present and future of end-user scie construction. We convert here and gain a few things: + A manifest format we can comment. + Abstraction over platforms and PBS details. + Support for embedded build provenanace information for more comprehensive black box debugability. + A clean path to publishing both "thin" and "fat" binaries and solving #154. --- package/pbt.lift.json | 115 ----------- package/pbt.toml | 57 ++++++ package/scie-pants.lift.json | 249 ------------------------ package/scie-pants.toml | 229 ++++++++++++++++++++++ package/src/main.rs | 74 +++---- package/src/scie_pants.rs | 76 +++++--- package/src/test.rs | 8 +- package/src/tools_pex.rs | 16 +- package/src/utils/build.rs | 90 +++------ package/src/utils/fs.rs | 7 + tools/src/scie_pants/configure_pants.py | 3 +- 11 files changed, 411 insertions(+), 513 deletions(-) delete mode 100644 package/pbt.lift.json create mode 100644 package/pbt.toml delete mode 100644 package/scie-pants.lift.json create mode 100644 package/scie-pants.toml diff --git a/package/pbt.lift.json b/package/pbt.lift.json deleted file mode 100644 index 3114295..0000000 --- a/package/pbt.lift.json +++ /dev/null @@ -1,115 +0,0 @@ -{ - "scie": { - "lift": { - "name": "pbt", - "description": "Python Build Tool: A BusyBox that provides `python`, `pip`, `pex`, `pex3` and `pex-tools`.", - "files": [ - { - "name": "ptex", - "executable": true - }, - { - "name": "cpython-3.8.15+20221106-aarch64-unknown-linux-gnu-install_only.tar.gz", - "key": "python-linux-aarch64", - "size": 25080793, - "hash": "886ab33ced13c84bf59ce8ff79eba6448365bfcafea1bf415bd1d75e21b690aa", - "type": "tar.gz", - "source": "ptex-fetch" - }, - { - "name": "cpython-3.8.15+20221106-x86_64-unknown-linux-gnu-install_only.tar.gz", - "key": "python-linux-x86_64", - "size": 26922922, - "hash": "e47edfb2ceaf43fc699e20c179ec428b6f3e497cf8e2dcd8e9c936d4b96b1e56", - "type": "tar.gz", - "source": "ptex-fetch" - }, - { - "name": "cpython-3.8.15+20221106-aarch64-apple-darwin-install_only.tar.gz", - "key": "python-macos-aarch64", - "size": 17603392, - "hash": "1e0a92d1a4f5e6d4a99f86b1cbf9773d703fe7fd032590f3e9c285c7a5eeb00a", - "type": "tar.gz", - "source": "ptex-fetch" - }, - { - "name": "cpython-3.8.15+20221106-x86_64-apple-darwin-install_only.tar.gz", - "key": "python-macos-x86_64", - "size": 17985182, - "hash": "70b57f28c2b5e1e3dd89f0d30edd5bc414e8b20195766cf328e1b26bed7890e1", - "type": "tar.gz", - "source": "ptex-fetch" - }, - { - "name": "cpython-3.8.15+20221106-x86_64-pc-windows-msvc-static-install_only.tar.gz", - "key": "python-windows-x86_64", - "size": 42772349, - "hash": "f9c799b74f258fb520f7afae548f87b65c535ee14d885a64cc935a291436ae50", - "type": "tar.gz", - "source": "ptex-fetch" - }, - { - "name": "pex", - "size": 4068845, - "hash": "a0633d2d84a20908c4ada92e01d4cd3e4ae76497a83f2bf0f2e6f0d5ca0098d5", - "type": "blob", - "source": "ptex-fetch" - } - ], - "boot": { - "commands": { - "pex": { - "exe": "{python-{scie.platform}}/python/bin/python3.8", - "args": [ - "{pex}" - ] - }, - "pex3": { - "env": { - "PEX_SCRIPT": "pex3" - }, - "exe": "{python-{scie.platform}}/python/bin/python3.8", - "args": [ - "{pex}" - ] - }, - "pex-tools": { - "env": { - "PEX_SCRIPT": "pex-tools" - }, - "exe": "{python-{scie.platform}}/python/bin/python3.8", - "args": [ - "{pex}" - ] - }, - "pip": { - "exe": "{python-{scie.platform}}/python/bin/python3.8", - "args": [ - "-m", - "pip" - ] - }, - "python": { - "exe": "{python-{scie.platform}}/python/bin/python3.8" - } - }, - "bindings": { - "ptex-fetch": { - "exe": "{ptex}", - "args": [ - "{scie.lift}" - ] - } - } - } - } - }, - "ptex": { - "cpython-3.8.15+20221106-aarch64-unknown-linux-gnu-install_only.tar.gz": "https://github.com/indygreg/python-build-standalone/releases/download/20221106/cpython-3.8.15+20221106-aarch64-unknown-linux-gnu-install_only.tar.gz", - "cpython-3.8.15+20221106-x86_64-unknown-linux-gnu-install_only.tar.gz": "https://github.com/indygreg/python-build-standalone/releases/download/20221106/cpython-3.8.15+20221106-x86_64-unknown-linux-gnu-install_only.tar.gz", - "cpython-3.8.15+20221106-aarch64-apple-darwin-install_only.tar.gz": "https://github.com/indygreg/python-build-standalone/releases/download/20221106/cpython-3.8.15+20221106-aarch64-apple-darwin-install_only.tar.gz", - "cpython-3.8.15+20221106-x86_64-apple-darwin-install_only.tar.gz": "https://github.com/indygreg/python-build-standalone/releases/download/20221106/cpython-3.8.15+20221106-x86_64-apple-darwin-install_only.tar.gz", - "cpython-3.8.15+20221106-x86_64-pc-windows-msvc-static-install_only.tar.gz": "https://github.com/indygreg/python-build-standalone/releases/download/20221106/cpython-3.8.15+20221106-x86_64-pc-windows-msvc-static-install_only.tar.gz", - "pex": "https://github.com/pantsbuild/pex/releases/download/v2.1.118/pex" - } -} diff --git a/package/pbt.toml b/package/pbt.toml new file mode 100644 index 0000000..759bdd3 --- /dev/null +++ b/package/pbt.toml @@ -0,0 +1,57 @@ +[lift] +name = "pbt" +description = """\ +Python Build Tool: A BusyBox that provides `python`, `pip`, `pex`, `pex3` and `pex-tools`.\ +""" + +[[lift.interpreters]] +id = "cpython" +provider = "PythonBuildStandalone" +release = "20230507" +lazy = true +version = "3.8.16" + +[[lift.files]] +name = "pex" +type = "blob" +digest = { size = 4098329, fingerprint = "faad51a6a108fba9d40b2a10e82a2646fccbaf8c3d9be47818f4bffae02d94b8" } +source = { url = "https://github.com/pantsbuild/pex/releases/download/v2.1.137/pex", lazy = true } + +[[lift.commands]] +name = "pex" +exe = "#{cpython:python}" +args = [ + "{pex}" +] + +[[lift.commands]] +name = "pex3" +exe = "#{cpython:python}" +args = [ + "{pex}" +] + +[lift.commands.env.replace] +PEX_SCRIPT = "pex3" + +[[lift.commands]] +name = "pex-tools" +exe = "#{cpython:python}" +args = [ + "{pex}" +] + +[lift.commands.env.replace] +PEX_SCRIPT = "pex-tools" + +[[lift.commands]] +name = "pip" +exe = "#{cpython:python}" +args = [ + "-m", + "pip" +] + +[[lift.commands]] +name = "python" +exe = "#{cpython:python}" diff --git a/package/scie-pants.lift.json b/package/scie-pants.lift.json deleted file mode 100644 index f1bd9ea..0000000 --- a/package/scie-pants.lift.json +++ /dev/null @@ -1,249 +0,0 @@ -{ - "scie": { - "lift": { - "name": "scie-pants", - "description": "Isolates your Pants from the elements.", - "load_dotenv": true, - "files": [ - { - "name": "scie-pants.bin", - "key": "scie-pants", - "executable": true - }, - { - "name": "ptex", - "executable": true - }, - { - "name": "tools.pex" - }, - { - "name": "cpython-3.8.15+20221106-aarch64-unknown-linux-gnu-install_only.tar.gz", - "key": "python3.8-linux-aarch64", - "size": 25080793, - "hash": "886ab33ced13c84bf59ce8ff79eba6448365bfcafea1bf415bd1d75e21b690aa", - "type": "tar.gz", - "source": "ptex-fetch" - }, - { - "name": "cpython-3.8.15+20221106-x86_64-unknown-linux-gnu-install_only.tar.gz", - "key": "python3.8-linux-x86_64", - "size": 26922922, - "hash": "e47edfb2ceaf43fc699e20c179ec428b6f3e497cf8e2dcd8e9c936d4b96b1e56", - "type": "tar.gz", - "source": "ptex-fetch" - }, - { - "name": "cpython-3.8.15+20221106-aarch64-apple-darwin-install_only.tar.gz", - "key": "python3.8-macos-aarch64", - "size": 17603392, - "hash": "1e0a92d1a4f5e6d4a99f86b1cbf9773d703fe7fd032590f3e9c285c7a5eeb00a", - "type": "tar.gz", - "source": "ptex-fetch" - }, - { - "name": "cpython-3.8.15+20221106-x86_64-apple-darwin-install_only.tar.gz", - "key": "python3.8-macos-x86_64", - "size": 17985182, - "hash": "70b57f28c2b5e1e3dd89f0d30edd5bc414e8b20195766cf328e1b26bed7890e1", - "type": "tar.gz", - "source": "ptex-fetch" - }, - { - "name": "cpython-3.9.15+20221106-aarch64-unknown-linux-gnu-install_only.tar.gz", - "key": "python3.9-linux-aarch64", - "size": 24524876, - "hash": "52a8c0a67fb919f80962d992da1bddb511cdf92faf382701ce7673e10a8ff98f", - "type": "tar.gz", - "source": "ptex-fetch" - }, - { - "name": "cpython-3.9.15+20221106-x86_64-unknown-linux-gnu-install_only.tar.gz", - "key": "python3.9-linux-x86_64", - "size": 26765574, - "hash": "cdc3a4cfddcd63b6cebdd75b14970e02d8ef0ac5be4d350e57ab5df56c19e85e", - "type": "tar.gz", - "source": "ptex-fetch" - }, - { - "name": "cpython-3.9.15+20221106-aarch64-apple-darwin-install_only.tar.gz", - "key": "python3.9-macos-aarch64", - "size": 16748283, - "hash": "64dc7e1013481c9864152c3dd806c41144c79d5e9cd3140e185c6a5060bdc9ab", - "type": "tar.gz", - "source": "ptex-fetch" - }, - { - "name": "cpython-3.9.15+20221106-x86_64-apple-darwin-install_only.tar.gz", - "key": "python3.9-macos-x86_64", - "size": 17148635, - "hash": "f2bcade6fc976c472f18f2b3204d67202d43ae55cf6f9e670f95e488f780da08", - "type": "tar.gz", - "source": "ptex-fetch" - } - ], - "boot": { - "commands": { - "": { - "description": "Detects the current Pants installation and launches it.", - "exe": "{scie-pants}" - }, - "pants": { - "description": "Runs a hermetic Pants installation.", - "env": { - "=PANTS_VERSION": "{scie.bindings.configure:PANTS_VERSION}", - "PANTS_BUILDROOT_OVERRIDE": "{scie.bindings.configure:PANTS_BUILDROOT_OVERRIDE}", - "=_PANTS_SERVER_EXE": "{scie.bindings.install:PANTS_SERVER_EXE}" - }, - "exe": "{scie.bindings.install:PANTS_CLIENT_EXE}", - "args": [ - "{scie.bindings.configure:PANTS_SHA_FIND_LINKS}" - ] - }, - "pants-debug": { - "description": "Runs a hermetic Pants installation with a debug server for debugging Pants code.", - "env": { - "=PANTS_VERSION": "{scie.bindings.configure:PANTS_VERSION}", - "PANTS_BUILDROOT_OVERRIDE": "{scie.bindings.configure:PANTS_BUILDROOT_OVERRIDE}" - }, - "exe": "{scie.bindings.install:VIRTUAL_ENV}/bin/python", - "args": [ - "-c", - "__import__(\"sys\").path.pop(0);print(\"Launching debugpy server at '127.0.0.1:5678' and waiting for client connection.\", file=__import__(\"sys\").stderr);__import__(\"debugpy.server.cli\").server.cli.main()", - "--listen", - "127.0.0.1:5678", - "--wait-for-client", - "{scie.bindings.install:VIRTUAL_ENV}/bin/pants", - "{scie.bindings.configure:PANTS_SHA_FIND_LINKS}" - ] - }, - "bootstrap-tools": { - "description": "Introspection tools for the Pants bootstrap process.", - "env": { - "PEX_.*": null, - "=PEX_ROOT": "{scie.bindings}/pex_root", - "=PEX_PYTHON_PATH": "{scie.files.{scie.bindings.configure:PYTHON}-{scie.platform}}/python/bin/{scie.bindings.configure:PYTHON}" - }, - "exe": "{scie.files.{scie.bindings.configure:PYTHON}-{scie.platform}}/python/bin/{scie.bindings.configure:PYTHON}", - "args": [ - "{tools.pex}", - "bootstrap-tools", - "--python-distribution-hash", - "{scie.files:hash.{scie.bindings.configure:PYTHON}-{scie.platform}}", - "--pants-version", - "{scie.bindings.configure:PANTS_VERSION}" - ] - }, - "update": { - "description": "Update scie-pants.", - "env": { - "PEX_.*": null, - "=PEX_ROOT": "{scie.bindings}/pex_root", - "=PEX_PYTHON_PATH": "{scie.files.python3.9-{scie.platform}}/python/bin/python3.9" - }, - "exe": "{scie.files.python3.9-{scie.platform}}/python/bin/python3.9", - "args": [ - "{tools.pex}", - "update-scie-pants", - "--ptex-path", - "{ptex}", - "--platform", - "{scie.platform}", - "--base-dir", - "{scie.bindings}", - "--scie", - "{scie}", - "--current-version", - "{scie.bindings.scie-pants-info:VERSION}", - "--github-api-bearer-token", - "{scie.env.PANTS_BOOTSTRAP_GITHUB_API_BEARER_TOKEN}" - ] - } - }, - "bindings": { - "ptex-fetch": { - "description": "Fetches hermetic CPython distributions for Pants internal use.", - "exe": "{ptex}", - "args": [ - "{scie.env.PANTS_BOOTSTRAP_URLS={scie.lift}}" - ] - }, - "scie-pants-info": { - "description": "Records information about the current scie-pants binary.", - "env": { - "PEX_.*": null, - "=PEX_ROOT": "{scie.bindings}/pex_root", - "=PEX_PYTHON_PATH": "{scie.files.python3.9-{scie.platform}}/python/bin/python3.9" - }, - "exe": "{scie.files.python3.9-{scie.platform}}/python/bin/python3.9", - "args": [ - "{tools.pex}", - "record-scie-pants-info", - "--base-dir", - "{scie.bindings}", - "--scie", - "{scie}" - ] - }, - "configure": { - "description": "Prompts the user for missing Pants configuration if needed.", - "env": { - "PEX_.*": null, - "=PEX_ROOT": "{scie.bindings}/pex_root", - "=PEX_PYTHON_PATH": "{scie.files.python3.9-{scie.platform}}/python/bin/python3.9", - "PANTS_VERSION_PROMPT_SALT": "{scie.env.PANTS_VERSION_PROMPT_SALT}" - }, - "exe": "{scie.files.python3.9-{scie.platform}}/python/bin/python3.9", - "args": [ - "{tools.pex}", - "configure-pants", - "--ptex-path", - "{ptex}", - "--pants-version", - "{scie.env.PANTS_VERSION}", - "--pants-sha", - "{scie.env.PANTS_SHA}", - "--pants-config", - "{scie.env.PANTS_TOML}", - "--github-api-bearer-token", - "{scie.env.PANTS_BOOTSTRAP_GITHUB_API_BEARER_TOKEN}", - "{scie.bindings}" - ] - }, - "install": { - "description": "Installs a hermetic Pants environment from PyPI or binaries.pantsbuild.org with optional debug support.", - "env": { - "PEX_.*": null, - "=PEX_ROOT": "{scie.bindings}/pex_root", - "=PEX_PYTHON_PATH": "{scie.files.{scie.bindings.configure:PYTHON}-{scie.platform}}/python/bin/{scie.bindings.configure:PYTHON}" - }, - "exe": "{scie.files.{scie.bindings.configure:PYTHON}-{scie.platform}}/python/bin/{scie.bindings.configure:PYTHON}", - "args": [ - "{tools.pex}", - "install-pants", - "--pants-version", - "{scie.bindings.configure:PANTS_VERSION}", - "--find-links", - "{scie.bindings.configure:FIND_LINKS}", - "--debug", - "{scie.env.PANTS_DEBUG}", - "--debugpy-requirement", - "{scie.env.PANTS_DEBUGPY_VERSION}", - "{scie.bindings}" - ] - } - } - } - } - }, - "ptex": { - "cpython-3.8.15+20221106-aarch64-unknown-linux-gnu-install_only.tar.gz": "https://github.com/indygreg/python-build-standalone/releases/download/20221106/cpython-3.8.15+20221106-aarch64-unknown-linux-gnu-install_only.tar.gz", - "cpython-3.8.15+20221106-x86_64-unknown-linux-gnu-install_only.tar.gz": "https://github.com/indygreg/python-build-standalone/releases/download/20221106/cpython-3.8.15+20221106-x86_64-unknown-linux-gnu-install_only.tar.gz", - "cpython-3.8.15+20221106-aarch64-apple-darwin-install_only.tar.gz": "https://github.com/indygreg/python-build-standalone/releases/download/20221106/cpython-3.8.15+20221106-aarch64-apple-darwin-install_only.tar.gz", - "cpython-3.8.15+20221106-x86_64-apple-darwin-install_only.tar.gz": "https://github.com/indygreg/python-build-standalone/releases/download/20221106/cpython-3.8.15+20221106-x86_64-apple-darwin-install_only.tar.gz", - "cpython-3.9.15+20221106-aarch64-unknown-linux-gnu-install_only.tar.gz": "https://github.com/indygreg/python-build-standalone/releases/download/20221106/cpython-3.9.15+20221106-aarch64-unknown-linux-gnu-install_only.tar.gz", - "cpython-3.9.15+20221106-x86_64-unknown-linux-gnu-install_only.tar.gz": "https://github.com/indygreg/python-build-standalone/releases/download/20221106/cpython-3.9.15+20221106-x86_64-unknown-linux-gnu-install_only.tar.gz", - "cpython-3.9.15+20221106-aarch64-apple-darwin-install_only.tar.gz": "https://github.com/indygreg/python-build-standalone/releases/download/20221106/cpython-3.9.15+20221106-aarch64-apple-darwin-install_only.tar.gz", - "cpython-3.9.15+20221106-x86_64-apple-darwin-install_only.tar.gz": "https://github.com/indygreg/python-build-standalone/releases/download/20221106/cpython-3.9.15+20221106-x86_64-apple-darwin-install_only.tar.gz" - } -} diff --git a/package/scie-pants.toml b/package/scie-pants.toml new file mode 100644 index 0000000..41d057a --- /dev/null +++ b/package/scie-pants.toml @@ -0,0 +1,229 @@ +[lift] +name = "scie-pants" +description = "Isolates your Pants from the elements." +load_dotenv = true + +[lift.app_info] +repo = "https://github.com/pantsbuild/scie-pants" + +[lift.ptex] +id = "ptex" +version = "0.7.0" +lazy_argv1 = "{scie.env.PANTS_BOOTSTRAP_URLS={scie.lift}}" + +[lift.scie-jump] +version = "0.11.0" + +[[lift.interpreters]] +id = "cpython38" +provider = "PythonBuildStandalone" +release = "20230507" +lazy = true +version = "3.8.16" + +[[lift.interpreters]] +id = "cpython39" +provider = "PythonBuildStandalone" +release = "20230507" +lazy = true +version = "3.9.16" + +[[lift.interpreter_groups]] +id = "cpython" +selector = "{scie.bindings.configure:PYTHON}" +members = [ + "cpython38", + "cpython39", +] + +[[lift.files]] +# N.B.: We name the scie-pants binary scie-pants.bin since the scie itself is named scie-pants +# which would conflict when packaging. +name = "scie-pants.bin" +executable = true + +[[lift.files]] +name = "tools.pex" + +# Boot +[[lift.commands]] +description = "Detects the current Pants installation and launches it." +exe = "{scie-pants.bin}" + +# Run Pants +[[lift.commands]] +name = "pants" +description = "Runs a hermetic Pants installation." +exe = "{scie.bindings.install:PANTS_CLIENT_EXE}" +args = [ + "{scie.bindings.configure:PANTS_SHA_FIND_LINKS}", +] + +[lift.commands.env.default] +PANTS_BUILDROOT_OVERRIDE = "{scie.bindings.configure:PANTS_BUILDROOT_OVERRIDE}" + +[lift.commands.env.replace] +PANTS_VERSION = "{scie.bindings.configure:PANTS_VERSION}" +_PANTS_SERVER_EXE = "{scie.bindings.install:PANTS_SERVER_EXE}" + +# Run Pants Debug Mode +[[lift.commands]] +name = "pants-debug" +description = "Runs a hermetic Pants installation with a debug server for debugging Pants code." +exe = "{scie.bindings.install:VIRTUAL_ENV}/bin/pants" +args = [ + "-c", + """\ +sys = __import__("sys") +sys.path.pop(0) +print( + "Launching debugpy server at '127.0.0.1:5678' and waiting for client connection.", + file=sys.stderr +) +del sys +__import__("debugpy.server.cli").server.cli.main() +""", + "--listen", + "127.0.0.1:5678", + "--wait-for-client", + "{scie.bindings.install:VIRTUAL_ENV}/bin/pants", + "{scie.bindings.configure:PANTS_SHA_FIND_LINKS}", +] + +[lift.commands.env.default] +PANTS_BUILDROOT_OVERRIDE = "{scie.bindings.configure:PANTS_BUILDROOT_OVERRIDE}" + +[lift.commands.env.replace] +PANTS_VERSION = "{scie.bindings.configure:PANTS_VERSION}" + + +# Bootstrap Tools +[[lift.commands]] +name = "bootstrap-tools" +description = "Introspection tools for the Pants bootstrap process." +exe = "#{cpython:python}" +args = [ + "{tools.pex}", + "bootstrap-tools", + "--python-distribution-hash", + "{scie.files:hash.#{cpython}}", + "--pants-version", + "{scie.bindings.configure:PANTS_VERSION}", +] +env.remove_re = [ + "PEX_.*", +] + +[lift.commands.env.replace] +PEX_ROOT = "{scie.bindings}/pex_root" +PEX_PYTHON_PATH = "#{cpython:python}" + +# Self Update +[[lift.commands]] +name = "update" +description = "Update scie-pants." +exe = "#{cpython39:python}" +args = [ + "{tools.pex}", + "update-scie-pants", + "--ptex-path", + "{ptex}", + "--platform", + "{scie.platform}", + "--base-dir", + "{scie.bindings}", + "--scie", + "{scie}", + "--current-version", + "{scie.bindings.scie-pants-info:VERSION}", + "--github-api-bearer-token", + "{scie.env.PANTS_BOOTSTRAP_GITHUB_API_BEARER_TOKEN}", +] +env.remove_re = [ + "PEX_.*", +] + +[lift.commands.env.replace] +PEX_ROOT = "{scie.bindings}/pex_root" +PEX_PYTHON_PATH = "#{cpython39:python}" + +# Self Info +[[lift.bindings]] +name = "scie-pants-info" +description = "Records information about the current scie-pants binary." +exe = "#{cpython39:python}" +args = [ + "{tools.pex}", + "record-scie-pants-info", + "--base-dir", + "{scie.bindings}", + "--scie", + "{scie}", +] +env.remove_re = [ + "PEX_.*", +] + +[lift.bindings.env.replace] +PEX_ROOT = "{scie.bindings}/pex_root" +PEX_PYTHON_PATH = "#{cpython39:python}" + +# Configure Pants +[[lift.bindings]] +name = "configure" +description = "Prompts the user for missing Pants configuration if needed." +exe = "#{cpython39:python}" +args = [ + "{tools.pex}", + "configure-pants", + "--ptex-path", + "{ptex}", + "--pants-version", + "{scie.env.PANTS_VERSION}", + "--pants-sha", + "{scie.env.PANTS_SHA}", + "--pants-config", + "{scie.env.PANTS_TOML}", + "--github-api-bearer-token", + "{scie.env.PANTS_BOOTSTRAP_GITHUB_API_BEARER_TOKEN}", + "{scie.bindings}", +] +env.remove_re = [ + "PEX_.*", +] + +[lift.bindings.env.default] +PANTS_VERSION_PROMPT_SALT = "{scie.env.PANTS_VERSION_PROMPT_SALT}" + +[lift.bindings.env.replace] +PEX_ROOT = "{scie.bindings}/pex_root" +PEX_PYTHON_PATH = "#{cpython39:python}" + +# Install Pants +[[lift.bindings]] +name = "install" +description = """\ +Installs a hermetic Pants environment from PyPI or binaries.pantsbuild.org with optional debug \ +support.\ +""" +exe = "#{cpython:python}" +args = [ + "{tools.pex}", + "install-pants", + "--pants-version", + "{scie.bindings.configure:PANTS_VERSION}", + "--find-links", + "{scie.bindings.configure:FIND_LINKS}", + "--debug", + "{scie.env.PANTS_DEBUG}", + "--debugpy-requirement", + "{scie.env.PANTS_DEBUGPY_VERSION}", + "{scie.bindings}", +] +env.remove_re = [ + "PEX_.*", +] + +[lift.bindings.env.replace] +PEX_ROOT = "{scie.bindings}/pex_root" +PEX_PYTHON_PATH = "#{cpython:python}" diff --git a/package/src/main.rs b/package/src/main.rs index 72b5362..c5ffe17 100644 --- a/package/src/main.rs +++ b/package/src/main.rs @@ -16,20 +16,20 @@ use std::io::Write; use std::ops::Deref; use std::path::{Path, PathBuf}; -use anyhow::{bail, Context, Result}; +use anyhow::{bail, Result}; use clap::{arg, command, Parser, Subcommand}; use termcolor::{Color, WriteColor}; +use utils::fs; -use crate::scie_pants::build_scie_pants_scie; +use crate::scie_pants::{build_scie_pants_scie, SciePantsBuild}; use crate::test::run_integration_tests; use crate::tools_pex::build_tools_pex; -use crate::utils::build::{check_sha256, fetch_skinny_scie_tools, fingerprint, BuildContext}; +use crate::utils::build::{check_sha256, fetch_skinny_scie_tools, BuildContext}; use crate::utils::fs::{canonicalize, copy, ensure_directory}; const BINARY: &str = "scie-pants"; -const PTEX_TAG: &str = "v0.7.0"; -const SCIE_JUMP_TAG: &str = "v0.11.0"; +const SCIENCE_TAG: &str = "v0.1.1"; #[derive(Clone)] struct SpecifiedPath(PathBuf); @@ -118,19 +118,11 @@ struct Args { #[arg( long, help = format!( - "Instead of using the released {PTEX_TAG} ptex, package ptex from the ptex project \ - repo at this directory." + "Instead of using the released {SCIENCE_TAG} science, package science from the science \ + project repo at this directory." ) )] - ptex: Option, - #[arg( - long, - help = format!( - "Instead of using the released {SCIE_JUMP_TAG} scie-jump, package the scie-jump from \ - the scie-jump project repo at this directory." - ) - )] - scie_jump: Option, + science: Option, #[arg( long, help = "Refresh the tools lock before building the tools.pex", @@ -147,7 +139,7 @@ struct Args { command: Commands, } -fn maybe_build(args: &Args, build_context: &BuildContext) -> Result> { +fn maybe_build(args: &Args, build_context: &BuildContext) -> Result> { match &args.command { Commands::Test { tools_pex: Some(tools_pex), @@ -192,7 +184,7 @@ fn maybe_build(args: &Args, build_context: &BuildContext) -> Result Result Result { let skinny_scie_tools = fetch_skinny_scie_tools(build_context)?; - Ok(Some(build_tools_pex( - build_context, - &skinny_scie_tools, - args.update_lock, - )?)) + build_tools_pex(build_context, &skinny_scie_tools, args.update_lock)?; + Ok(None) } } } @@ -259,35 +248,18 @@ fn main() -> Result<()> { ); } - let build_context = BuildContext::new( - args.target.as_deref(), - args.ptex.as_deref(), - args.scie_jump.as_deref(), - )?; - if let Some(output_file) = maybe_build(&args, &build_context)? { - let dest_file_name = output_file - .file_name() - .with_context(|| format!("Failed to determine the basename of {output_file:?}"))? - .to_str() - .with_context(|| { - format!("Failed to interpret the basename of {output_file:?} as a UTF-8 string") - })?; - let dest_file = dest_dir.join(dest_file_name); + let build_context = BuildContext::new(args.target.as_deref(), args.science.as_deref())?; + if let Some(scie_pants) = maybe_build(&args, &build_context)? { ensure_directory(dest_dir, false)?; - copy(&output_file, &dest_file)?; - let fingerprint_file = dest_file.with_file_name(format!("{dest_file_name}.sha256")); - let dest_file_digest = fingerprint(&dest_file)?; - std::fs::write( - &fingerprint_file, - format!("{dest_file_digest} *{dest_file_name}\n"), - ) - .with_context(|| { - format!( - "Failed to write fingerprint file {fingerprint_file}", - fingerprint_file = fingerprint_file.display() - ) - })?; + let dest_file_name = fs::base_name(&scie_pants.exe)?; + let dest_file = dest_dir.join(dest_file_name); + copy(&scie_pants.exe, &dest_file)?; + copy( + &scie_pants.sha256, + &dest_dir.join(fs::base_name(&scie_pants.sha256)?), + )?; + check_sha256(&dest_file)?; log!( diff --git a/package/src/scie_pants.rs b/package/src/scie_pants.rs index a51c146..7b70049 100644 --- a/package/src/scie_pants.rs +++ b/package/src/scie_pants.rs @@ -1,4 +1,3 @@ -use std::env; use std::io::Write; use std::path::{Path, PathBuf}; use std::process::Command; @@ -8,45 +7,70 @@ use termcolor::WriteColor; use crate::utils::build::{BuildContext, SkinnyScieTools}; use crate::utils::exe::{binary_full_name, execute}; -use crate::utils::fs::{ensure_directory, hardlink, rename}; +use crate::utils::fs::ensure_directory; use crate::{build_step, BINARY}; +pub(crate) struct SciePantsBuild { + pub(crate) exe: PathBuf, + pub(crate) sha256: PathBuf, +} + pub(crate) fn build_scie_pants_scie( build_context: &BuildContext, skinny_scie_tools: &SkinnyScieTools, tools_pex_file: &Path, -) -> Result { +) -> Result { build_step!("Building the scie-pants Rust binary."); let scie_pants_exe = build_context.build_scie_pants()?; build_step!("Building the `scie-pants` scie"); - // Setup the scie-pants boot-pack. + // Setup the scie-pants science build. let scie_pants_package_dir = build_context.cargo_output_root.join("scie-pants"); ensure_directory(&scie_pants_package_dir, true)?; - let scie_jump_dst = scie_pants_package_dir.join("scie-jump"); - let ptex_dst = scie_pants_package_dir.join("ptex"); - // N.B.: We name the scie-pants binary scie-pants.bin since the scie itself is named scie-pants - // which would conflict when packaging. - let scie_pants_dst = scie_pants_package_dir.join("scie-pants.bin"); - let tools_pex_dst = scie_pants_package_dir.join("tools.pex"); let scie_pants_manifest = build_context .package_crate_root - .join("scie-pants.lift.json"); - let scie_pants_manifest_dst = scie_pants_package_dir.join("lift.json"); - hardlink(&skinny_scie_tools.scie_jump, &scie_jump_dst)?; - hardlink(&skinny_scie_tools.ptex, &ptex_dst)?; - hardlink(&scie_pants_exe, &scie_pants_dst)?; - hardlink(tools_pex_file, &tools_pex_dst)?; - hardlink(&scie_pants_manifest, &scie_pants_manifest_dst)?; - - // Run the boot-pack. - execute(Command::new(&scie_jump_dst).current_dir(&scie_pants_package_dir))?; - let scie_pants_scie = scie_pants_package_dir - .join(BINARY) - .with_extension(env::consts::EXE_EXTENSION); - let scie_pants_scie_with_platform = scie_pants_package_dir.join(binary_full_name(BINARY)); - rename(&scie_pants_scie, &scie_pants_scie_with_platform)?; - Ok(scie_pants_scie_with_platform) + .join("scie-pants.toml") + .strip_prefix(&build_context.workspace_root)? + .to_owned(); + + // N.B.: We name the scie-pants binary scie-pants.bin since the scie itself is named scie-pants + // which would conflict when packaging. + execute( + Command::new(&skinny_scie_tools.science) + .args([ + "lift", + "--include-provenance", + "--file", + &format!( + "scie-pants.bin={scie_pants_exe}", + scie_pants_exe = scie_pants_exe.display() + ), + "--file", + &format!( + "tools.pex={tools_pex}", + tools_pex = tools_pex_file.display() + ), + "build", + "--dest-dir", + &format!( + "{scie_pants_package_dir}", + scie_pants_package_dir = scie_pants_package_dir.display() + ), + "--use-platform-suffix", + "--hash", + "sha256", + &format!( + "{scie_pants_manifest}", + scie_pants_manifest = scie_pants_manifest.display() + ), + ]) + .current_dir(&build_context.workspace_root), + )?; + let exe_full_name = binary_full_name(BINARY); + Ok(SciePantsBuild { + exe: scie_pants_package_dir.join(exe_full_name.clone()), + sha256: scie_pants_package_dir.join(format!("{exe_full_name}.sha256")), + }) } diff --git a/package/src/test.rs b/package/src/test.rs index 8b3e12b..1b82797 100644 --- a/package/src/test.rs +++ b/package/src/test.rs @@ -654,11 +654,15 @@ fn test_self_downgrade(scie_pants_scie: &Path) { integration_test!("Verifying downgrade works"); // Additionally, we exercise using a relative path to the scie-jump binary which triggered // https://github.com/pantsbuild/scie-pants/issues/38 in the past. + let tmpdir = create_tempdir().unwrap(); + let scie_pants_basename = scie_pants_scie.file_name().unwrap(); + let scie_pants = tmpdir.path().join(scie_pants_basename); + copy(scie_pants_scie, &scie_pants).unwrap(); execute( - Command::new(PathBuf::from(".").join(scie_pants_scie.file_name().unwrap())) + Command::new(PathBuf::from(".").join(scie_pants_basename)) .env("SCIE_BOOT", "update") .arg("0.1.8") - .current_dir(scie_pants_scie.parent().unwrap()), + .current_dir(tmpdir.path()), ) .unwrap(); } diff --git a/package/src/tools_pex.rs b/package/src/tools_pex.rs index 9b6d6ec..aa0bf2d 100644 --- a/package/src/tools_pex.rs +++ b/package/src/tools_pex.rs @@ -19,22 +19,22 @@ pub(crate) fn build_tools_pex( skinny_scie_tools: &SkinnyScieTools, update_lock: bool, ) -> Result { - build_step!("Executing scie-jump boot-pack of the `pbt` helper binary"); + build_step!("Executing science build of the `pbt` helper binary"); let pbt_package_dir = build_context.cargo_output_root.join("pbt"); ensure_directory(&pbt_package_dir, true)?; let pbt_exe = pbt_package_dir .join("pbt") .with_extension(env::consts::EXE_EXTENSION); - let scie_jump_dst = pbt_package_dir.join(skinny_scie_tools.scie_jump.file_name().unwrap()); - let ptex_dst = pbt_package_dir.join(skinny_scie_tools.ptex.file_name().unwrap()); - let pbt_manifest = build_context.package_crate_root.join("pbt.lift.json"); - let pbt_manifest_dst = pbt_package_dir.join("lift.json"); - hardlink(&skinny_scie_tools.scie_jump, &scie_jump_dst)?; - hardlink(&skinny_scie_tools.ptex, &ptex_dst)?; + let pbt_manifest = build_context.package_crate_root.join("pbt.toml"); + let pbt_manifest_dst = pbt_package_dir.join("lift.toml"); hardlink(&pbt_manifest, &pbt_manifest_dst)?; - execute(Command::new(&skinny_scie_tools.scie_jump).current_dir(&pbt_package_dir))?; + execute( + Command::new(&skinny_scie_tools.science) + .args(["lift", "build"]) + .current_dir(&pbt_package_dir), + )?; let tools_path = build_context.workspace_root.join("tools"); let lock_path = tools_path.join("lock.json"); diff --git a/package/src/utils/build.rs b/package/src/utils/build.rs index 66920bb..25a6ed9 100644 --- a/package/src/utils/build.rs +++ b/package/src/utils/build.rs @@ -17,7 +17,9 @@ use url::Url; use crate::utils::exe::{binary_full_name, execute, prepare_exe}; use crate::utils::fs::{copy, ensure_directory, path_as_str, rename}; use crate::utils::os::PATHSEP; -use crate::{build_step, BINARY, PTEX_TAG, SCIE_JUMP_TAG}; +use crate::{build_step, BINARY, SCIENCE_TAG}; + +const BOOTSTRAP_PTEX_TAG: &str = "v0.7.0"; const CARGO: &str = env!("CARGO"); const CARGO_MANIFEST_DIR: &str = env!("CARGO_MANIFEST_DIR"); @@ -78,17 +80,12 @@ pub(crate) struct BuildContext { pub(crate) cargo_output_root: PathBuf, target: String, target_prepared: Cell, - ptex_repo: Option, - scie_jump_repo: Option, + science_repo: Option, cargo_output_bin_dir: PathBuf, } impl BuildContext { - pub(crate) fn new( - target: Option<&str>, - ptex_repo: Option<&Path>, - scie_jump_repo: Option<&Path>, - ) -> Result { + pub(crate) fn new(target: Option<&str>, science_repo: Option<&Path>) -> Result { let target = target.unwrap_or(TARGET).to_string(); let package_crate_root = PathBuf::from(CARGO_MANIFEST_DIR); let workspace_root = package_crate_root @@ -104,8 +101,7 @@ impl BuildContext { cargo_output_root: output_root, target, target_prepared: Cell::new(false), - ptex_repo: ptex_repo.map(Path::to_path_buf), - scie_jump_repo: scie_jump_repo.map(Path::to_path_buf), + science_repo: science_repo.map(Path::to_path_buf), cargo_output_bin_dir: output_bin_dir, }) } @@ -122,40 +118,30 @@ impl BuildContext { Ok(()) } - pub(crate) fn obtain_ptex(&self, dest_dir: &Path) -> Result { - if let Some(ref ptex_from) = self.ptex_repo { + pub(crate) fn obtain_science(&self, dest_dir: &Path) -> Result { + if let Some(ref science_from) = self.science_repo { self.ensure_target()?; build_step!( - "Building the `ptex` binary from the source at {ptex_from}", - ptex_from = ptex_from.display() + "Building the `science` binary from the source at {science_from}", + science_from = science_from.display() ); - build_a_scie_project(ptex_from, &self.target, dest_dir)?; - } else { - fetch_a_scie_project(self, "ptex", PTEX_TAG, "ptex", dest_dir)?; - } - let ptex_exe_path = dest_dir.join(binary_full_name("ptex")); - prepare_exe(&ptex_exe_path)?; - let ptex_exe = dest_dir.join("ptex"); - rename(&ptex_exe_path, &ptex_exe)?; - Ok(ptex_exe) - } - - pub(crate) fn obtain_scie_jump(&self, dest_dir: &Path) -> Result { - if let Some(ref scie_jump_from) = self.scie_jump_repo { - self.ensure_target()?; - build_step!( - "Building the `scie-jump` binary from the source at {scie_jump_from}", - scie_jump_from = scie_jump_from.display() - ); - build_a_scie_project(scie_jump_from, &self.target, dest_dir)?; + execute( + Command::new("nox") + .args(["-e", "package"]) + .env( + "SCIENCE_LIFT_BUILD_DEST_DIR", + format!("{dest_dir}", dest_dir = dest_dir.display()), + ) + .current_dir(science_from), + )?; } else { - fetch_a_scie_project(self, "jump", SCIE_JUMP_TAG, "scie-jump", dest_dir)?; + fetch_a_scie_project(self, "lift", SCIENCE_TAG, "science", dest_dir)?; } - let scie_jump_exe_path = dest_dir.join(binary_full_name("scie-jump")); - prepare_exe(&scie_jump_exe_path)?; - let scie_jump_exe = dest_dir.join("scie_jump"); - rename(&scie_jump_exe_path, &scie_jump_exe)?; - Ok(scie_jump_exe) + let science_exe_path = dest_dir.join(binary_full_name("science")); + prepare_exe(&science_exe_path)?; + let science_exe = dest_dir.join("science"); + rename(&science_exe_path, &science_exe)?; + Ok(science_exe) } pub(crate) fn build_scie_pants(&self) -> Result { @@ -184,23 +170,7 @@ impl BuildContext { } pub(crate) struct SkinnyScieTools { - pub(crate) ptex: PathBuf, - pub(crate) scie_jump: PathBuf, -} - -fn build_a_scie_project(a_scie_project_repo: &Path, target: &str, dest_dir: &Path) -> Result<()> { - execute(Command::new(CARGO).args([ - "run", - "--manifest-path", - path_as_str(&a_scie_project_repo.join("Cargo.toml"))?, - "-p", - "package", - "--", - "--target", - target, - path_as_str(dest_dir)?, - ])) - .map(|_| ()) + pub(crate) science: PathBuf, } fn fetch_a_scie_project( @@ -240,7 +210,7 @@ fn fetch_a_scie_project( "--git", "https://github.com/a-scie/ptex", "--tag", - PTEX_TAG, + BOOTSTRAP_PTEX_TAG, "--root", path_as_str(&build_context.cargo_output_root)?, "--target", @@ -283,10 +253,8 @@ fn fetch_a_scie_project( pub(crate) fn fetch_skinny_scie_tools(build_context: &BuildContext) -> Result { let skinny_scies = build_context.cargo_output_root.join("skinny-scies"); ensure_directory(&skinny_scies, true)?; - let ptex_exe = build_context.obtain_ptex(&skinny_scies)?; - let scie_jump_exe = build_context.obtain_scie_jump(&skinny_scies)?; + let science_exe = build_context.obtain_science(&skinny_scies)?; Ok(SkinnyScieTools { - ptex: ptex_exe, - scie_jump: scie_jump_exe, + science: science_exe, }) } diff --git a/package/src/utils/fs.rs b/package/src/utils/fs.rs index bc2dff1..445c88d 100644 --- a/package/src/utils/fs.rs +++ b/package/src/utils/fs.rs @@ -142,6 +142,13 @@ pub(crate) fn canonicalize(path: &Path) -> Result { }) } +pub(crate) fn base_name(path: &Path) -> Result<&str> { + path.file_name() + .with_context(|| format!("Failed to determine the basename of {path:?}"))? + .to_str() + .with_context(|| format!("Failed to interpret the basename of {path:?} as a UTF-8 string")) +} + pub(crate) fn dev_cache_dir() -> Result { if let Ok(cache_dir) = env::var("SCIE_PANTS_DEV_CACHE") { let cache_path = PathBuf::from(cache_dir); diff --git a/tools/src/scie_pants/configure_pants.py b/tools/src/scie_pants/configure_pants.py index a240a88..b7c3c42 100644 --- a/tools/src/scie_pants/configure_pants.py +++ b/tools/src/scie_pants/configure_pants.py @@ -115,7 +115,8 @@ def main() -> NoReturn: finalizers.append(configure_version) version = resolve_info.stable_version - python = "python3.8" if version < Version("2.5") else "python3.9" + # N.B.: These values must match the lift TOML interpreter ids. + python = "cpython38" if version < Version("2.5") else "cpython39" for finalizer in finalizers: finalizer()