Add --scie option to produce native PEX exes.

pex/bin/pex.py

@@ -1305,6 +1305,7 @@ def do_main(
             url_fetcher = URLFetcher(
                 network_configuration=resolver_configuration.network_configuration,
                 password_entries=resolver_configuration.repos_configuration.password_entries,
+                handle_file_urls=True,
             )
             with TRACER.timed("Building scie(s)"):
                 for par_info in scie.build(

pex/scie/__init__.py

@@ -3,8 +3,10 @@
 
 from __future__ import absolute_import
 
+import os.path
 from argparse import Namespace, _ActionsContainer
 
+from pex.compatibility import urlparse
 from pex.fetcher import URLFetcher
 from pex.orderedset import OrderedSet
 from pex.pep_440 import Version
@@ -17,6 +19,7 @@
     ScieStyle,
     ScieTarget,
 )
+from pex.scie.science import SCIENCE_RELEASES_URL, SCIENCE_REQUIREMENT
 from pex.typing import TYPE_CHECKING, cast
 from pex.variables import ENV, Variables
 
@@ -39,6 +42,7 @@ def register_options(parser):
 
     parser.add_argument(
         "--scie",
+        "--par",
         dest="scie_style",
         default=None,
         type=ScieStyle.for_value,
@@ -106,6 +110,19 @@ def register_options(parser):
             "to the patch level."
         ),
     )
+    parser.add_argument(
+        "--scie-science-binary",
+        dest="scie_science_binary",
+        default=None,
+        type=str,
+        help=(
+            "The file path of a `science` binary or a URL to use to fetch the `science` binary "
+            "when there is no `science` on the PATH with a version matching {science_requirement}. "
+            "Pex uses the official `science` releases at {science_releases_url} by default.".format(
+                science_requirement=SCIENCE_REQUIREMENT, science_releases_url=SCIENCE_RELEASES_URL
+            )
+        ),
+    )
 
 
 def render_options(options):
@@ -121,6 +138,9 @@ def render_options(options):
     if options.python_version:
         args.append("--scie-python-version")
         args.append(".".join(map(str, options.python_version)))
+    if options.science_binary_url:
+        args.append("--scie-science-binary")
+        args.append(options.science_binary_url)
     return " ".join(args)
 
 
@@ -138,7 +158,7 @@ def extract_options(options):
         ):
             raise ValueError(
                 "Invalid Python version: '{python_version}'.\n"
-                "Must be in the form `<major>.<minor>` or `<major>.<minor>.<release>`".format(
+                "Must be in the form `<major>.<minor>` or `<major>.<minor>.<patch>`".format(
                     python_version=options.scie_python_version
                 )
             )
@@ -156,11 +176,18 @@ def extract_options(options):
                 )
             )
 
+    science_binary_url = options.scie_science_binary
+    if science_binary_url:
+        url_info = urlparse.urlparse(options.scie_science_binary)
+        if not url_info.scheme and url_info.path and os.path.isfile(url_info.path):
+            science_binary_url = "file://{path}".format(path=os.path.abspath(url_info.path))
+
     return ScieOptions(
         style=options.scie_style,
         platforms=tuple(OrderedSet(options.scie_platforms)),
         pbs_release=options.scie_pbs_release,
         python_version=python_version,
+        science_binary_url=science_binary_url,
     )
 
 

pex/scie/model.py

@@ -133,6 +133,7 @@ class ScieOptions(object):
     python_version = attr.ib(
         default=None
     )  # type: Optional[Union[Tuple[int, int], Tuple[int, int, int]]]
+    science_binary_url = attr.ib(default=None)  # type: Optional[str]
 
     def create_configuration(self, targets):
         # type: (Targets) -> ScieConfiguration

pex/scie/science.py

@@ -11,22 +11,25 @@
 from subprocess import CalledProcessError
 
 from pex.atomic_directory import atomic_directory
-from pex.common import is_exe, pluralize, safe_mkdtemp, safe_open
+from pex.common import chmod_plus_x, is_exe, pluralize, safe_mkdtemp, safe_open
 from pex.compatibility import shlex_quote
 from pex.exceptions import production_assert
 from pex.fetcher import URLFetcher
 from pex.hashing import Sha256
 from pex.layout import Layout
 from pex.pep_440 import Version
 from pex.pex_info import PexInfo
+from pex.result import Error, try_
 from pex.scie.model import ScieConfiguration, ScieInfo, SciePlatform, ScieStyle, ScieTarget
+from pex.third_party.packaging.specifiers import SpecifierSet
 from pex.third_party.packaging.version import InvalidVersion
+from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING, cast
 from pex.util import CacheHelper
 from pex.variables import ENV, Variables, unzip_dir_relpath
 
 if TYPE_CHECKING:
-    from typing import Any, Dict, Iterator, Optional, cast
+    from typing import Any, Dict, Iterator, Optional, Union, cast
 
     import attr  # vendor:skip
     import toml  # vendor:skip
@@ -48,7 +51,21 @@ def qualified_binary_name(self, binary_name):
         return self.target.platform.qualified_binary_name(binary_name)
 
 
+SCIENCE_RELEASES_URL = "https://github.com/a-scie/lift/releases"
 MIN_SCIENCE_VERSION = Version("0.3.0")
+SCIENCE_REQUIREMENT = SpecifierSet("~={min_version}".format(min_version=MIN_SCIENCE_VERSION))
+
+
+def _science_binary_url(suffix=""):
+    # type: (str) -> str
+    return "{science_releases_url}/download/v{version}/{binary}{suffix}".format(
+        science_releases_url=SCIENCE_RELEASES_URL,
+        version=MIN_SCIENCE_VERSION.raw,
+        binary=_qualified_science_fat_binary_name(),
+        suffix=suffix,
+    )
+
+
 PTEX_VERSION = "1.1.1"
 SCIE_JUMP_VERSION = "1.1.1"
 
@@ -149,18 +166,47 @@ def _science_dir(
     *components  # type: str
 ):
     # type: (...) -> str
-    return os.path.join(env.PEX_ROOT, "scies", "science", str(MIN_SCIENCE_VERSION), *components)
+    return os.path.join(env.PEX_ROOT, "scies", "science", MIN_SCIENCE_VERSION.raw, *components)
 
 
-def _qualified_science_binary_name():
+def _qualified_science_fat_binary_name():
     # type: () -> str
-    return SciePlatform.current().qualified_binary_name("science")
+    return SciePlatform.current().qualified_binary_name("science-fat")
 
 
 def _science_binary_names():
     # type: () -> Iterator[str]
     yield "science"
-    yield _qualified_science_binary_name()
+    yield _qualified_science_fat_binary_name()
+    yield SciePlatform.current().qualified_binary_name("science")
+
+
+def _is_compatible_science_binary(
+    binary,  # type: str
+    source=None,  # type: Optional[str]
+):
+    # type: (...) -> Union[Version, Error]
+    try:
+        version = Version(
+            subprocess.check_output(args=[binary, "--version"]).decode("utf-8").strip()
+        )
+    except (CalledProcessError, InvalidVersion) as e:
+        return Error(
+            "Failed to determine --version of science binary at {source}: {err}".format(
+                source=source or binary, err=e
+            )
+        )
+    else:
+        if version.raw in SCIENCE_REQUIREMENT:
+            return version
+        return Error(
+            "The science binary at {source} is version {version} which does not match Pex's "
+            "science requirement of {science_requirement}.".format(
+                source=source or binary,
+                version=version.raw,
+                science_requirement=SCIENCE_REQUIREMENT,
+            )
+        )
 
 
 def _path_science():
@@ -171,29 +217,15 @@ def _path_science():
         ):
             if not is_exe(binary):
                 continue
-            try:
-                if (
-                    Version(subprocess.check_output(args=[binary, "--version"]).decode("utf-8"))
-                    < MIN_SCIENCE_VERSION
-                ):
-                    continue
-            except (CalledProcessError, InvalidVersion):
+            if isinstance(_is_compatible_science_binary(binary), Error):
                 continue
             return binary
     return None
 
 
-def _science_binary_url(suffix=""):
-    # type: (str) -> str
-    return "https://github.com/a-scie/science/releases/download/v{version}/{binary}{suffix}".format(
-        version=MIN_SCIENCE_VERSION,
-        binary=_qualified_science_binary_name(),
-        suffix=suffix,
-    )
-
-
 def _ensure_science(
     url_fetcher=None,  # type: Optional[URLFetcher]
+    science_binary_url=None,  # type: Optional[str]
     env=ENV,  # type: Variables
 ):
     # type: (...) -> str
@@ -207,27 +239,40 @@ def _ensure_science(
                 shutil.copy(path_science, target_science)
             else:
                 fetcher = url_fetcher or URLFetcher()
-                science_binary_url = _science_binary_url()
                 with open(target_science, "wb") as write_fp, fetcher.get_body_stream(
-                    science_binary_url
+                    science_binary_url or _science_binary_url()
                 ) as read_fp:
                     shutil.copyfileobj(read_fp, write_fp)
+                chmod_plus_x(target_science)
 
-                science_sha256_url = _science_binary_url(".sha256")
-                with fetcher.get_body_stream(science_sha256_url) as fp:
-                    expected_sha256, _, _ = fp.read().decode("utf-8").partition(" ")
-                actual_sha256 = CacheHelper.hash(target_science, hasher=Sha256)
-                if expected_sha256 != actual_sha256:
-                    raise ValueError(
-                        "The science binary downloaded from {science_binary_url} does not match "
-                        "the expected SHA-256 fingerprint recorded in {science_sha256_url}.\n"
-                        "Expected {expected_sha256} but found {actual_sha256}.".format(
-                            science_binary_url=science_binary_url,
-                            science_sha256_url=science_sha256_url,
-                            expected_sha256=expected_sha256,
-                            actual_sha256=actual_sha256,
+                if science_binary_url:
+                    custom_science_binary_version = try_(
+                        _is_compatible_science_binary(target_science, source=science_binary_url)
+                    )
+                    TRACER.log(
+                        "Using custom science binary from {source} with version {version}.".format(
+                            source=science_binary_url, version=custom_science_binary_version.raw
                         )
                     )
+                else:
+                    # Since we used the canonical GitHub Releases URL, we know a checksum file is
+                    # available we can use to verify.
+                    science_sha256_url = _science_binary_url(".sha256")
+                    with fetcher.get_body_stream(science_sha256_url) as fp:
+                        expected_sha256, _, _ = fp.read().decode("utf-8").partition(" ")
+                    actual_sha256 = CacheHelper.hash(target_science, hasher=Sha256)
+                    if expected_sha256 != actual_sha256:
+                        raise ValueError(
+                            "The science binary downloaded from {science_binary_url} does not "
+                            "match the expected SHA-256 fingerprint recorded in "
+                            "{science_sha256_url}.\n"
+                            "Expected {expected_sha256} but found {actual_sha256}.".format(
+                                science_binary_url=science_binary_url,
+                                science_sha256_url=science_sha256_url,
+                                expected_sha256=expected_sha256,
+                                actual_sha256=actual_sha256,
+                            )
+                        )
     return os.path.join(target_dir, "science")
 
 
@@ -243,7 +288,11 @@ def build(
 ):
     # type: (...) -> Iterator[ScieInfo]
 
-    science = _ensure_science(url_fetcher=url_fetcher, env=env)
+    science = _ensure_science(
+        url_fetcher=url_fetcher,
+        science_binary_url=configuration.options.science_binary_url,
+        env=env,
+    )
     name = re.sub(r"\.pex$", "", os.path.basename(pex_file), flags=re.IGNORECASE)
     pex_info = PexInfo.from_pex(pex_file)
     layout = Layout.identify(pex_file)