Migrated to official science/lift releases (#88)

- Bumped to 0.1.1 of official release, and applied breaking changes
- Have a flaky, but working plugin using lift files

examples/python/hellofastapi/BUILD.pants

@@ -26,10 +26,12 @@ pex_binary(
     name="hellofastapi-pex",
     entry_point="hellofastapi.main",
     dependencies=[":libhellofastapi"],
+    include_tools=True,
 )
 
 scie_binary(
     name="hellofastapi-scie",
     dependencies=[":hellofastapi-pex"],
-    platforms=["linux-x86_64", "macos-aarch64"]
+    #platforms=["linux-x86_64", "macos-aarch64"]
+    lift="hellofastapi.toml"
 )

examples/python/hellofastapi/hellofastapi.toml

@@ -0,0 +1,43 @@
+[lift]
+name = "hellofastapi"
+description = "An example FastAPI Lift application including using an external uvicorn server"
+platforms = [
+    "linux-aarch64",
+    "linux-x86_64",
+    "macos-aarch64",
+    "macos-x86_64",
+]
+
+[[lift.interpreters]]
+id = "cpython"
+provider = "PythonBuildStandalone"
+release = "20230507"
+version = "3.11.3"
+
+[[lift.files]]
+name = ":hellofastapi-pex"
+
+[[lift.commands]]
+exe = "{scie.bindings.venv}/venv/bin/uvicorn"
+args = ["hellofastapi.main:app", "--port", "7999"]
+description = "The FastAPI executable."
+
+[[lift.bindings]]
+name = "venv"
+description = "Installs HelloFastAPI into a venv and pre-compiles .pyc"
+exe = "#{cpython:python}"
+args = [
+    "{:hellofastapi-pex}",
+    "venv",
+    "--bin-path",
+    "prepend",
+    "--compile",
+    "--rm",
+    "all",
+    "{scie.bindings}/venv",
+]
+
+[lift.bindings.env.default]
+"=PATH" = "{cpython}/python/bin:{scie.env.PATH}"
+"PEX_TOOLS" = "1"
+"PEX_ROOT" = "{scie.bindings}/pex_root"

examples/python/helloworld/BUILD.pants

@@ -32,7 +32,8 @@ pex_binary(
 scie_binary(
     name="helloworld-scie",
     dependencies=[":helloworld-pex"],
-    platforms=["linux-x86_64", "macos-aarch64"]
+    #platforms=["linux-x86_64", "macos-aarch64"]
+    lift="lift.toml"
 )
 
 # Testing multi-dependency pyoxidizer cases below

examples/python/helloworld/lift.toml

@@ -0,0 +1,26 @@
+[lift]
+name = "helloworld"
+description = "An example HelloWorld Lift application"
+platforms = [
+    "linux-aarch64",
+    "linux-x86_64",
+    "macos-aarch64",
+    "macos-x86_64",
+]
+
+[[lift.interpreters]]
+id = "cpython"
+provider = "PythonBuildStandalone"
+release = "20230507"
+version = "3.11.3"
+
+[[lift.files]]
+name = ":helloworld-pex"
+
+[[lift.commands]]
+# This selects the `python` binary from within the CPython 3.11 interpreter distribution
+# (a tarball) provided by PythonBuildStandalone above.
+exe = "#{cpython:python}"
+args = [
+    "{:helloworld-pex}",
+]

pants-plugins/experimental/scie/config.py

@@ -4,25 +4,56 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
+import logging
+import toml
+
+logger = logging.getLogger(__name__)
 
 @dataclass(frozen=True)
 class Config:
-    science: ScienceConfig
+    lift: LiftConfig
+
+    def __post_init__(self):
+        if isinstance(self.lift, dict):
+            logger.warning(f"LIFT: {self.lift}")
+            object.__setattr__(self, 'lift', LiftConfig(**self.lift))
+
+    @classmethod
+    def from_toml(cls, file_path):
+        with open(file_path) as f:
+            data = toml.load(f)
+        return cls(**data)
 
 @dataclass(frozen=True)
-class ScienceConfig:
+class LiftConfig:
+    """
+    This configuration is a subset of the configuration that can be found here:
+    https://github.com/a-scie/lift/blob/main/science/model.py
+    """
     name: str
     description: str
     platforms: list[str]
     interpreters: list[Interpreter] 
     files: list[File]
     commands: list[Command]
+    bindings: frozenset[Command] = frozenset()
+
+    def __post_init__(self):
+        if any(isinstance(i, dict) for i in self.interpreters):
+            object.__setattr__(self, 'interpreters', [Interpreter(**i) for i in self.interpreters]) # type: ignore
+        if any(isinstance(f, dict) for f in self.files):
+            object.__setattr__(self, 'files', [File(**f) for f in self.files]) # type: ignore
+        if any(isinstance(c, dict) for c in self.commands):
+            object.__setattr__(self, 'commands', [Command(**c) for c in self.commands]) # type: ignore
+        if any(isinstance(b, dict) for b in self.bindings):
+            object.__setattr__(self, 'bindings', [Command(**b) for b in self.bindings]) # type: ignore
 
 @dataclass(frozen=True)
 class Interpreter:
     version: str
     id: str = "cpython"
     provider: str = "PythonBuildStandalone"
+    release: str = "3.11.3"
     lazy: bool = True
 
 @dataclass(frozen=True)
@@ -33,3 +64,6 @@ class File:
 class Command:
     exe: str
     args: list[str]
+    env: dict[str, str] | None = None
+    name: str | None = None
+    description: str | None = None

pants-plugins/experimental/scie/rules.py

@@ -2,37 +2,43 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 from __future__ import annotations
+from ast import arg
 
 import logging
 import os
-from dataclasses import asdict, dataclass
-from typing import Iterable, Mapping
+from dataclasses import asdict, dataclass, replace
+from typing import Final, Iterable, Mapping
 from pathlib import PurePath
 
 from pants.backend.python.target_types import PythonSourceField
 from pants.backend.python.util_rules.pex_from_targets import InterpreterConstraintsRequest, interpreter_constraints_for_targets
+from pants.backend.visibility.glob import PathGlob
+from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior
 from pants.core.target_types import EnvironmentAwarePackageRequest, RemovePrefix
 from pants.init.plugin_resolver import InterpreterConstraints
 
-from experimental.scie.config import Config, ScienceConfig, Interpreter, File, Command
-from experimental.scie.subsystems.science import Science
+from experimental.scie.config import Config, LiftConfig, Interpreter, File, Command
+from experimental.scie.subsystems import Science
 from experimental.scie.target_types import (
     ScieBinaryNameField,
     ScieDependenciesField,
-    SciePlatformField
+    SciePlatformField,
+    ScieLiftSourceField
 )
 import toml
-from experimental.scie.subsystems.standalone import PythonStandaloneInterpreter
 from pants.core.goals.package import BuiltPackage, BuiltPackageArtifact, PackageFieldSet
 from pants.core.goals.run import RunFieldSet, RunRequest, RunInSandboxBehavior
 from pants.core.util_rules.external_tool import DownloadedExternalTool, ExternalToolRequest
 from pants.engine.fs import (
+    EMPTY_DIGEST,
     CreateDigest,
     Digest,
+    DigestContents,
     DigestEntries,
     DownloadFile,
     FileContent,
     MergeDigests,
+    PathGlobs,
     Snapshot,
 )
 from pants.engine.platform import Platform
@@ -43,13 +49,18 @@
     DescriptionField,
     FieldSetsPerTarget,
     FieldSetsPerTargetRequest,
+    HydrateSourcesRequest,
+    HydratedSources,
+    SingleSourceField,
+    SourcesField,
     Targets,
 )
 from pants.engine.unions import UnionRule
 from pants.util.logging import LogLevel
 
 logger = logging.getLogger(__name__)
 
+DEFAULT_LIFT_PATH: Final[str] = "lift.toml"
 
 @dataclass(frozen=True)
 class ScieFieldSet(PackageFieldSet, RunFieldSet):
@@ -60,6 +71,7 @@ class ScieFieldSet(PackageFieldSet, RunFieldSet):
     description: DescriptionField
     dependencies: ScieDependenciesField
     platforms: SciePlatformField
+    lift: ScieLiftSourceField
 
 
 @rule_helper
@@ -85,6 +97,17 @@ def _get_files_config(built_packages: Iterable[BuiltPackage]) -> Iterable[File]:
 def _contains_pex(built_package: BuiltPackage) -> bool:
     return any(artifact.relpath is not None and artifact.relpath.endswith(".pex") for artifact in built_package.artifacts)
 
+@rule_helper
+async def _parse_lift_source(source: ScieLiftSourceField) -> Config:
+    hydrated_source = await Get(HydratedSources, HydrateSourcesRequest(source))
+    digest_contents = await Get(DigestContents, Digest, hydrated_source.snapshot.digest)
+    content = digest_contents[0].content.decode("utf-8")
+    lift_toml = toml.loads(content)
+    logger.error(lift_toml)
+    return Config(**lift_toml)
+
+# def _post_process_lift_config(config: Config, field_set: ScieFieldSet) -> Config:
+#     # 
 
 @rule(level=LogLevel.DEBUG)
 async def scie_binary(
@@ -119,9 +142,6 @@ async def scie_binary(
     pex_artifact_path = PurePath(pex_artifact.relpath)
     pex_filename = pex_artifact_path.name
 
-    # Strip the prefix from the .pex file (workaround for https://github.com/a-scie/lift/issues/2)
-    stripped_pex_digest = await Get(Digest, RemovePrefix(pex_package.digest, str(pex_artifact_path.parent)))
-
     # Prepare the configuration toml for the Science tool
     binary_name = field_set.binary_name.value or field_set.address.target_name
     assert science.default_url_platform_mapping is not None
@@ -131,7 +151,7 @@ async def scie_binary(
 
     # Create a toml configuration from the input targets and the minimum_version, and place that into a Digest for later usage
     config = Config(
-        science=ScienceConfig(
+        lift=LiftConfig(
             name=binary_name,
             description=field_set.description.value or "",
             platforms=list(target_platforms),
@@ -140,8 +160,16 @@ async def scie_binary(
             commands=[Command(exe="#{cpython:python}", args=[f"{{{pex_filename}}}"])],
         )
     )
-    config_filename = "config.toml"
-    config_digest = await Get(Digest, CreateDigest([FileContent(config_filename, toml.dumps(asdict(config)).encode())]))
+
+    lift_digest = EMPTY_DIGEST
+    lift_path = DEFAULT_LIFT_PATH
+    if field_set.lift.value is not None:
+        config = await _parse_lift_source(field_set.lift)
+        assert field_set.lift.file_path is not None
+        lift_path = field_set.lift.file_path
+
+    config_content = toml.dumps(asdict(config)).encode()
+    lift_digest = await Get(Digest, CreateDigest([FileContent(lift_path, config_content)]))
 
     # Download the Science tool for this platform
     downloaded_tool = await Get(
@@ -153,20 +181,27 @@ async def scie_binary(
         Digest,
         MergeDigests(
             (
-                config_digest,
+                lift_digest,
                 downloaded_tool.digest,
                 *(pkg.digest for pkg in non_pex_packages),
-                stripped_pex_digest,
+                pex_package.digest,
             )
         ),
     )
 
-    # The output files are the binary name followed by each of the platforms (if specified), otherwise just the binary name for native-only
-    output_files = [binary_name] + [f"{binary_name}-{platform}" for platform in target_platforms]
+    # The output files are based on the config.lift.name key and each of the platforms (if specified), otherwise just the config.lift.name for native-only
+    output_files = [config.lift.name] + [f"{config.lift.name}-{platform}" for platform in config.lift.platforms]
 
+    # If any of the config filenames start with `:` then add a filemapping command line arg in the form --file NAME=LOCATION
+    file_mappings = [f"--file {file.name}={pex_artifact_path}" for file in config.lift.files if file.name.startswith(":")]
+    # Split each file mapping into a list of arguments
+    file_mappings = [arg for mapping in file_mappings for arg in mapping.split(" ")]
+    logger.warning(file_mappings)
+
     # Run science to generate the scie binaries (depending on the `platforms` setting)
+    argv = (downloaded_tool.exe, "lift", *file_mappings, "build", "--use-platform-suffix" if config.lift.platforms else "", lift_path)
     process = Process(
-        argv=(downloaded_tool.exe, "build", config_filename),
+        argv=argv,
         input_digest=input_digest,
         description="Run science on the input digests",
         output_files=output_files,

pants-plugins/experimental/scie/subsystems/science.py → pants-plugins/experimental/scie/subsystems.py

@@ -15,16 +15,16 @@ class Science(TemplatedExternalTool):
     options_scope = "science"
     help = softwrap("""A high level tool to build scies with.""")
 
-    default_version = "0.1.0"
+    default_version = "0.1.1"
     default_known_versions = [
-        "0.1.0|linux_arm64|ecb4c8d7d882df2a7c15b9960765b2acb6bdb52e6c65bc2a94b6ce53196ba34c|6692875",
-        "0.1.0|linux_x86_64|ea76ca3475275fd0a7c7fa8437b487f4de193a62c99eb88d9280d597265c6849|7575320",
-        "0.1.0|macos_arm64|5041ecbdcccd61417a01668b33be58a085b0ec84519c6de37b9fec021f8fddb2|4557956",
-        "0.1.0|macos_x86_64|3fcfba52d7a92dbdd93fb61ef3fd412e66e85249e9d2acb2259d4d104bf10aaa|4692688",
+        "0.1.1|linux_arm64|f2082538b6dcd65326cf20ac5aee832f1743f278e36fba9af09fcf81597190ac|5570863",
+        "0.1.1|linux_x86_64|edfd24effab7c4ff07c581d278e363aa5e64e9f5e397f13596194ce4583adb3c|6465812",
+        "0.1.1|macos_arm64|68a8a09a4792e578da763e621d37695b0e7876dab3b10b54c970722875f05e9a|3455268",
+        "0.1.1|macos_x86_64|defcd967c9d51272a749030fcbb57871070cfb7e8943d9391d3a537da69251f0|3573076",
     ]
 
     default_url_template = (
-        "https://github.com/sureshjoshi/scie-lift/releases/download/{version}/science-{platform}"
+        "https://github.com/a-scie/lift/releases/download/v{version}/science-{platform}"
     )
 
     default_url_platform_mapping = {