Add plugin support for building PyOxidizer apps

pants.toml

@@ -24,6 +24,7 @@ backend_packages.add = [
   "pants.backend.experimental.java.lint.google_java_format",
   "pants.backend.experimental.java.debug_goals",
   "pants.backend.experimental.python",
+  "pants.backend.experimental.python.packaging.pyoxidizer",
   "pants.backend.experimental.scala",
   "pants.backend.experimental.scala.lint.scalafmt",
   "pants.backend.experimental.codegen.avro.java",

src/python/pants/backend/experimental/python/packaging/BUILD

@@ -0,0 +1,4 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_sources()

src/python/pants/backend/experimental/python/packaging/pyoxidizer/BUILD

@@ -0,0 +1,4 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_sources()

src/python/pants/backend/experimental/python/packaging/pyoxidizer/register.py

@@ -0,0 +1,14 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from pants.backend.python.packaging.pyoxidizer import subsystem
+from pants.backend.python.packaging.pyoxidizer.rules import rules as pyoxidizer_rules
+from pants.backend.python.packaging.pyoxidizer.target_types import PyOxidizerTarget
+
+
+def rules():
+    return [*pyoxidizer_rules(), *subsystem.rules()]
+
+
+def target_types():
+    return [PyOxidizerTarget]

src/python/pants/backend/python/packaging/BUILD

@@ -0,0 +1,4 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_sources()

src/python/pants/backend/python/packaging/pyoxidizer/BUILD

@@ -0,0 +1,25 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_sources()
+
+python_tests(
+    name="config_test",
+    sources=["config_test.py"],
+)
+
+python_tests(
+    name="rules_integration_test",
+    sources=["rules_integration_test.py"],
+    timeout=480,
+)
+
+python_tests(
+    name="subsystem_integration_test",
+    sources=["subsystem_integration_test.py"],
+)
+
+python_tests(
+    name="target_types_integration_test",
+    sources=["target_types_integration_test.py"],
+)

src/python/pants/backend/python/packaging/pyoxidizer/config.py

@@ -0,0 +1,88 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from dataclasses import dataclass
+from string import Template
+from textwrap import indent
+from typing import List, Optional
+
+DEFAULT_TEMPLATE = """
+def make_exe():
+    dist = default_python_distribution()
+    policy = dist.make_python_packaging_policy()
+
+    # Note: Adding this for pydanic and libs that have the "unable to load from memory" error
+    # https://github.com/indygreg/PyOxidizer/issues/438
+    policy.resources_location_fallback = "filesystem-relative:lib"
+
+    python_config = dist.make_python_interpreter_config()
+    $RUN_MODULE
+
+    exe = dist.to_python_executable(
+        name="$NAME",
+        packaging_policy=policy,
+        config=python_config,
+    )
+
+    # pip_download requires that wheels are available for each dep
+    # exe.add_python_resources(exe.pip_download($WHEELS))
+    exe.add_python_resources(exe.pip_install($WHEELS))
+    $UNCLASSIFIED_RESOURCE_INSTALLATION
+
+    return exe
+
+def make_embedded_resources(exe):
+    return exe.to_embedded_resources()
+
+def make_install(exe):
+    # Create an object that represents our installed application file layout.
+    files = FileManifest()
+    # Add the generated executable to our install layout in the root directory.
+    files.add_python_resource(".", exe)
+    return files
+
+register_target("exe", make_exe)
+register_target("resources", make_embedded_resources, depends=["exe"], default_build_script=True)
+register_target("install", make_install, depends=["exe"], default=True)
+resolve_targets()
+"""
+
+UNCLASSIFIED_RESOURCES_TEMPLATE = """
+for resource in exe.pip_install($UNCLASSIFIED_RESOURCES):
+    resource.add_location = "filesystem-relative:lib"
+    exe.add_python_resource(resource)
+"""
+
+
+@dataclass(frozen=True, unsafe_hash=True)
+class PyOxidizerConfig:
+    executable_name: str
+    wheels: List[str]
+    entry_point: Optional[str] = None
+    template: Optional[str] = None
+    unclassified_resources: Optional[List[str]] = None
+
+    @property
+    def run_module(self) -> str:
+        return (
+            f"python_config.run_module = '{self.entry_point}'"
+            if self.entry_point is not None
+            else ""
+        )
+
+    def render(self) -> str:
+        unclassified_resource_snippet = ""
+        if self.unclassified_resources is not None:
+            unclassified_resource_snippet = Template(
+                UNCLASSIFIED_RESOURCES_TEMPLATE
+            ).safe_substitute(UNCLASSIFIED_RESOURCES=self.unclassified_resources)
+
+            unclassified_resource_snippet = indent(unclassified_resource_snippet, "    ")
+
+        template = Template(self.template or DEFAULT_TEMPLATE)
+        return template.safe_substitute(
+            NAME=self.executable_name,
+            WHEELS=self.wheels,
+            RUN_MODULE=self.run_module,
+            UNCLASSIFIED_RESOURCE_INSTALLATION=unclassified_resource_snippet,
+        )

src/python/pants/backend/python/packaging/pyoxidizer/config_test.py

@@ -0,0 +1,65 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from pants.backend.python.packaging.pyoxidizer.config import PyOxidizerConfig
+
+
+def test_run_module_without_entry_point():
+    config = PyOxidizerConfig(executable_name="my-output", wheels=[], entry_point=None)
+    assert config.run_module == ""
+
+
+def test_run_module_with_entry_point():
+    config = PyOxidizerConfig(executable_name="my-output", wheels=[], entry_point="helloworld.main")
+    assert config.run_module == "python_config.run_module = 'helloworld.main'"
+
+
+def test_render_without_template_uses_default():
+    config = PyOxidizerConfig(
+        executable_name="my-output",
+        wheels=["wheel1", "wheel2"],
+        entry_point="helloworld.main",
+        unclassified_resources=["resource1", "resource2"],
+    )
+
+    rendered_config = config.render()
+    assert "resolve_targets" in rendered_config
+    assert all(
+        [
+            item in rendered_config
+            for item in (
+                "my-output",
+                "wheel1",
+                "wheel2",
+                "helloworld.main",
+                "resource1",
+                "resource2",
+            )
+        ]
+    )
+
+
+def test_render_with_template():
+    config = PyOxidizerConfig(
+        executable_name="my-output",
+        wheels=["wheel1", "wheel2"],
+        entry_point="helloworld.main",
+        unclassified_resources=["resource1", "resource2"],
+        template="$NAME | $WHEELS | $RUN_MODULE | $UNCLASSIFIED_RESOURCE_INSTALLATION",
+    )
+
+    rendered_config = config.render()
+    assert "resolve_targets" not in rendered_config
+    assert all(
+        [
+            item in rendered_config
+            for item in (
+                "my-output",
+                "wheel1",
+                "wheel2",
+                "helloworld.main",
+                "resource1",
+                "resource2",
+            )
+        ]
+    )

src/python/pants/backend/python/packaging/pyoxidizer/rules.py

@@ -0,0 +1,146 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import logging
+from dataclasses import dataclass
+
+from pants.backend.python.packaging.pyoxidizer.config import PyOxidizerConfig
+from pants.backend.python.packaging.pyoxidizer.subsystem import PyOxidizer
+from pants.backend.python.packaging.pyoxidizer.target_types import (
+    PyOxidizerConfigSourceField,
+    PyOxidizerDependenciesField,
+    PyOxidizerEntryPointField,
+    PyOxidizerUnclassifiedResources,
+)
+from pants.backend.python.util_rules.pex import Pex, PexProcess, PexRequest
+from pants.core.goals.package import BuiltPackage, BuiltPackageArtifact, PackageFieldSet
+from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
+from pants.engine.fs import (
+    CreateDigest,
+    Digest,
+    DigestContents,
+    FileContent,
+    MergeDigests,
+    Snapshot,
+)
+from pants.engine.process import ProcessResult
+from pants.engine.rules import Get, MultiGet, collect_rules, rule
+from pants.engine.target import (
+    DependenciesRequest,
+    FieldSetsPerTarget,
+    FieldSetsPerTargetRequest,
+    Targets,
+)
+from pants.engine.unions import UnionRule
+from pants.util.logging import LogLevel
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class PyOxidizerFieldSet(PackageFieldSet):
+    required_fields = (PyOxidizerDependenciesField,)
+
+    entry_point: PyOxidizerEntryPointField
+    dependencies: PyOxidizerDependenciesField
+    unclassified_resources: PyOxidizerUnclassifiedResources
+    template: PyOxidizerConfigSourceField
+
+
+@rule(level=LogLevel.DEBUG)
+async def package_pyoxidizer_binary(
+    pyoxidizer: PyOxidizer, field_set: PyOxidizerFieldSet
+) -> BuiltPackage:
+    logger.debug(f"Incoming package_pyoxidizer_binary field set: {field_set}")
+    targets = await Get(Targets, DependenciesRequest(field_set.dependencies))
+    target = targets[0]
+
+    logger.debug(f"Received these targets inside pyox targets: {target.address.target_name}")
+
+    packages = await Get(
+        FieldSetsPerTarget,
+        FieldSetsPerTargetRequest(PackageFieldSet, [target]),
+    )
+    logger.debug(f"Retrieved the following FieldSetsPerTarget {packages}")
+
+    built_packages = await MultiGet(
+        Get(BuiltPackage, PackageFieldSet, field_set) for field_set in packages.field_sets
+    )
+
+    wheels = [
+        artifact.relpath
+        for wheel in built_packages
+        for artifact in wheel.artifacts
+        if artifact.relpath is not None
+    ]
+    logger.debug(f"This is the built package retrieved {built_packages}")
+
+    # Pulling this merged digests idea from the Docker plugin
+    built_package_digests = [built_package.digest for built_package in built_packages]
+
+    # Pip install pyoxidizer
+    pyoxidizer_pex = await Get(
+        Pex,
+        PexRequest(
+            output_filename="pyoxidizer.pex",
+            internal_only=True,
+            requirements=pyoxidizer.pex_requirements(),
+            interpreter_constraints=pyoxidizer.interpreter_constraints,
+            main=pyoxidizer.main,
+        ),
+    )
+
+    config_template = None
+    if field_set.template.value is not None:
+        config_template_source = await Get(SourceFiles, SourceFilesRequest([field_set.template]))
+
+        digest_contents = await Get(DigestContents, Digest, config_template_source.snapshot.digest)
+        config_template = digest_contents[0].content.decode("utf-8")
+
+    config = PyOxidizerConfig(
+        executable_name=field_set.address.target_name,
+        entry_point=field_set.entry_point.value,
+        wheels=wheels,
+        template=config_template,
+        unclassified_resources=None
+        if not field_set.unclassified_resources.value
+        else list(field_set.unclassified_resources.value),
+    )
+
+    rendered_config = config.render()
+    logger.debug(f"Rendered configuation to use -> {rendered_config}")
+    config_digest = await Get(
+        Digest,
+        CreateDigest([FileContent("pyoxidizer.bzl", rendered_config.encode("utf-8"))]),
+    )
+
+    all_digests = (config_digest, *built_package_digests)
+    merged_digest = await Get(Digest, MergeDigests(d for d in all_digests if d))
+    merged_digest_snapshot = await Get(Snapshot, Digest, merged_digest)
+    logger.debug(merged_digest_snapshot)
+
+    result = await Get(
+        ProcessResult,
+        PexProcess(
+            pyoxidizer_pex,
+            argv=["build", *pyoxidizer.args],
+            description="Running PyOxidizer build (...this can take a minute...)",
+            input_digest=merged_digest,
+            level=LogLevel.DEBUG,
+            output_directories=["build"],
+        ),
+    )
+
+    snapshot = await Get(Snapshot, Digest, result.output_digest)
+    artifacts = [BuiltPackageArtifact(file) for file in snapshot.files]
+    return BuiltPackage(
+        result.output_digest,
+        artifacts=tuple(artifacts),
+    )
+
+
+def rules():
+    return (
+        *collect_rules(),
+        UnionRule(PackageFieldSet, PyOxidizerFieldSet),
+    )