setup: Build per-platform wheels. (#5787)

* setup: Build per-platform wheels.

Build separate wheels for each platform containing only
the correct bootloaders for said platform with no C
source code/build scripts. Set the platform tag so that
the correct wheel will be selected automatically by pip.

To facilitate cross compiling (or rather cross platform wheel
building), add new cross-build wheel commands to the setup.py and
a "build the lot" command (bdist_wheels) to run them all:

    $ python setup.py --help-commands
    ...
    Extra commands:
        ...
        wheel_windows_64bit  Create a Windows-64bit wheel
        wheel_windows_32bit  Create a Windows-32bit wheel
        wheel_linux_64bit    Create a Linux-64bit wheel
        wheel_linux_32bit    Create a Linux-32bit wheel
        wheel_darwin_64bit   Create a Darwin-64bit wheel
        ...
        bdist_wheels         Build all available wheel types
        ...

With the above in place, it no longer makes sense to include
bootloaders in source distributions (sdists) as the only time
pip will choose the sdist over the wheels is if it is
installing on a platform which is incompatible with all the
wheels' platforms and therefore all of the pre-built bootloaders.
Therefore, strip all bootloaders (and various other things that
shouldn't have been there anyway) from the MANIFEST.in.

Co-authored-by: Rok Mandeljc <rok.mandeljc@gmail.com>

MANIFEST.in

@@ -1,33 +1,4 @@
-graft bootloader
-graft doc
-graft PyInstaller
-graft tests
-
-include *.rst
-include *.txt
-include *.toml
-include pyinstaller.py
-include pyinstaller-gui.py
-
-prune bootloader/build
-prune bootloader/.waf-*
-prune bootloader/.waf3-*
-prune bootloader/waf-*
-prune bootloader/waf3-*
-prune bootloader/_sdks
-prune bootloader/.vagrant
-exclude bootloader/.lock-waf*
-
-prune doc/source
-prune doc/_build
-recursive-exclude doc *.tmp
-include news/_template.rst
-prune news
-
-prune old
-prune scripts
-prune tests/scripts
-prune .github
-
-exclude .* *.yml *~ .directory
-global-exclude *.py[co]
+recursive-include bootloader *.c *.h wscript waf strip.py
+recursive-exclude PyInstaller/bootloader *
+recursive-include PyInstaller/bootloader/images *
+include pyproject.toml

setup.cfg

@@ -52,8 +52,7 @@ classifiers =
 
 [options]
 zip_safe = False
-packages = PyInstaller
-include_package_data = True
+include_package_data = False
 python_requires = >=3.6
 ## IMPORTANT: Keep aligned with requirements.txt
 install_requires =
@@ -74,15 +73,6 @@ hook_testing =
 encryption =
     tinyaes>=1.0.0
 
-[options.package_data]
-# This includes precompiled bootloaders and icons for bootloaders
-PyInstaller: bootloader/*/*
-# This file is necessary for rthooks (runtime hooks)
-PyInstaller.hooks: rthooks.dat
-# Needed for tests discovered by entry points;
-# see ``PyInstaller/utils/run_tests.py``.
-PyInstaller.utils: pytest.ini
-
 [options.entry_points]
 console_scripts =
 	pyinstaller = PyInstaller.__main__:run
@@ -102,6 +92,10 @@ formats=gztar
 # dependencies per platforms and version and includes compiled binaries.
 #universal = MUST NOT
 
+[clean]
+# Always fully clean. Otherwise, bootloaders for one platform remain in the
+# build cache and end up blindly copied into wheels for another platform.
+all=1
 
 [zest.releaser]
 python-file-with-version = PyInstaller/__init__.py

setup.py

@@ -12,7 +12,11 @@
 
 import sys
 import os
-from setuptools import setup
+import subprocess
+from typing import Type
+
+from setuptools import setup, find_packages
+
 
 # Hack required to allow compat to not fail when pypiwin32 isn't found
 os.environ["PYINSTALLER_NO_PYWIN32_FAILURE"] = "1"
@@ -23,6 +27,12 @@
 from distutils.core import Command
 from distutils.command.build import build
 
+try:
+    from wheel.bdist_wheel import bdist_wheel
+except ImportError:
+    raise SystemExit("Error: Building wheels requires the 'wheel' package. "
+                     "Please `pip install wheel` then try again.")
+
 
 class build_bootloader(Command):
     """
@@ -70,11 +80,127 @@ def run(self):
         self.run_command('build_bootloader')
         build.run(self)
 
+
+# --- Builder classes for separate per-platform wheels. ---
+
+
+class Wheel(bdist_wheel):
+    """Base class for building a wheel for one platform, collecting only the
+    relevant bootloaders for that platform."""
+
+    # The setuptools platform tag.
+    PLAT_NAME = "manylinux2014_x86_64"
+    # The folder of bootloaders from PyInstaller/bootloaders to include.
+    PYI_PLAT_NAME = "Linux-64bit"
+
+    def finalize_options(self):
+        # Inject the platform name.
+        self.plat_name = self.PLAT_NAME
+        self.plat_name_supplied = True
+
+        if not self.has_bootloaders():
+            raise SystemExit(
+                f"Error: No bootloaders for {self.PLAT_NAME} found in "
+                f"{self.bootloaders_dir()}. See "
+                f"https://pyinstaller.readthedocs.io/en/stable/"
+                f"bootloader-building.html for how to compile them.")
+
+        # And add the correct bootloaders as data files.
+        self.distribution.package_data = {
+            "PyInstaller": [f"bootloader/{self.PYI_PLAT_NAME}/*",
+                            "bootloader/images/*"],
+        }
+        super().finalize_options()
+
+    def run(self):
+        # Note that 'clean' relies on clean::all=1 being set in the
+        # `setup.cfg` or the build cache "leaks" into subsequently built
+        # wheels.
+        self.run_command("clean")
+        super().run()
+
+    @classmethod
+    def bootloaders_dir(cls):
+        """Locate the bootloader folder inside the PyInstaller package."""
+        return f"PyInstaller/bootloader/{cls.PYI_PLAT_NAME}"
+
+    @classmethod
+    def has_bootloaders(cls):
+        """Does the bootloader folder exist and is there anything in it?"""
+        dir = cls.bootloaders_dir()
+        return os.path.exists(dir) and len(os.listdir(dir))
+
+
+# Map PyInstaller platform names to their setuptools counterparts.
+# Other OSs can be added as and when we start shipping wheels for them.
+PLATFORMS = {
+    "Windows-64bit": "win_amd64",
+    "Windows-32bit": "win32",
+    # The manylinux version tag depends on the glibc version compiled against.
+    # If we ever change the docker image used to build the bootloaders then we
+    # must check/update this tag.
+    "Linux-64bit":  "manylinux2014_x86_64",
+    "Linux-32bit": "manylinux2014_i686",
+    # The macOS version must be kept in sync with the -mmacosx-version-min in
+    # the waf build script.
+    # TODO: Once we start shipping universal2 bootloaders and PyPA have
+    #       decided what the wheel tag should be, we will need to set it here.
+    "Darwin-64bit": "macosx_10_13_x86_64",
+}
+
+# Create a subclass of Wheel() for each platform.
+wheel_commands = {}
+for (pyi_plat_name, plat_name) in PLATFORMS.items():
+    # This is the name it will have on the setup.py command line.
+    command_name = "wheel_" + pyi_plat_name.replace("-", "_").lower()
+
+    # Create and register the subclass, overriding the PLAT_NAME and
+    # PYI_PLAT_NAME attributes.
+    platform = {"PLAT_NAME": plat_name, "PYI_PLAT_NAME": pyi_plat_name}
+    command: Type[Wheel] = type(command_name, (Wheel,), platform)
+    command.description = f"Create a {command.PYI_PLAT_NAME} wheel"
+    wheel_commands[command_name] = command
+
+
+class bdist_wheels(Command):
+    """Build a wheel for every platform listed in the PLATFORMS dict which has
+    bootloaders available in `PyInstaller/bootloaders/[platform-name]`.
+    """
+    description = "Build all available wheel types"
+
+    # Overload these to keep the abstract metaclass happy.
+    user_options = []
+    def initialize_options(self): pass
+    def finalize_options(self): pass
+
+    def run(self) -> None:
+        command: Type[Wheel]
+        for (name, command) in wheel_commands.items():
+            if not command.has_bootloaders():
+                print("Skipping", name, "because no bootloaders were found in",
+                      command.bootloaders_dir())
+                continue
+
+            print("running", name)
+            # This should be `self.run_command(name)` but there is some
+            # aggressive caching from distutils which has to be suppressed
+            # by us using forced cleaning. One distutils behaviour that
+            # seemingly can't be disabled is that each command should only
+            # run once - this is at odds with what we want because we need
+            # to run 'build' for every platform.
+            # The only way I can get it not to skip subsequent builds is to
+            # isolate the processes completely using subprocesses...
+            subprocess.run([sys.executable, __file__, "-q", name])
+
+
 #--
 
 setup(
     setup_requires = ["setuptools >= 39.2.0"],
     cmdclass = {'build_bootloader': build_bootloader,
                 'build': MyBuild,
+                **wheel_commands,
+                'bdist_wheels': bdist_wheels,
                 },
+    packages=find_packages(include=["PyInstaller", "PyInstaller.*"]),
 )