1660120: Make Python bindings installable from source

.circleci/config.yml

@@ -815,6 +815,24 @@ jobs:
           root: build/
           paths: docs/python
 
+  pypi-source-release:
+    docker:
+      - image: circleci/python:3.8.2
+    steps:
+      - install-rustup
+      - setup-rust-toolchain
+      - checkout
+      - run:
+          name: Setup default Python version
+          command: |
+            echo "export PATH=/opt/python/cp38-cp38/bin:$PATH" >> $BASH_ENV
+      - run:
+          name: Build Python extension
+          command: |
+            make python-setup
+            .venv3.8/bin/python3 setup.py sdist
+            .venv3.8/bin/python3 -m twine upload dist/*
+
   pypi-linux-release:
     docker:
       # The official docker image for building manylinux1 wheels
@@ -1135,6 +1153,10 @@ workflows:
     jobs:
       - Python 3_8 tests:
           filters: *release-filters
+      - pypi-source-release:
+          requires:
+            - Python 3_8 tests
+          filters: *release-filters
       - pypi-linux-release:
           requires:
             - Python 3_8 tests

.gitignore

@@ -28,6 +28,10 @@ Carthage
 .DS_Store
 *.dSYM
 
+# Python stuff
+*.egg-info
+dist/
+
 # C# stuff
 *.suo
 *.user

CHANGELOG.md

@@ -6,6 +6,8 @@
   * Track the size of the database file at startup ([#1141](https://github.com/mozilla/glean/pull/1141)).
 * iOS
   * Disabled code coverage in release builds ([#1195](https://github.com/mozilla/glean/issues/1195)).
+* Python
+  * Glean now ships a source package to pip install on platforms where wheels aren't provided.
 
 # v32.3.0 (2020-08-27)
 

Makefile

@@ -48,8 +48,8 @@ build-swift: ## Build all Swift code
 build-apk: build-kotlin ## Build an apk of the Glean sample app
 	./gradlew glean-sample-app:build
 
-build-python: python-setup build-rust ## Build the Python bindings
-	$(GLEAN_PYENV)/bin/python3 glean-core/python/setup.py install
+build-python: python-setup ## Build the Python bindings
+	$(GLEAN_PYENV)/bin/python3 glean-core/python/setup.py build install
 
 build-csharp: ## Build the C# bindings
 	dotnet build glean-core/csharp/csharp.sln

bin/prepare-release.sh

@@ -120,6 +120,14 @@ run $SED -i.bak -E \
     "${WORKSPACE_ROOT}/${FILE}"
 run rm "${WORKSPACE_ROOT}/${FILE}.bak"
 
+# Update the glean-python version
+
+FILE=glean-core/python/setup.py
+run $SED -i.bak -E \
+    -e "s/^version = \"[0-9a-z.-]+\"/version = \"${NEW_VERSION}\"/" \
+    "${WORKSPACE_ROOT}/${FILE}"
+run rm "${WORKSPACE_ROOT}/${FILE}.bak"
+
 ### Update Cargo.lock
 
 cargo update -p glean-core -p glean-ffi

docs/dev/python/setting-up-python-build-environment.md

@@ -75,12 +75,11 @@ By default, the `Makefile` installs the latest version available of each of Glea
 
 ### Manual method
 
-First, rebuild the Rust core if any Rust changes were made:
+Building the Python bindings also builds the Rust shared object for the Glean SDK core.
 
 ```bash
-  $ cargo build  # If there were Rust changes
   $ cd glean-core/python
-  $ python setup.py install
+  $ python setup.py build install
 ```
 
 ### Makefile method

docs/user/adding-glean-to-your-project.md

@@ -120,15 +120,16 @@ For integration with the build system you can follow the [Carthage Quick Start s
 We recommend using a virtual environment for your work to isolate the dependencies for your project. There are many popular abstractions on top of virtual environments in the Python ecosystem which can help manage your project dependencies.
 
 The Glean SDK Python bindings currently have [prebuilt wheels on PyPI for Windows (i686 and x86_64), Linux (x86_64) and macOS (x86_64)](https://pypi.org/project/glean-sdk/#files).
+For other platforms, the `glean_sdk` package will be built from source on your machine. 
+This requires that Cargo and Rust are already installed. 
+The easiest way to do this is through [rustup](https://rustup.rs/).
 
-If you're running one of those platforms and have your virtual environment set up and activated, you can install the Glean SDK into it using:
+Once you have your virtual environment set up and activated, you can install the Glean SDK into it using:
 
 ```bash
 $ python -m pip install glean_sdk
 ```
 
-If you are not on one of these platforms, you will need to build the Glean SDK Python bindings from source using [these instructions](../dev/python/setting-up-python-build-environment.html).
-
 The Glean SDK Python bindings make extensive use of type annotations to catch type related errors at build time. We highly recommend adding [mypy](https://mypy-lang.org) to your continuous integration workflow to catch errors related to type mismatches early.
 
 </div>

glean-core/python/ffi_build.py

@@ -12,9 +12,15 @@
 """
 
 
+from pathlib import Path
+
+
 import cffi
 
 
+ROOT = Path(__file__).parent.absolute()
+
+
 def _load_header(path: str) -> str:
     """
     Load a C header file and convert it to something parseable by cffi.
@@ -28,7 +34,7 @@ def _load_header(path: str) -> str:
 
 ffibuilder = cffi.FFI()
 ffibuilder.set_source("glean._glean_ffi", None)
-ffibuilder.cdef(_load_header("../ffi/glean.h"))
+ffibuilder.cdef(_load_header(ROOT.parent / "ffi" / "glean.h"))
 
 
 if __name__ == "__main__":

glean-core/python/requirements_dev.txt

@@ -11,6 +11,5 @@ pip
 pytest-localserver==0.5.0
 pytest-runner==5.2
 pytest==6.0.1
-toml==0.10.1
 twine==3.2.0
 wheel==0.34.2

glean-core/python/setup.py

@@ -4,8 +4,10 @@
 
 """The setup script."""
 
+from distutils.command.build import build as _build
 import os
 import shutil
+import subprocess
 import sys
 
 from setuptools import setup, Distribution, find_packages
@@ -30,21 +32,24 @@
     sys.exit(1)
 
 from pathlib import Path  # noqa
-import toml  # noqa
 
-ROOT = Path(__file__).parent.absolute()
+# Path to the directory containing this file
+PYTHON_ROOT = Path(__file__).parent.absolute()
 
-os.chdir(str(ROOT))
+# Relative path to this directory from cwd.
+FROM_TOP = PYTHON_ROOT.relative_to(Path.cwd())
 
-with (ROOT.parent.parent / "README.md").open() as readme_file:
+# Path to the root of the git checkout
+SRC_ROOT = PYTHON_ROOT.parents[1]
+
+with (SRC_ROOT / "README.md").open() as readme_file:
     readme = readme_file.read()
 
-with (ROOT.parent.parent / "CHANGELOG.md").open() as history_file:
+with (SRC_ROOT / "CHANGELOG.md").open() as history_file:
     history = history_file.read()
 
-with (ROOT.parent / "Cargo.toml").open() as cargo:
-    parsed_toml = toml.load(cargo)
-    version = parsed_toml["package"]["version"]
+# glean version. Automatically updated by the bin/prepare_release.sh script
+version = "32.2.0"
 
 requirements = [
     "cffi>=1",
@@ -58,11 +63,11 @@
 buildvariant = os.environ.get("GLEAN_BUILD_VARIANT", "debug")
 
 if mingw_arch == "i686":
-    shared_object_build_dir = "../../target/i686-pc-windows-gnu"
+    shared_object_build_dir = SRC_ROOT / "target" / "i686-pc-windows-gnu"
 elif mingw_arch == "x86_64":
-    shared_object_build_dir = "../../target/x86_64-pc-windows-gnu"
+    shared_object_build_dir = SRC_ROOT / "target" / "x86_64-pc-windows-gnu"
 else:
-    shared_object_build_dir = "../../target"
+    shared_object_build_dir = SRC_ROOT / "target"
 
 
 if platform == "linux":
@@ -76,19 +81,6 @@
 else:
     raise ValueError(f"The platform {sys.platform} is not supported.")
 
-shared_object_path = f"{shared_object_build_dir}/{buildvariant}/{shared_object}"
-
-shutil.copyfile("../metrics.yaml", "glean/metrics.yaml")
-shutil.copyfile("../pings.yaml", "glean/pings.yaml")
-# When running inside of `requirements-builder`, the Rust shared object may not
-# yet exist, so ignore the exception when trying to copy it. Under normal
-# circumstances, this will still show up as an error when running the `build`
-# command as a missing `package_data` file.
-try:
-    shutil.copyfile(shared_object_path, "glean/" + shared_object)
-except FileNotFoundError:
-    pass
-
 
 class BinaryDistribution(Distribution):
     def is_pure(self):
@@ -126,6 +118,37 @@ def finalize_options(self):
             self.install_lib = self.install_platlib
 
 
+class build(_build):
+    def run(self):
+        try:
+            subprocess.run(["cargo"])
+        except subprocess.CalledProcessError:
+            print("Install Rust and Cargo through Rustup: https://rustup.rs/.")
+            print(
+                "Need help installing the glean_sdk? https://github.com/mozilla/glean/#contact"
+            )
+            sys.exit(1)
+
+        command = ["cargo", "build", "--package", "glean-ffi"]
+        if buildvariant != "debug":
+            command.append(f"--{buildvariant}")
+
+        subprocess.run(command, cwd=SRC_ROOT)
+        shutil.copyfile(
+            shared_object_build_dir / buildvariant / shared_object,
+            PYTHON_ROOT / "glean" / shared_object,
+        )
+
+        shutil.copyfile(
+            PYTHON_ROOT.parent / "metrics.yaml", PYTHON_ROOT / "glean" / "metrics.yaml"
+        )
+        shutil.copyfile(
+            PYTHON_ROOT.parent / "pings.yaml", PYTHON_ROOT / "glean" / "pings.yaml"
+        )
+
+        return _build.run(self)
+
+
 setup(
     author="The Glean Team",
     author_email="glean-team@mozilla.com",
@@ -145,12 +168,25 @@ def finalize_options(self):
     keywords="glean",
     name="glean-sdk",
     version=version,
-    packages=find_packages(include=["glean", "glean.*"]),
+    packages=[
+        "glean",
+        "glean._subprocess",
+        "glean.metrics",
+        "glean.net",
+        "glean.testing",
+    ],
+    package_dir={
+        "glean": FROM_TOP / "glean",
+        "glean._subprocess": FROM_TOP / "glean" / "_subprocess",
+        "glean.metrics": FROM_TOP / "glean" / "metrics",
+        "glean.net": FROM_TOP / "glean" / "net",
+        "glean.testing": FROM_TOP / "glean" / "testing",
+    },
     setup_requires=setup_requirements,
-    cffi_modules=["ffi_build.py:ffibuilder"],
+    cffi_modules=[str(PYTHON_ROOT / "ffi_build.py:ffibuilder")],
     url="https://github.com/mozilla/glean",
     zip_safe=False,
     package_data={"glean": [shared_object, "metrics.yaml", "pings.yaml"]},
     distclass=BinaryDistribution,
-    cmdclass={"install": InstallPlatlib, "bdist_wheel": bdist_wheel},
+    cmdclass={"install": InstallPlatlib, "bdist_wheel": bdist_wheel, "build": build},
 )

setup.py

@@ -0,0 +1,16 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+A basic top-level setup.py script that delegates to the real one in
+glean-core/python/setup.py
+
+This is used to generate the source package for glean_sdk on PyPI.
+"""
+
+from pathlib import Path
+import sys
+
+sys.path.insert(0, str((Path(__file__).parent / "glean-core" / "python").resolve()))
+from setup import *