From e0ae0775eb1c978b55fb6af3ffc36af8b8e60052 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 8 Sep 2020 15:42:10 -0400 Subject: [PATCH 1/5] 1660120: Make Python bindings installable from source --- .circleci/config.yml | 22 +++++++ .gitignore | 4 ++ Makefile | 4 +- bin/prepare-release.sh | 8 +++ glean-core/python/ffi_build.py | 8 ++- glean-core/python/requirements_dev.txt | 1 - glean-core/python/setup.py | 81 +++++++++++++++++--------- setup.py | 16 +++++ 8 files changed, 114 insertions(+), 30 deletions(-) create mode 100644 setup.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 9fb11af062..d2ad1d0e9c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -815,6 +815,24 @@ jobs: root: build/ paths: docs/python + pypi-source-release: + docker: + - image: circleci/python:3.8.2 + steps: + - install-rustup + - setup-rust-toolchain + - checkout + - run: + name: Setup default Python version + command: | + echo "export PATH=/opt/python/cp38-cp38/bin:$PATH" >> $BASH_ENV + - run: + name: Build Python extension + command: | + make python-setup + .venv3.8/bin/python3 setup.py sdist + .venv3.8/bin/python3 -m twine upload dist/* + pypi-linux-release: docker: # The official docker image for building manylinux1 wheels @@ -1135,6 +1153,10 @@ workflows: jobs: - Python 3_8 tests: filters: *release-filters + - pypi-source-release: + requires: + - Python 3_8 tests + filters: *release-filters - pypi-linux-release: requires: - Python 3_8 tests diff --git a/.gitignore b/.gitignore index 5b79c0707b..b404822f0c 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,10 @@ Carthage .DS_Store *.dSYM +# Python stuff +*.egg-info +dist/ + # C# stuff *.suo *.user diff --git a/Makefile b/Makefile index e9b66fdedd..dd312d19e0 100644 --- a/Makefile +++ b/Makefile @@ -48,8 +48,8 @@ build-swift: ## Build all Swift code build-apk: build-kotlin ## Build an apk of the Glean sample app ./gradlew glean-sample-app:build -build-python: python-setup build-rust ## Build the Python bindings - $(GLEAN_PYENV)/bin/python3 glean-core/python/setup.py install +build-python: python-setup ## Build the Python bindings + $(GLEAN_PYENV)/bin/python3 glean-core/python/setup.py build install build-csharp: ## Build the C# bindings dotnet build glean-core/csharp/csharp.sln diff --git a/bin/prepare-release.sh b/bin/prepare-release.sh index 5b24a73390..fba0208bc5 100755 --- a/bin/prepare-release.sh +++ b/bin/prepare-release.sh @@ -120,6 +120,14 @@ run $SED -i.bak -E \ "${WORKSPACE_ROOT}/${FILE}" run rm "${WORKSPACE_ROOT}/${FILE}.bak" +# Update the glean-python version + +FILE=glean-core/python/setup.py +run $SED -i.bak -E \ + -e "s/^version = \"[0-9a-z.-]+\"/version = \"${NEW_VERSION}\"/" \ + "${WORKSPACE_ROOT}/${FILE}" +run rm "${WORKSPACE_ROOT}/${FILE}.bak" + ### Update Cargo.lock cargo update -p glean-core -p glean-ffi diff --git a/glean-core/python/ffi_build.py b/glean-core/python/ffi_build.py index 94e64df54b..d30e88bf21 100644 --- a/glean-core/python/ffi_build.py +++ b/glean-core/python/ffi_build.py @@ -12,9 +12,15 @@ """ +from pathlib import Path + + import cffi +ROOT = Path(__file__).parent.absolute() + + def _load_header(path: str) -> str: """ Load a C header file and convert it to something parseable by cffi. @@ -28,7 +34,7 @@ def _load_header(path: str) -> str: ffibuilder = cffi.FFI() ffibuilder.set_source("glean._glean_ffi", None) -ffibuilder.cdef(_load_header("../ffi/glean.h")) +ffibuilder.cdef(_load_header(ROOT.parent / "ffi" / "glean.h")) if __name__ == "__main__": diff --git a/glean-core/python/requirements_dev.txt b/glean-core/python/requirements_dev.txt index a7155d181c..f8aa478b2a 100644 --- a/glean-core/python/requirements_dev.txt +++ b/glean-core/python/requirements_dev.txt @@ -11,6 +11,5 @@ pip pytest-localserver==0.5.0 pytest-runner==5.2 pytest==6.0.1 -toml==0.10.1 twine==3.2.0 wheel==0.34.2 diff --git a/glean-core/python/setup.py b/glean-core/python/setup.py index ec0bc598cb..ba2dc8e840 100644 --- a/glean-core/python/setup.py +++ b/glean-core/python/setup.py @@ -4,8 +4,10 @@ """The setup script.""" +from distutils.command.build import build as _build import os import shutil +import subprocess import sys from setuptools import setup, Distribution, find_packages @@ -30,21 +32,24 @@ sys.exit(1) from pathlib import Path # noqa -import toml # noqa +# Path to the directory containing this file ROOT = Path(__file__).parent.absolute() -os.chdir(str(ROOT)) +# Relative path to this directory from cwd. +FROM_TOP = ROOT.relative_to(Path.cwd()) -with (ROOT.parent.parent / "README.md").open() as readme_file: +# Path to the root of the git checkout +GIT_ROOT = ROOT.parents[1] + +with (GIT_ROOT / "README.md").open() as readme_file: readme = readme_file.read() -with (ROOT.parent.parent / "CHANGELOG.md").open() as history_file: +with (GIT_ROOT / "CHANGELOG.md").open() as history_file: history = history_file.read() -with (ROOT.parent / "Cargo.toml").open() as cargo: - parsed_toml = toml.load(cargo) - version = parsed_toml["package"]["version"] +# glean version. Automatically updated by the bin/prepare_release.sh script +version = "32.2.0" requirements = [ "cffi>=1", @@ -58,11 +63,11 @@ buildvariant = os.environ.get("GLEAN_BUILD_VARIANT", "debug") if mingw_arch == "i686": - shared_object_build_dir = "../../target/i686-pc-windows-gnu" + shared_object_build_dir = GIT_ROOT / "target" / "i686-pc-windows-gnu" elif mingw_arch == "x86_64": - shared_object_build_dir = "../../target/x86_64-pc-windows-gnu" + shared_object_build_dir = GIT_ROOT / "target" / "x86_64-pc-windows-gnu" else: - shared_object_build_dir = "../../target" + shared_object_build_dir = GIT_ROOT / "target" if platform == "linux": @@ -76,19 +81,6 @@ else: raise ValueError(f"The platform {sys.platform} is not supported.") -shared_object_path = f"{shared_object_build_dir}/{buildvariant}/{shared_object}" - -shutil.copyfile("../metrics.yaml", "glean/metrics.yaml") -shutil.copyfile("../pings.yaml", "glean/pings.yaml") -# When running inside of `requirements-builder`, the Rust shared object may not -# yet exist, so ignore the exception when trying to copy it. Under normal -# circumstances, this will still show up as an error when running the `build` -# command as a missing `package_data` file. -try: - shutil.copyfile(shared_object_path, "glean/" + shared_object) -except FileNotFoundError: - pass - class BinaryDistribution(Distribution): def is_pure(self): @@ -126,6 +118,30 @@ def finalize_options(self): self.install_lib = self.install_platlib +class build(_build): + def run(self): + try: + subprocess.run(["cargo"]) + except subprocess.CalledProcessError: + print("glean_sdk requires 'cargo' to build its Rust extension.") + sys.exit(1) + + command = ["cargo", "build", "--all"] + if buildvariant != "debug": + command.append(f"--{buildvariant}") + + subprocess.run(command, cwd=GIT_ROOT) + shutil.copyfile( + shared_object_build_dir / buildvariant / shared_object, + ROOT / "glean" / shared_object, + ) + + shutil.copyfile(ROOT.parent / "metrics.yaml", ROOT / "glean" / "metrics.yaml") + shutil.copyfile(ROOT.parent / "pings.yaml", ROOT / "glean" / "pings.yaml") + + return _build.run(self) + + setup( author="The Glean Team", author_email="glean-team@mozilla.com", @@ -145,12 +161,25 @@ def finalize_options(self): keywords="glean", name="glean-sdk", version=version, - packages=find_packages(include=["glean", "glean.*"]), + packages=[ + "glean", + "glean._subprocess", + "glean.metrics", + "glean.net", + "glean.testing", + ], + package_dir={ + "glean": FROM_TOP / "glean", + "glean._subprocess": FROM_TOP / "glean" / "_subprocess", + "glean.metrics": FROM_TOP / "glean" / "metrics", + "glean.net": FROM_TOP / "glean" / "net", + "glean.testing": FROM_TOP / "glean" / "testing", + }, setup_requires=setup_requirements, - cffi_modules=["ffi_build.py:ffibuilder"], + cffi_modules=[str(ROOT / "ffi_build.py:ffibuilder")], url="https://github.com/mozilla/glean", zip_safe=False, package_data={"glean": [shared_object, "metrics.yaml", "pings.yaml"]}, distclass=BinaryDistribution, - cmdclass={"install": InstallPlatlib, "bdist_wheel": bdist_wheel}, + cmdclass={"install": InstallPlatlib, "bdist_wheel": bdist_wheel, "build": build}, ) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000..798f4760b8 --- /dev/null +++ b/setup.py @@ -0,0 +1,16 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +A basic top-level setup.py script that delegates to the real one in +glean-core/python/setup.py + +This is used to generate the source package for glean_sdk on PyPI. +""" + +from pathlib import Path +import sys + +sys.path.insert(0, str((Path(__file__).parent / "glean-core" / "python").resolve())) +from setup import * From 5269edcd68b5fd805d503fd184c9f4ac15c14932 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 8 Sep 2020 15:44:41 -0400 Subject: [PATCH 2/5] Add CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21fd861ba2..5d0df9b98e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ * Track the size of the database file at startup ([#1141](https://github.com/mozilla/glean/pull/1141)). * iOS * Disabled code coverage in release builds ([#1195](https://github.com/mozilla/glean/issues/1195)). +* Python + * Glean now ships a source package to pip install on platforms where wheels aren't provided. # v32.3.0 (2020-08-27) From acb01007975b20bb40da110bf7f2bcfd3831b372 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 8 Sep 2020 15:52:15 -0400 Subject: [PATCH 3/5] Improve variable names --- glean-core/python/setup.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/glean-core/python/setup.py b/glean-core/python/setup.py index ba2dc8e840..53938ba6bf 100644 --- a/glean-core/python/setup.py +++ b/glean-core/python/setup.py @@ -34,18 +34,18 @@ from pathlib import Path # noqa # Path to the directory containing this file -ROOT = Path(__file__).parent.absolute() +PYTHON_ROOT = Path(__file__).parent.absolute() # Relative path to this directory from cwd. -FROM_TOP = ROOT.relative_to(Path.cwd()) +FROM_TOP = PYTHON_ROOT.relative_to(Path.cwd()) # Path to the root of the git checkout -GIT_ROOT = ROOT.parents[1] +SRC_ROOT = PYTHON_ROOT.parents[1] -with (GIT_ROOT / "README.md").open() as readme_file: +with (SRC_ROOT / "README.md").open() as readme_file: readme = readme_file.read() -with (GIT_ROOT / "CHANGELOG.md").open() as history_file: +with (SRC_ROOT / "CHANGELOG.md").open() as history_file: history = history_file.read() # glean version. Automatically updated by the bin/prepare_release.sh script @@ -63,11 +63,11 @@ buildvariant = os.environ.get("GLEAN_BUILD_VARIANT", "debug") if mingw_arch == "i686": - shared_object_build_dir = GIT_ROOT / "target" / "i686-pc-windows-gnu" + shared_object_build_dir = SRC_ROOT / "target" / "i686-pc-windows-gnu" elif mingw_arch == "x86_64": - shared_object_build_dir = GIT_ROOT / "target" / "x86_64-pc-windows-gnu" + shared_object_build_dir = SRC_ROOT / "target" / "x86_64-pc-windows-gnu" else: - shared_object_build_dir = GIT_ROOT / "target" + shared_object_build_dir = SRC_ROOT / "target" if platform == "linux": @@ -130,14 +130,18 @@ def run(self): if buildvariant != "debug": command.append(f"--{buildvariant}") - subprocess.run(command, cwd=GIT_ROOT) + subprocess.run(command, cwd=SRC_ROOT) shutil.copyfile( shared_object_build_dir / buildvariant / shared_object, - ROOT / "glean" / shared_object, + PYTHON_ROOT / "glean" / shared_object, ) - shutil.copyfile(ROOT.parent / "metrics.yaml", ROOT / "glean" / "metrics.yaml") - shutil.copyfile(ROOT.parent / "pings.yaml", ROOT / "glean" / "pings.yaml") + shutil.copyfile( + PYTHON_ROOT.parent / "metrics.yaml", PYTHON_ROOT / "glean" / "metrics.yaml" + ) + shutil.copyfile( + PYTHON_ROOT.parent / "pings.yaml", PYTHON_ROOT / "glean" / "pings.yaml" + ) return _build.run(self) @@ -176,7 +180,7 @@ def run(self): "glean.testing": FROM_TOP / "glean" / "testing", }, setup_requires=setup_requirements, - cffi_modules=[str(ROOT / "ffi_build.py:ffibuilder")], + cffi_modules=[str(PYTHON_ROOT / "ffi_build.py:ffibuilder")], url="https://github.com/mozilla/glean", zip_safe=False, package_data={"glean": [shared_object, "metrics.yaml", "pings.yaml"]}, From 8017f0c798e5696e418c6b0fb10d545c6e5a387a Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 9 Sep 2020 08:49:41 -0400 Subject: [PATCH 4/5] Address comments in PR --- glean-core/python/setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/glean-core/python/setup.py b/glean-core/python/setup.py index 53938ba6bf..0c9ffb980c 100644 --- a/glean-core/python/setup.py +++ b/glean-core/python/setup.py @@ -123,10 +123,13 @@ def run(self): try: subprocess.run(["cargo"]) except subprocess.CalledProcessError: - print("glean_sdk requires 'cargo' to build its Rust extension.") + print("Install Rust and Cargo through Rustup: https://rustup.rs/.") + print( + "Need help installing the glean_sdk? https://github.com/mozilla/glean/#contact" + ) sys.exit(1) - command = ["cargo", "build", "--all"] + command = ["cargo", "build", "--package", "glean-ffi"] if buildvariant != "debug": command.append(f"--{buildvariant}") From dd5fc299b7d07c12150e48a12007449ea6724d21 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 9 Sep 2020 08:49:47 -0400 Subject: [PATCH 5/5] Update docs --- docs/dev/python/setting-up-python-build-environment.md | 5 ++--- docs/user/adding-glean-to-your-project.md | 7 ++++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/dev/python/setting-up-python-build-environment.md b/docs/dev/python/setting-up-python-build-environment.md index 2253adee94..df22dab75b 100644 --- a/docs/dev/python/setting-up-python-build-environment.md +++ b/docs/dev/python/setting-up-python-build-environment.md @@ -75,12 +75,11 @@ By default, the `Makefile` installs the latest version available of each of Glea ### Manual method -First, rebuild the Rust core if any Rust changes were made: +Building the Python bindings also builds the Rust shared object for the Glean SDK core. ```bash - $ cargo build # If there were Rust changes $ cd glean-core/python - $ python setup.py install + $ python setup.py build install ``` ### Makefile method diff --git a/docs/user/adding-glean-to-your-project.md b/docs/user/adding-glean-to-your-project.md index d897268958..6e471bb184 100644 --- a/docs/user/adding-glean-to-your-project.md +++ b/docs/user/adding-glean-to-your-project.md @@ -120,15 +120,16 @@ For integration with the build system you can follow the [Carthage Quick Start s We recommend using a virtual environment for your work to isolate the dependencies for your project. There are many popular abstractions on top of virtual environments in the Python ecosystem which can help manage your project dependencies. The Glean SDK Python bindings currently have [prebuilt wheels on PyPI for Windows (i686 and x86_64), Linux (x86_64) and macOS (x86_64)](https://pypi.org/project/glean-sdk/#files). +For other platforms, the `glean_sdk` package will be built from source on your machine. +This requires that Cargo and Rust are already installed. +The easiest way to do this is through [rustup](https://rustup.rs/). -If you're running one of those platforms and have your virtual environment set up and activated, you can install the Glean SDK into it using: +Once you have your virtual environment set up and activated, you can install the Glean SDK into it using: ```bash $ python -m pip install glean_sdk ``` -If you are not on one of these platforms, you will need to build the Glean SDK Python bindings from source using [these instructions](../dev/python/setting-up-python-build-environment.html). - The Glean SDK Python bindings make extensive use of type annotations to catch type related errors at build time. We highly recommend adding [mypy](https://mypy-lang.org) to your continuous integration workflow to catch errors related to type mismatches early.