From 8110fafabc64192de627807da3f4425779c37969 Mon Sep 17 00:00:00 2001 From: Ansh Dadwal Date: Sun, 31 Mar 2024 20:05:12 +0530 Subject: [PATCH] recipes: Introduce `RustCompiledComponentsRecipe`, add `pydantic-core` and update `cryptography` (#2962) --- Makefile | 4 + pythonforandroid/recipe.py | 176 +++++++++++++++++- .../recipes/cryptography/__init__.py | 25 +-- pythonforandroid/recipes/numpy/__init__.py | 20 +- .../recipes/pydantic-core/__init__.py | 12 ++ pythonforandroid/recipes/pydantic/__init__.py | 12 -- .../recipes/setuptools/__init__.py | 2 +- 7 files changed, 221 insertions(+), 30 deletions(-) create mode 100644 pythonforandroid/recipes/pydantic-core/__init__.py delete mode 100644 pythonforandroid/recipes/pydantic/__init__.py diff --git a/Makefile b/Makefile index 8ea28c5973..19a2379ab2 100644 --- a/Makefile +++ b/Makefile @@ -23,8 +23,12 @@ virtualenv: $(VIRTUAL_ENV) test: $(TOX) -- tests/ --ignore tests/test_pythonpackage.py +# Also install and configure rust rebuild_updated_recipes: virtualenv . $(ACTIVATE) && \ + curl https://sh.rustup.rs -sSf | sh -s -- -y && \ + . "$(HOME)/.cargo/env" && \ + rustup target list && \ ANDROID_SDK_HOME=$(ANDROID_SDK_HOME) ANDROID_NDK_HOME=$(ANDROID_NDK_HOME) \ $(PYTHON) ci/rebuild_updated_recipes.py $(REBUILD_UPDATED_RECIPES_EXTRA_ARGS) diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index bbd61e603d..d70571ad08 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -1,4 +1,4 @@ -from os.path import basename, dirname, exists, isdir, isfile, join, realpath, split +from os.path import basename, dirname, exists, isdir, isfile, join, realpath, split, sep import glob import hashlib @@ -7,6 +7,7 @@ import sh import shutil import fnmatch +import zipfile import urllib.request from urllib.request import urlretrieve from os import listdir, unlink, environ, curdir, walk @@ -20,7 +21,7 @@ import packaging.version from pythonforandroid.logger import ( - logger, info, warning, debug, shprint, info_main) + logger, info, warning, debug, shprint, info_main, error) from pythonforandroid.util import ( current_directory, ensure_dir, BuildInterruptingException, rmdir, move, touch) @@ -175,6 +176,7 @@ def download_file(self, url, target, cwd=None): """ if not url: return + info('Downloading {} from {}'.format(self.name, url)) if cwd: @@ -458,7 +460,6 @@ def unpack(self, arch): # apparently happens sometimes with # github zips pass - import zipfile fileh = zipfile.ZipFile(extraction_filename, 'r') root_directory = fileh.filelist[0].filename.split('/')[0] if root_directory != basename(directory_name): @@ -837,6 +838,9 @@ class PythonRecipe(Recipe): on python2 or python3 which can break the dependency graph ''' + hostpython_prerequisites = [] + '''List of hostpython packages required to build a recipe''' + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -930,6 +934,7 @@ def should_build(self, arch): def build_arch(self, arch): '''Install the Python module by calling setup.py install with the target Python dir.''' + self.install_hostpython_prerequisites() super().build_arch(arch) self.install_python_package(arch) @@ -958,9 +963,13 @@ def install_python_package(self, arch, name=None, env=None, is_dir=True): def get_hostrecipe_env(self, arch): env = environ.copy() - env['PYTHONPATH'] = join(dirname(self.real_hostpython_location), 'Lib', 'site-packages') + env['PYTHONPATH'] = self.hostpython_site_dir return env + @property + def hostpython_site_dir(self): + return join(dirname(self.real_hostpython_location), 'Lib', 'site-packages') + def install_hostpython_package(self, arch): env = self.get_hostrecipe_env(arch) real_hostpython = sh.Command(self.real_hostpython_location) @@ -969,6 +978,27 @@ def install_hostpython_package(self, arch): '--install-lib=Lib/site-packages', _env=env, *self.setup_extra_args) + @property + def python_version(self): + return Recipe.get_recipe("python3", self.ctx).version + + def install_hostpython_prerequisites(self, force_upgrade=True): + if len(self.hostpython_prerequisites) == 0: + return + pip_options = [ + "install", + *self.hostpython_prerequisites, + "--target", self.hostpython_site_dir, "--python-version", + self.python_version, + # Don't use sources, instead wheels + "--only-binary=:all:", + "--no-deps" + ] + if force_upgrade: + pip_options.append("--upgrade") + # Use system's pip + shprint(sh.pip, *pip_options) + class CompiledComponentsPythonRecipe(PythonRecipe): pre_build_ext = False @@ -1127,6 +1157,144 @@ def get_recipe_env(self, arch, with_flags_in_cc=True): return env +class RustCompiledComponentsRecipe(PythonRecipe): + # Rust toolchain codes + # https://doc.rust-lang.org/nightly/rustc/platform-support.html + RUST_ARCH_CODES = { + "arm64-v8a": "aarch64-linux-android", + "armeabi-v7a": "armv7-linux-androideabi", + "x86_64": "x86_64-linux-android", + "x86": "i686-linux-android", + } + + # Build python wheel using `maturin` instead + # of default `python -m build [...]` + use_maturin = False + + # Directory where to find built wheel + # For normal build: "dist/*.whl" + # For maturin: "target/wheels/*-linux_*.whl" + built_wheel_pattern = None + + call_hostpython_via_targetpython = False + + def __init__(self, *arg, **kwargs): + super().__init__(*arg, **kwargs) + self.append_deps_if_absent(["python3"]) + self.set_default_hostpython_deps() + if not self.built_wheel_pattern: + self.built_wheel_pattern = ( + "target/wheels/*-linux_*.whl" + if self.use_maturin + else "dist/*.whl" + ) + + def set_default_hostpython_deps(self): + if not self.use_maturin: + self.hostpython_prerequisites += ["build", "setuptools_rust", "wheel", "pyproject_hooks"] + else: + self.hostpython_prerequisites += ["maturin"] + + def append_deps_if_absent(self, deps): + for dep in deps: + if dep not in self.depends: + self.depends.append(dep) + + def get_recipe_env(self, arch): + env = super().get_recipe_env(arch) + + # Set rust build target + build_target = self.RUST_ARCH_CODES[arch.arch] + cargo_linker_name = "CARGO_TARGET_{}_LINKER".format( + build_target.upper().replace("-", "_") + ) + env["CARGO_BUILD_TARGET"] = build_target + env[cargo_linker_name] = join( + self.ctx.ndk.llvm_prebuilt_dir, + "bin", + "{}{}-clang".format( + # NDK's Clang format + build_target.replace("7", "7a") + if build_target.startswith("armv7") + else build_target, + self.ctx.ndk_api, + ), + ) + realpython_dir = Recipe.get_recipe("python3", self.ctx).get_build_dir(arch.arch) + + env["RUSTFLAGS"] = "-Clink-args=-L{} -L{}".format( + self.ctx.get_libs_dir(arch.arch), join(realpython_dir, "android-build") + ) + + env["PYO3_CROSS_LIB_DIR"] = realpath(glob.glob(join( + realpython_dir, "android-build", "build", + "lib.linux-*-{}/".format(self.get_python_formatted_version()), + ))[0]) + + info_main("Ensuring rust build toolchain") + shprint(sh.rustup, "target", "add", build_target) + + # Add host python to PATH + env["PATH"] = ("{hostpython_dir}:{old_path}").format( + hostpython_dir=Recipe.get_recipe( + "hostpython3", self.ctx + ).get_path_to_python(), + old_path=env["PATH"], + ) + return env + + def get_python_formatted_version(self): + parsed_version = packaging.version.parse(self.python_version) + return f"{parsed_version.major}.{parsed_version.minor}" + + def check_host_deps(self): + if not hasattr(sh, "rustup"): + error( + "`rustup` was not found on host system." + "Please install it using :" + "\n`curl https://sh.rustup.rs -sSf | sh`\n" + ) + exit(1) + + def build_arch(self, arch): + self.check_host_deps() + self.install_hostpython_prerequisites() + build_dir = self.get_build_dir(arch.arch) + env = self.get_recipe_env(arch) + built_wheel = None + + # Copy the exec with version info + hostpython_exec = join( + sep, + *self.hostpython_location.split(sep)[:-1], + "python{}".format(self.get_python_formatted_version()), + ) + shprint(sh.cp, self.hostpython_location, hostpython_exec) + + with current_directory(build_dir): + if self.use_maturin: + shprint( + sh.Command(join(self.hostpython_site_dir, "bin", "maturin")), + "build", "--interpreter", hostpython_exec, "--skip-auditwheel", + _env=env, + ) + else: + shprint( + sh.Command(hostpython_exec), + "-m", "build", "--no-isolation", "--skip-dependency-check", "--wheel", + _env=env, + ) + # Find the built wheel + built_wheel = realpath(glob.glob(self.built_wheel_pattern)[0]) + + info("Unzipping built wheel '{}'".format(basename(built_wheel))) + + # Unzip .whl file into site-packages + with zipfile.ZipFile(built_wheel, "r") as zip_ref: + zip_ref.extractall(self.ctx.get_python_install_dir(arch.arch)) + info("Successfully installed '{}'".format(basename(built_wheel))) + + class TargetPythonRecipe(Recipe): '''Class for target python recipes. Sets ctx.python_recipe to point to itself, so as to know later what kind of Python was built or used.''' diff --git a/pythonforandroid/recipes/cryptography/__init__.py b/pythonforandroid/recipes/cryptography/__init__.py index 182c745996..8e476b3129 100644 --- a/pythonforandroid/recipes/cryptography/__init__.py +++ b/pythonforandroid/recipes/cryptography/__init__.py @@ -1,21 +1,24 @@ -from pythonforandroid.recipe import CompiledComponentsPythonRecipe, Recipe +from pythonforandroid.recipe import RustCompiledComponentsRecipe +from os.path import join -class CryptographyRecipe(CompiledComponentsPythonRecipe): +class CryptographyRecipe(RustCompiledComponentsRecipe): + name = 'cryptography' - version = '2.8' - url = 'https://github.com/pyca/cryptography/archive/{version}.tar.gz' + version = '42.0.1' + url = 'https://github.com/pyca/cryptography/archive/refs/tags/{version}.tar.gz' depends = ['openssl', 'six', 'setuptools', 'cffi'] - call_hostpython_via_targetpython = False + # recipe built cffi does not work on apple M1 + hostpython_prerequisites = ["semantic_version", "cffi"] def get_recipe_env(self, arch): env = super().get_recipe_env(arch) - - openssl_recipe = Recipe.get_recipe('openssl', self.ctx) - env['CFLAGS'] += openssl_recipe.include_flags(arch) - env['LDFLAGS'] += openssl_recipe.link_dirs_flags(arch) - env['LIBS'] = openssl_recipe.link_libs_flags() - + openssl_build_dir = self.get_recipe('openssl', self.ctx).get_build_dir(arch.arch) + build_target = self.RUST_ARCH_CODES[arch.arch].upper().replace("-", "_") + openssl_include = "{}_OPENSSL_INCLUDE_DIR".format(build_target) + openssl_libs = "{}_OPENSSL_LIB_DIR".format(build_target) + env[openssl_include] = join(openssl_build_dir, 'include') + env[openssl_libs] = join(openssl_build_dir) return env diff --git a/pythonforandroid/recipes/numpy/__init__.py b/pythonforandroid/recipes/numpy/__init__.py index 55a0279770..7e51eba66b 100644 --- a/pythonforandroid/recipes/numpy/__init__.py +++ b/pythonforandroid/recipes/numpy/__init__.py @@ -1,4 +1,4 @@ -from pythonforandroid.recipe import CompiledComponentsPythonRecipe +from pythonforandroid.recipe import CompiledComponentsPythonRecipe, Recipe from pythonforandroid.logger import shprint, info from pythonforandroid.util import current_directory from multiprocessing import cpu_count @@ -13,7 +13,11 @@ class NumpyRecipe(CompiledComponentsPythonRecipe): version = '1.22.3' url = 'https://pypi.python.org/packages/source/n/numpy/numpy-{version}.zip' site_packages_name = 'numpy' - depends = ['setuptools', 'cython'] + depends = ["cython"] + + # This build specifically requires setuptools version 59.2.0 + hostpython_prerequisites = ["setuptools==59.2.0"] + install_in_hostpython = True call_hostpython_via_targetpython = False @@ -36,6 +40,18 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): return env + def build_arch(self, arch): + self.hostpython_prerequisites = ["setuptools==59.2.0"] + self.install_hostpython_prerequisites() + + super().build_arch(arch) + + # Post build step to restore setuptools version + self.hostpython_prerequisites = ["setuptools=={}".format( + Recipe.get_recipe("setuptools", self.ctx).version) + ] + self.install_hostpython_prerequisites() + def _build_compiled_components(self, arch): info('Building compiled components in {}'.format(self.name)) diff --git a/pythonforandroid/recipes/pydantic-core/__init__.py b/pythonforandroid/recipes/pydantic-core/__init__.py new file mode 100644 index 0000000000..8702dfc002 --- /dev/null +++ b/pythonforandroid/recipes/pydantic-core/__init__.py @@ -0,0 +1,12 @@ +from pythonforandroid.recipe import RustCompiledComponentsRecipe + + +class PydanticcoreRecipe(RustCompiledComponentsRecipe): + version = "2.16.1" + url = "https://github.com/pydantic/pydantic-core/archive/refs/tags/v{version}.tar.gz" + use_maturin = True + hostpython_prerequisites = ["typing_extensions"] + site_packages_name = "pydantic_core" + + +recipe = PydanticcoreRecipe() diff --git a/pythonforandroid/recipes/pydantic/__init__.py b/pythonforandroid/recipes/pydantic/__init__.py deleted file mode 100644 index 16e61e1b61..0000000000 --- a/pythonforandroid/recipes/pydantic/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -from pythonforandroid.recipe import PythonRecipe - - -class PydanticRecipe(PythonRecipe): - version = '1.10.4' - url = 'https://github.com/pydantic/pydantic/archive/refs/tags/v{version}.zip' - depends = ['setuptools'] - python_depends = ['Cython', 'devtools', 'email-validator', 'typing-extensions', 'python-dotenv'] - call_hostpython_via_targetpython = False - - -recipe = PydanticRecipe() diff --git a/pythonforandroid/recipes/setuptools/__init__.py b/pythonforandroid/recipes/setuptools/__init__.py index 8190f8efd1..02b205023d 100644 --- a/pythonforandroid/recipes/setuptools/__init__.py +++ b/pythonforandroid/recipes/setuptools/__init__.py @@ -2,7 +2,7 @@ class SetuptoolsRecipe(PythonRecipe): - version = '51.3.3' + version = '69.2.0' url = 'https://pypi.python.org/packages/source/s/setuptools/setuptools-{version}.tar.gz' call_hostpython_via_targetpython = False install_in_hostpython = True