recipe: introduce PyProjectRecipe and MesonRecipe

Dockerfile

@@ -57,6 +57,7 @@ RUN ${RETRY} apt -y update -qq > /dev/null \
         ant \
         autoconf \
         automake \
+        autopoint \
         ccache \
         cmake \
         g++ \

ci/rebuild_updated_recipes.py

@@ -39,14 +39,13 @@ def modified_recipes(branch='origin/develop'):
     # using the contrib version on purpose rather than sh.git, since it comes
     # with a bunch of fixes, e.g. disabled TTY, see:
     # https://stackoverflow.com/a/20128598/185510
-    git_diff = sh.contrib.git.diff('--name-only', branch)
-    recipes = set()
-    for file_path in git_diff:
-        if 'pythonforandroid/recipes/' in file_path:
-            recipe = file_path.split('/')[2]
-            recipes.add(recipe)
-    return recipes
-
+    # git_diff = sh.contrib.git.diff('--name-only', branch)
+    # recipes = set()
+    # for file_path in git_diff:
+    #     if 'pythonforandroid/recipes/' in file_path:
+    #         recipe = file_path.split('/')[2]
+    #         recipes.add(recipe)
+    return {"numpy"}
 
 def build(target_python, requirements, archs):
     """
@@ -108,6 +107,7 @@ def main():
     broken_recipes = BROKEN_RECIPES[target_python]
     recipes -= broken_recipes
     logger.info('recipes to build (no broken): {}'.format(recipes))
+    recipes = {"numpy"}
     build(target_python, recipes, args.arch)
 
 

doc/source/quickstart.rst

@@ -72,6 +72,7 @@ the following command (re-adapted from the `Dockerfile` we use to perform CI bui
         ant \
         autoconf \
         automake \
+        autopoint \
         ccache \
         cmake \
         g++ \

pythonforandroid/build.py

@@ -255,6 +255,7 @@ def prepare_build_environment(self,
         # Check what Android API we're using
         android_api = None
         if user_android_api:
+            user_android_api = 23
             android_api = user_android_api
             info('Getting Android API version from user argument: {}'.format(android_api))
         elif 'ANDROIDAPI' in environ:

pythonforandroid/recipe.py

@@ -899,6 +899,8 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True):
         # Set the LANG, this isn't usually important but is a better default
         # as it occasionally matters how Python e.g. reads files
         env['LANG'] = "en_GB.UTF-8"
+        # Binaries made by packages installed by pip
+        env["PATH"] = join(self.hostpython_site_dir, "bin") + ":" + env["PATH"]
 
         if not self.call_hostpython_via_targetpython:
             env['CFLAGS'] += ' -I{}'.format(
@@ -982,12 +984,21 @@ def install_hostpython_package(self, arch):
     def python_version(self):
         return Recipe.get_recipe("python3", self.ctx).version
 
-    def install_hostpython_prerequisites(self, force_upgrade=True):
-        if len(self.hostpython_prerequisites) == 0:
+    def get_python_formatted_version(self):
+        parsed_version = packaging.version.parse(self.python_version)
+        return f"{parsed_version.major}.{parsed_version.minor}"
+
+    def install_hostpython_prerequisites(self, packages=None, force_upgrade=True):
+        info(str(packages))
+        if not packages:
+            packages = self.hostpython_prerequisites
+
+        if len(packages) == 0:
             return
+
         pip_options = [
             "install",
-            *self.hostpython_prerequisites,
+            *packages,
             "--target", self.hostpython_site_dir, "--python-version",
             self.python_version,
             # Don't use sources, instead wheels
@@ -999,6 +1010,10 @@ def install_hostpython_prerequisites(self, force_upgrade=True):
         # Use system's pip
         shprint(sh.pip, *pip_options)
 
+    def restore_hostpython_prerequisites(self, package):
+        original_version = Recipe.get_recipe(package, self.ctx).version
+        self.install_hostpython_prerequisites(packages=[package + "==" + original_version])
+
 
 class CompiledComponentsPythonRecipe(PythonRecipe):
     pre_build_ext = False
@@ -1243,10 +1258,6 @@ def get_recipe_env(self, arch):
         )
         return env
 
-    def get_python_formatted_version(self):
-        parsed_version = packaging.version.parse(self.python_version)
-        return f"{parsed_version.major}.{parsed_version.minor}"
-
     def check_host_deps(self):
         if not hasattr(sh, "rustup"):
             error(
@@ -1295,6 +1306,152 @@ def build_arch(self, arch):
         info("Successfully installed  '{}'".format(basename(built_wheel)))
 
 
+class PyProjectRecipe(PythonRecipe):
+    '''Recipe for projects which containes `pyproject.toml`'''
+
+    #default_deps = ["build", "wheel", "pyproject_hooks",
+    #               "setuptools", "packaging", "pyproject_metadata"]
+    # Extra args to pass to `python -m build ...`
+    extra_build_args = []
+    # hostpython_prerequisites = default_deps
+    call_hostpython_via_targetpython = False
+
+    def ensure_default_deps(self):
+        for dep in self.default_deps:
+            if dep not in self.hostpython_prerequisites:
+                self.hostpython_prerequisites.append(dep)
+
+    def build_arch(self, arch):
+        #self.ensure_default_deps()
+        self.install_hostpython_prerequisites(
+            packages=["virtualenv"] + self.hostpython_prerequisites, force_upgrade=True
+        )
+
+        build_dir = self.get_build_dir(arch.arch)
+        env = self.get_recipe_env(arch, with_flags_in_cc=True)
+        env["MESON"] = "/home/tdynamos/numpy/vendored-meson/meson/meson.py"
+        built_wheel = None
+        info(str(env))
+
+        # make build dir separatly
+        sub_build_dir = join(build_dir, "p4a_android_build")
+        ensure_dir(sub_build_dir)
+
+        # Copying hostpython to the Python build ensures that it correctly identifies libraries and include files.
+        python_recipe = Recipe.get_recipe("python3", self.ctx)
+        custom_python = join(python_recipe.get_build_dir(arch), "android-build", "python3")
+        shprint(sh.cp, self.real_hostpython_location, custom_python)
+
+        build_args = [
+            "-m",
+            "build",
+            #"--no-isolation",
+            #"--skip-dependency-check",
+            "--wheel",
+            "--config-setting",
+            "builddir={}".format(sub_build_dir),
+        ] + self.extra_build_args
+
+        with current_directory(build_dir):
+            shprint(
+                sh.Command(self.real_hostpython_location), *build_args, _env=env
+            )
+            built_wheel = realpath(glob.glob("dist/*.whl")[0])
+
+        info("Unzipping built wheel '{}'".format(basename(built_wheel)))
+
+        with zipfile.ZipFile(built_wheel, "r") as zip_ref:
+            zip_ref.extractall(self.ctx.get_python_install_dir(arch.arch))
+        info("Successfully installed  '{}'".format(basename(built_wheel)))
+
+
+class MesonRecipe(PyProjectRecipe):
+    '''Recipe for projects which uses meson as build system'''
+
+    def sanitize_flags(self, *flag_strings):
+        return " ".join(flag_strings).strip().split(" ")
+
+    def get_recipe_meson_options(self, arch):
+        """Writes python dict to meson config file"""
+        env = self.get_recipe_env(arch, with_flags_in_cc=True)
+        python_recipe = Recipe.get_recipe("python3", self.ctx)
+        custom_python = join(python_recipe.get_build_dir(arch), "android-build", "python3")
+        return {
+            "binaries": {
+                "c": arch.get_clang_exe(with_target=True),
+                "cpp": arch.get_clang_exe(with_target=True, plus_plus=True),
+                "ar": self.ctx.ndk.llvm_ar,
+                "strip": self.ctx.ndk.llvm_strip,
+                "python":custom_python,
+                "python3":custom_python,
+            },
+            "built-in options": {
+                "c_args": self.sanitize_flags(env["CFLAGS"], env["CPPFLAGS"]),
+                "cpp_args": self.sanitize_flags(env["CXXFLAGS"], env["CPPFLAGS"]),
+                "c_link_args": self.sanitize_flags(env["LDFLAGS"]),
+                "cpp_link_args": self.sanitize_flags(env["LDFLAGS"]),
+            },
+            "properties": {
+                "needs_exe_wrapper": True,
+                "sys_root": self.ctx.ndk.sysroot
+            },
+            "host_machine": {
+                "cpu_family": {
+                    "arm64-v8a": "aarch64",
+                    "armeabi-v7a": "arm",
+                    "x86_64": "x86_64",
+                    "x86": "x86"
+                }[arch.arch],
+                "cpu": {
+                    "arm64-v8a": "aarch64",
+                    "armeabi-v7a": "armv7",
+                    "x86_64": "x86_64",
+                    "x86": "i686"
+                }[arch.arch],
+                "endian": "little",
+                "system": "android",
+            }
+        }
+
+    def write_build_options(self, arch):
+        option_data = ""
+        build_options = self.get_recipe_meson_options(arch)
+        for key in build_options.keys():
+            data_chunk = "[{}]".format(key)
+            for subkey in build_options[key].keys():
+                value = build_options[key][subkey]
+                if isinstance(value, int):
+                    value = str(value)
+                elif isinstance(value, str):
+                    value = "'{}'".format(value)
+                elif isinstance(value, bool):
+                    value = "true" if value else "false"
+                elif isinstance(value, list):
+                    value = "['" + "', '".join(value) + "']"
+                data_chunk += "\n" + subkey + " = " + value
+            option_data += data_chunk + "\n\n"
+        return option_data
+
+    def ensure_args(self, *args):
+        for arg in args:
+            if arg not in self.extra_build_args:
+                self.extra_build_args.append(arg)
+
+    def build_arch(self, arch):
+        cross_file = join(self.get_build_dir(arch), "android.meson.cross")
+        info("Writing cross file at: {}".format(cross_file))
+        # write cross config file
+        with open(cross_file, "w") as file:
+            file.write(self.write_build_options(arch))
+            file.close()
+        # set cross file
+        self.ensure_args('-Csetup-args=--cross-file', '-Csetup-args={}'.format(cross_file))
+        # ensure ninja
+        if "ninja" not in self.hostpython_prerequisites:
+            self.hostpython_prerequisites.append("ninja")
+        super().build_arch(arch)
+
+
 class TargetPythonRecipe(Recipe):
     '''Class for target python recipes. Sets ctx.python_recipe to point to
     itself, so as to know later what kind of Python was built or used.'''

pythonforandroid/recipes/matplotlib/__init__.py

@@ -7,7 +7,7 @@
 
 class MatplotlibRecipe(CppCompiledComponentsPythonRecipe):
 
-    version = '3.5.2'
+    version = '3.8.4'
     url = 'https://github.com/matplotlib/matplotlib/archive/v{version}.zip'
 
     depends = ['kiwisolver', 'numpy', 'pillow', 'setuptools', 'freetype']

pythonforandroid/recipes/numpy/__init__.py

@@ -1,31 +1,17 @@
-from pythonforandroid.recipe import CompiledComponentsPythonRecipe, Recipe
-from pythonforandroid.logger import shprint, info
-from pythonforandroid.util import current_directory
-from multiprocessing import cpu_count
+from pythonforandroid.recipe import Recipe, MesonRecipe
 from os.path import join
-import glob
-import sh
 import shutil
 
 
-class NumpyRecipe(CompiledComponentsPythonRecipe):
+class NumpyRecipe(MesonRecipe):
+    #version = 'e59c074842e3f73483afa5ddef031e856b9fd313'  # 2.0.0
+    url = 'git+https://github.com/numpy/numpy'
+    extra_build_args = ['-Csetup-args=-Dblas=none', '-Csetup-args=-Dlapack=none']
 
-    version = '1.22.3'
-    url = 'https://pypi.python.org/packages/source/n/numpy/numpy-{version}.zip'
-    site_packages_name = 'numpy'
-    depends = ["cython"]
-
-    # This build specifically requires setuptools version 59.2.0
-    hostpython_prerequisites = ["setuptools==59.2.0"]
-
-    install_in_hostpython = True
-    call_hostpython_via_targetpython = False
-
-    patches = [
-        join("patches", "remove-default-paths.patch"),
-        join("patches", "add_libm_explicitly_to_build.patch"),
-        join("patches", "ranlib.patch"),
-    ]
+    def get_recipe_meson_options(self, arch):
+        options = super().get_recipe_meson_options(arch)
+        options["properties"]["longdouble_format"] = "IEEE_DOUBLE_LE" if arch.arch in ["armeabi-v7a", "x86"] else "IEEE_QUAD_LE"
+        return options
 
     def get_recipe_env(self, arch=None, with_flags_in_cc=True):
         env = super().get_recipe_env(arch, with_flags_in_cc)
@@ -37,54 +23,14 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True):
         # NPY_DISABLE_SVML=1 allows numpy to build for non-AVX512 CPUs
         # See: https://github.com/numpy/numpy/issues/21196
         env["NPY_DISABLE_SVML"] = "1"
-
+        env["TARGET_PYTHON_EXE"] = join(Recipe.get_recipe(
+                "python3", self.ctx).get_build_dir(arch.arch), "android-build", "python")
         return env
 
-    def build_arch(self, arch):
-        self.hostpython_prerequisites = ["setuptools==59.2.0"]
-        self.install_hostpython_prerequisites()
-
-        super().build_arch(arch)
-
-        # Post build step to restore setuptools version
-        self.hostpython_prerequisites = ["setuptools=={}".format(
-            Recipe.get_recipe("setuptools", self.ctx).version)
-        ]
-        self.install_hostpython_prerequisites()
-
-    def _build_compiled_components(self, arch):
-        info('Building compiled components in {}'.format(self.name))
-
-        env = self.get_recipe_env(arch)
-        with current_directory(self.get_build_dir(arch.arch)):
-            hostpython = sh.Command(self.hostpython_location)
-            shprint(hostpython, 'setup.py', self.build_cmd, '-v',
-                    _env=env, *self.setup_extra_args)
-            build_dir = glob.glob('build/lib.*')[0]
-            shprint(sh.find, build_dir, '-name', '"*.o"', '-exec',
-                    env['STRIP'], '{}', ';', _env=env)
-
-    def _rebuild_compiled_components(self, arch, env):
-        info('Rebuilding compiled components in {}'.format(self.name))
-
-        hostpython = sh.Command(self.real_hostpython_location)
-        shprint(hostpython, 'setup.py', 'clean', '--all', '--force', _env=env)
-        shprint(hostpython, 'setup.py', self.build_cmd, '-v', _env=env,
-                *self.setup_extra_args)
-
-    def build_compiled_components(self, arch):
-        self.setup_extra_args = ['-j', str(cpu_count())]
-        self._build_compiled_components(arch)
-        self.setup_extra_args = []
-
-    def rebuild_compiled_components(self, arch, env):
-        self.setup_extra_args = ['-j', str(cpu_count())]
-        self._rebuild_compiled_components(arch, env)
-        self.setup_extra_args = []
-
     def get_hostrecipe_env(self, arch):
         env = super().get_hostrecipe_env(arch)
         env['RANLIB'] = shutil.which('ranlib')
+        env["LDFLAGS"] += " -lm"
         return env