From 87d099f8094d5fb2c78729adfc6df9c68f68b450 Mon Sep 17 00:00:00 2001 From: opacam Date: Thu, 15 Aug 2019 11:19:04 +0200 Subject: [PATCH 01/72] [recipes] Fix compilation for regex recipe The error was: build/other_builds/hostpython3/desktop/hostpython3/Include/Python.h:39:19: fatal error: crypt.h: No such file or directory --- pythonforandroid/recipes/regex/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pythonforandroid/recipes/regex/__init__.py b/pythonforandroid/recipes/regex/__init__.py index 9533905303..7acc481ef6 100644 --- a/pythonforandroid/recipes/regex/__init__.py +++ b/pythonforandroid/recipes/regex/__init__.py @@ -7,6 +7,7 @@ class RegexRecipe(CompiledComponentsPythonRecipe): url = 'https://pypi.python.org/packages/d1/23/5fa829706ee1d4452552eb32e0bfc1039553e01f50a8754c6f7152e85c1b/regex-{version}.tar.gz' depends = ['setuptools'] + call_hostpython_via_targetpython = False recipe = RegexRecipe() From 200550890953f5b59c7c759beadf2b9f4fa885ce Mon Sep 17 00:00:00 2001 From: opacam Date: Thu, 15 Aug 2019 11:20:24 +0200 Subject: [PATCH 02/72] [recipes] Update regex's recipe --- pythonforandroid/recipes/regex/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pythonforandroid/recipes/regex/__init__.py b/pythonforandroid/recipes/regex/__init__.py index 7acc481ef6..6ac914845f 100644 --- a/pythonforandroid/recipes/regex/__init__.py +++ b/pythonforandroid/recipes/regex/__init__.py @@ -3,8 +3,8 @@ class RegexRecipe(CompiledComponentsPythonRecipe): name = 'regex' - version = '2017.07.28' - url = 'https://pypi.python.org/packages/d1/23/5fa829706ee1d4452552eb32e0bfc1039553e01f50a8754c6f7152e85c1b/regex-{version}.tar.gz' + version = '2019.06.08' + url = 'https://pypi.python.org/packages/source/r/regex/regex-{version}.tar.gz' # noqa depends = ['setuptools'] call_hostpython_via_targetpython = False From cb6035a16c016d8ba9f32be0781ccb35303b462d Mon Sep 17 00:00:00 2001 From: Gabriel Pettier Date: Sun, 18 Aug 2019 00:48:48 +0200 Subject: [PATCH 03/72] put the sdk manager command in a code block (#1956) to avoid quotes being replaced by fancy ones --- doc/source/quickstart.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 1f50579dde..74161d7d8e 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -125,10 +125,11 @@ First, install an API platform to target. **The recommended *target* API level is 27**, you can replace it with a different number but keep in mind other API versions are less well-tested and older devices are still supported down to the **recommended specified *minimum* -API/NDK API level 21**: +API/NDK API level 21**:: $SDK_DIR/tools/bin/sdkmanager "platforms;android-27" + Second, install the build-tools. You can use ``$SDK_DIR/tools/bin/sdkmanager --list`` to see all the possibilities, but 28.0.2 is the latest version at the time of writing:: From 982469a5015ce10fb0a178a8bf4113e64e251cf5 Mon Sep 17 00:00:00 2001 From: Alexander Taylor Date: Mon, 19 Aug 2019 21:55:53 +0100 Subject: [PATCH 04/72] Updated version number for develop branch following 2019.08.09 release (#1960) --- pythonforandroid/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonforandroid/__init__.py b/pythonforandroid/__init__.py index ee4d0cdbb6..db365eba80 100644 --- a/pythonforandroid/__init__.py +++ b/pythonforandroid/__init__.py @@ -1 +1 @@ -__version__ = '2019.08.09' +__version__ = '2019.08.09.1.dev0' From eb8437226bc53544f3f016c4965b912409f4ccbd Mon Sep 17 00:00:00 2001 From: Pol Canelles Date: Fri, 23 Aug 2019 21:28:47 +0200 Subject: [PATCH 05/72] Fix build for case-insensitive FS and add CI test for OSX (#1951) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [python] Fix build for case-insensitive FS :apple: It turns out that the generated python binary for some builds are named `python.exe` instead of `python`. This depends on the File System where the build happens. It will be named `python.exe` when the FS is case-insensitive (Mac OSX and Cygwin), but it will be named `python` when the FS is case-sensitive (most GNU Linux distributions). The proposed solution consists in make a copy of the generated python binary with a given name (python3 or python2) so this way To achieve this goal we refactor a little our `HostPythonRecipe`:   - add private property `HostPythonRecipe._exe_name` (the name of the python executable based on major version)   - add public property `HostPythonRecipe.python_exe` (the full path of the python executable)   - implement `HostPythonRecipe.should_build` And also it's affected the `GuestPythonRecipe`, because we need to use the generated python executable by `HostPythonRecipe`:   - add private property `GuestPythonRecipe._libpython` (python's library name with extension...hardcoded for now...)   - implement `GuestPythonRecipe.should_build`... to check the library instead of the executable so we avoid conflicts with case-insensitive FS We also need: - fix `PythonRecipe.real_hostpython_location` because the name of our host python executable will depend on major version   - fix python2 interpreter (fix-interpreter-version.patch) Note: the variation of the name of the python's executable is mentioned at python's build docs (https://github.com/python/cpython/blob/3.7/README.rst#build-instructions) Note: @TheSin- , ¡¡¡thanks for your debugging sessions!!! * [ci] Add Mac OSX CI's test & refactor android's NDK/SDK installation To do so we:   - create a makefile to install the necessary dependencies for Mac OS X: `ci/makefiles/osx.mk`   - create a makefile to install android's SDK/NDK: `ci/makefiles/android.mk`   - refactor docker files: make use of android's makefile - change OS, from `linux` to `osx`, for CI test `Python 3 armeabi-v7a`, so we don't increase the overall build time and jobs   - rename the `Python 2` test to `Python 2 armeabi-v7a (with numpy)` to reflect the build arch and numpy's build. --- .travis.yml | 39 +++++--- Dockerfile.py2 | 62 +++---------- Dockerfile.py3 | 61 ++---------- ci/makefiles/android.mk | 68 ++++++++++++++ ci/makefiles/osx.mk | 23 +++++ pythonforandroid/python.py | 93 +++++++++++++------ pythonforandroid/recipe.py | 4 +- pythonforandroid/recipes/python2/__init__.py | 4 +- .../patches/fix-interpreter-version.patch | 11 +++ 9 files changed, 221 insertions(+), 144 deletions(-) create mode 100644 ci/makefiles/android.mk create mode 100644 ci/makefiles/osx.mk create mode 100644 pythonforandroid/recipes/python2/patches/fix-interpreter-version.patch diff --git a/.travis.yml b/.travis.yml index acbb91eb4f..317ad33283 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,11 +16,6 @@ before_install: - git remote set-branches --add origin develop - git fetch -env: - global: - - ANDROID_SDK_HOME=/opt/android/android-sdk - - ANDROID_NDK_HOME=/opt/android/android-ndk - jobs: include: - &linting @@ -65,19 +60,41 @@ jobs: # case that the travis log doesn't produce any output for more than 10 minutes - while sleep 540; do echo "==== Still running (travis, don't kill me) ===="; done & script: - - docker run -e CI -e TRAVIS_JOB_ID="$TRAVIS_JOB_ID" -e TRAVIS_BRANCH="$TRAVIS_BRANCH" p4a /bin/sh -c "$COMMAND" + - > + docker run + -e CI + -e TRAVIS_JOB_ID + -e TRAVIS_BRANCH + -e ANDROID_SDK_HOME="/home/user/.android/android-sdk" + -e ANDROID_NDK_HOME="/home/user/.android/android-ndk" + p4a /bin/sh -c "$COMMAND" after_script: # kill the background process started before run docker - kill %1 - name: Python 3 armeabi-v7a + name: Python 3 arm64-v8a # overrides requirements to skip `peewee` pure python module, see: # https://github.com/kivy/python-for-android/issues/1263#issuecomment-390421054 - env: COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools' --arch=armeabi-v7a + env: + COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools' --arch=arm64-v8a - <<: *testing - name: Python 3 arm64-v8a - env: COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools' --arch=arm64-v8a + name: Python 3 armeabi-v7a + os: osx + osx_image: xcode11 # since xcode1.3, python3 is the default interpreter + before_script: + # installs java 1.8, android's SDK/NDK and p4a + - make -f ci/makefiles/osx.mk + - export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home + # Run a background process (like we do with linux tests) + - while sleep 540; do echo "==== Still running (travis, don't kill me) ===="; done & + script: + - > + cd testapps && python3 setup_testapp_python3_sqlite_openssl.py apk + --sdk-dir $HOME/.android/android-sdk + --ndk-dir $HOME/.android/android-ndk + --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools + --arch=armeabi-v7a - <<: *testing - name: Python 2 basic + name: Python 2 armeabi-v7a (with numpy) env: COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements sdl2,pyjnius,kivy,python2,openssl,requests,sqlite3,setuptools,numpy' - <<: *testing name: Rebuild updated recipes diff --git a/Dockerfile.py2 b/Dockerfile.py2 index 5c9202c6df..19418329ca 100644 --- a/Dockerfile.py2 +++ b/Dockerfile.py2 @@ -37,64 +37,17 @@ ENV RETRY="retry -t 3 --" RUN curl https://raw.githubusercontent.com/kadwanev/retry/1.0.1/retry \ --output /usr/local/bin/retry && chmod +x /usr/local/bin/retry -ENV ANDROID_NDK_HOME="${ANDROID_HOME}/android-ndk" -ENV ANDROID_NDK_VERSION="17c" -ENV ANDROID_NDK_HOME_V="${ANDROID_NDK_HOME}-r${ANDROID_NDK_VERSION}" - -# get the latest version from https://developer.android.com/ndk/downloads/index.html -ENV ANDROID_NDK_ARCHIVE="android-ndk-r${ANDROID_NDK_VERSION}-linux-x86_64.zip" -ENV ANDROID_NDK_DL_URL="https://dl.google.com/android/repository/${ANDROID_NDK_ARCHIVE}" - -# download and install Android NDK -RUN ${RETRY} curl --location --progress-bar --insecure \ - "${ANDROID_NDK_DL_URL}" \ - --output "${ANDROID_NDK_ARCHIVE}" \ - && mkdir --parents "${ANDROID_NDK_HOME_V}" \ - && unzip -q "${ANDROID_NDK_ARCHIVE}" -d "${ANDROID_HOME}" \ - && ln -sfn "${ANDROID_NDK_HOME_V}" "${ANDROID_NDK_HOME}" \ - && rm -rf "${ANDROID_NDK_ARCHIVE}" - - -ENV ANDROID_SDK_HOME="${ANDROID_HOME}/android-sdk" - -# get the latest version from https://developer.android.com/studio/index.html -ENV ANDROID_SDK_TOOLS_VERSION="4333796" -ENV ANDROID_SDK_BUILD_TOOLS_VERSION="28.0.2" -ENV ANDROID_SDK_TOOLS_ARCHIVE="sdk-tools-linux-${ANDROID_SDK_TOOLS_VERSION}.zip" -ENV ANDROID_SDK_TOOLS_DL_URL="https://dl.google.com/android/repository/${ANDROID_SDK_TOOLS_ARCHIVE}" - -# download and install Android SDK -RUN ${RETRY} curl --location --progress-bar --insecure \ - "${ANDROID_SDK_TOOLS_DL_URL}" \ - --output "${ANDROID_SDK_TOOLS_ARCHIVE}" \ - && mkdir --parents "${ANDROID_SDK_HOME}" \ - && unzip -q "${ANDROID_SDK_TOOLS_ARCHIVE}" -d "${ANDROID_SDK_HOME}" \ - && rm -rf "${ANDROID_SDK_TOOLS_ARCHIVE}" - -# update Android SDK, install Android API, Build Tools... -RUN mkdir --parents "${ANDROID_SDK_HOME}/.android/" \ - && echo '### User Sources for Android SDK Manager' \ - > "${ANDROID_SDK_HOME}/.android/repositories.cfg" - -# Download and accept Android licenses (JDK necessary!) -RUN ${RETRY} apt -y install -qq --no-install-recommends openjdk-8-jdk \ - && apt -y autoremove -RUN yes | "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "build-tools;${ANDROID_SDK_BUILD_TOOLS_VERSION}" > /dev/null -RUN yes | "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-27" > /dev/null - -# Set avdmanager permissions (executable) -RUN chmod +x "${ANDROID_SDK_HOME}/tools/bin/avdmanager" - - ENV USER="user" ENV HOME_DIR="/home/${USER}" +ENV ANDROID_HOME="${HOME_DIR}/.android" ENV WORK_DIR="${HOME_DIR}" \ PATH="${HOME_DIR}/.local/bin:${PATH}" # install system dependencies RUN ${RETRY} apt -y install -qq --no-install-recommends \ python virtualenv python-pip wget lbzip2 patch sudo \ - && apt -y autoremove + && apt -y autoremove \ + && apt -y clean # build dependencies # https://buildozer.readthedocs.io/en/latest/installation.html#android-on-ubuntu-16-04-64bit @@ -113,6 +66,10 @@ RUN ${RETRY} apt -y install -qq --no-install-recommends \ && apt -y autoremove \ && apt -y clean +# Install Java, set JAVA_HOME (to accept android's SDK licenses) and clean +RUN ${RETRY} apt -y install -qq --no-install-recommends openjdk-8-jdk \ + && apt -y autoremove && apt -y clean +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 # prepare non root env RUN useradd --create-home --shell /bin/bash ${USER} @@ -124,9 +81,12 @@ RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers WORKDIR ${WORK_DIR} COPY --chown=user:user . ${WORK_DIR} -RUN chown --recursive ${USER} ${ANDROID_SDK_HOME} +RUN mkdir ${ANDROID_HOME} && chown --recursive ${USER} ${ANDROID_HOME} USER ${USER} +# Download and install android's NDK/SDK +RUN make -f ci/makefiles/android.mk target_os=linux + # install python-for-android from current branch RUN virtualenv --python=python venv \ && . venv/bin/activate \ diff --git a/Dockerfile.py3 b/Dockerfile.py3 index 5a3b40a878..70c9417180 100644 --- a/Dockerfile.py3 +++ b/Dockerfile.py3 @@ -17,8 +17,6 @@ FROM ubuntu:18.04 -ENV ANDROID_HOME="/opt/android" - # configure locale RUN apt update -qq > /dev/null && apt install -qq --yes --no-install-recommends \ locales && \ @@ -37,57 +35,9 @@ ENV RETRY="retry -t 3 --" RUN curl https://raw.githubusercontent.com/kadwanev/retry/1.0.1/retry \ --output /usr/local/bin/retry && chmod +x /usr/local/bin/retry -ENV ANDROID_NDK_HOME="${ANDROID_HOME}/android-ndk" -ENV ANDROID_NDK_VERSION="17c" -ENV ANDROID_NDK_HOME_V="${ANDROID_NDK_HOME}-r${ANDROID_NDK_VERSION}" - -# get the latest version from https://developer.android.com/ndk/downloads/index.html -ENV ANDROID_NDK_ARCHIVE="android-ndk-r${ANDROID_NDK_VERSION}-linux-x86_64.zip" -ENV ANDROID_NDK_DL_URL="https://dl.google.com/android/repository/${ANDROID_NDK_ARCHIVE}" - -# download and install Android NDK -RUN ${RETRY} curl --location --progress-bar --insecure \ - "${ANDROID_NDK_DL_URL}" \ - --output "${ANDROID_NDK_ARCHIVE}" \ - && mkdir --parents "${ANDROID_NDK_HOME_V}" \ - && unzip -q "${ANDROID_NDK_ARCHIVE}" -d "${ANDROID_HOME}" \ - && ln -sfn "${ANDROID_NDK_HOME_V}" "${ANDROID_NDK_HOME}" \ - && rm -rf "${ANDROID_NDK_ARCHIVE}" - - -ENV ANDROID_SDK_HOME="${ANDROID_HOME}/android-sdk" - -# get the latest version from https://developer.android.com/studio/index.html -ENV ANDROID_SDK_TOOLS_VERSION="4333796" -ENV ANDROID_SDK_BUILD_TOOLS_VERSION="28.0.2" -ENV ANDROID_SDK_TOOLS_ARCHIVE="sdk-tools-linux-${ANDROID_SDK_TOOLS_VERSION}.zip" -ENV ANDROID_SDK_TOOLS_DL_URL="https://dl.google.com/android/repository/${ANDROID_SDK_TOOLS_ARCHIVE}" - -# download and install Android SDK -RUN ${RETRY} curl --location --progress-bar --insecure \ - "${ANDROID_SDK_TOOLS_DL_URL}" \ - --output "${ANDROID_SDK_TOOLS_ARCHIVE}" \ - && mkdir --parents "${ANDROID_SDK_HOME}" \ - && unzip -q "${ANDROID_SDK_TOOLS_ARCHIVE}" -d "${ANDROID_SDK_HOME}" \ - && rm -rf "${ANDROID_SDK_TOOLS_ARCHIVE}" - -# update Android SDK, install Android API, Build Tools... -RUN mkdir --parents "${ANDROID_SDK_HOME}/.android/" \ - && echo '### User Sources for Android SDK Manager' \ - > "${ANDROID_SDK_HOME}/.android/repositories.cfg" - -# Download and accept Android licenses (JDK necessary!) -RUN ${RETRY} apt -y install -qq --no-install-recommends openjdk-8-jdk \ - && apt -y autoremove -RUN yes | "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "build-tools;${ANDROID_SDK_BUILD_TOOLS_VERSION}" > /dev/null -RUN yes | "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-27" > /dev/null - -# Set avdmanager permissions (executable) -RUN chmod +x "${ANDROID_SDK_HOME}/tools/bin/avdmanager" - - ENV USER="user" ENV HOME_DIR="/home/${USER}" +ENV ANDROID_HOME="${HOME_DIR}/.android" ENV WORK_DIR="${HOME_DIR}" \ PATH="${HOME_DIR}/.local/bin:${PATH}" @@ -114,6 +64,10 @@ RUN ${RETRY} apt -y install -qq --no-install-recommends \ && apt -y autoremove \ && apt -y clean +# Install Java and set JAVA_HOME (to accept android's SDK licenses) +RUN ${RETRY} apt -y install -qq --no-install-recommends openjdk-8-jdk \ + && apt -y autoremove && apt -y clean +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 # prepare non root env RUN useradd --create-home --shell /bin/bash ${USER} @@ -127,9 +81,12 @@ RUN pip2 install --upgrade Cython==0.28.6 WORKDIR ${WORK_DIR} COPY --chown=user:user . ${WORK_DIR} -RUN chown --recursive ${USER} ${ANDROID_SDK_HOME} +RUN mkdir ${ANDROID_HOME} && chown --recursive ${USER} ${ANDROID_HOME} USER ${USER} +# Download and install android's NDK/SDK +RUN make -f ci/makefiles/android.mk target_os=linux + # install python-for-android from current branch RUN virtualenv --python=python3 venv \ && . venv/bin/activate \ diff --git a/ci/makefiles/android.mk b/ci/makefiles/android.mk new file mode 100644 index 0000000000..44d705838f --- /dev/null +++ b/ci/makefiles/android.mk @@ -0,0 +1,68 @@ +# Downloads and installs the Android SDK depending on supplied platform: darwin or linux + +# We must provide a platform (darwin or linux) and we need JAVA_HOME defined +ifndef target_os + $(error target_os is not set...aborted!) +endif + +# Those android NDK/SDK variables can be override when running the file +ANDROID_NDK_VERSION ?= 17c +ANDROID_SDK_TOOLS_VERSION ?= 4333796 +ANDROID_SDK_BUILD_TOOLS_VERSION ?= 28.0.2 +ANDROID_HOME ?= $(HOME)/.android +ANDROID_API_LEVEL ?= 27 + +ANDROID_SDK_HOME=$(ANDROID_HOME)/android-sdk +ANDROID_SDK_TOOLS_ARCHIVE=sdk-tools-$(target_os)-$(ANDROID_SDK_TOOLS_VERSION).zip +ANDROID_SDK_TOOLS_DL_URL=https://dl.google.com/android/repository/$(ANDROID_SDK_TOOLS_ARCHIVE) + +ANDROID_NDK_HOME=$(ANDROID_HOME)/android-ndk +ANDROID_NDK_FOLDER=$(ANDROID_HOME)/android-ndk-r$(ANDROID_NDK_VERSION) +ANDROID_NDK_ARCHIVE=android-ndk-r$(ANDROID_NDK_VERSION)-$(target_os)-x86_64.zip +ANDROID_NDK_DL_URL=https://dl.google.com/android/repository/$(ANDROID_NDK_ARCHIVE) + +$(info Target install OS is : $(target_os)) +$(info Android SDK home is : $(ANDROID_SDK_HOME)) +$(info Android NDK home is : $(ANDROID_NDK_HOME)) +$(info Android SDK download url is : $(ANDROID_SDK_TOOLS_DL_URL)) +$(info Android NDK download url is : $(ANDROID_NDK_DL_URL)) +$(info Android API level is : $(ANDROID_API_LEVEL)) +$(info Android NDK version is : $(ANDROID_NDK_VERSION)) +$(info JAVA_HOME is : $(JAVA_HOME)) + +all: install_sdk install_ndk + +install_sdk: download_android_sdk extract_android_sdk update_android_sdk + +install_ndk: download_android_ndk extract_android_ndk + +download_android_sdk: + curl --location --progress-bar --continue-at - \ + $(ANDROID_SDK_TOOLS_DL_URL) --output $(ANDROID_SDK_TOOLS_ARCHIVE) + +download_android_ndk: + curl --location --progress-bar --continue-at - \ + $(ANDROID_NDK_DL_URL) --output $(ANDROID_NDK_ARCHIVE) + +# Extract android SDK and remove the compressed file +extract_android_sdk: + mkdir -p $(ANDROID_SDK_HOME) \ + && unzip -q $(ANDROID_SDK_TOOLS_ARCHIVE) -d $(ANDROID_SDK_HOME) \ + && rm -f $(ANDROID_SDK_TOOLS_ARCHIVE) + + +# Extract android NDK and remove the compressed file +extract_android_ndk: + mkdir -p $(ANDROID_NDK_FOLDER) \ + && unzip -q $(ANDROID_NDK_ARCHIVE) -d $(ANDROID_HOME) \ + && ln -sfn $(ANDROID_NDK_FOLDER) $(ANDROID_NDK_HOME) \ + && rm -f $(ANDROID_NDK_ARCHIVE) + +# updates Android SDK, install Android API, Build Tools and accept licenses +update_android_sdk: + touch $(ANDROID_HOME)/repositories.cfg + yes | $(ANDROID_SDK_HOME)/tools/bin/sdkmanager --licenses > /dev/null + $(ANDROID_SDK_HOME)/tools/bin/sdkmanager "build-tools;$(ANDROID_SDK_BUILD_TOOLS_VERSION)" > /dev/null + $(ANDROID_SDK_HOME)/tools/bin/sdkmanager "platforms;android-$(ANDROID_API_LEVEL)" > /dev/null + # Set avdmanager permissions (executable) + chmod +x $(ANDROID_SDK_HOME)/tools/bin/avdmanager diff --git a/ci/makefiles/osx.mk b/ci/makefiles/osx.mk new file mode 100644 index 0000000000..0b274a6ab1 --- /dev/null +++ b/ci/makefiles/osx.mk @@ -0,0 +1,23 @@ +# installs java 1.8, android's SDK/NDK, cython and p4a + +# The following variable/s can be override when running the file +ANDROID_HOME ?= $(HOME)/.android + +all: install_java upgrade_cython install_android_ndk_sdk install_p4a + +install_java: + brew tap adoptopenjdk/openjdk + brew cask install adoptopenjdk8 + /usr/libexec/java_home -V + +upgrade_cython: + pip3 install --upgrade Cython==0.28.6 + +install_android_ndk_sdk: + mkdir -p $(ANDROID_HOME) + make -f ci/makefiles/android.mk target_os=darwin JAVA_HOME=`/usr/libexec/java_home -v 1.8` + +install_p4a: + # check python version and install p4a + python3 --version + pip3 install -e . diff --git a/pythonforandroid/python.py b/pythonforandroid/python.py index ab9035e11a..4604aa2bed 100755 --- a/pythonforandroid/python.py +++ b/pythonforandroid/python.py @@ -3,7 +3,7 @@ build our python3 and python2 recipes and his corresponding hostpython recipes. ''' -from os.path import dirname, exists, join +from os.path import dirname, exists, join, isfile from multiprocessing import cpu_count from shutil import copy2 from os import environ @@ -205,6 +205,17 @@ def add_flags(include_flags, link_dirs, link_libs): recipe.link_dirs_flags(arch), recipe.link_libs_flags()) return env + @property + def _libpython(self): + '''return the python's library name (with extension)''' + py_version = self.major_minor_version_string + if self.major_minor_version_string[0] == '3': + py_version += 'm' + return 'libpython{version}.so'.format(version=py_version) + + def should_build(self, arch): + return not isfile(join(self.link_root(arch.arch), self._libpython)) + def prebuild_arch(self, arch): super(TargetPythonRecipe, self).prebuild_arch(arch) self.ctx.python_recipe = self @@ -243,13 +254,11 @@ def build_arch(self, arch): exec_prefix=sys_exec_prefix)).split(' '), _env=env) - if not exists('python'): - py_version = self.major_minor_version_string - if self.major_minor_version_string[0] == '3': - py_version += 'm' - shprint(sh.make, 'all', '-j', str(cpu_count()), - 'INSTSONAME=libpython{version}.so'.format( - version=py_version), _env=env) + shprint( + sh.make, 'all', '-j', str(cpu_count()), + 'INSTSONAME={lib_name}'.format(lib_name=self._libpython), + _env=env + ) # TODO: Look into passing the path to pyconfig.h in a # better way, although this is probably acceptable @@ -382,6 +391,30 @@ class HostPythonRecipe(Recipe): '''The default url to download our host python recipe. This url will change depending on the python version set in attribute :attr:`version`.''' + @property + def _exe_name(self): + ''' + Returns the name of the python executable depending on the version. + ''' + if not self.version: + raise BuildInterruptingException( + 'The hostpython recipe must have set version' + ) + version = self.version.split('.')[0] + return 'python{major_version}'.format(major_version=version) + + @property + def python_exe(self): + '''Returns the full path of the hostpython executable.''' + return join(self.get_path_to_python(), self._exe_name) + + def should_build(self, arch): + if exists(self.python_exe): + # no need to build, but we must set hostpython for our Context + self.ctx.hostpython = self.python_exe + return False + return True + def get_build_container_dir(self, arch=None): choices = self.check_recipe_choices() dir_name = '-'.join([self.name] + choices) @@ -404,22 +437,28 @@ def build_arch(self, arch): build_dir = join(recipe_build_dir, self.build_subdir) ensure_dir(build_dir) - if not exists(join(build_dir, 'python')): - with current_directory(recipe_build_dir): - # Configure the build - with current_directory(build_dir): - if not exists('config.status'): - shprint( - sh.Command(join(recipe_build_dir, 'configure'))) - - # Create the Setup file. This copying from Setup.dist - # seems to be the normal and expected procedure. - shprint(sh.cp, join('Modules', 'Setup.dist'), - join(build_dir, 'Modules', 'Setup')) - - shprint(sh.make, '-j', str(cpu_count()), '-C', build_dir) - else: - info('Skipping {name} ({version}) build, as it has already ' - 'been completed'.format(name=self.name, version=self.version)) - - self.ctx.hostpython = join(build_dir, 'python') + with current_directory(recipe_build_dir): + # Configure the build + with current_directory(build_dir): + if not exists('config.status'): + shprint(sh.Command(join(recipe_build_dir, 'configure'))) + + # Create the Setup file. This copying from Setup.dist + # seems to be the normal and expected procedure. + shprint(sh.cp, join('Modules', 'Setup.dist'), + join(build_dir, 'Modules', 'Setup')) + + shprint(sh.make, '-j', str(cpu_count()), '-C', build_dir) + + # make a copy of the python executable giving it the name we want, + # because we got different python's executable names depending on + # the fs being case-insensitive (Mac OS X, Cygwin...) or + # case-sensitive (linux)...so this way we will have an unique name + # for our hostpython, regarding the used fs + for exe_name in ['python.exe', 'python']: + exe = join(self.get_path_to_python(), exe_name) + if isfile(exe): + shprint(sh.cp, exe, self.python_exe) + break + + self.ctx.hostpython = self.python_exe diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 7d2c4ac4e8..ec0d7510ea 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -773,9 +773,9 @@ def clean_build(self, arch=None): @property def real_hostpython_location(self): host_name = 'host{}'.format(self.ctx.python_recipe.name) - host_build = Recipe.get_recipe(host_name, self.ctx).get_build_dir() if host_name in ['hostpython2', 'hostpython3']: - return join(host_build, 'native-build', 'python') + python_recipe = Recipe.get_recipe(host_name, self.ctx) + return python_recipe.python_exe else: python_recipe = self.ctx.python_recipe return 'python{}'.format(python_recipe.version) diff --git a/pythonforandroid/recipes/python2/__init__.py b/pythonforandroid/recipes/python2/__init__.py index e99697f215..25c6e714ea 100644 --- a/pythonforandroid/recipes/python2/__init__.py +++ b/pythonforandroid/recipes/python2/__init__.py @@ -32,7 +32,9 @@ class Python2Recipe(GuestPythonRecipe): 'patches/fix-gethostbyaddr.patch', 'patches/fix-posix-declarations.patch', 'patches/fix-pwd-gecos.patch', - 'patches/fix-ctypes-util-find-library.patch'] + 'patches/fix-ctypes-util-find-library.patch', + 'patches/fix-interpreter-version.patch', + ] configure_args = ('--host={android_host}', '--build={android_build}', diff --git a/pythonforandroid/recipes/python2/patches/fix-interpreter-version.patch b/pythonforandroid/recipes/python2/patches/fix-interpreter-version.patch new file mode 100644 index 0000000000..a1828bf582 --- /dev/null +++ b/pythonforandroid/recipes/python2/patches/fix-interpreter-version.patch @@ -0,0 +1,11 @@ +--- Python-2.7.15/configure.orig 2018-04-30 00:47:33.000000000 +0200 ++++ Python-2.7.15/configure 2019-08-16 14:09:48.696030207 +0200 +@@ -2903,7 +2903,7 @@ case $host_os in *\ *) host_os=`echo "$h + # pybuilddir.txt will be created by --generate-posix-vars in the Makefile + rm -f pybuilddir.txt + +-for ac_prog in python$PACKAGE_VERSION python3 python ++for ac_prog in python$PACKAGE_VERSION python2 python + do + # Extract the first word of "$ac_prog", so it can be a program name with args. + set dummy $ac_prog; ac_word=$2 From c3145cf9b427cd8e15779a951ce809995d0aaf25 Mon Sep 17 00:00:00 2001 From: Philipp Auersperg-Castell Date: Fri, 23 Aug 2019 23:43:51 +0200 Subject: [PATCH 06/72] customizability (#1869) * customizability options - added compile-options * apptheme option for all bootstraps --- .../bootstraps/common/build/build.py | 18 ++++ .../common/build/templates/build.tmpl.gradle | 96 +++++++++++-------- .../build/templates/AndroidManifest.tmpl.xml | 4 +- .../build/templates/AndroidManifest.tmpl.xml | 2 +- .../build/templates/AndroidManifest.tmpl.xml | 2 +- 5 files changed, 77 insertions(+), 45 deletions(-) diff --git a/pythonforandroid/bootstraps/common/build/build.py b/pythonforandroid/bootstraps/common/build/build.py index ed5e708892..8068c6f53a 100644 --- a/pythonforandroid/bootstraps/common/build/build.py +++ b/pythonforandroid/bootstraps/common/build/build.py @@ -663,6 +663,24 @@ def parse_args(args=None): 'https://developer.android.com/guide/' 'topics/manifest/' 'activity-element.html')) + + ap.add_argument('--android-entrypoint', dest='android_entrypoint', + default='org.kivy.android.PythonActivity', + help='Defines which java class will be used for startup, usually a subclass of PythonActivity') + ap.add_argument('--android-apptheme', dest='android_apptheme', + default='@android:style/Theme.NoTitleBar', + help='Defines which app theme should be selected for the main activity') + ap.add_argument('--add-compile-option', dest='compile_options', default=[], + action='append', help='add compile options to gradle.build') + ap.add_argument('--add-gradle-repository', dest='gradle_repositories', + default=[], + action='append', + help='Ddd a repository for gradle') + ap.add_argument('--add-packaging-option', dest='packaging_options', + default=[], + action='append', + help='Dndroid packaging options') + ap.add_argument('--wakelock', dest='wakelock', action='store_true', help=('Indicate if the application needs the device ' 'to stay on')) diff --git a/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle b/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle index 32bd091b72..fe78dda5a8 100644 --- a/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle +++ b/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle @@ -13,68 +13,82 @@ allprojects { repositories { google() jcenter() - flatDir { - dirs 'libs' - } + {%- for repo in args.gradle_repositories %} + {{repo}} + {%- endfor %} + flatDir { + dirs 'libs' + } } } apply plugin: 'com.android.application' android { - compileSdkVersion {{ android_api }} - buildToolsVersion '{{ build_tools_version }}' - defaultConfig { - minSdkVersion {{ args.min_sdk_version }} - targetSdkVersion {{ android_api }} - versionCode {{ args.numeric_version }} - versionName '{{ args.version }}' - } + compileSdkVersion {{ android_api }} + buildToolsVersion '{{ build_tools_version }}' + defaultConfig { + minSdkVersion {{ args.min_sdk_version }} + targetSdkVersion {{ android_api }} + versionCode {{ args.numeric_version }} + versionName '{{ args.version }}' + } - {% if args.sign -%} - signingConfigs { - release { - storeFile file(System.getenv("P4A_RELEASE_KEYSTORE")) - keyAlias System.getenv("P4A_RELEASE_KEYALIAS") - storePassword System.getenv("P4A_RELEASE_KEYSTORE_PASSWD") - keyPassword System.getenv("P4A_RELEASE_KEYALIAS_PASSWD") - } - } + {% if args.sign -%} + signingConfigs { + release { + storeFile file(System.getenv("P4A_RELEASE_KEYSTORE")) + keyAlias System.getenv("P4A_RELEASE_KEYALIAS") + storePassword System.getenv("P4A_RELEASE_KEYSTORE_PASSWD") + keyPassword System.getenv("P4A_RELEASE_KEYALIAS_PASSWD") + } + } {%- endif %} - buildTypes { - debug { - } - release { - {% if args.sign -%} - signingConfig signingConfigs.release - {%- endif %} - } - } + {% if args.packaging_options -%} + packagingOptions { + {%- for option in args.packaging_options %} + {{option}} + {%- endfor %} + } + {%- endif %} + + buildTypes { + debug { + } + release { + {% if args.sign -%} + signingConfig signingConfigs.release + {%- endif %} + } + } compileOptions { sourceCompatibility JavaVersion.VERSION_1_7 targetCompatibility JavaVersion.VERSION_1_7 + {%- for option in args.compile_options %} + {{option}} + {%- endfor %} } sourceSets { main { jniLibs.srcDir 'libs' - } + } } } dependencies { - {%- for aar in aars %} - compile(name: '{{ aar }}', ext: 'aar') - {%- endfor -%} - {%- for jar in jars %} - compile files('src/main/libs/{{ jar }}') - {%- endfor -%} - {%- if args.depends -%} - {%- for depend in args.depends %} - compile '{{ depend }}' - {%- endfor %} - {%- endif %} + {%- for aar in aars %} + compile(name: '{{ aar }}', ext: 'aar') + {%- endfor -%} + {%- for jar in jars %} + compile files('src/main/libs/{{ jar }}') + {%- endfor -%} + {%- if args.depends -%} + {%- for depend in args.depends %} + compile '{{ depend }}' + {%- endfor %} + {%- endif %} } diff --git a/pythonforandroid/bootstraps/sdl2/build/templates/AndroidManifest.tmpl.xml b/pythonforandroid/bootstraps/sdl2/build/templates/AndroidManifest.tmpl.xml index 119a4daefd..6f9b03903d 100644 --- a/pythonforandroid/bootstraps/sdl2/build/templates/AndroidManifest.tmpl.xml +++ b/pythonforandroid/bootstraps/sdl2/build/templates/AndroidManifest.tmpl.xml @@ -54,7 +54,7 @@ {% for l in args.android_used_libs %} @@ -64,7 +64,7 @@ {% endfor %} - {% for l in args.android_used_libs %} diff --git a/pythonforandroid/bootstraps/webview/build/templates/AndroidManifest.tmpl.xml b/pythonforandroid/bootstraps/webview/build/templates/AndroidManifest.tmpl.xml index 2e1d58f6d3..d71985c66e 100644 --- a/pythonforandroid/bootstraps/webview/build/templates/AndroidManifest.tmpl.xml +++ b/pythonforandroid/bootstraps/webview/build/templates/AndroidManifest.tmpl.xml @@ -48,7 +48,7 @@ {% for l in args.android_used_libs %} From 3887d2b680c7cae03084d352ec56c8fc0931b8c0 Mon Sep 17 00:00:00 2001 From: Pol Canelles Date: Sat, 24 Aug 2019 21:08:24 +0200 Subject: [PATCH 07/72] Initial migration to NDK r19 (Part I - The core) (#1722) * [ndk19] Rework of the arch environment to work with ndk r19+ Things done here: - Modifies target so we can use the toolchains `in-place` (as if we were using a standalone toolchain) - Removes some cflags that we don't need anymore - Move `macros` and `includes` from `CFLAGS` to `CPPFLAGS` (because this is the way it's supposed to be) - Move libraries from `LDFLAGS` to `LDLIBS` (again, the same as above) - Refactor `Arch`'s variables into class attributes, to make `get_env` smaller and to be more readable/maintainable: + class attributes for flags: * common_cflags * common_cppflags * common_ldflags * common_ldlibs * common_ldshared * arch_cflags + class attributes for compiler: * clang_path * clang_exe * clang_exe_cxx - Remove hardcoded cpu count in `MAKE` - Remove `gcc` of environment (because it has been partially removed in ndk r18) - Shorten long line to be pep8 friendly (line where we define env['BUILDLIB_PATH']) - Commented lines for `ArchAarch_64` that it seems tha we don't need anymore (not removed because I'm not sure if we need them) Note: This will force us to use a minimal ndk version of 19 See also: https://developer.android.com/ndk/guides/other_build_systems * [ndk19] Update `test_archs` to ndk's r19+ build system Also add some more tests. Now we check if some of the functions we mock, `glob` and `find_executable`, are called with the expected arguments * [ndk19] Update `test_bootstrap` to ndk's r19+ build system Because `clang` is the new default compiler and the affected tests were made thinking in `gcc` compiler * [ndk19] Remove `clang` of `get_recipe_env` Because as per ndk r19 the default compiler is clang and gcc has been partially removed * [ndk19] Adapt python recipes to ndk's r19+ build system We move python recipe to use a minimal ndk version of 19 because it will simplify the python's cross compilation (this way we can remove some environment variables that we don't need anymore). The only side effect detected is that the python's `zlib` module isn't built anymore. This is caused because python3 and python2 build systems contains libz's hardcoded version, which does not match the version used in ndk 19 (may be this the cause of some `zip/unzip` errors reported in python-for-android's issues?). We will take care later of this zlib problem, or we will get a runtime crash...so it seems that zlib is an `essential module` to successfully run a python-for-android app * [ndk19] Update libffi to version post `v3.3RC0` and adapt it to be ndk r19+ compatible * [ndk19] Fix build of python's zlib module Because python has a hardcoded zlib version which does not match android's ndk r19, and that prevents the build of zlib's module Note: it seems that we have to point to the right lib and headers, otherwise we won't succeed in building python's zlib module * [ndk19] Fix build of python2's _ctypes module for ndk r19+ * [ndk19] Fix numpy for ndk r19+ Note: because of the new build system recently introduced, we can safely remove some of the patches that we needed because it seems that it builds and runs fine without them * [ndk19] Update docs with the new ndk requirements * [ndk19] Set min/max/recommended android's NDK * [ndk19] Make CI tests with android's NDK `r19b` Because it's our minimum supported android NDK * [ndk19] Fix png recipe for NDK r19+ We remove `get_recipe_env` because we don't need anymore to specify the flags set in there. Note: As per new NDK r19+ build system implemented in p4a, the removed `flags` should be automatically set by the android's NDK * [ndk19] Add `-fPIC` to `armeabi-v7a`'s arch cflags... ...force `Position-Independent Code` To solve the kivy's build error detected in travis `OSX` test (https://travis-ci.org/kivy/python-for-android/jobs/576204088#L1105). But also detected the same error in my local tests (linux os in this case) for `freetype` and `harfbuzz` recipes...so I have the feeling that this is better to set as global :crossed_fingers:..is it? --- README.md | 8 +- ci/makefiles/android.mk | 2 +- doc/source/quickstart.rst | 4 +- pythonforandroid/archs.py | 299 +++++++++++------- pythonforandroid/python.py | 100 +++--- pythonforandroid/recipe.py | 4 +- pythonforandroid/recipes/jpeg/__init__.py | 2 +- pythonforandroid/recipes/libffi/__init__.py | 31 +- .../recipes/libffi/fix-includedir.patch | 34 -- .../recipes/libffi/remove-version-info.patch | 19 +- pythonforandroid/recipes/numpy/__init__.py | 27 +- .../recipes/numpy/patches/ar.patch | 41 --- .../patches/do_not_use_system_libs.patch | 2 +- .../patches/fix_environment_detection.patch | 48 --- .../patches/fix_setup_dependencies.patch | 40 --- pythonforandroid/recipes/openssl/__init__.py | 2 +- pythonforandroid/recipes/png/__init__.py | 9 - pythonforandroid/recipes/pycrypto/__init__.py | 2 +- pythonforandroid/recipes/python2/__init__.py | 3 +- .../patches/fix-missing-extensions.patch | 12 - .../python2/patches/fix-zlib-version.patch | 12 + pythonforandroid/recipes/python3/__init__.py | 3 +- .../python3/patches/fix-zlib-version.patch | 12 + pythonforandroid/recommendations.py | 6 +- tests/test_archs.py | 161 +++++++--- tests/test_bootstrap.py | 10 +- 26 files changed, 432 insertions(+), 461 deletions(-) delete mode 100644 pythonforandroid/recipes/libffi/fix-includedir.patch delete mode 100644 pythonforandroid/recipes/numpy/patches/ar.patch delete mode 100644 pythonforandroid/recipes/numpy/patches/fix_environment_detection.patch delete mode 100644 pythonforandroid/recipes/numpy/patches/fix_setup_dependencies.patch create mode 100644 pythonforandroid/recipes/python2/patches/fix-zlib-version.patch create mode 100644 pythonforandroid/recipes/python3/patches/fix-zlib-version.patch diff --git a/README.md b/README.md index 0504c58765..82f7c1d9d8 100644 --- a/README.md +++ b/README.md @@ -108,8 +108,12 @@ require to target to a **minimum api level of 21**, api level below 21, you must use an old version of python-for-android (<=0.7.1). All this work has been done using android ndk version r17c, and your build should success with that version...but be aware that the -project is in constant development so...the ndk version will change at -some time. +project is in constant development so, as per time of writing, +``we recommend to use android's NDK r19b`` because the toolchains installed by +default with the NDK may be used *in-place* and the python-for-android project +has been adapted to that feature, so, it's mandatory to use, at least, ndk +version r19 (be aware that more modern versions of the +android's ndk may not work). Those mentioned changes has been done this way to make easier the transition between python3 and python2. We will slowly phase out python2 support diff --git a/ci/makefiles/android.mk b/ci/makefiles/android.mk index 44d705838f..d2d4648007 100644 --- a/ci/makefiles/android.mk +++ b/ci/makefiles/android.mk @@ -6,7 +6,7 @@ ifndef target_os endif # Those android NDK/SDK variables can be override when running the file -ANDROID_NDK_VERSION ?= 17c +ANDROID_NDK_VERSION ?= 19b ANDROID_SDK_TOOLS_VERSION ?= 4333796 ANDROID_SDK_BUILD_TOOLS_VERSION ?= 28.0.2 ANDROID_HOME ?= $(HOME)/.android diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 74161d7d8e..b70558d4c2 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -110,7 +110,7 @@ named ``tools``, and you will need to run extra commands to install the SDK packages needed. For Android NDK, note that modern releases will only work on a 64-bit -operating system. **The minimal, and recommended, NDK version to use is r17c:** +operating system. **The minimal, and recommended, NDK version to use is r19b:** - `Go to ndk downloads page `_ - Windows users should create a virtual machine with an GNU Linux os @@ -144,7 +144,7 @@ variables necessary for building on android:: # Adjust the paths! export ANDROIDSDK="$HOME/Documents/android-sdk-27" - export ANDROIDNDK="$HOME/Documents/android-ndk-r17c" + export ANDROIDNDK="$HOME/Documents/android-ndk-r19b" export ANDROIDAPI="27" # Target API version of your application export NDKAPI="21" # Minimum supported API version of your application export ANDROIDNDKVER="r10e" # Version of the NDK you installed diff --git a/pythonforandroid/archs.py b/pythonforandroid/archs.py index 42f143ed21..bf716db3b5 100644 --- a/pythonforandroid/archs.py +++ b/pythonforandroid/archs.py @@ -1,6 +1,7 @@ from distutils.spawn import find_executable from os import environ -from os.path import (exists, join, dirname, split) +from os.path import join, split +from multiprocessing import cpu_count from glob import glob from pythonforandroid.recipe import Recipe @@ -15,6 +16,35 @@ class Arch(object): command_prefix = None '''The prefix for NDK commands such as gcc.''' + arch = "" + '''Name of the arch such as: `armeabi-v7a`, `arm64-v8a`, `x86`...''' + + arch_cflags = [] + '''Specific arch `cflags`, expect to be overwrote in subclass if needed.''' + + common_cflags = [ + '-target {target}', + '-fomit-frame-pointer' + ] + + common_cppflags = [ + '-DANDROID', + '-D__ANDROID_API__={ctx.ndk_api}', + '-I{ctx.ndk_dir}/sysroot/usr/include/{command_prefix}', + '-I{python_includes}', + ] + + common_ldflags = ['-L{ctx_libs_dir}'] + + common_ldlibs = ['-lm'] + + common_ldshared = [ + '-pthread', + '-shared', + '-Wl,-O1', + '-Wl,-Bsymbolic-functions', + ] + def __init__(self, ctx): super(Arch, self).__init__() self.ctx = ctx @@ -38,125 +68,159 @@ def include_dirs(self): @property def target(self): - target_data = self.command_prefix.split('-') - return '-'.join( - [target_data[0], 'none', target_data[1], target_data[2]]) - - def get_env(self, with_flags_in_cc=True, clang=False): - env = {} - - cflags = [ - '-DANDROID', - '-fomit-frame-pointer', - '-D__ANDROID_API__={}'.format(self.ctx.ndk_api)] - if not clang: - cflags.append('-mandroid') - else: - cflags.append('-target ' + self.target) - toolchain = '{android_host}-{toolchain_version}'.format( - android_host=self.ctx.toolchain_prefix, - toolchain_version=self.ctx.toolchain_version) - toolchain = join(self.ctx.ndk_dir, 'toolchains', toolchain, - 'prebuilt', build_platform) - cflags.append('-gcc-toolchain {}'.format(toolchain)) + # As of NDK r19, the toolchains installed by default with the + # NDK may be used in-place. The make_standalone_toolchain.py script + # is no longer needed for interfacing with arbitrary build systems. + # See: https://developer.android.com/ndk/guides/other_build_systems + return '{triplet}{ndk_api}'.format( + triplet=self.command_prefix, ndk_api=self.ctx.ndk_api + ) - env['CFLAGS'] = ' '.join(cflags) + @property + def clang_path(self): + """Full path of the clang compiler""" + llvm_dirname = split( + glob(join(self.ctx.ndk_dir, 'toolchains', 'llvm*'))[-1] + )[-1] + return join( + self.ctx.ndk_dir, + 'toolchains', + llvm_dirname, + 'prebuilt', + build_platform, + 'bin', + ) - # Link the extra global link paths first before anything else - # (such that overriding system libraries with them is possible) - env['LDFLAGS'] = ' ' + " ".join([ - "-L'" + l.replace("'", "'\"'\"'") + "'" # no shlex.quote in py2 - for l in self.extra_global_link_paths - ]) + ' ' - - sysroot = join(self.ctx._ndk_dir, 'sysroot') - if exists(sysroot): - # post-15 NDK per - # https://android.googlesource.com/platform/ndk/+/ndk-r15-release/docs/UnifiedHeaders.md - env['CFLAGS'] += ' -isystem {}/sysroot/usr/include/{}'.format( - self.ctx.ndk_dir, self.ctx.toolchain_prefix) - env['CFLAGS'] += ' -I{}/sysroot/usr/include/{}'.format( - self.ctx.ndk_dir, self.command_prefix) - else: - sysroot = self.ctx.ndk_platform - env['CFLAGS'] += ' -I{}'.format(self.ctx.ndk_platform) - env['CFLAGS'] += ' -isysroot {} '.format(sysroot) - env['CFLAGS'] += '-I' + join(self.ctx.get_python_install_dir(), - 'include/python{}'.format( - self.ctx.python_recipe.version[0:3]) - ) + @property + def clang_exe(self): + """Full path of the clang compiler depending on the android's ndk + version used.""" + return self.get_clang_exe() - env['LDFLAGS'] += '--sysroot={} '.format(self.ctx.ndk_platform) + @property + def clang_exe_cxx(self): + """Full path of the clang++ compiler depending on the android's ndk + version used.""" + return self.get_clang_exe(plus_plus=True) + + def get_clang_exe(self, with_target=False, plus_plus=False): + """Returns the full path of the clang/clang++ compiler, supports two + kwargs: + + - `with_target`: prepend `target` to clang + - `plus_plus`: will return the clang++ compiler (defaults to `False`) + """ + compiler = 'clang' + if with_target: + compiler = '{target}-{compiler}'.format( + target=self.target, compiler=compiler + ) + if plus_plus: + compiler += '++' + return join(self.clang_path, compiler) + + def get_env(self, with_flags_in_cc=True): + env = {} - env["CXXFLAGS"] = env["CFLAGS"] + # CFLAGS/CXXFLAGS: the processor flags + env['CFLAGS'] = ' '.join(self.common_cflags).format(target=self.target) + if self.arch_cflags: + # each architecture may have has his own CFLAGS + env['CFLAGS'] += ' ' + ' '.join(self.arch_cflags) + env['CXXFLAGS'] = env['CFLAGS'] - env["LDFLAGS"] += " ".join(['-lm', '-L' + self.ctx.get_libs_dir(self.arch)]) + # CPPFLAGS (for macros and includes) + env['CPPFLAGS'] = ' '.join(self.common_cppflags).format( + ctx=self.ctx, + command_prefix=self.command_prefix, + python_includes=join( + self.ctx.get_python_install_dir(), + 'include/python{}'.format(self.ctx.python_recipe.version[0:3]), + ), + ) - toolchain_prefix = self.ctx.toolchain_prefix - toolchain_version = self.ctx.toolchain_version - command_prefix = self.command_prefix + # LDFLAGS: Link the extra global link paths first before anything else + # (such that overriding system libraries with them is possible) + env['LDFLAGS'] = ( + ' ' + + " ".join( + [ + "-L'" + + l.replace("'", "'\"'\"'") + + "'" # no shlex.quote in py2 + for l in self.extra_global_link_paths + ] + ) + + ' ' + ' '.join(self.common_ldflags).format( + ctx_libs_dir=self.ctx.get_libs_dir(self.arch) + ) + ) - env['TOOLCHAIN_PREFIX'] = toolchain_prefix - env['TOOLCHAIN_VERSION'] = toolchain_version + # LDLIBS: Library flags or names given to compilers when they are + # supposed to invoke the linker. + env['LDLIBS'] = ' '.join(self.common_ldlibs) + # CCACHE ccache = '' if self.ctx.ccache and bool(int(environ.get('USE_CCACHE', '1'))): # print('ccache found, will optimize builds') ccache = self.ctx.ccache + ' ' env['USE_CCACHE'] = '1' env['NDK_CCACHE'] = self.ctx.ccache - env.update({k: v for k, v in environ.items() if k.startswith('CCACHE_')}) - - if clang: - llvm_dirname = split( - glob(join(self.ctx.ndk_dir, 'toolchains', 'llvm*'))[-1])[-1] - clang_path = join(self.ctx.ndk_dir, 'toolchains', llvm_dirname, - 'prebuilt', build_platform, 'bin') - environ['PATH'] = '{clang_path}:{path}'.format( - clang_path=clang_path, path=environ['PATH']) - exe = join(clang_path, 'clang') - execxx = join(clang_path, 'clang++') - else: - exe = '{command_prefix}-gcc'.format(command_prefix=command_prefix) - execxx = '{command_prefix}-g++'.format(command_prefix=command_prefix) + env.update( + {k: v for k, v in environ.items() if k.startswith('CCACHE_')} + ) - cc = find_executable(exe, path=environ['PATH']) + # Compiler: `CC` and `CXX` (and make sure that the compiler exists) + environ['PATH'] = '{clang_path}:{path}'.format( + clang_path=self.clang_path, path=environ['PATH'] + ) + cc = find_executable(self.clang_exe, path=environ['PATH']) if cc is None: print('Searching path are: {!r}'.format(environ['PATH'])) raise BuildInterruptingException( 'Couldn\'t find executable for CC. This indicates a ' 'problem locating the {} executable in the Android ' 'NDK, not that you don\'t have a normal compiler ' - 'installed. Exiting.'.format(exe)) + 'installed. Exiting.'.format(self.clang_exe)) if with_flags_in_cc: env['CC'] = '{ccache}{exe} {cflags}'.format( - exe=exe, + exe=self.clang_exe, ccache=ccache, cflags=env['CFLAGS']) env['CXX'] = '{ccache}{execxx} {cxxflags}'.format( - execxx=execxx, + execxx=self.clang_exe_cxx, ccache=ccache, cxxflags=env['CXXFLAGS']) else: env['CC'] = '{ccache}{exe}'.format( - exe=exe, + exe=self.clang_exe, ccache=ccache) env['CXX'] = '{ccache}{execxx}'.format( - execxx=execxx, + execxx=self.clang_exe_cxx, ccache=ccache) + # Android's binaries + command_prefix = self.command_prefix env['AR'] = '{}-ar'.format(command_prefix) env['RANLIB'] = '{}-ranlib'.format(command_prefix) - env['LD'] = '{}-ld'.format(command_prefix) - env['LDSHARED'] = env["CC"] + " -pthread -shared " +\ - "-Wl,-O1 -Wl,-Bsymbolic-functions " - env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix) - env['MAKE'] = 'make -j5' + env['MAKE'] = 'make -j{}'.format(str(cpu_count())) env['READELF'] = '{}-readelf'.format(command_prefix) env['NM'] = '{}-nm'.format(command_prefix) + env['LD'] = '{}-ld'.format(command_prefix) + # Android's arch/toolchain + env['ARCH'] = self.arch + env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api)) + env['TOOLCHAIN_PREFIX'] = self.ctx.toolchain_prefix + env['TOOLCHAIN_VERSION'] = self.ctx.toolchain_version + + # Custom linker options + env['LDSHARED'] = env['CC'] + ' ' + ' '.join(self.common_ldshared) + + # Host python (used by some recipes) hostpython_recipe = Recipe.get_recipe( 'host' + self.ctx.python_recipe.name, self.ctx) env['BUILDLIB_PATH'] = join( @@ -171,9 +235,6 @@ def get_env(self, with_flags_in_cc=True, clang=False): env['PATH'] = environ['PATH'] - env['ARCH'] = self.arch - env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api)) - return env @@ -186,20 +247,21 @@ class ArchARM(Arch): @property def target(self): target_data = self.command_prefix.split('-') - return '-'.join( - ['armv7a', 'none', target_data[1], target_data[2]]) + return '{triplet}{ndk_api}'.format( + triplet='-'.join(['armv7a', target_data[1], target_data[2]]), + ndk_api=self.ctx.ndk_api, + ) class ArchARMv7_a(ArchARM): arch = 'armeabi-v7a' - - def get_env(self, with_flags_in_cc=True, clang=False): - env = super(ArchARMv7_a, self).get_env(with_flags_in_cc, clang=clang) - env['CFLAGS'] = (env['CFLAGS'] + - (' -march=armv7-a -mfloat-abi=softfp ' - '-mfpu=vfp -mthumb')) - env['CXXFLAGS'] = env['CFLAGS'] - return env + arch_cflags = [ + '-march=armv7-a', + '-mfloat-abi=softfp', + '-mfpu=vfp', + '-mthumb', + '-fPIC', + ] class Archx86(Arch): @@ -207,13 +269,13 @@ class Archx86(Arch): toolchain_prefix = 'x86' command_prefix = 'i686-linux-android' platform_dir = 'arch-x86' - - def get_env(self, with_flags_in_cc=True, clang=False): - env = super(Archx86, self).get_env(with_flags_in_cc, clang=clang) - env['CFLAGS'] = (env['CFLAGS'] + - ' -march=i686 -mtune=intel -mssse3 -mfpmath=sse -m32') - env['CXXFLAGS'] = env['CFLAGS'] - return env + arch_cflags = [ + '-march=i686', + '-mtune=intel', + '-mssse3', + '-mfpmath=sse', + '-m32', + ] class Archx86_64(Arch): @@ -221,13 +283,13 @@ class Archx86_64(Arch): toolchain_prefix = 'x86_64' command_prefix = 'x86_64-linux-android' platform_dir = 'arch-x86_64' - - def get_env(self, with_flags_in_cc=True, clang=False): - env = super(Archx86_64, self).get_env(with_flags_in_cc, clang=clang) - env['CFLAGS'] = (env['CFLAGS'] + - ' -march=x86-64 -msse4.2 -mpopcnt -m64 -mtune=intel') - env['CXXFLAGS'] = env['CFLAGS'] - return env + arch_cflags = [ + '-march=x86-64', + '-msse4.2', + '-mpopcnt', + '-m64', + '-mtune=intel', + ] class ArchAarch_64(Arch): @@ -235,14 +297,17 @@ class ArchAarch_64(Arch): toolchain_prefix = 'aarch64-linux-android' command_prefix = 'aarch64-linux-android' platform_dir = 'arch-arm64' - - def get_env(self, with_flags_in_cc=True, clang=False): - env = super(ArchAarch_64, self).get_env(with_flags_in_cc, clang=clang) - incpath = ' -I' + join(dirname(__file__), 'includes', 'arm64-v8a') - env['EXTRA_CFLAGS'] = incpath - env['CFLAGS'] += incpath - env['CXXFLAGS'] += incpath - if with_flags_in_cc: - env['CC'] += incpath - env['CXX'] += incpath - return env + arch_cflags = [ + '-march=armv8-a', + # '-I' + join(dirname(__file__), 'includes', 'arm64-v8a'), + ] + + # Note: This `EXTRA_CFLAGS` below should target the commented `include` + # above in `arch_cflags`. The original lines were added during the Sdl2's + # bootstrap creation, and modified/commented during the migration to the + # NDK r19 build system, because it seems that we don't need it anymore, + # do we need them? + # def get_env(self, with_flags_in_cc=True): + # env = super(ArchAarch_64, self).get_env(with_flags_in_cc) + # env['EXTRA_CFLAGS'] = self.arch_cflags[-1] + # return env diff --git a/pythonforandroid/python.py b/pythonforandroid/python.py index 4604aa2bed..c4c4ebcc19 100755 --- a/pythonforandroid/python.py +++ b/pythonforandroid/python.py @@ -12,10 +12,13 @@ import sh from pythonforandroid.recipe import Recipe, TargetPythonRecipe -from pythonforandroid.logger import logger, info, shprint +from pythonforandroid.logger import info, warning, shprint from pythonforandroid.util import ( - current_directory, ensure_dir, walk_valid_filens, - BuildInterruptingException, build_platform) + current_directory, + ensure_dir, + walk_valid_filens, + BuildInterruptingException, +) class GuestPythonRecipe(TargetPythonRecipe): @@ -105,26 +108,9 @@ def __init__(self, *args, **kwargs): def get_recipe_env(self, arch=None, with_flags_in_cc=True): env = environ.copy() + env['HOSTARCH'] = arch.command_prefix - android_host = env['HOSTARCH'] = arch.command_prefix - toolchain = '{toolchain_prefix}-{toolchain_version}'.format( - toolchain_prefix=self.ctx.toolchain_prefix, - toolchain_version=self.ctx.toolchain_version) - toolchain = join(self.ctx.ndk_dir, 'toolchains', - toolchain, 'prebuilt', build_platform) - - env['CC'] = ( - '{clang} -target {target} -gcc-toolchain {toolchain}').format( - clang=join(self.ctx.ndk_dir, 'toolchains', 'llvm', 'prebuilt', - build_platform, 'bin', 'clang'), - target=arch.target, - toolchain=toolchain) - env['AR'] = join(toolchain, 'bin', android_host) + '-ar' - env['LD'] = join(toolchain, 'bin', android_host) + '-ld' - env['RANLIB'] = join(toolchain, 'bin', android_host) + '-ranlib' - env['READELF'] = join(toolchain, 'bin', android_host) + '-readelf' - env['STRIP'] = join(toolchain, 'bin', android_host) + '-strip' - env['STRIP'] += ' --strip-debug --strip-unneeded' + env['CC'] = arch.get_clang_exe(with_target=True) env['PATH'] = ( '{hostpython_dir}:{old_path}').format( @@ -132,43 +118,22 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): 'host' + self.name, self.ctx).get_path_to_python(), old_path=env['PATH']) - ndk_flags = ( - '-fPIC --sysroot={ndk_sysroot} -D__ANDROID_API__={android_api} ' - '-isystem {ndk_android_host} -I{ndk_include}').format( - ndk_sysroot=join(self.ctx.ndk_dir, 'sysroot'), - android_api=self.ctx.ndk_api, - ndk_android_host=join( - self.ctx.ndk_dir, 'sysroot', 'usr', 'include', android_host), - ndk_include=join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include')) - sysroot = self.ctx.ndk_platform - env['CFLAGS'] = env.get('CFLAGS', '') + ' ' + ndk_flags - env['CPPFLAGS'] = env.get('CPPFLAGS', '') + ' ' + ndk_flags - env['LDFLAGS'] = env.get('LDFLAGS', '') + ' --sysroot={} -L{}'.format( - sysroot, join(sysroot, 'usr', 'lib')) - - # Manually add the libs directory, and copy some object - # files to the current directory otherwise they aren't - # picked up. This seems necessary because the --sysroot - # setting in LDFLAGS is overridden by the other flags. - # TODO: Work out why this doesn't happen in the original - # bpo-30386 Makefile system. - logger.warning('Doing some hacky stuff to link properly') - lib_dir = join(sysroot, 'usr', 'lib') - if arch.arch == 'x86_64': - lib_dir = join(sysroot, 'usr', 'lib64') - env['LDFLAGS'] += ' -L{}'.format(lib_dir) - shprint(sh.cp, join(lib_dir, 'crtbegin_so.o'), './') - shprint(sh.cp, join(lib_dir, 'crtend_so.o'), './') - - env['SYSROOT'] = sysroot + env['CFLAGS'] = ' '.join( + [ + '-fPIC', + '-DANDROID', + '-D__ANDROID_API__={}'.format(self.ctx.ndk_api), + ] + ) + env['LDFLAGS'] = env.get('LDFLAGS', '') if sh.which('lld') is not None: # Note: The -L. is to fix a bug in python 3.7. # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409 - env["LDFLAGS"] += ' -L. -fuse-ld=lld' + env['LDFLAGS'] += ' -L. -fuse-ld=lld' else: - logger.warning('lld not found, linking without it. ' + - 'Consider installing lld if linker errors occur.') + warning('lld not found, linking without it. ' + 'Consider installing lld if linker errors occur.') return env @@ -203,6 +168,33 @@ def add_flags(include_flags, link_dirs, link_libs): recipe = Recipe.get_recipe('openssl', self.ctx) add_flags(recipe.include_flags(arch), recipe.link_dirs_flags(arch), recipe.link_libs_flags()) + + # python build system contains hardcoded zlib version which prevents + # the build of zlib module, here we search for android's zlib version + # and sets the right flags, so python can be build with android's zlib + info("Activating flags for android's zlib") + zlib_lib_path = join(self.ctx.ndk_platform, 'usr', 'lib') + zlib_includes = join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include') + zlib_h = join(zlib_includes, 'zlib.h') + try: + with open(zlib_h) as fileh: + zlib_data = fileh.read() + except IOError: + raise BuildInterruptingException( + "Could not determine android's zlib version, no zlib.h ({}) in" + " the NDK dir includes".format(zlib_h) + ) + for line in zlib_data.split('\n'): + if line.startswith('#define ZLIB_VERSION '): + break + else: + raise BuildInterruptingException( + 'Could not parse zlib.h...so we cannot find zlib version,' + 'required by python build,' + ) + env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '') + add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz') + return env @property diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index ec0d7510ea..f165ce474c 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -430,12 +430,12 @@ def unpack(self, arch): else: info('{} is already unpacked, skipping'.format(self.name)) - def get_recipe_env(self, arch=None, with_flags_in_cc=True, clang=False): + def get_recipe_env(self, arch=None, with_flags_in_cc=True): """Return the env specialized for the recipe """ if arch is None: arch = self.filtered_archs[0] - return arch.get_env(with_flags_in_cc=with_flags_in_cc, clang=clang) + return arch.get_env(with_flags_in_cc=with_flags_in_cc) def prebuild_arch(self, arch): '''Run any pre-build tasks for the Recipe. By default, this checks if diff --git a/pythonforandroid/recipes/jpeg/__init__.py b/pythonforandroid/recipes/jpeg/__init__.py index 1969d2c1c5..507ba775e4 100644 --- a/pythonforandroid/recipes/jpeg/__init__.py +++ b/pythonforandroid/recipes/jpeg/__init__.py @@ -63,7 +63,7 @@ def build_arch(self, arch): for lib in glob(join(build_dir, '*.a')): shprint(sh.cp, '-L', lib, self.ctx.libs_dir) - def get_recipe_env(self, arch=None, with_flags_in_cc=False, clang=True): + def get_recipe_env(self, arch=None, with_flags_in_cc=False): env = environ.copy() build_platform = '{system}-{machine}'.format( diff --git a/pythonforandroid/recipes/libffi/__init__.py b/pythonforandroid/recipes/libffi/__init__.py index 31ed9c69b5..3608df6eb0 100644 --- a/pythonforandroid/recipes/libffi/__init__.py +++ b/pythonforandroid/recipes/libffi/__init__.py @@ -12,16 +12,24 @@ class LibffiRecipe(Recipe): - `automake` for the `aclocal` binary - `autoconf` for the `autoreconf` binary - `libltdl-dev` which defines the `LT_SYS_SYMBOL_USCORE` macro + + .. note:: + Some notes about libffi version: + + - v3.2.1 it's from year 2014...it's a little outdated and has + problems with clang (see issue #1525) + - v3.3-rc0 it was released at april 2018 (it's a pre-release), and + it lacks some commits that we are interested, specially those + ones that fixes specific issues for Arm64, you can check those + commits at (search for commit `8fa8837` and look at the below + commits): https://github.com/libffi/libffi/commits/master """ name = 'libffi' - version = '3.2.1' - url = 'https://github.com/libffi/libffi/archive/v{version}.tar.gz' + # Version pinned to post `v3.3RC0` + version = '8fa8837' + url = 'https://github.com/libffi/libffi/archive/{version}.tar.gz' - patches = ['remove-version-info.patch', - # This patch below is already included into libffi's master - # branch and included in the pre-release 3.3rc0...so we should - # remove this when we update the version number for libffi - 'fix-includedir.patch'] + patches = ['remove-version-info.patch'] def should_build(self, arch): return not exists(join(self.ctx.get_libs_dir(arch.arch), 'libffi.so')) @@ -37,14 +45,11 @@ def build_arch(self, arch): '--prefix=' + self.get_build_dir(arch.arch), '--disable-builddir', '--enable-shared', _env=env) - shprint(sh.make, '-j', str(cpu_count()), 'libffi.la', _env=env) - - host_build = self.get_build_dir(arch.arch) ensure_dir(self.ctx.get_libs_dir(arch.arch)) - shprint(sh.cp, - join(host_build, '.libs', 'libffi.so'), - self.ctx.get_libs_dir(arch.arch)) + self.install_libs( + arch, join(self.get_build_dir(arch.arch), '.libs', 'libffi.so') + ) def get_include_dirs(self, arch): return [join(self.get_build_dir(arch.arch), 'include')] diff --git a/pythonforandroid/recipes/libffi/fix-includedir.patch b/pythonforandroid/recipes/libffi/fix-includedir.patch deleted file mode 100644 index 0dc35c70ca..0000000000 --- a/pythonforandroid/recipes/libffi/fix-includedir.patch +++ /dev/null @@ -1,34 +0,0 @@ -From 982b89c01aca99c7bc229914fc1521f96930919b Mon Sep 17 00:00:00 2001 -From: Yen Chi Hsuan -Date: Sun, 13 Nov 2016 19:17:19 +0800 -Subject: [PATCH] Install public headers in the standard path - ---- - include/Makefile.am | 3 +-- - libffi.pc.in | 2 +- - 2 files changed, 2 insertions(+), 3 deletions(-) - -diff --git a/include/Makefile.am b/include/Makefile.am -index bb241e88..c59df9fb 100644 ---- a/include/Makefile.am -+++ b/include/Makefile.am -@@ -6,5 +6,4 @@ DISTCLEANFILES=ffitarget.h - noinst_HEADERS=ffi_common.h ffi_cfi.h - EXTRA_DIST=ffi.h.in - --includesdir = $(libdir)/@PACKAGE_NAME@-@PACKAGE_VERSION@/include --nodist_includes_HEADERS = ffi.h ffitarget.h -+nodist_include_HEADERS = ffi.h ffitarget.h -diff --git a/libffi.pc.in b/libffi.pc.in -index edf6fde5..6fad83b4 100644 ---- a/libffi.pc.in -+++ b/libffi.pc.in -@@ -2,7 +2,7 @@ prefix=@prefix@ - exec_prefix=@exec_prefix@ - libdir=@libdir@ - toolexeclibdir=@toolexeclibdir@ --includedir=${libdir}/@PACKAGE_NAME@-@PACKAGE_VERSION@/include -+includedir=@includedir@ - - Name: @PACKAGE_NAME@ - Description: Library supporting Foreign Function Interfaces diff --git a/pythonforandroid/recipes/libffi/remove-version-info.patch b/pythonforandroid/recipes/libffi/remove-version-info.patch index 7bdc11a641..0a32b7e614 100644 --- a/pythonforandroid/recipes/libffi/remove-version-info.patch +++ b/pythonforandroid/recipes/libffi/remove-version-info.patch @@ -1,12 +1,11 @@ -diff -Naur libffi/Makefile.am b/Makefile.am ---- libffi/Makefile.am 2014-11-12 06:00:59.000000000 -0600 -+++ b/Makefile.am 2015-12-23 15:57:10.363148806 -0600 -@@ -249,7 +249,7 @@ - AM_CFLAGS += -DFFI_DEBUG - endif - --libffi_la_LDFLAGS = -no-undefined -version-info `grep -v '^\#' $(srcdir)/libtool-version` $(LTLDFLAGS) $(AM_LTLDFLAGS) +--- libffi/Makefile.am.orig 2018-12-21 16:11:26.159181262 +0100 ++++ libffi/Makefile.am 2018-12-21 16:14:44.075179374 +0100 +@@ -156,7 +156,7 @@ + libffi.map: $(top_srcdir)/libffi.map.in + $(COMPILE) -D$(TARGET) -E -x assembler-with-cpp -o $@ $< + +-libffi_la_LDFLAGS = -no-undefined $(libffi_version_info) $(libffi_version_script) $(LTLDFLAGS) $(AM_LTLDFLAGS) +libffi_la_LDFLAGS = -no-undefined -avoid-version $(LTLDFLAGS) $(AM_LTLDFLAGS) - + libffi_la_DEPENDENCIES = $(libffi_la_LIBADD) $(libffi_version_dep) + AM_CPPFLAGS = -I. -I$(top_srcdir)/include -Iinclude -I$(top_srcdir)/src - AM_CCASFLAGS = $(AM_CPPFLAGS) diff --git a/pythonforandroid/recipes/numpy/__init__.py b/pythonforandroid/recipes/numpy/__init__.py index 4e47e9d890..027b5465fe 100644 --- a/pythonforandroid/recipes/numpy/__init__.py +++ b/pythonforandroid/recipes/numpy/__init__.py @@ -8,14 +8,12 @@ class NumpyRecipe(CompiledComponentsPythonRecipe): version = '1.16.4' url = 'https://pypi.python.org/packages/source/n/numpy/numpy-{version}.zip' site_packages_name = 'numpy' + depends = ['setuptools'] patches = [ join('patches', 'add_libm_explicitly_to_build.patch'), join('patches', 'do_not_use_system_libs.patch'), join('patches', 'remove_unittest_call.patch'), - join('patches', 'ar.patch'), - join('patches', 'fix_setup_dependencies.patch'), - join('patches', 'fix_environment_detection.patch'), ] call_hostpython_via_targetpython = False @@ -30,28 +28,5 @@ def rebuild_compiled_components(self, arch, env): super(NumpyRecipe, self).rebuild_compiled_components(arch, env) self.setup_extra_args = [] - def get_recipe_env(self, arch): - env = super(NumpyRecipe, self).get_recipe_env(arch) - - flags = " -L{} --sysroot={}".format( - join(self.ctx.ndk_platform, 'usr', 'lib'), - self.ctx.ndk_platform - ) - - py_ver = self.ctx.python_recipe.major_minor_version_string - py_inc_dir = self.ctx.python_recipe.include_root(arch.arch) - py_lib_dir = self.ctx.python_recipe.link_root(arch.arch) - flags += ' -I{}'.format(py_inc_dir) - flags += ' -L{} -lpython{}'.format(py_lib_dir, py_ver) - if 'python3' in self.ctx.python_recipe.name: - flags += 'm' - - if flags not in env['CC']: - env['CC'] += flags - if flags not in env['LD']: - env['LD'] += flags + ' -shared' - - return env - recipe = NumpyRecipe() diff --git a/pythonforandroid/recipes/numpy/patches/ar.patch b/pythonforandroid/recipes/numpy/patches/ar.patch deleted file mode 100644 index c806636dc1..0000000000 --- a/pythonforandroid/recipes/numpy/patches/ar.patch +++ /dev/null @@ -1,41 +0,0 @@ -diff --git a/numpy/core/code_generators/generate_umath.py b/numpy/core/code_generators/generate_umath.py -index 0fac9b0..94be92a 100644 ---- a/numpy/core/code_generators/generate_umath.py -+++ b/numpy/core/code_generators/generate_umath.py -@@ -982,6 +982,7 @@ def make_arrays(funcdict): - funclist.append('%s_%s' % (tname, name)) - if t.simd is not None: - for vt in t.simd: -+ continue - code2list.append(textwrap.dedent("""\ - #ifdef HAVE_ATTRIBUTE_TARGET_{ISA} - if (npy_cpu_supports("{isa}")) {{ -diff --git a/numpy/distutils/ccompiler.py b/numpy/distutils/ccompiler.py -index 14451fa..dfd65da 100644 ---- a/numpy/distutils/ccompiler.py -+++ b/numpy/distutils/ccompiler.py -@@ -295,6 +295,7 @@ def CCompiler_compile(self, sources, output_dir=None, macros=None, - self._setup_compile(output_dir, macros, include_dirs, sources, - depends, extra_postargs) - cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) -+ cc_args += os.environ['CFLAGS'].split() - display = "compile options: '%s'" % (' '.join(cc_args)) - if extra_postargs: - display += "\nextra options: '%s'" % (' '.join(extra_postargs)) -@@ -795,4 +796,3 @@ for _cc in ['msvc9', 'msvc', '_msvc', 'bcpp', 'cygwinc', 'emxc', 'unixc']: - _m = sys.modules.get('distutils.' + _cc + 'compiler') - if _m is not None: - setattr(_m, 'gen_lib_options', gen_lib_options) -- -diff --git a/numpy/distutils/unixccompiler.py b/numpy/distutils/unixccompiler.py -index 11b2cce..c3e9f10 100644 ---- a/numpy/distutils/unixccompiler.py -+++ b/numpy/distutils/unixccompiler.py -@@ -111,6 +111,7 @@ def UnixCCompiler_create_static_lib(self, objects, output_libname, - while tmp_objects: - objects = tmp_objects[:50] - tmp_objects = tmp_objects[50:] -+ self.archiver[0] = os.environ['AR'] - display = '%s: adding %d object files to %s' % ( - os.path.basename(self.archiver[0]), - len(objects), output_filename) diff --git a/pythonforandroid/recipes/numpy/patches/do_not_use_system_libs.patch b/pythonforandroid/recipes/numpy/patches/do_not_use_system_libs.patch index cdcc41086b..13c1f4bab0 100644 --- a/pythonforandroid/recipes/numpy/patches/do_not_use_system_libs.patch +++ b/pythonforandroid/recipes/numpy/patches/do_not_use_system_libs.patch @@ -4,7 +4,7 @@ index 806f4f7..0d51cfa 100644 +++ b/numpy/distutils/system_info.py @@ -734,6 +734,7 @@ class system_info(object): return self.get_paths(self.section, key) - + def get_libs(self, key, default): + return [] try: diff --git a/pythonforandroid/recipes/numpy/patches/fix_environment_detection.patch b/pythonforandroid/recipes/numpy/patches/fix_environment_detection.patch deleted file mode 100644 index 3c7251eaa1..0000000000 --- a/pythonforandroid/recipes/numpy/patches/fix_environment_detection.patch +++ /dev/null @@ -1,48 +0,0 @@ -commit 9a09edac303c534a38c5d829d8537176f8a8dfb9 -Author: Alexander Taylor -Date: Fri Jun 28 22:50:45 2019 +0100 - - fix_environment_detection.patch - -diff --git a/numpy/core/include/numpy/npy_common.h b/numpy/core/include/numpy/npy_common.h -index 64aaaac..e6293f9 100644 ---- a/numpy/core/include/numpy/npy_common.h -+++ b/numpy/core/include/numpy/npy_common.h -@@ -164,12 +164,12 @@ extern long long __cdecl _ftelli64(FILE *); - #endif - #else - #ifdef HAVE_FSEEKO -- #define npy_fseek fseeko -+ #define npy_fseek fseek - #else - #define npy_fseek fseek - #endif - #ifdef HAVE_FTELLO -- #define npy_ftell ftello -+ #define npy_ftell ftell - #else - #define npy_ftell ftell - #endif -@@ -321,13 +321,15 @@ typedef unsigned char npy_bool; - #define NPY_TRUE 1 - - --#if NPY_SIZEOF_LONGDOUBLE == NPY_SIZEOF_DOUBLE -- typedef double npy_longdouble; -- #define NPY_LONGDOUBLE_FMT "g" --#else -- typedef long double npy_longdouble; -- #define NPY_LONGDOUBLE_FMT "Lg" --#endif -+/* #if NPY_SIZEOF_LONGDOUBLE == NPY_SIZEOF_DOUBLE */ -+/* typedef double npy_longdouble; */ -+/* #define NPY_LONGDOUBLE_FMT "g" */ -+/* #else */ -+/* typedef long double npy_longdouble; */ -+/* #define NPY_LONGDOUBLE_FMT "Lg" */ -+/* #endif */ -+typedef long double npy_longdouble; -+#define NPY_LONGDOUBLE_FMT "Lg" - - #ifndef Py_USING_UNICODE - #error Must use Python with unicode enabled. diff --git a/pythonforandroid/recipes/numpy/patches/fix_setup_dependencies.patch b/pythonforandroid/recipes/numpy/patches/fix_setup_dependencies.patch deleted file mode 100644 index 1c38bc6068..0000000000 --- a/pythonforandroid/recipes/numpy/patches/fix_setup_dependencies.patch +++ /dev/null @@ -1,40 +0,0 @@ -diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py -index 42374ac..67fcd98 100644 ---- a/numpy/distutils/misc_util.py -+++ b/numpy/distutils/misc_util.py -@@ -9,7 +9,6 @@ import atexit - import tempfile - import subprocess - import shutil --import multiprocessing - - import distutils - from distutils.errors import DistutilsError -@@ -94,11 +93,7 @@ def get_num_build_jobs(): - - """ - from numpy.distutils.core import get_distribution -- try: -- cpu_count = len(os.sched_getaffinity(0)) -- except AttributeError: -- cpu_count = multiprocessing.cpu_count() -- cpu_count = min(cpu_count, 8) -+ cpu_count = 1 - envjobs = int(os.environ.get("NPY_NUM_BUILD_JOBS", cpu_count)) - dist = get_distribution() - # may be None during configuration -diff --git a/setup.py b/setup.py -index 8b2ded1..431c1b8 100755 ---- a/setup.py -+++ b/setup.py -@@ -389,9 +389,8 @@ def setup_package(): - # Raise errors for unsupported commands, improve help output, etc. - run_build = parse_setuppy_commands() - -- from setuptools import setup -+ from numpy.distutils.core import setup - if run_build: -- from numpy.distutils.core import setup - cwd = os.path.abspath(os.path.dirname(__file__)) - if not os.path.exists(os.path.join(cwd, 'PKG-INFO')): - # Generate Cython sources, unless building from source release diff --git a/pythonforandroid/recipes/openssl/__init__.py b/pythonforandroid/recipes/openssl/__init__.py index d3033a3594..a4aad3df16 100644 --- a/pythonforandroid/recipes/openssl/__init__.py +++ b/pythonforandroid/recipes/openssl/__init__.py @@ -90,7 +90,7 @@ def should_build(self, arch): 'libcrypto' + self.version + '.so') def get_recipe_env(self, arch=None): - env = super(OpenSSLRecipe, self).get_recipe_env(arch, clang=True) + env = super(OpenSSLRecipe, self).get_recipe_env(arch) env['OPENSSL_VERSION'] = self.version env['MAKE'] = 'make' # This removes the '-j5', which isn't safe env['ANDROID_NDK'] = self.ctx.ndk_dir diff --git a/pythonforandroid/recipes/png/__init__.py b/pythonforandroid/recipes/png/__init__.py index 1ad49cc1ca..a6e281a570 100644 --- a/pythonforandroid/recipes/png/__init__.py +++ b/pythonforandroid/recipes/png/__init__.py @@ -16,15 +16,6 @@ def should_build(self, arch): join(self.get_build_dir(arch.arch), '.libs', 'libpng16.so') ) - def get_recipe_env(self, arch=None): - env = super(PngRecipe, self).get_recipe_env(arch) - ndk_lib_dir = join(self.ctx.ndk_platform, 'usr', 'lib') - ndk_include_dir = join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include') - env['CFLAGS'] += ' -I{}'.format(ndk_include_dir) - env['LDFLAGS'] += ' -L{}'.format(ndk_lib_dir) - env['LDFLAGS'] += ' --sysroot={}'.format(self.ctx.ndk_platform) - return env - def build_arch(self, arch): super(PngRecipe, self).build_arch(arch) build_dir = self.get_build_dir(arch.arch) diff --git a/pythonforandroid/recipes/pycrypto/__init__.py b/pythonforandroid/recipes/pycrypto/__init__.py index e8bfab2666..8f70df28e9 100644 --- a/pythonforandroid/recipes/pycrypto/__init__.py +++ b/pythonforandroid/recipes/pycrypto/__init__.py @@ -15,7 +15,7 @@ class PyCryptoRecipe(CompiledComponentsPythonRecipe): call_hostpython_via_targetpython = False patches = ['add_length.patch'] - def get_recipe_env(self, arch=None, clang=True): + def get_recipe_env(self, arch=None): env = super(PyCryptoRecipe, self).get_recipe_env(arch) openssl_recipe = Recipe.get_recipe('openssl', self.ctx) env['CC'] = env['CC'] + openssl_recipe.include_flags(arch) diff --git a/pythonforandroid/recipes/python2/__init__.py b/pythonforandroid/recipes/python2/__init__.py index 25c6e714ea..0f14e5e617 100644 --- a/pythonforandroid/recipes/python2/__init__.py +++ b/pythonforandroid/recipes/python2/__init__.py @@ -20,7 +20,7 @@ class Python2Recipe(GuestPythonRecipe): url = 'https://www.python.org/ftp/python/{version}/Python-{version}.tgz' name = 'python2' - depends = ['hostpython2'] + depends = ['hostpython2', 'libffi'] conflicts = ['python3'] patches = [ @@ -34,6 +34,7 @@ class Python2Recipe(GuestPythonRecipe): 'patches/fix-pwd-gecos.patch', 'patches/fix-ctypes-util-find-library.patch', 'patches/fix-interpreter-version.patch', + 'patches/fix-zlib-version.patch', ] configure_args = ('--host={android_host}', diff --git a/pythonforandroid/recipes/python2/patches/fix-missing-extensions.patch b/pythonforandroid/recipes/python2/patches/fix-missing-extensions.patch index a098b25634..cb777d3b1f 100644 --- a/pythonforandroid/recipes/python2/patches/fix-missing-extensions.patch +++ b/pythonforandroid/recipes/python2/patches/fix-missing-extensions.patch @@ -1,15 +1,3 @@ -diff -Naurp Python-2.7.15/Modules/Setup.dist.orig Python-2.7.15/Modules/Setup.dist ---- Python-2.7.15/Modules/Setup.dist.orig 2018-04-30 00:47:33.000000000 +0200 -+++ Python-2.7.15/Modules/Setup.dist 2018-11-17 20:40:20.153518694 +0100 -@@ -464,7 +464,7 @@ - # Andrew Kuchling's zlib module. - # This require zlib 1.1.3 (or later). - # See http://www.gzip.org/zlib/ --#zlib zlibmodule.c -I$(prefix)/include -L$(exec_prefix)/lib -lz -+zlib zlibmodule.c -I$(prefix)/include -L$(exec_prefix)/lib -lz - - # Interface to the Expat XML parser - # diff -Naurp Python-2.7.15.orig/Makefile.pre.in Python-2.7.15/Makefile.pre.in --- Python-2.7.15.orig/Makefile.pre.in 2018-04-30 00:47:33.000000000 +0200 +++ Python-2.7.15/Makefile.pre.in 2018-11-18 00:43:58.777379280 +0100 diff --git a/pythonforandroid/recipes/python2/patches/fix-zlib-version.patch b/pythonforandroid/recipes/python2/patches/fix-zlib-version.patch new file mode 100644 index 0000000000..b7c14dbf80 --- /dev/null +++ b/pythonforandroid/recipes/python2/patches/fix-zlib-version.patch @@ -0,0 +1,12 @@ +--- Python-3.7.1/setup.py.orig 2018-10-20 08:04:19.000000000 +0200 ++++ Python-3.7.1/setup.py 2019-02-17 00:24:30.715904412 +0100 +@@ -1416,7 +1416,8 @@ class PyBuildExt(build_ext): + if zlib_inc is not None: + zlib_h = zlib_inc[0] + '/zlib.h' + version = '"0.0.0"' +- version_req = '"1.1.3"' ++ version_req = '"{}"'.format( ++ os.environ.get('ZLIB_VERSION', '1.1.3')) + if host_platform == 'darwin' and is_macosx_sdk_path(zlib_h): + zlib_h = os.path.join(macosx_sdk_root(), zlib_h[1:]) + with open(zlib_h) as fp: diff --git a/pythonforandroid/recipes/python3/__init__.py b/pythonforandroid/recipes/python3/__init__.py index 963fad635f..57765b642d 100644 --- a/pythonforandroid/recipes/python3/__init__.py +++ b/pythonforandroid/recipes/python3/__init__.py @@ -22,7 +22,8 @@ class Python3Recipe(GuestPythonRecipe): url = 'https://www.python.org/ftp/python/{version}/Python-{version}.tgz' name = 'python3' - patches = ["patches/fix-ctypes-util-find-library.patch"] + patches = ['patches/fix-ctypes-util-find-library.patch', + 'patches/fix-zlib-version.patch'] if sh.which('lld') is not None: patches = patches + ["patches/remove-fix-cortex-a8.patch"] diff --git a/pythonforandroid/recipes/python3/patches/fix-zlib-version.patch b/pythonforandroid/recipes/python3/patches/fix-zlib-version.patch new file mode 100644 index 0000000000..0dbffae246 --- /dev/null +++ b/pythonforandroid/recipes/python3/patches/fix-zlib-version.patch @@ -0,0 +1,12 @@ +--- Python-3.7.1/setup.py.orig 2018-10-20 08:04:19.000000000 +0200 ++++ Python-3.7.1/setup.py 2019-02-17 00:24:30.715904412 +0100 +@@ -1410,7 +1410,8 @@ class PyBuildExt(build_ext): + if zlib_inc is not None: + zlib_h = zlib_inc[0] + '/zlib.h' + version = '"0.0.0"' +- version_req = '"1.1.3"' ++ version_req = '"{}"'.format( ++ os.environ.get('ZLIB_VERSION', '1.1.3')) + if host_platform == 'darwin' and is_macosx_sdk_path(zlib_h): + zlib_h = os.path.join(macosx_sdk_root(), zlib_h[1:]) + with open(zlib_h) as fp: diff --git a/pythonforandroid/recommendations.py b/pythonforandroid/recommendations.py index 6cf18eceb9..a4a0531558 100644 --- a/pythonforandroid/recommendations.py +++ b/pythonforandroid/recommendations.py @@ -8,10 +8,10 @@ from pythonforandroid.util import BuildInterruptingException # We only check the NDK major version -MIN_NDK_VERSION = 17 -MAX_NDK_VERSION = 17 +MIN_NDK_VERSION = 19 +MAX_NDK_VERSION = 20 -RECOMMENDED_NDK_VERSION = "17c" +RECOMMENDED_NDK_VERSION = "19b" NDK_DOWNLOAD_URL = "https://developer.android.com/ndk/downloads/" # Important log messages diff --git a/tests/test_archs.py b/tests/test_archs.py index 39bf261654..7cbe5421ad 100644 --- a/tests/test_archs.py +++ b/tests/test_archs.py @@ -1,6 +1,6 @@ import os - import unittest +from os import environ try: from unittest import mock @@ -8,6 +8,7 @@ # `Python 2` or lower than `Python 3.3` does not # have the `unittest.mock` module built-in import mock + from pythonforandroid.bootstrap import Bootstrap from pythonforandroid.distribution import Distribution from pythonforandroid.recipe import Recipe @@ -53,6 +54,7 @@ class ArchSetUpBaseClass(object): """ ctx = None + expected_compiler = "" def setUp(self): self.ctx = Context() @@ -66,6 +68,12 @@ def setUp(self): self.ctx, name="sdl2", recipes=["python3", "kivy"] ) self.ctx.python_recipe = Recipe.get_recipe("python3", self.ctx) + # Here we define the expected compiler, which, as per ndk >= r19, + # should be the same for all the tests (no more gcc compiler) + self.expected_compiler = ( + "/opt/android/android-ndk/toolchains/" + "llvm/prebuilt/linux-x86_64/bin/clang" + ) class TestArch(ArchSetUpBaseClass, unittest.TestCase): @@ -77,16 +85,8 @@ class TestArch(ArchSetUpBaseClass, unittest.TestCase): def test_arch(self): arch = Arch(self.ctx) - with self.assertRaises(AttributeError) as e1: - arch.__str__() - self.assertEqual( - e1.exception.args[0], "'Arch' object has no attribute 'arch'" - ) - with self.assertRaises(AttributeError) as e2: - getattr(arch, "target") - self.assertEqual( - e2.exception.args[0], "'NoneType' object has no attribute 'split'" - ) + self.assertEqual(arch.__str__(), arch.arch) + self.assertEqual(arch.target, "None21") self.assertIsNone(arch.toolchain_prefix) self.assertIsNone(arch.command_prefix) self.assertIsInstance(arch.include_dirs, list) @@ -98,9 +98,10 @@ class TestArchARM(ArchSetUpBaseClass, unittest.TestCase): will be used to perform tests for :class:`~pythonforandroid.archs.ArchARM`. """ + @mock.patch("pythonforandroid.archs.glob") @mock.patch("pythonforandroid.archs.find_executable") @mock.patch("pythonforandroid.build.ensure_dir") - def test_arch_arm(self, mock_ensure_dir, mock_find_executable): + def test_arch_arm(self, mock_ensure_dir, mock_find_executable, mock_glob): """ Test that class :class:`~pythonforandroid.archs.ArchARM` returns some expected attributes and environment variables. @@ -115,15 +116,16 @@ def test_arch_arm(self, mock_ensure_dir, mock_find_executable): not exist) """ - mock_find_executable.return_value = "arm-linux-androideabi-gcc" + mock_find_executable.return_value = self.expected_compiler mock_ensure_dir.return_value = True + mock_glob.return_value = ["llvm"] arch = ArchARM(self.ctx) self.assertEqual(arch.arch, "armeabi") self.assertEqual(arch.__str__(), "armeabi") self.assertEqual(arch.toolchain_prefix, "arm-linux-androideabi") self.assertEqual(arch.command_prefix, "arm-linux-androideabi") - self.assertEqual(arch.target, "armv7a-none-linux-androideabi") + self.assertEqual(arch.target, "armv7a-linux-androideabi21") self.assertEqual(arch.platform_dir, "arch-arm") arch = ArchARM(self.ctx) @@ -134,9 +136,20 @@ def test_arch_arm(self, mock_ensure_dir, mock_find_executable): expected_env_gcc_keys, set(env.keys()) & expected_env_gcc_keys ) + # check glob and find_executable calls + self.assertEqual(mock_glob.call_count, 4) + for glob_call, kw in mock_glob.call_args_list: + self.assertEqual( + glob_call[0], + "{ndk_dir}/toolchains/llvm*".format(ndk_dir=self.ctx._ndk_dir), + ) + mock_find_executable.assert_called_once_with( + self.expected_compiler, path=environ["PATH"] + ) + # check gcc compilers - self.assertEqual(env["CC"].split()[0], "arm-linux-androideabi-gcc") - self.assertEqual(env["CXX"].split()[0], "arm-linux-androideabi-g++") + self.assertEqual(env["CC"].split()[0], self.expected_compiler) + self.assertEqual(env["CXX"].split()[0], self.expected_compiler + "++") # check android binaries self.assertEqual(env["AR"], "arm-linux-androideabi-ar") self.assertEqual(env["LD"], "arm-linux-androideabi-ld") @@ -166,9 +179,9 @@ def test_arch_arm(self, mock_ensure_dir, mock_find_executable): self.assertEqual( e.exception.args[0], "Couldn't find executable for CC. This indicates a problem " - "locating the arm-linux-androideabi-gcc executable in the Android " + "locating the {expected_compiler} executable in the Android " "NDK, not that you don't have a normal compiler installed. " - "Exiting.", + "Exiting.".format(expected_compiler=self.expected_compiler), ) @@ -197,7 +210,7 @@ def test_arch_armv7a( tests the `get_env` with clang """ - mock_find_executable.return_value = "arm-linux-androideabi-gcc" + mock_find_executable.return_value = self.expected_compiler mock_ensure_dir.return_value = True mock_glob.return_value = ["llvm"] @@ -206,10 +219,21 @@ def test_arch_armv7a( self.assertEqual(arch.__str__(), "armeabi-v7a") self.assertEqual(arch.toolchain_prefix, "arm-linux-androideabi") self.assertEqual(arch.command_prefix, "arm-linux-androideabi") - self.assertEqual(arch.target, "armv7a-none-linux-androideabi") + self.assertEqual(arch.target, "armv7a-linux-androideabi21") self.assertEqual(arch.platform_dir, "arch-arm") - env = arch.get_env(clang=True) + env = arch.get_env() + # check glob and find_executable calls + self.assertEqual(mock_glob.call_count, 4) + for glob_call, kw in mock_glob.call_args_list: + self.assertEqual( + glob_call[0], + "{ndk_dir}/toolchains/llvm*".format(ndk_dir=self.ctx._ndk_dir), + ) + mock_find_executable.assert_called_once_with( + self.expected_compiler, path=environ["PATH"] + ) + # check clang build_platform = "{system}-{machine}".format( system=os.uname()[0], machine=os.uname()[-1] @@ -242,29 +266,46 @@ class TestArchX86(ArchSetUpBaseClass, unittest.TestCase): will be used to perform tests for :class:`~pythonforandroid.archs.Archx86`. """ + @mock.patch("pythonforandroid.archs.glob") @mock.patch("pythonforandroid.archs.find_executable") @mock.patch("pythonforandroid.build.ensure_dir") - def test_arch_x86(self, mock_ensure_dir, mock_find_executable): + def test_arch_x86(self, mock_ensure_dir, mock_find_executable, mock_glob): """ Test that class :class:`~pythonforandroid.archs.Archx86` returns some expected attributes and environment variables. - .. note:: Here we mock the same functions than - :meth:`TestArchARM.test_arch_arm` + .. note:: + Here we mock the same functions than + :meth:`TestArchARM.test_arch_arm` plus `glob`, so we make sure that + the glob result is the expected even if the folder doesn't exist, + which is probably the case. This has to be done because here we + tests the `get_env` with clang """ - mock_find_executable.return_value = "arm-linux-androideabi-gcc" + mock_find_executable.return_value = self.expected_compiler mock_ensure_dir.return_value = True + mock_glob.return_value = ["llvm"] arch = Archx86(self.ctx) self.assertEqual(arch.arch, "x86") self.assertEqual(arch.__str__(), "x86") self.assertEqual(arch.toolchain_prefix, "x86") self.assertEqual(arch.command_prefix, "i686-linux-android") - self.assertEqual(arch.target, "i686-none-linux-android") + self.assertEqual(arch.target, "i686-linux-android21") self.assertEqual(arch.platform_dir, "arch-x86") - # For x86 we expect some extra cflags in our `environment` env = arch.get_env() + # check glob and find_executable calls + self.assertEqual(mock_glob.call_count, 4) + for glob_call, kw in mock_glob.call_args_list: + self.assertEqual( + glob_call[0], + "{ndk_dir}/toolchains/llvm*".format(ndk_dir=self.ctx._ndk_dir), + ) + mock_find_executable.assert_called_once_with( + self.expected_compiler, path=environ["PATH"] + ) + + # For x86 we expect some extra cflags in our `environment` self.assertIn( " -march=i686 -mtune=intel -mssse3 -mfpmath=sse -m32", env["CFLAGS"], @@ -278,29 +319,49 @@ class TestArchX86_64(ArchSetUpBaseClass, unittest.TestCase): :class:`~pythonforandroid.archs.Archx86_64`. """ + @mock.patch("pythonforandroid.archs.glob") @mock.patch("pythonforandroid.archs.find_executable") @mock.patch("pythonforandroid.build.ensure_dir") - def test_arch_x86_64(self, mock_ensure_dir, mock_find_executable): + def test_arch_x86_64( + self, mock_ensure_dir, mock_find_executable, mock_glob + ): """ Test that class :class:`~pythonforandroid.archs.Archx86_64` returns some expected attributes and environment variables. - .. note:: Here we mock the same functions than - :meth:`TestArchARM.test_arch_arm` + .. note:: + Here we mock the same functions than + :meth:`TestArchARM.test_arch_arm` plus `glob`, so we make sure that + the glob result is the expected even if the folder doesn't exist, + which is probably the case. This has to be done because here we + tests the `get_env` with clang """ - mock_find_executable.return_value = "arm-linux-androideabi-gcc" + mock_find_executable.return_value = self.expected_compiler mock_ensure_dir.return_value = True + mock_glob.return_value = ["llvm"] arch = Archx86_64(self.ctx) self.assertEqual(arch.arch, "x86_64") self.assertEqual(arch.__str__(), "x86_64") self.assertEqual(arch.toolchain_prefix, "x86_64") self.assertEqual(arch.command_prefix, "x86_64-linux-android") - self.assertEqual(arch.target, "x86_64-none-linux-android") + self.assertEqual(arch.target, "x86_64-linux-android21") self.assertEqual(arch.platform_dir, "arch-x86_64") - # For x86_64 we expect some extra cflags in our `environment` env = arch.get_env() + # check glob and find_executable calls + self.assertEqual(mock_glob.call_count, 4) + for glob_call, kw in mock_glob.call_args_list: + self.assertEqual( + glob_call[0], + "{ndk_dir}/toolchains/llvm*".format(ndk_dir=self.ctx._ndk_dir), + ) + mock_find_executable.assert_called_once_with( + self.expected_compiler, path=environ["PATH"] + ) + + # For x86_64 we expect some extra cflags in our `environment` + mock_find_executable.assert_called_once() self.assertIn( " -march=x86-64 -msse4.2 -mpopcnt -m64 -mtune=intel", env["CFLAGS"] ) @@ -313,27 +374,47 @@ class TestArchAArch64(ArchSetUpBaseClass, unittest.TestCase): :class:`~pythonforandroid.archs.ArchAarch_64`. """ + @mock.patch("pythonforandroid.archs.glob") @mock.patch("pythonforandroid.archs.find_executable") @mock.patch("pythonforandroid.build.ensure_dir") - def test_arch_aarch_64(self, mock_ensure_dir, mock_find_executable): + def test_arch_aarch_64( + self, mock_ensure_dir, mock_find_executable, mock_glob + ): """ Test that class :class:`~pythonforandroid.archs.ArchAarch_64` returns some expected attributes and environment variables. - .. note:: Here we mock the same functions than - :meth:`TestArchARM.test_arch_arm` + .. note:: + Here we mock the same functions than + :meth:`TestArchARM.test_arch_arm` plus `glob`, so we make sure that + the glob result is the expected even if the folder doesn't exist, + which is probably the case. This has to be done because here we + tests the `get_env` with clang """ - mock_find_executable.return_value = "arm-linux-androideabi-gcc" + mock_find_executable.return_value = self.expected_compiler mock_ensure_dir.return_value = True + mock_glob.return_value = ["llvm"] arch = ArchAarch_64(self.ctx) self.assertEqual(arch.arch, "arm64-v8a") self.assertEqual(arch.__str__(), "arm64-v8a") self.assertEqual(arch.toolchain_prefix, "aarch64-linux-android") self.assertEqual(arch.command_prefix, "aarch64-linux-android") - self.assertEqual(arch.target, "aarch64-none-linux-android") + self.assertEqual(arch.target, "aarch64-linux-android21") self.assertEqual(arch.platform_dir, "arch-arm64") - # For x86_64 we expect to find an extra key in`environment` env = arch.get_env() - self.assertIn("EXTRA_CFLAGS", env.keys()) + # check glob and find_executable calls + self.assertEqual(mock_glob.call_count, 4) + for glob_call, kw in mock_glob.call_args_list: + self.assertEqual( + glob_call[0], + "{ndk_dir}/toolchains/llvm*".format(ndk_dir=self.ctx._ndk_dir), + ) + mock_find_executable.assert_called_once_with( + self.expected_compiler, path=environ["PATH"] + ) + + # For x86_64 we expect to find an extra key in`environment` + for flag in {"CFLAGS", "CXXFLAGS", "CC", "CXX"}: + self.assertIn("-march=armv8-a", env[flag]) diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index 6f66925818..e3e75b945f 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -533,15 +533,23 @@ def reset_mocks(): @mock.patch("pythonforandroid.bootstrap.shprint") @mock.patch("pythonforandroid.bootstrap.sh.Command") @mock.patch("pythonforandroid.build.ensure_dir") + @mock.patch("pythonforandroid.archs.glob") @mock.patch("pythonforandroid.archs.find_executable") def test_bootstrap_strip( self, mock_find_executable, + mock_glob, mock_ensure_dir, mock_sh_command, mock_sh_print, ): - mock_find_executable.return_value = "arm-linux-androideabi-gcc" + mock_find_executable.return_value = os.path.join( + self.ctx._ndk_dir, + "toolchains/llvm/prebuilt/linux-x86_64/bin/clang", + ) + mock_glob.return_value = [ + os.path.join(self.ctx._ndk_dir, "toolchains", "llvm") + ] # prepare arch, bootstrap, distribution and PythonRecipe arch = ArchARMv7_a(self.ctx) bs = Bootstrap().get_bootstrap(self.bootstrap_name, self.ctx) From 98018653a3195368922a443620b80af38b1f6225 Mon Sep 17 00:00:00 2001 From: Pol Canelles Date: Sun, 25 Aug 2019 14:55:28 +0200 Subject: [PATCH 08/72] [LIBS - PART I] Initial refactor of library recipes (#1944) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit will introduce a new cls attribute: `built_libraries`. This `built_libraries` attribute must be a dictionary. The keys are the full library name, e.g.: `libffi.so`. And the values must be the relative path of the library, e.g: `.libs`. So the this cls attribute for libffi recipe it would look like: ```built_libraries = {'libffi.so': '.libs'}``` This new cls attribute will allow us to detect library recipes and to refactor common operations that we do for those kind of recipes: - copy library into right location - implement `should_build` for library recipes Those two things will make that our rebuilds to be more consistent and I hope that it will be easier to maintain. **So, once explained a little this PR, in here we do:** - [x] Add attribute `built_libraries` - [x] Add methods to refactor common library operations: - [x] install_libraries - [x] get_libraries - [x] implement `should_build` for library recipes - [x] implement basic tests for the newly introduced `Recipe` methods - [x] Make use of those Library attributes/methods for some basic set of recipes: - [x] libffi - [x] openssl - [x] png - [x] jpeg - [x] freetype - [x] harfbuzz - [x] libcurl - [x] libzbar - [x] libiconv - [x] libexpat - [x] libogg - [x] libxml2 - [x] libxslt - [x] libshine - [x] libx264 - [x] libglob - [x] libsodium - [x] libsecp256k1 **Things to do in separate PRs**, to be easy to review and because those points will start to conflict with the `NDK r19 migration` thing just merged: - Make use of Library attributes/methods introduced in this PR for un covered library recipes in here. Here we have two situations: - library recipes that depends on android's STL library (almost all the work done in here will depend of the `NDK r19 migration` thing) - all remaining library recipes, which are not STL lib dependent and that are not implemented in here (because I was unable to perform a successful build with them using arch `arm64-v8a`...so I think it would be better to deal with those recipes in separate PRs and later...with the `NDK r19 migration` thing merged) **Notes about changed recipes:** all the recipes touched in here almost have the same changes: - remove `should_build` method (to make use of the one implemented in base class for library recipes) - remove the copy of the library (because that will be done automatically by the method `install_libraries` implemented in base class) - fixed the imports due to refactoring * [recipe-lib] Add attribute Recipe.built_libraries so we can refactor some essential operations performed for library recipes (like copy the library into the right location, or decide if we should build the library recipe again) * [recipe-lib] Add method Recipe.get_libraries This function will allows us to get a list of the built/context libraries. This will be useful to check if we have to built a library recipe or which library should be rebuild * [recipe-lib] Make `Recipe.should_build` decide the library build In case that the attribute built_libraries has been set, we will check the need of the recipe build, otherwise we will act as usual, forcing the build unless overwrote in subclass * [recipe-lib] Add method `Recipe.install_libraries` So we can: - control the copy of the library recipe - refactor the install of the lib for all library recipes * [recipe-lib] Make libffi a library recipe * [recipe-lib] Make openssl a library recipe and ... also make the imports from the right module * [recipe-lib] Make png a library recipe * [recipe-lib] Make jpeg a library recipe * [recipe-lib] Make freetype a library recipe and ... also make the imports from the right module * [recipe-lib] Make harfbuzz a library recipe and ... also make the imports from the right module * [recipe-lib] Make libcurl a library recipe and ... also make the imports from the right module * [recipe-lib] Make libzbar a library recipe and ... also make the imports from the right module * [recipe-lib] Make libiconv a library recipe and ... also make the imports from the right module * [recipe-lib] Make libexpat a library recipe and ... also: - make the imports from the right module - remove hardcoded arch * [recipe-lib] Make libogg a library recipe * [recipe-lib] Make libxml2 a library recipe and ... also make the imports from the right module * [recipe-lib] Make libxslt a library recipe and ... also make the imports from the right module * [recipe-lib] Make libshine a library recipe and ... also: - make the imports from the right module - remove the hardcoded cpu count when compiling * [recipe-lib] Make libx264 a library recipe and ... also:   - make the imports from the right module   - remove the hardcoded cpu count when compiling * [recipe-lib] Make libglob a library recipe * [recipe-lib] Make libsodium a library recipe and ... also:   - make the imports from the right module   - fix hardcoded arch   - enable cpu count when compiling * [recipe-lib] Make libsecp256k1 a library recipe and ... also make the imports from the right module * [tests] Add tests for library recipe * [NDK19] Fix libglob for android NDK r19 - change the `ARG_MAX` define, because it's already defined at `sysroot/usr/include/linux/limits.h` - Replaced `size_t` by Including the header. Because found the solution at stackoverflow: As per C99, §7.17, size_t is not a builtin type but defined in See also: - https://travis-ci.org/kivy/python-for-android/jobs/576392711#L5992-L6013 - https://stackoverflow.com/questions/26410466/gcc-linaro-compiler-throws-error-unknown-type-name-size-t --- pythonforandroid/build.py | 1 + pythonforandroid/recipe.py | 60 ++++++++++++++++++- pythonforandroid/recipes/freetype/__init__.py | 28 ++++----- pythonforandroid/recipes/harfbuzz/__init__.py | 16 ++--- pythonforandroid/recipes/jpeg/__init__.py | 13 +--- pythonforandroid/recipes/libcurl/__init__.py | 16 ++--- pythonforandroid/recipes/libexpat/__init__.py | 18 ++---- pythonforandroid/recipes/libffi/__init__.py | 9 +-- pythonforandroid/recipes/libglob/__init__.py | 6 +- pythonforandroid/recipes/libglob/glob.patch | 6 +- pythonforandroid/recipes/libiconv/__init__.py | 13 ++-- pythonforandroid/recipes/libogg/__init__.py | 9 +-- .../recipes/libsecp256k1/__init__.py | 8 +-- pythonforandroid/recipes/libshine/__init__.py | 13 ++-- .../recipes/libsodium/__init__.py | 24 ++++---- pythonforandroid/recipes/libx264/__init__.py | 14 ++--- pythonforandroid/recipes/libxml2/__init__.py | 15 ++--- pythonforandroid/recipes/libxslt/__init__.py | 18 ++---- pythonforandroid/recipes/libzbar/__init__.py | 10 +--- pythonforandroid/recipes/openssl/__init__.py | 16 ++--- pythonforandroid/recipes/png/__init__.py | 9 +-- tests/test_recipe.py | 55 +++++++++++++++++ 22 files changed, 214 insertions(+), 163 deletions(-) diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index 16cc459b4c..cc6cea1406 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -575,6 +575,7 @@ def build_recipes(build_order, python_modules, ctx, project_dir, info_main('Building {} for {}'.format(recipe.name, arch.arch)) if recipe.should_build(arch): recipe.build_arch(arch) + recipe.install_libraries(arch) else: info('{} said it is already built, skipping' .format(recipe.name)) diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index f165ce474c..0edca4a514 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -106,6 +106,25 @@ class Recipe(with_metaclass(RecipeMeta)): archs = ['armeabi'] # Not currently implemented properly + built_libraries = {} + """Each recipe that builds a system library (e.g.:libffi, openssl, etc...) + should contain a dict holding the relevant information of the library. The + keys should be the generated libraries and the values the relative path of + the library inside his build folder. This dict will be used to perform + different operations: + - copy the library into the right location, depending on if it's shared + or static) + - check if we have to rebuild the library + + Here an example of how it would look like for `libffi` recipe: + + - `built_libraries = {'libffi.so': '.libs'}` + + .. note:: in case that the built library resides in recipe's build + directory, you can set the following values for the relative + path: `'.', None or ''` + """ + @property def version(self): key = 'VERSION_' + self.name @@ -479,9 +498,14 @@ def apply_patches(self, arch, build_dir=None): def should_build(self, arch): '''Should perform any necessary test and return True only if it needs - building again. + building again. Per default we implement a library test, in case that + we detect so. ''' + if self.built_libraries: + return not all( + exists(lib) for lib in self.get_libraries(arch.arch) + ) return True def build_arch(self, arch): @@ -492,6 +516,19 @@ def build_arch(self, arch): if hasattr(self, build): getattr(self, build)() + def install_libraries(self, arch): + '''This method is always called after `build_arch`. In case that we + detect a library recipe, defined by the class attribute + `built_libraries`, we will copy all defined libraries into the + right location. + ''' + if not self.built_libraries: + return + shared_libs = [ + lib for lib in self.get_libraries(arch) if lib.endswith(".so") + ] + self.install_libs(arch, *shared_libs) + def postbuild_arch(self, arch): '''Run any post-build tasks for the Recipe. By default, this checks if any postbuild_archname methods exist for the archname of the @@ -554,6 +591,27 @@ def install_libs(self, arch, *libs): def has_libs(self, arch, *libs): return all(map(lambda l: self.ctx.has_lib(arch.arch, l), libs)) + def get_libraries(self, arch_name, in_context=False): + """Return the full path of the library depending on the architecture. + Per default, the build library path it will be returned, unless + `get_libraries` has been called with kwarg `in_context` set to + True. + + .. note:: this method should be used for library recipes only + """ + recipe_libs = set() + if not self.built_libraries: + return recipe_libs + for lib, rel_path in self.built_libraries.items(): + if not in_context: + abs_path = join(self.get_build_dir(arch_name), rel_path, lib) + if rel_path in {".", "", None}: + abs_path = join(self.get_build_dir(arch_name), lib) + else: + abs_path = join(self.ctx.get_libs_dir(arch_name), lib) + recipe_libs.add(abs_path) + return recipe_libs + @classmethod def recipe_dirs(cls, ctx): recipe_dirs = [] diff --git a/pythonforandroid/recipes/freetype/__init__.py b/pythonforandroid/recipes/freetype/__init__.py index bea70cdafc..cf8de5ee85 100644 --- a/pythonforandroid/recipes/freetype/__init__.py +++ b/pythonforandroid/recipes/freetype/__init__.py @@ -1,7 +1,7 @@ -from pythonforandroid.toolchain import Recipe +from pythonforandroid.recipe import Recipe from pythonforandroid.logger import shprint, info from pythonforandroid.util import current_directory -from os.path import exists, join +from os.path import join, exists from multiprocessing import cpu_count import sh @@ -26,16 +26,7 @@ class FreetypeRecipe(Recipe): version = '2.5.5' url = 'http://download.savannah.gnu.org/releases/freetype/freetype-{version}.tar.gz' # noqa - - def should_build(self, arch): - return not exists( - join( - self.get_build_dir(arch.arch), - 'objs', - '.libs', - 'libfreetype.so', - ) - ) + built_libraries = {'libfreetype.so': 'objs/.libs'} def get_recipe_env(self, arch=None, with_harfbuzz=False): env = super(FreetypeRecipe, self).get_recipe_env(arch) @@ -111,11 +102,14 @@ def build_arch(self, arch, with_harfbuzz=False): # First build, install the compiled lib, and clean build env shprint(sh.make, 'install', _env=env) shprint(sh.make, 'distclean', _env=env) - else: - # Second build (or the first if harfbuzz not enabled), now we - # copy definitive libs to libs collection. Be sure to link your - # recipes to the definitive library, located at: objs/.libs - self.install_libs(arch, 'objs/.libs/libfreetype.so') + + def install_libraries(self, arch): + # This library it's special because the first time we built it may not + # generate the expected library, because it can depend on harfbuzz, so + # we will make sure to only install it when the library exists + if not exists(list(self.get_libraries(arch))[0]): + return + self.install_libs(arch, *self.get_libraries(arch)) recipe = FreetypeRecipe() diff --git a/pythonforandroid/recipes/harfbuzz/__init__.py b/pythonforandroid/recipes/harfbuzz/__init__.py index c2922540ac..17c72a27e2 100644 --- a/pythonforandroid/recipes/harfbuzz/__init__.py +++ b/pythonforandroid/recipes/harfbuzz/__init__.py @@ -1,8 +1,8 @@ -from pythonforandroid.toolchain import Recipe +from pythonforandroid.recipe import Recipe from pythonforandroid.util import current_directory from pythonforandroid.logger import shprint from multiprocessing import cpu_count -from os.path import exists, join +from os.path import join import sh @@ -23,13 +23,7 @@ class HarfbuzzRecipe(Recipe): version = '0.9.40' url = 'http://www.freedesktop.org/software/harfbuzz/release/harfbuzz-{version}.tar.bz2' # noqa opt_depends = ['freetype'] - - def should_build(self, arch): - return not exists( - join( - self.get_build_dir(arch.arch), 'src', '.libs', 'libharfbuzz.so' - ) - ) + built_libraries = {'libharfbuzz.so': 'src/.libs'} def get_recipe_env(self, arch=None): env = super(HarfbuzzRecipe, self).get_recipe_env(arch) @@ -68,12 +62,12 @@ def build_arch(self, arch): _env=env, ) shprint(sh.make, '-j', str(cpu_count()), _env=env) - self.install_libs(arch, join('src', '.libs', 'libharfbuzz.so')) if 'freetype' in self.ctx.recipe_build_order: - # Rebuild freetype with harfbuzz support + # Rebuild/install freetype with harfbuzz support freetype = self.get_recipe('freetype', self.ctx) freetype.build_arch(arch, with_harfbuzz=True) + freetype.install_libraries(arch) recipe = HarfbuzzRecipe() diff --git a/pythonforandroid/recipes/jpeg/__init__.py b/pythonforandroid/recipes/jpeg/__init__.py index 507ba775e4..1937ecb1d8 100644 --- a/pythonforandroid/recipes/jpeg/__init__.py +++ b/pythonforandroid/recipes/jpeg/__init__.py @@ -1,9 +1,8 @@ from pythonforandroid.recipe import Recipe from pythonforandroid.logger import shprint from pythonforandroid.util import current_directory -from os.path import join, exists +from os.path import join from os import environ, uname -from glob import glob import sh @@ -16,15 +15,11 @@ class JpegRecipe(Recipe): name = 'jpeg' version = '2.0.1' url = 'https://github.com/libjpeg-turbo/libjpeg-turbo/archive/{version}.tar.gz' # noqa + built_libraries = {'libjpeg.a': '.', 'libturbojpeg.a': '.'} # we will require this below patch to build the shared library # patches = ['remove-version.patch'] - def should_build(self, arch): - return not exists(join(self.get_build_dir(arch.arch), - 'libturbojpeg.a')) - def build_arch(self, arch): - super(JpegRecipe, self).build_arch(arch) build_dir = self.get_build_dir(arch.arch) # TODO: Fix simd/neon @@ -59,10 +54,6 @@ def build_arch(self, arch): _env=env) shprint(sh.make, _env=env) - # copy static libs to libs collection - for lib in glob(join(build_dir, '*.a')): - shprint(sh.cp, '-L', lib, self.ctx.libs_dir) - def get_recipe_env(self, arch=None, with_flags_in_cc=False): env = environ.copy() diff --git a/pythonforandroid/recipes/libcurl/__init__.py b/pythonforandroid/recipes/libcurl/__init__.py index e8cc86042d..27b142e071 100644 --- a/pythonforandroid/recipes/libcurl/__init__.py +++ b/pythonforandroid/recipes/libcurl/__init__.py @@ -1,20 +1,18 @@ import sh -from pythonforandroid.toolchain import Recipe, shprint, shutil, current_directory -from os.path import exists, join +from pythonforandroid.recipe import Recipe +from pythonforandroid.util import current_directory +from pythonforandroid.logger import shprint +from os.path import join from multiprocessing import cpu_count class LibcurlRecipe(Recipe): version = '7.55.1' url = 'https://curl.haxx.se/download/curl-7.55.1.tar.gz' + built_libraries = {'libcurl.so': 'dist/lib'} depends = ['openssl'] - def should_build(self, arch): - super(LibcurlRecipe, self).should_build(arch) - return not exists(join(self.ctx.get_libs_dir(arch.arch), 'libcurl.so')) - def build_arch(self, arch): - super(LibcurlRecipe, self).build_arch(arch) env = self.get_recipe_env(arch) r = self.get_recipe('openssl', self.ctx) @@ -31,10 +29,6 @@ def build_arch(self, arch): _env=env) shprint(sh.make, '-j', str(cpu_count()), _env=env) shprint(sh.make, 'install', _env=env) - shutil.copyfile('{}/lib/libcurl.so'.format(dst_dir), - join( - self.ctx.get_libs_dir(arch.arch), - 'libcurl.so')) recipe = LibcurlRecipe() diff --git a/pythonforandroid/recipes/libexpat/__init__.py b/pythonforandroid/recipes/libexpat/__init__.py index ecf5265db3..614b0df0ff 100644 --- a/pythonforandroid/recipes/libexpat/__init__.py +++ b/pythonforandroid/recipes/libexpat/__init__.py @@ -1,38 +1,32 @@ import sh -from pythonforandroid.toolchain import Recipe, shprint, shutil, current_directory -from os.path import exists, join +from pythonforandroid.recipe import Recipe +from pythonforandroid.util import current_directory +from pythonforandroid.logger import shprint +from os.path import join from multiprocessing import cpu_count class LibexpatRecipe(Recipe): version = 'master' url = 'https://github.com/libexpat/libexpat/archive/{version}.zip' + built_libraries = {'libexpat.so': 'dist/lib'} depends = [] - def should_build(self, arch): - super(LibexpatRecipe, self).should_build(arch) - return not exists( - join(self.ctx.get_libs_dir(arch.arch), 'libexpat.so')) - def build_arch(self, arch): - super(LibexpatRecipe, self).build_arch(arch) env = self.get_recipe_env(arch) with current_directory(join(self.get_build_dir(arch.arch), 'expat')): dst_dir = join(self.get_build_dir(arch.arch), 'dist') shprint(sh.Command('./buildconf.sh'), _env=env) shprint( sh.Command('./configure'), - '--host=arm-linux-androideabi', + '--host={}'.format(arch.command_prefix), '--enable-shared', '--without-xmlwf', '--prefix={}'.format(dst_dir), _env=env) shprint(sh.make, '-j', str(cpu_count()), _env=env) shprint(sh.make, 'install', _env=env) - shutil.copyfile( - '{}/lib/libexpat.so'.format(dst_dir), - join(self.ctx.get_libs_dir(arch.arch), 'libexpat.so')) recipe = LibexpatRecipe() diff --git a/pythonforandroid/recipes/libffi/__init__.py b/pythonforandroid/recipes/libffi/__init__.py index 3608df6eb0..0d26ec2c43 100644 --- a/pythonforandroid/recipes/libffi/__init__.py +++ b/pythonforandroid/recipes/libffi/__init__.py @@ -2,7 +2,7 @@ from multiprocessing import cpu_count from pythonforandroid.recipe import Recipe from pythonforandroid.logger import shprint -from pythonforandroid.util import current_directory, ensure_dir +from pythonforandroid.util import current_directory import sh @@ -31,8 +31,7 @@ class LibffiRecipe(Recipe): patches = ['remove-version-info.patch'] - def should_build(self, arch): - return not exists(join(self.ctx.get_libs_dir(arch.arch), 'libffi.so')) + built_libraries = {'libffi.so': '.libs'} def build_arch(self, arch): env = self.get_recipe_env(arch) @@ -46,10 +45,6 @@ def build_arch(self, arch): '--disable-builddir', '--enable-shared', _env=env) shprint(sh.make, '-j', str(cpu_count()), 'libffi.la', _env=env) - ensure_dir(self.ctx.get_libs_dir(arch.arch)) - self.install_libs( - arch, join(self.get_build_dir(arch.arch), '.libs', 'libffi.so') - ) def get_include_dirs(self, arch): return [join(self.get_build_dir(arch.arch), 'include')] diff --git a/pythonforandroid/recipes/libglob/__init__.py b/pythonforandroid/recipes/libglob/__init__.py index e0fccfecfe..4c69657c87 100644 --- a/pythonforandroid/recipes/libglob/__init__.py +++ b/pythonforandroid/recipes/libglob/__init__.py @@ -3,13 +3,13 @@ available via '-lglob' LDFLAG """ from os.path import exists, join -from pythonforandroid.recipe import CompiledComponentsPythonRecipe +from pythonforandroid.recipe import Recipe from pythonforandroid.toolchain import current_directory from pythonforandroid.logger import info, shprint import sh -class LibGlobRecipe(CompiledComponentsPythonRecipe): +class LibGlobRecipe(Recipe): """Make a glob.h and glob.so for the python_install_dir()""" version = '0.0.1' url = None @@ -20,6 +20,7 @@ class LibGlobRecipe(CompiledComponentsPythonRecipe): # https://raw.githubusercontent.com/white-gecko/TokyoCabinet/master/glob.c # and pushed in via patch name = 'libglob' + built_libraries = {'libglob.so': '.'} depends = [('hostpython2', 'hostpython3')] patches = ['glob.patch'] @@ -60,7 +61,6 @@ def build_arch(self, arch): cflags.extend(['-shared', '-I.', 'glob.o', '-o', 'libglob.so']) cflags.extend(env['LDFLAGS'].split()) shprint(cc, *cflags, _env=env) - shprint(sh.cp, 'libglob.so', join(self.ctx.libs_dir, arch.arch)) recipe = LibGlobRecipe() diff --git a/pythonforandroid/recipes/libglob/glob.patch b/pythonforandroid/recipes/libglob/glob.patch index c7fe81738f..ee71719a1e 100644 --- a/pythonforandroid/recipes/libglob/glob.patch +++ b/pythonforandroid/recipes/libglob/glob.patch @@ -911,7 +911,7 @@ diff -Nur /tmp/x/glob.c libglob/glob.c diff -Nur /tmp/x/glob.h libglob/glob.h --- /tmp/x/glob.h 1969-12-31 19:00:00.000000000 -0500 +++ libglob/glob.h 2017-08-19 15:22:18.367109399 -0400 -@@ -0,0 +1,102 @@ +@@ -0,0 +1,104 @@ +/* + * Copyright (c) 1989, 1993 + * The Regents of the University of California. All rights reserved. @@ -952,10 +952,12 @@ diff -Nur /tmp/x/glob.h libglob/glob.h + +#include +#include ++#ifndef ARG_MAX +#define ARG_MAX 6553 ++#endif + +#ifndef _SIZE_T_DECLARED -+typedef __size_t size_t; ++#include +#define _SIZE_T_DECLARED +#endif + diff --git a/pythonforandroid/recipes/libiconv/__init__.py b/pythonforandroid/recipes/libiconv/__init__.py index 4a64669202..6bde196e56 100644 --- a/pythonforandroid/recipes/libiconv/__init__.py +++ b/pythonforandroid/recipes/libiconv/__init__.py @@ -1,5 +1,5 @@ -import os -from pythonforandroid.toolchain import shprint, current_directory +from pythonforandroid.logger import shprint +from pythonforandroid.util import current_directory from pythonforandroid.recipe import Recipe from multiprocessing import cpu_count import sh @@ -11,14 +11,11 @@ class LibIconvRecipe(Recipe): url = 'https://ftp.gnu.org/pub/gnu/libiconv/libiconv-{version}.tar.gz' - patches = ['libiconv-1.15-no-gets.patch'] + built_libraries = {'libiconv.so': 'lib/.libs'} - def should_build(self, arch): - return not os.path.exists( - os.path.join(self.ctx.get_libs_dir(arch.arch), 'libiconv.so')) + patches = ['libiconv-1.15-no-gets.patch'] def build_arch(self, arch): - super(LibIconvRecipe, self).build_arch(arch) env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): shprint( @@ -27,8 +24,6 @@ def build_arch(self, arch): '--prefix=' + self.ctx.get_python_install_dir(), _env=env) shprint(sh.make, '-j' + str(cpu_count()), _env=env) - libs = ['lib/.libs/libiconv.so'] - self.install_libs(arch, *libs) recipe = LibIconvRecipe() diff --git a/pythonforandroid/recipes/libogg/__init__.py b/pythonforandroid/recipes/libogg/__init__.py index 064189eb7d..a96eca9c34 100644 --- a/pythonforandroid/recipes/libogg/__init__.py +++ b/pythonforandroid/recipes/libogg/__init__.py @@ -1,14 +1,12 @@ -from pythonforandroid.recipe import NDKRecipe +from pythonforandroid.recipe import Recipe from pythonforandroid.toolchain import current_directory, shprint -from os.path import join import sh -class OggRecipe(NDKRecipe): +class OggRecipe(Recipe): version = '1.3.3' url = 'http://downloads.xiph.org/releases/ogg/libogg-{version}.tar.gz' - - generated_libraries = ['libogg.so'] + built_libraries = {'libogg.so': 'src/.libs'} def build_arch(self, arch): with current_directory(self.get_build_dir(arch.arch)): @@ -20,7 +18,6 @@ def build_arch(self, arch): configure = sh.Command('./configure') shprint(configure, *flags, _env=env) shprint(sh.make, _env=env) - self.install_libs(arch, join('src', '.libs', 'libogg.so')) recipe = OggRecipe() diff --git a/pythonforandroid/recipes/libsecp256k1/__init__.py b/pythonforandroid/recipes/libsecp256k1/__init__.py index a8552577eb..caa5a6fc37 100644 --- a/pythonforandroid/recipes/libsecp256k1/__init__.py +++ b/pythonforandroid/recipes/libsecp256k1/__init__.py @@ -1,4 +1,5 @@ -from pythonforandroid.toolchain import shprint, current_directory +from pythonforandroid.logger import shprint +from pythonforandroid.util import current_directory from pythonforandroid.recipe import Recipe from multiprocessing import cpu_count from os.path import exists @@ -7,10 +8,11 @@ class LibSecp256k1Recipe(Recipe): + built_libraries = {'libsecp256k1.so': '.libs'} + url = 'https://github.com/bitcoin-core/secp256k1/archive/master.zip' def build_arch(self, arch): - super(LibSecp256k1Recipe, self).build_arch(arch) env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): if not exists('configure'): @@ -25,8 +27,6 @@ def build_arch(self, arch): '--enable-module-ecdh', _env=env) shprint(sh.make, '-j' + str(cpu_count()), _env=env) - libs = ['.libs/libsecp256k1.so'] - self.install_libs(arch, *libs) recipe = LibSecp256k1Recipe() diff --git a/pythonforandroid/recipes/libshine/__init__.py b/pythonforandroid/recipes/libshine/__init__.py index fe9b5b589c..7d114d25a1 100644 --- a/pythonforandroid/recipes/libshine/__init__.py +++ b/pythonforandroid/recipes/libshine/__init__.py @@ -1,5 +1,8 @@ -from pythonforandroid.toolchain import Recipe, current_directory, shprint -from os.path import exists, join, realpath +from pythonforandroid.recipe import Recipe +from pythonforandroid.util import current_directory +from pythonforandroid.logger import shprint +from multiprocessing import cpu_count +from os.path import realpath import sh @@ -7,9 +10,7 @@ class LibShineRecipe(Recipe): version = 'c72aba9031bde18a0995e7c01c9b53f2e08a0e46' url = 'https://github.com/toots/shine/archive/{version}.zip' - def should_build(self, arch): - build_dir = self.get_build_dir(arch.arch) - return not exists(join(build_dir, 'lib', 'libshine.a')) + built_libraries = {'libshine.a': 'lib'} def build_arch(self, arch): with current_directory(self.get_build_dir(arch.arch)): @@ -23,7 +24,7 @@ def build_arch(self, arch): '--enable-static', '--prefix={}'.format(realpath('.')), _env=env) - shprint(sh.make, '-j4', _env=env) + shprint(sh.make, '-j', str(cpu_count()), _env=env) shprint(sh.make, 'install', _env=env) diff --git a/pythonforandroid/recipes/libsodium/__init__.py b/pythonforandroid/recipes/libsodium/__init__.py index 9911e36baa..8165ebc57e 100644 --- a/pythonforandroid/recipes/libsodium/__init__.py +++ b/pythonforandroid/recipes/libsodium/__init__.py @@ -1,5 +1,7 @@ -from pythonforandroid.toolchain import Recipe, shprint, shutil, current_directory -from os.path import exists, join +from pythonforandroid.recipe import Recipe +from pythonforandroid.util import current_directory +from pythonforandroid.logger import shprint +from multiprocessing import cpu_count import sh @@ -8,19 +10,21 @@ class LibsodiumRecipe(Recipe): url = 'https://github.com/jedisct1/libsodium/releases/download/{version}/libsodium-{version}.tar.gz' depends = [] patches = ['size_max_fix.patch'] - - def should_build(self, arch): - super(LibsodiumRecipe, self).should_build(arch) - return not exists(join(self.ctx.get_libs_dir(arch.arch), 'libsodium.so')) + built_libraries = {'libsodium.so': 'src/libsodium/.libs'} def build_arch(self, arch): - super(LibsodiumRecipe, self).build_arch(arch) env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): bash = sh.Command('bash') - shprint(bash, 'configure', '--disable-soname-versions', '--host=arm-linux-androideabi', '--enable-shared', _env=env) - shprint(sh.make, _env=env) - shutil.copyfile('src/libsodium/.libs/libsodium.so', join(self.ctx.get_libs_dir(arch.arch), 'libsodium.so')) + shprint( + bash, + 'configure', + '--disable-soname-versions', + '--host={}'.format(arch.command_prefix), + '--enable-shared', + _env=env, + ) + shprint(sh.make, '-j', str(cpu_count()), _env=env) def get_recipe_env(self, arch): env = super(LibsodiumRecipe, self).get_recipe_env(arch) diff --git a/pythonforandroid/recipes/libx264/__init__.py b/pythonforandroid/recipes/libx264/__init__.py index 89d48c8410..13ba31f066 100644 --- a/pythonforandroid/recipes/libx264/__init__.py +++ b/pythonforandroid/recipes/libx264/__init__.py @@ -1,15 +1,15 @@ -from pythonforandroid.toolchain import Recipe, current_directory, shprint -from os.path import exists, join, realpath +from pythonforandroid.recipe import Recipe +from pythonforandroid.util import current_directory +from pythonforandroid.logger import shprint +from multiprocessing import cpu_count +from os.path import realpath import sh class LibX264Recipe(Recipe): version = 'x264-snapshot-20171218-2245-stable' # using mirror url since can't use ftp url = 'http://mirror.yandex.ru/mirrors/ftp.videolan.org/x264/snapshots/{version}.tar.bz2' - - def should_build(self, arch): - build_dir = self.get_build_dir(arch.arch) - return not exists(join(build_dir, 'lib', 'libx264.a')) + built_libraries = {'libx264.a': 'lib'} def build_arch(self, arch): with current_directory(self.get_build_dir(arch.arch)): @@ -29,7 +29,7 @@ def build_arch(self, arch): '--enable-static', '--prefix={}'.format(realpath('.')), _env=env) - shprint(sh.make, '-j4', _env=env) + shprint(sh.make, '-j', str(cpu_count()), _env=env) shprint(sh.make, 'install', _env=env) diff --git a/pythonforandroid/recipes/libxml2/__init__.py b/pythonforandroid/recipes/libxml2/__init__.py index cdeaf88d95..bcccc4ebd8 100644 --- a/pythonforandroid/recipes/libxml2/__init__.py +++ b/pythonforandroid/recipes/libxml2/__init__.py @@ -1,6 +1,7 @@ from pythonforandroid.recipe import Recipe -from pythonforandroid.toolchain import shprint, shutil, current_directory -from os.path import exists, join +from pythonforandroid.util import current_directory +from pythonforandroid.logger import shprint +from os.path import exists import sh @@ -9,14 +10,9 @@ class Libxml2Recipe(Recipe): url = 'http://xmlsoft.org/sources/libxml2-{version}.tar.gz' depends = [] patches = ['add-glob.c.patch'] - - def should_build(self, arch): - super(Libxml2Recipe, self).should_build(arch) - return not exists( - join(self.get_build_dir(arch.arch), '.libs', 'libxml2.a')) + built_libraries = {'libxml2.a': '.libs'} def build_arch(self, arch): - super(Libxml2Recipe, self).build_arch(arch) env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): @@ -46,9 +42,6 @@ def build_arch(self, arch): # we'll need the glob dependency which is a big headache shprint(sh.make, "libxml2.la", _env=env) - shutil.copyfile('.libs/libxml2.a', - join(self.ctx.libs_dir, 'libxml2.a')) - def get_recipe_env(self, arch): env = super(Libxml2Recipe, self).get_recipe_env(arch) env['CONFIG_SHELL'] = '/bin/bash' diff --git a/pythonforandroid/recipes/libxslt/__init__.py b/pythonforandroid/recipes/libxslt/__init__.py index 076d6cc6a1..b22e2b328a 100644 --- a/pythonforandroid/recipes/libxslt/__init__.py +++ b/pythonforandroid/recipes/libxslt/__init__.py @@ -1,5 +1,6 @@ from pythonforandroid.recipe import Recipe -from pythonforandroid.toolchain import shprint, shutil, current_directory +from pythonforandroid.util import current_directory +from pythonforandroid.logger import shprint from os.path import exists, join import sh @@ -9,16 +10,14 @@ class LibxsltRecipe(Recipe): url = 'http://xmlsoft.org/sources/libxslt-{version}.tar.gz' depends = ['libxml2'] patches = ['fix-dlopen.patch'] + built_libraries = { + 'libxslt.a': 'libxslt/.libs', + 'libexslt.a': 'libexslt/.libs' + } call_hostpython_via_targetpython = False - def should_build(self, arch): - return not exists( - join(self.get_build_dir(arch.arch), - 'libxslt', '.libs', 'libxslt.a')) - def build_arch(self, arch): - super(LibxsltRecipe, self).build_arch(arch) env = self.get_recipe_env(arch) build_dir = self.get_build_dir(arch.arch) with current_directory(build_dir): @@ -45,11 +44,6 @@ def build_arch(self, arch): _env=env) shprint(sh.make, "V=1", _env=env) - shutil.copyfile('libxslt/.libs/libxslt.a', - join(self.ctx.libs_dir, 'libxslt.a')) - shutil.copyfile('libexslt/.libs/libexslt.a', - join(self.ctx.libs_dir, 'libexslt.a')) - def get_recipe_env(self, arch): env = super(LibxsltRecipe, self).get_recipe_env(arch) env['CONFIG_SHELL'] = '/bin/bash' diff --git a/pythonforandroid/recipes/libzbar/__init__.py b/pythonforandroid/recipes/libzbar/__init__.py index 43ae34cc9d..8d7d48e2b7 100644 --- a/pythonforandroid/recipes/libzbar/__init__.py +++ b/pythonforandroid/recipes/libzbar/__init__.py @@ -1,6 +1,7 @@ import os -from pythonforandroid.toolchain import shprint, current_directory from pythonforandroid.recipe import Recipe +from pythonforandroid.util import current_directory +from pythonforandroid.logger import shprint from multiprocessing import cpu_count import sh @@ -15,9 +16,7 @@ class LibZBarRecipe(Recipe): patches = ["werror.patch"] - def should_build(self, arch): - return not os.path.exists( - os.path.join(self.ctx.get_libs_dir(arch.arch), 'libzbar.so')) + built_libraries = {'libzbar.so': 'zbar/.libs'} def get_recipe_env(self, arch=None, with_flags_in_cc=True): env = super(LibZBarRecipe, self).get_recipe_env(arch, with_flags_in_cc) @@ -28,7 +27,6 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): return env def build_arch(self, arch): - super(LibZBarRecipe, self).build_arch(arch) env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): shprint(sh.Command('autoreconf'), '-vif', _env=env) @@ -50,8 +48,6 @@ def build_arch(self, arch): '--enable-static=no', _env=env) shprint(sh.make, '-j' + str(cpu_count()), _env=env) - libs = ['zbar/.libs/libzbar.so'] - self.install_libs(arch, *libs) recipe = LibZBarRecipe() diff --git a/pythonforandroid/recipes/openssl/__init__.py b/pythonforandroid/recipes/openssl/__init__.py index a4aad3df16..a2ecb2c549 100644 --- a/pythonforandroid/recipes/openssl/__init__.py +++ b/pythonforandroid/recipes/openssl/__init__.py @@ -1,6 +1,8 @@ from os.path import join -from pythonforandroid.toolchain import Recipe, shprint, current_directory +from pythonforandroid.recipe import Recipe +from pythonforandroid.util import current_directory +from pythonforandroid.logger import shprint import sh @@ -50,6 +52,11 @@ class OpenSSLRecipe(Recipe): url = 'https://www.openssl.org/source/openssl-{url_version}.tar.gz' + built_libraries = { + 'libcrypto{version}.so'.format(version=version): '.', + 'libssl{version}.so'.format(version=version): '.', + } + @property def versioned_url(self): if self.url is None: @@ -85,10 +92,6 @@ def link_flags(self, arch): in the format: `-L -l`''' return self.link_dirs_flags(arch) + self.link_libs_flags() - def should_build(self, arch): - return not self.has_libs(arch, 'libssl' + self.version + '.so', - 'libcrypto' + self.version + '.so') - def get_recipe_env(self, arch=None): env = super(OpenSSLRecipe, self).get_recipe_env(arch) env['OPENSSL_VERSION'] = self.version @@ -129,8 +132,5 @@ def build_arch(self, arch): shprint(sh.make, 'build_libs', _env=env) - self.install_libs(arch, 'libssl' + self.version + '.so', - 'libcrypto' + self.version + '.so') - recipe = OpenSSLRecipe() diff --git a/pythonforandroid/recipes/png/__init__.py b/pythonforandroid/recipes/png/__init__.py index a6e281a570..a49f28ea2c 100644 --- a/pythonforandroid/recipes/png/__init__.py +++ b/pythonforandroid/recipes/png/__init__.py @@ -2,7 +2,6 @@ from pythonforandroid.logger import shprint from pythonforandroid.util import current_directory from multiprocessing import cpu_count -from os.path import join, exists import sh @@ -10,14 +9,9 @@ class PngRecipe(Recipe): name = 'png' version = 'v1.6.37' url = 'https://github.com/glennrp/libpng/archive/{version}.zip' - - def should_build(self, arch): - return not exists( - join(self.get_build_dir(arch.arch), '.libs', 'libpng16.so') - ) + built_libraries = {'libpng16.so': '.libs'} def build_arch(self, arch): - super(PngRecipe, self).build_arch(arch) build_dir = self.get_build_dir(arch.arch) with current_directory(build_dir): env = self.get_recipe_env(arch) @@ -37,7 +31,6 @@ def build_arch(self, arch): _env=env, ) shprint(sh.make, '-j', str(cpu_count()), _env=env) - self.install_libs(arch, join(build_dir, '.libs', 'libpng16.so')) recipe = PngRecipe() diff --git a/tests/test_recipe.py b/tests/test_recipe.py index 9685b15d68..fe4d8c853a 100644 --- a/tests/test_recipe.py +++ b/tests/test_recipe.py @@ -7,6 +7,9 @@ from backports import tempfile from pythonforandroid.build import Context from pythonforandroid.recipe import Recipe, import_recipe +from pythonforandroid.archs import ArchAarch_64 +from pythonforandroid.bootstrap import Bootstrap +from test_bootstrap import BaseClassSetupBootstrap def patch_logger(level): @@ -176,3 +179,55 @@ def test_download_file_scheme_https_oserror(self): assert m_urlretrieve.call_args_list == expected_call_args_list expected_call_args_list = [mock.call(1)] * (retry - 1) assert m_sleep.call_args_list == expected_call_args_list + + +class TestLibraryRecipe(BaseClassSetupBootstrap, unittest.TestCase): + def setUp(self): + """ + Initialize a Context with a Bootstrap and a Distribution to properly + test an library recipe, to do so we reuse `BaseClassSetupBootstrap` + """ + super(TestLibraryRecipe, self).setUp() + self.ctx.bootstrap = Bootstrap().get_bootstrap('sdl2', self.ctx) + self.setUp_distribution_with_bootstrap(self.ctx.bootstrap) + + def test_built_libraries(self): + """The openssl recipe is a library recipe, so it should have set the + attribute `built_libraries`, but not the case of `pyopenssl` recipe. + """ + recipe = Recipe.get_recipe('openssl', self.ctx) + self.assertTrue(recipe.built_libraries) + + recipe = Recipe.get_recipe('pyopenssl', self.ctx) + self.assertFalse(recipe.built_libraries) + + @mock.patch('pythonforandroid.recipe.exists') + def test_should_build(self, mock_exists): + arch = ArchAarch_64(self.ctx) + recipe = Recipe.get_recipe('openssl', self.ctx) + recipe.ctx = self.ctx + self.assertFalse(recipe.should_build(arch)) + + mock_exists.return_value = False + self.assertTrue(recipe.should_build(arch)) + + @mock.patch('pythonforandroid.recipe.Recipe.get_libraries') + @mock.patch('pythonforandroid.recipe.Recipe.install_libs') + def test_install_libraries(self, mock_install_libs, mock_get_libraries): + mock_get_libraries.return_value = { + '/build_lib/libsample1.so', + '/build_lib/libsample2.so', + } + self.ctx.recipe_build_order = [ + "hostpython3", + "openssl", + "python3", + "sdl2", + "kivy", + ] + arch = ArchAarch_64(self.ctx) + recipe = Recipe.get_recipe('openssl', self.ctx) + recipe.install_libraries(arch) + mock_install_libs.assert_called_once_with( + arch, *mock_get_libraries.return_value + ) From be8566dd58c18aa23344b04c5521518ab45a9c41 Mon Sep 17 00:00:00 2001 From: Pol Canelles Date: Sun, 25 Aug 2019 23:43:59 +0200 Subject: [PATCH 09/72] [NDK19] Remove `--sysroot` from LDFLAGS for cffi and pymunk (#1965) Because `--sysroot` flag is not needed anymore and make it fails the build since we migrated to new android's build system (implemented in NDK r19) See also: - https://developer.android.com/ndk/guides/other_build_systems --- pythonforandroid/recipes/cffi/__init__.py | 1 - pythonforandroid/recipes/pymunk/__init__.py | 1 - 2 files changed, 2 deletions(-) diff --git a/pythonforandroid/recipes/cffi/__init__.py b/pythonforandroid/recipes/cffi/__init__.py index 50458e55f6..aa291dca46 100644 --- a/pythonforandroid/recipes/cffi/__init__.py +++ b/pythonforandroid/recipes/cffi/__init__.py @@ -38,7 +38,6 @@ def get_recipe_env(self, arch=None): ndk_dir = self.ctx.ndk_platform ndk_lib_dir = os.path.join(ndk_dir, 'usr', 'lib') env['LDFLAGS'] += ' -L{}'.format(ndk_lib_dir) - env['LDFLAGS'] += " --sysroot={}".format(self.ctx.ndk_platform) env['PYTHONPATH'] = ':'.join([ self.ctx.get_site_packages_dir(), env['BUILDLIB_PATH'], diff --git a/pythonforandroid/recipes/pymunk/__init__.py b/pythonforandroid/recipes/pymunk/__init__.py index bb33b275a5..5f027ab0fb 100644 --- a/pythonforandroid/recipes/pymunk/__init__.py +++ b/pythonforandroid/recipes/pymunk/__init__.py @@ -14,7 +14,6 @@ def get_recipe_env(self, arch): env['PYTHON_ROOT'] = self.ctx.get_python_install_dir() env['LDFLAGS'] += " -shared -llog" env['LDFLAGS'] += ' -L{}'.format(join(self.ctx.ndk_platform, 'usr', 'lib')) - env['LDFLAGS'] += " --sysroot={}".format(self.ctx.ndk_platform) env['LIBS'] = env.get('LIBS', '') + ' -landroid' return env From f6df2bd61376f85e1bd2ff287cf718ea5311c11f Mon Sep 17 00:00:00 2001 From: Alexander Taylor Date: Mon, 26 Aug 2019 15:01:19 +0100 Subject: [PATCH 10/72] Added libffi headers troubleshooting note to doc (#1972) --- doc/source/troubleshooting.rst | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/doc/source/troubleshooting.rst b/doc/source/troubleshooting.rst index 87194de98e..f213e8d9eb 100644 --- a/doc/source/troubleshooting.rst +++ b/doc/source/troubleshooting.rst @@ -24,7 +24,7 @@ get help with any problems using the same channels as Kivy itself: - by email to the `kivy-users Google group `_ - on `#support Discord channel `_ - + If you find a bug, you can also post an issue on the `python-for-android Github page `_. @@ -58,7 +58,7 @@ grepping this). When your app crashes, you'll see the normal Python traceback here, as well as the output of any print statements etc. that your app runs. Use these to diagnose the problem just as normal. - + The adb command passes its arguments straight to adb itself, so you can also do other debugging tasks such as ``python-for-android adb devices`` to get the list of connected devices. @@ -88,7 +88,7 @@ At the top level, this will always contain the same set of files:: The Python distribution is in the assets folder:: $ cd assets - $ ls + $ ls private.mp3 ``private.mp3`` is actually a tarball containing all your packaged @@ -169,7 +169,7 @@ fix it, change your code to reference websocket-client: if you see errors relating to 'SSL not available' ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Ensure you have the package backports.ssl-match-hostname in the buildozer requirements, since Kivy targets python 2.7.x - + You may also need sslopt={"cert_reqs": ssl.CERT_NONE} as a parameter to ws.run_forever() if you get an error relating to host verification Requested API target 19 is not available, install it with the SDK android tool @@ -183,3 +183,10 @@ version). If using buildozer this should be done automatically, but as a workaround you can run these from ``~/.buildozer/android/platform/android-sdk-20/tools/android``. + +ModuleNotFoundError: No module named '_ctypes' +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You do not have the libffi headers available to python-for-android, so you need to install them. On Ubuntu and derivatives these come from the `libffi-dev` package. + +After installing the headers, clean the build (`p4a clean builds`, or with buildozer delete the `.buildozer` directory within your app directory) and run python-for-android again. From 7ec64aaafd72267c832f9ad888bfd42e48b80c5d Mon Sep 17 00:00:00 2001 From: Pol Canelles Date: Thu, 29 Aug 2019 22:50:18 +0200 Subject: [PATCH 11/72] [LIBS - PART II] Part II of NDK r19 migration - Initial STL lib migration (#1947) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [recipe-stl] Add android's STL lib support to `Recipe` To allow us to refactor some common operations that we use in our recipes that depends on android's STL library. Note: This commit will allow us to begin the migration to `c++_shared`. This is a must when we move to android's NDK r19+, because as for android NDK >= 18 is the only one supported STL library. * [recipe-stl] Make CppCompiledComponentsPythonRecipe use Recipe's STL support * [recipe-stl] Make icu a library recipe with STL support (rework) Also done here:   - Remove hardcoded version in url   - Disable versioned shared libraries   - Make it to be build as a shared libraries (instead of static)   - Disable the build of static libraries (because we build them as shared ones, so we avoid to link with them without our consents)   - Shorten long lines to be pep8's friendly   - Remove icu from ci/constants   - Remove `RuntimeError` about the need to use NDK api <= 19 (because that is not the case anymore)   - consider host's number of cpu to perform the build * [recipe-stl] Rework pyicu recipe to match latest icu changes Also done here:   - Remove icu.patch because now we don't have the version in our icu libraries   - Shorten long lines to be pep8's friendly * [tests] Add tests for recipe with STL support * [tests] Add tests for icu recipe * [tests] Add tests for pyicu recipe --- ci/constants.py | 1 - pythonforandroid/recipe.py | 90 ++++++---- pythonforandroid/recipes/icu/__init__.py | 123 +++++-------- .../recipes/icu/disable-libs-version.patch | 66 +++++++ pythonforandroid/recipes/pyicu/__init__.py | 55 ++---- pythonforandroid/recipes/pyicu/icu.patch | 19 -- tests/recipes/test_icu.py | 167 ++++++++++++++++++ tests/recipes/test_pyicu.py | 88 +++++++++ tests/test_recipe.py | 122 +++++++++++++ 9 files changed, 563 insertions(+), 168 deletions(-) create mode 100644 pythonforandroid/recipes/icu/disable-libs-version.patch delete mode 100644 pythonforandroid/recipes/pyicu/icu.patch create mode 100644 tests/recipes/test_icu.py create mode 100644 tests/recipes/test_pyicu.py diff --git a/ci/constants.py b/ci/constants.py index c5ab61099d..6699d24cd3 100644 --- a/ci/constants.py +++ b/ci/constants.py @@ -65,7 +65,6 @@ class TargetPython(Enum): # IndexError: list index out of range 'secp256k1', 'ffpyplayer', - 'icu', # requires `libpq-dev` system dependency e.g. for `pg_config` binary 'psycopg2', 'protobuf_cpp', diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 0edca4a514..3d814d5c4e 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -125,6 +125,46 @@ class Recipe(with_metaclass(RecipeMeta)): path: `'.', None or ''` """ + need_stl_shared = False + '''Some libraries or python packages may need to be linked with android's + stl. We can automatically do this for any recipe if we set this property to + `True`''' + + stl_lib_name = 'c++_shared' + ''' + The default STL shared lib to use: `c++_shared`. + + .. note:: Android NDK version > 17 only supports 'c++_shared', because + starting from NDK r18 the `gnustl_shared` lib has been deprecated. + ''' + + stl_lib_source = '{ctx.ndk_dir}/sources/cxx-stl/llvm-libc++' + ''' + The source directory of the selected stl lib, defined in property + `stl_lib_name` + ''' + + @property + def stl_include_dir(self): + return join(self.stl_lib_source.format(ctx=self.ctx), 'include') + + def get_stl_lib_dir(self, arch): + return join( + self.stl_lib_source.format(ctx=self.ctx), 'libs', arch.arch + ) + + def get_stl_library(self, arch): + return join( + self.get_stl_lib_dir(arch), + 'lib{name}.so'.format(name=self.stl_lib_name), + ) + + def install_stl_lib(self, arch): + if not self.ctx.has_lib( + arch.arch, 'lib{name}.so'.format(name=self.stl_lib_name) + ): + self.install_libs(arch, self.get_stl_library(arch)) + @property def version(self): key = 'VERSION_' + self.name @@ -454,7 +494,22 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): """ if arch is None: arch = self.filtered_archs[0] - return arch.get_env(with_flags_in_cc=with_flags_in_cc) + env = arch.get_env(with_flags_in_cc=with_flags_in_cc) + + if self.need_stl_shared: + env['CPPFLAGS'] = env.get('CPPFLAGS', '') + env['CPPFLAGS'] += ' -I{}'.format(self.stl_include_dir) + + env['CXXFLAGS'] = env['CFLAGS'] + ' -frtti -fexceptions' + + if with_flags_in_cc: + env['CXX'] += ' -frtti -fexceptions' + + env['LDFLAGS'] += ' -L{}'.format(self.get_stl_lib_dir(arch)) + env['LIBS'] = env.get('LIBS', '') + " -l{}".format( + self.stl_lib_name + ) + return env def prebuild_arch(self, arch): '''Run any pre-build tasks for the Recipe. By default, this checks if @@ -538,6 +593,9 @@ def postbuild_arch(self, arch): if hasattr(self, postbuild): getattr(self, postbuild)() + if self.need_stl_shared: + self.install_stl_lib(arch) + def prepare_build_dir(self, arch): '''Copies the recipe data into a build dir for the given arch. By default, this unpacks a downloaded recipe. You should override @@ -982,35 +1040,7 @@ def rebuild_compiled_components(self, arch, env): class CppCompiledComponentsPythonRecipe(CompiledComponentsPythonRecipe): """ Extensions that require the cxx-stl """ call_hostpython_via_targetpython = False - - def get_recipe_env(self, arch): - env = super(CppCompiledComponentsPythonRecipe, self).get_recipe_env(arch) - keys = dict( - ctx=self.ctx, - arch=arch, - arch_noeabi=arch.arch.replace('eabi', '') - ) - env['LDSHARED'] = env['CC'] + ' -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions' - env['CFLAGS'] += ( - " -I{ctx.ndk_dir}/platforms/android-{ctx.android_api}/arch-{arch_noeabi}/usr/include" + - " -I{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/include" + - " -I{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}/include").format(**keys) - env['CXXFLAGS'] = env['CFLAGS'] + ' -frtti -fexceptions' - env['LDFLAGS'] += ( - " -L{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}" + - " -lgnustl_shared").format(**keys) - - return env - - def build_compiled_components(self, arch): - super(CppCompiledComponentsPythonRecipe, self).build_compiled_components(arch) - - # Copy libgnustl_shared.so - with current_directory(self.get_build_dir(arch.arch)): - sh.cp( - "{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}/libgnustl_shared.so".format(ctx=self.ctx, arch=arch), - self.ctx.get_libs_dir(arch.arch) - ) + need_stl_shared = True class CythonRecipe(PythonRecipe): diff --git a/pythonforandroid/recipes/icu/__init__.py b/pythonforandroid/recipes/icu/__init__.py index 4bb2de0c99..43c5ac9eb8 100644 --- a/pythonforandroid/recipes/icu/__init__.py +++ b/pythonforandroid/recipes/icu/__init__.py @@ -1,33 +1,54 @@ import sh import os -from os.path import join, isdir -from pythonforandroid.recipe import NDKRecipe +from os.path import join, isdir, exists +from multiprocessing import cpu_count +from pythonforandroid.recipe import Recipe from pythonforandroid.toolchain import shprint from pythonforandroid.util import current_directory, ensure_dir -class ICURecipe(NDKRecipe): +class ICURecipe(Recipe): name = 'icu4c' version = '57.1' - url = 'http://download.icu-project.org/files/icu4c/57.1/icu4c-57_1-src.tgz' + major_version = version.split('.')[0] + url = ('http://download.icu-project.org/files/icu4c/' + '{version}/icu4c-{version_underscore}-src.tgz') depends = [('hostpython2', 'hostpython3')] # installs in python - generated_libraries = [ - 'libicui18n.so', 'libicuuc.so', 'libicudata.so', 'libicule.so'] - - def get_lib_dir(self, arch): - lib_dir = join(self.ctx.get_python_install_dir(), "lib") - ensure_dir(lib_dir) - return lib_dir - - def prepare_build_dir(self, arch): - if self.ctx.android_api > 19: - # greater versions do not have /usr/include/sys/exec_elf.h - raise RuntimeError("icu needs an android api <= 19") - - super(ICURecipe, self).prepare_build_dir(arch) - - def build_arch(self, arch, *extra_args): + patches = ['disable-libs-version.patch'] + + built_libraries = { + 'libicui18n{}.so'.format(major_version): 'build_icu_android/lib', + 'libicuuc{}.so'.format(major_version): 'build_icu_android/lib', + 'libicudata{}.so'.format(major_version): 'build_icu_android/lib', + 'libicule{}.so'.format(major_version): 'build_icu_android/lib', + 'libicuio{}.so'.format(major_version): 'build_icu_android/lib', + 'libicutu{}.so'.format(major_version): 'build_icu_android/lib', + 'libiculx{}.so'.format(major_version): 'build_icu_android/lib', + } + need_stl_shared = True + + @property + def versioned_url(self): + if self.url is None: + return None + return self.url.format( + version=self.version, + version_underscore=self.version.replace('.', '_')) + + def get_recipe_dir(self): + """ + .. note:: We need to overwrite `Recipe.get_recipe_dir` due to the + mismatch name between the recipe's folder (icu) and the value + of `ICURecipe.name` (icu4c). + """ + if self.ctx.local_recipes is not None: + local_recipe_dir = join(self.ctx.local_recipes, 'icu') + if exists(local_recipe_dir): + return local_recipe_dir + return join(self.ctx.root_dir, 'recipes', 'icu') + + def build_arch(self, arch): env = self.get_recipe_env(arch).copy() build_root = self.get_build_dir(arch.arch) @@ -60,11 +81,11 @@ def make_build_dest(dest): "--prefix="+icu_build, "--enable-extras=no", "--enable-strict=no", - "--enable-static", + "--enable-static=no", "--enable-tests=no", "--enable-samples=no", _env=host_env) - shprint(sh.make, "-j5", _env=host_env) + shprint(sh.make, "-j", str(cpu_count()), _env=host_env) shprint(sh.make, "install", _env=host_env) build_android, exists = make_build_dest("build_icu_android") @@ -72,62 +93,23 @@ def make_build_dest(dest): configure = sh.Command(join(build_root, "source", "configure")) - include = ( - " -I{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/include/" - " -I{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/" - "{arch}/include") - include = include.format(ndk=self.ctx.ndk_dir, - version=env["TOOLCHAIN_VERSION"], - arch=arch.arch) - env["CPPFLAGS"] = env["CXXFLAGS"] + " " - env["CPPFLAGS"] += host_env["CPPFLAGS"] - env["CPPFLAGS"] += include - - lib = "{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/{arch}" - lib = lib.format(ndk=self.ctx.ndk_dir, - version=env["TOOLCHAIN_VERSION"], - arch=arch.arch) - env["LDFLAGS"] += " -lgnustl_shared -L"+lib - - env.pop("CFLAGS", None) - env.pop("CXXFLAGS", None) - with current_directory(build_android): shprint( configure, "--with-cross-build="+build_linux, "--enable-extras=no", "--enable-strict=no", - "--enable-static", + "--enable-static=no", "--enable-tests=no", "--enable-samples=no", "--host="+env["TOOLCHAIN_PREFIX"], "--prefix="+icu_build, _env=env) - shprint(sh.make, "-j5", _env=env) + shprint(sh.make, "-j", str(cpu_count()), _env=env) shprint(sh.make, "install", _env=env) - self.copy_files(arch) - - def copy_files(self, arch): - env = self.get_recipe_env(arch) - - lib = "{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/{arch}" - lib = lib.format(ndk=self.ctx.ndk_dir, - version=env["TOOLCHAIN_VERSION"], - arch=arch.arch) - stl_lib = join(lib, "libgnustl_shared.so") - dst_dir = join(self.ctx.get_site_packages_dir(), "..", "lib-dynload") - shprint(sh.cp, stl_lib, dst_dir) - - src_lib = join(self.get_build_dir(arch.arch), "icu_build", "lib") - dst_lib = self.get_lib_dir(arch) - - src_suffix = "." + self.version - dst_suffix = "." + self.version.split(".")[0] # main version - for lib in self.generated_libraries: - shprint(sh.cp, join(src_lib, lib+src_suffix), - join(dst_lib, lib+dst_suffix)) + def install_libraries(self, arch): + super(ICURecipe, self).install_libraries(arch) src_include = join( self.get_build_dir(arch.arch), "icu_build", "include") @@ -137,16 +119,5 @@ def copy_files(self, arch): shprint(sh.cp, "-r", join(src_include, "layout"), dst_include) shprint(sh.cp, "-r", join(src_include, "unicode"), dst_include) - # copy stl library - lib = "{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/{arch}" - lib = lib.format(ndk=self.ctx.ndk_dir, - version=env["TOOLCHAIN_VERSION"], - arch=arch.arch) - stl_lib = join(lib, "libgnustl_shared.so") - - dst_dir = join(self.ctx.get_python_install_dir(), "lib") - ensure_dir(dst_dir) - shprint(sh.cp, stl_lib, dst_dir) - recipe = ICURecipe() diff --git a/pythonforandroid/recipes/icu/disable-libs-version.patch b/pythonforandroid/recipes/icu/disable-libs-version.patch new file mode 100644 index 0000000000..872abe01e4 --- /dev/null +++ b/pythonforandroid/recipes/icu/disable-libs-version.patch @@ -0,0 +1,66 @@ +diff -aur icu4c-org/source/config/Makefile.inc.in icu4c/source/config/Makefile.inc.in +--- icu/source/config/Makefile.inc.in.orig 2016-03-23 21:50:50.000000000 +0100 ++++ icu/source/config/Makefile.inc.in 2019-02-15 17:59:28.331749766 +0100 +@@ -142,8 +142,8 @@ + LDLIBRARYPATH_ENVVAR = LD_LIBRARY_PATH + + # Versioned target for a shared library +-FINAL_SO_TARGET = $(SO_TARGET).$(SO_TARGET_VERSION) +-MIDDLE_SO_TARGET = $(SO_TARGET).$(SO_TARGET_VERSION_MAJOR) ++FINAL_SO_TARGET = $(SO_TARGET).$(SO_TARGET_VERSION) ++MIDDLE_SO_TARGET = $(SO_TARGET) + + # Access to important ICU tools. + # Use as follows: $(INVOKE) $(GENRB) arguments .. +diff -aur icu4c-org/source/config/mh-linux icu4c/source/config/mh-linux +--- icu4c-org/source/config/mh-linux 2017-07-05 13:23:06.000000000 +0200 ++++ icu4c/source/config/mh-linux 2017-07-06 14:02:52.275016858 +0200 +@@ -24,9 +24,17 @@ + + ## Compiler switch to embed a library name + # The initial tab in the next line is to prevent icu-config from reading it. +- LD_SONAME = -Wl,-soname -Wl,$(notdir $(MIDDLE_SO_TARGET)) ++ LD_SONAME = -Wl,-soname -Wl,$(notdir $(SO_TARGET)) ++ DATA_STUBNAME = data$(SO_TARGET_VERSION_MAJOR) ++ COMMON_STUBNAME = uc$(SO_TARGET_VERSION_MAJOR) ++ I18N_STUBNAME = i18n$(SO_TARGET_VERSION_MAJOR) ++ LAYOUT_STUBNAME = le$(SO_TARGET_VERSION_MAJOR) ++ LAYOUTEX_STUBNAME = lx$(SO_TARGET_VERSION_MAJOR) ++ IO_STUBNAME = io$(SO_TARGET_VERSION_MAJOR) ++ TOOLUTIL_STUBNAME = tu$(SO_TARGET_VERSION_MAJOR) ++ CTESTFW_STUBNAME = test$(SO_TARGET_VERSION_MAJOR) + #SH# # We can't depend on MIDDLE_SO_TARGET being set. +-#SH# LD_SONAME= ++#SH# LD_SONAME=$(SO_TARGET) + + ## Shared library options + LD_SOOPTIONS= -Wl,-Bsymbolic +@@ -64,10 +64,10 @@ + + ## Versioned libraries rules + +-%.$(SO).$(SO_TARGET_VERSION_MAJOR): %.$(SO).$(SO_TARGET_VERSION) +- $(RM) $@ && ln -s ${ Date: Thu, 29 Aug 2019 22:51:17 +0200 Subject: [PATCH 12/72] [WIP][LIBS - PART III] Rework of pyleveldb, leveldb and snappy (#1966) * [recipe-lib] Make snappy a library recipe * [recipe-stl] Make leveldb a library recipe and... make it work with the reworked snappy recipe * [recipe-stl] Fix pyleveldb for reworked leveldb/snappy --- pythonforandroid/recipes/leveldb/__init__.py | 71 ++++--- .../recipes/leveldb/disable-so-version.patch | 10 - .../recipes/leveldb/find-snappy.patch | 11 - .../recipes/pyleveldb/__init__.py | 22 +- .../recipes/pyleveldb/bindings-only.patch | 196 ++++++++++-------- pythonforandroid/recipes/snappy/__init__.py | 27 ++- 6 files changed, 180 insertions(+), 157 deletions(-) delete mode 100644 pythonforandroid/recipes/leveldb/disable-so-version.patch delete mode 100644 pythonforandroid/recipes/leveldb/find-snappy.patch diff --git a/pythonforandroid/recipes/leveldb/__init__.py b/pythonforandroid/recipes/leveldb/__init__.py index e7ebe71606..7f65a55a42 100644 --- a/pythonforandroid/recipes/leveldb/__init__.py +++ b/pythonforandroid/recipes/leveldb/__init__.py @@ -1,47 +1,46 @@ -from pythonforandroid.toolchain import Recipe, shprint, shutil, current_directory +from pythonforandroid.logger import shprint +from pythonforandroid.util import current_directory +from pythonforandroid.recipe import Recipe +from multiprocessing import cpu_count from os.path import join import sh class LevelDBRecipe(Recipe): - version = '1.18' - url = 'https://github.com/google/leveldb/archive/v{version}.tar.gz' - opt_depends = ['snappy'] - patches = ['disable-so-version.patch', 'find-snappy.patch'] - - def should_build(self, arch): - return not self.has_libs(arch, 'libleveldb.so', 'libgnustl_shared.so') + version = '1.22' + url = 'https://github.com/google/leveldb/archive/{version}.tar.gz' + depends = ['snappy'] + built_libraries = {'libleveldb.so': '.'} + need_stl_shared = True def build_arch(self, arch): - super(LevelDBRecipe, self).build_arch(arch) env = self.get_recipe_env(arch) - with current_directory(self.get_build_dir(arch.arch)): - if 'snappy' in recipe.ctx.recipe_build_order: - # Copy source from snappy recipe - sh.cp('-rf', self.get_recipe('snappy', self.ctx).get_build_dir(arch.arch), 'snappy') - # Build - shprint(sh.make, _env=env) - # Copy the shared library - shutil.copyfile('libleveldb.so', join(self.ctx.get_libs_dir(arch.arch), 'libleveldb.so')) - # Copy stl - shutil.copyfile(self.ctx.ndk_dir + '/sources/cxx-stl/gnu-libstdc++/' + self.ctx.toolchain_version + '/libs/' + arch.arch + '/libgnustl_shared.so', - join(self.ctx.get_libs_dir(arch.arch), 'libgnustl_shared.so')) - - def get_recipe_env(self, arch): - env = super(LevelDBRecipe, self).get_recipe_env(arch) - env['TARGET_OS'] = 'OS_ANDROID_CROSSCOMPILE' - if 'snappy' in recipe.ctx.recipe_build_order: - env['CFLAGS'] += ' -DSNAPPY' + \ - ' -I./snappy' - env['CFLAGS'] += ' -I' + self.ctx.ndk_dir + '/platforms/android-' + str(self.ctx.android_api) + '/arch-' + arch.arch.replace('eabi', '') + '/usr/include' + \ - ' -I' + self.ctx.ndk_dir + '/sources/cxx-stl/gnu-libstdc++/' + self.ctx.toolchain_version + '/include' + \ - ' -I' + self.ctx.ndk_dir + '/sources/cxx-stl/gnu-libstdc++/' + self.ctx.toolchain_version + '/libs/' + arch.arch + '/include' - env['CXXFLAGS'] = env['CFLAGS'] - env['CXXFLAGS'] += ' -frtti' - env['CXXFLAGS'] += ' -fexceptions' - env['LDFLAGS'] += ' -L' + self.ctx.ndk_dir + '/sources/cxx-stl/gnu-libstdc++/' + self.ctx.toolchain_version + '/libs/' + arch.arch + \ - ' -lgnustl_shared' - return env + source_dir = self.get_build_dir(arch.arch) + with current_directory(source_dir): + snappy_recipe = self.get_recipe('snappy', self.ctx) + snappy_build = snappy_recipe.get_build_dir(arch.arch) + + shprint(sh.cmake, source_dir, + '-DANDROID_ABI={}'.format(arch.arch), + '-DANDROID_NATIVE_API_LEVEL={}'.format(self.ctx.ndk_api), + '-DANDROID_STL=' + self.stl_lib_name, + + '-DCMAKE_TOOLCHAIN_FILE={}'.format( + join(self.ctx.ndk_dir, 'build', 'cmake', + 'android.toolchain.cmake')), + '-DCMAKE_BUILD_TYPE=Release', + + '-DBUILD_SHARED_LIBS=1', + + '-DHAVE_SNAPPY=1', + '-DCMAKE_CXX_FLAGS=-I{path}'.format(path=snappy_build), + '-DCMAKE_SHARED_LINKER_FLAGS=-L{path} -lsnappy'.format( + path=snappy_build), + '-DCMAKE_EXE_LINKER_FLAGS=-L{path} -lsnappy'.format( + path=snappy_build), + + _env=env) + shprint(sh.make, '-j' + str(cpu_count()), _env=env) recipe = LevelDBRecipe() diff --git a/pythonforandroid/recipes/leveldb/disable-so-version.patch b/pythonforandroid/recipes/leveldb/disable-so-version.patch deleted file mode 100644 index 0f6a7e7280..0000000000 --- a/pythonforandroid/recipes/leveldb/disable-so-version.patch +++ /dev/null @@ -1,10 +0,0 @@ ---- leveldb/build_detect_platform 2014-09-16 23:19:52.000000000 +0200 -+++ leveldb-patch/build_detect_platform 2016-03-01 20:25:04.074484399 +0100 -@@ -124,6 +124,7 @@ - ;; - OS_ANDROID_CROSSCOMPILE) - PLATFORM=OS_ANDROID -+ PLATFORM_SHARED_VERSIONED= - COMMON_FLAGS="$MEMCMP_FLAG -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX" - PLATFORM_LDFLAGS="" # All pthread features are in the Android C library - PORT_FILE=port/port_posix.cc diff --git a/pythonforandroid/recipes/leveldb/find-snappy.patch b/pythonforandroid/recipes/leveldb/find-snappy.patch deleted file mode 100644 index ae978d7b07..0000000000 --- a/pythonforandroid/recipes/leveldb/find-snappy.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- leveldb/build_detect_platform 2014-09-16 23:19:52.000000000 +0200 -+++ leveldb-patch/build_detect_platform 2016-03-01 21:56:04.926650079 +0100 -@@ -156,7 +157,7 @@ - # except for the test and benchmark files. By default, find will output a list - # of all files matching either rule, so we need to append -print to make the - # prune take effect. --DIRS="$PREFIX/db $PREFIX/util $PREFIX/table" -+DIRS="$PREFIX/snappy $PREFIX/db $PREFIX/util $PREFIX/table" - - set -f # temporarily disable globbing so that our patterns aren't expanded - PRUNE_TEST="-name *test*.cc -prune" diff --git a/pythonforandroid/recipes/pyleveldb/__init__.py b/pythonforandroid/recipes/pyleveldb/__init__.py index 61477092f6..60b86bb18b 100644 --- a/pythonforandroid/recipes/pyleveldb/__init__.py +++ b/pythonforandroid/recipes/pyleveldb/__init__.py @@ -2,12 +2,26 @@ class PyLevelDBRecipe(CppCompiledComponentsPythonRecipe): - version = '0.193' - url = 'https://pypi.python.org/packages/source/l/leveldb/leveldb-{version}.tar.gz' - depends = ['snappy', 'leveldb', ('hostpython2', 'hostpython3'), 'setuptools'] + version = '0.194' + url = ('https://pypi.python.org/packages/source/l/leveldb/' + 'leveldb-{version}.tar.gz') + depends = ['snappy', 'leveldb', 'setuptools'] patches = ['bindings-only.patch'] - call_hostpython_via_targetpython = False # Due to setuptools site_packages_name = 'leveldb' + def get_recipe_env(self, arch): + env = super(PyLevelDBRecipe, self).get_recipe_env(arch) + + snappy_recipe = self.get_recipe('snappy', self.ctx) + leveldb_recipe = self.get_recipe('leveldb', self.ctx) + + env["LDFLAGS"] += " -L" + snappy_recipe.get_build_dir(arch.arch) + env["LDFLAGS"] += " -L" + leveldb_recipe.get_build_dir(arch.arch) + + env["SNAPPY_BUILD_PATH"] = snappy_recipe.get_build_dir(arch.arch) + env["LEVELDB_BUILD_PATH"] = leveldb_recipe.get_build_dir(arch.arch) + + return env + recipe = PyLevelDBRecipe() diff --git a/pythonforandroid/recipes/pyleveldb/bindings-only.patch b/pythonforandroid/recipes/pyleveldb/bindings-only.patch index 2899f4efaa..9f7027abb0 100644 --- a/pythonforandroid/recipes/pyleveldb/bindings-only.patch +++ b/pythonforandroid/recipes/pyleveldb/bindings-only.patch @@ -1,103 +1,119 @@ ---- pyleveldb/setup.py 2014-03-28 02:51:24.000000000 +0100 -+++ pyleveldb-patch/setup.py 2016-03-02 11:52:13.780678586 +0100 -@@ -7,41 +7,22 @@ - # - # See LICENSE for details. - --import glob --import platform --import sys -- +This patch force to only build the python bindings, and to do so, we modify +the setup.py file in oder that finds our compiled libraries (libleveldb.so and +libsnappy.so) +--- leveldb-0.194/setup.py.orig 2016-09-17 02:05:55.000000000 +0200 ++++ leveldb-0.194/setup.py 2019-02-26 16:57:40.997435911 +0100 +@@ -11,44 +11,25 @@ import platform + import sys + from setuptools import setup, Extension - --system,node,release,version,machine,processor = platform.uname() ++from os import environ + + system, node, release, version, machine, processor = platform.uname() -common_flags = [ +- '-I./leveldb/include', +- '-I./leveldb', +- '-I./snappy', +extra_compile_args = [ - '-I./leveldb/include', - '-I./leveldb', -- '-I./snappy', -+ '-I./leveldb/snappy', - '-I.', -- '-fno-builtin-memcmp', - '-O2', - '-fPIC', - '-DNDEBUG', - '-DSNAPPY', --] -- ++ '-I{}/include'.format(environ.get('LEVELDB_BUILD_PATH')), ++ '-I{}'.format(environ.get('LEVELDB_BUILD_PATH')), ++ '-I{}'.format(environ.get('SNAPPY_BUILD_PATH')), ++ '-I.', + '-I.', +- '-fno-builtin-memcmp', + '-O2', + '-fPIC', + '-DNDEBUG', + '-DSNAPPY', ++ '-pthread', ++ '-Wall', ++ '-D_REENTRANT', ++ '-DOS_ANDROID', + ] + -if system == 'Darwin': -- extra_compile_args = common_flags + [ -- '-DOS_MACOSX', -+ '-Wall', - '-DLEVELDB_PLATFORM_POSIX', -- '-Wno-error=unused-command-line-argument-hard-error-in-future', -- ] +- extra_compile_args = common_flags + [ +- '-DOS_MACOSX', +- '-DLEVELDB_PLATFORM_POSIX', +- '-Wno-error=unused-command-line-argument-hard-error-in-future', +- ] -elif system == 'Linux': +- extra_compile_args = common_flags + [ +- '-pthread', +- '-Wall', +- '-DOS_LINUX', +- '-DLEVELDB_PLATFORM_POSIX', +- ] +-elif system == 'SunOS': - extra_compile_args = common_flags + [ - '-pthread', -- '-Wall', -- '-DOS_LINUX', +- '-Wall', +- '-DOS_SOLARIS', - '-DLEVELDB_PLATFORM_POSIX', - ] -else: -- print >>sys.stderr, "Don't know how to compile leveldb for %s!" % system -- sys.exit(0) -+ '-D_REENTRANT', -+ '-DOS_ANDROID', -+] - +- sys.stderr.write("Don't know how to compile leveldb for %s!\n" % system) +- sys.exit(1) +- setup( - name = 'leveldb', -@@ -75,52 +56,6 @@ - ext_modules = [ - Extension('leveldb', - sources = [ -- # snappy -- './snappy/snappy.cc', -- './snappy/snappy-stubs-internal.cc', -- './snappy/snappy-sinksource.cc', -- './snappy/snappy-c.cc', + name = 'leveldb', + version = '0.194', +@@ -81,57 +62,11 @@ setup( + ext_modules = [ + Extension('leveldb', + sources = [ +- # snappy +- './snappy/snappy.cc', +- './snappy/snappy-stubs-internal.cc', +- './snappy/snappy-sinksource.cc', +- './snappy/snappy-c.cc', - -- #leveldb -- 'leveldb/db/builder.cc', -- 'leveldb/db/c.cc', -- 'leveldb/db/db_impl.cc', -- 'leveldb/db/db_iter.cc', -- 'leveldb/db/dbformat.cc', -- 'leveldb/db/filename.cc', -- 'leveldb/db/log_reader.cc', -- 'leveldb/db/log_writer.cc', -- 'leveldb/db/memtable.cc', -- 'leveldb/db/repair.cc', -- 'leveldb/db/table_cache.cc', -- 'leveldb/db/version_edit.cc', -- 'leveldb/db/version_set.cc', -- 'leveldb/db/write_batch.cc', -- 'leveldb/table/block.cc', -- 'leveldb/table/block_builder.cc', -- 'leveldb/table/filter_block.cc', -- 'leveldb/table/format.cc', -- 'leveldb/table/iterator.cc', -- 'leveldb/table/merger.cc', -- 'leveldb/table/table.cc', -- 'leveldb/table/table_builder.cc', -- 'leveldb/table/two_level_iterator.cc', -- 'leveldb/util/arena.cc', -- 'leveldb/util/bloom.cc', -- 'leveldb/util/cache.cc', -- 'leveldb/util/coding.cc', -- 'leveldb/util/comparator.cc', -- 'leveldb/util/crc32c.cc', -- 'leveldb/util/env.cc', -- 'leveldb/util/env_posix.cc', -- 'leveldb/util/filter_policy.cc', -- 'leveldb/util/hash.cc', -- 'leveldb/util/histogram.cc', -- 'leveldb/util/logging.cc', -- 'leveldb/util/options.cc', -- 'leveldb/util/status.cc', -- 'leveldb/port/port_posix.cc', +- #leveldb +- 'leveldb/db/builder.cc', +- 'leveldb/db/c.cc', +- 'leveldb/db/db_impl.cc', +- 'leveldb/db/db_iter.cc', +- 'leveldb/db/dbformat.cc', +- 'leveldb/db/filename.cc', +- 'leveldb/db/log_reader.cc', +- 'leveldb/db/log_writer.cc', +- 'leveldb/db/memtable.cc', +- 'leveldb/db/repair.cc', +- 'leveldb/db/table_cache.cc', +- 'leveldb/db/version_edit.cc', +- 'leveldb/db/version_set.cc', +- 'leveldb/db/write_batch.cc', +- 'leveldb/table/block.cc', +- 'leveldb/table/block_builder.cc', +- 'leveldb/table/filter_block.cc', +- 'leveldb/table/format.cc', +- 'leveldb/table/iterator.cc', +- 'leveldb/table/merger.cc', +- 'leveldb/table/table.cc', +- 'leveldb/table/table_builder.cc', +- 'leveldb/table/two_level_iterator.cc', +- 'leveldb/util/arena.cc', +- 'leveldb/util/bloom.cc', +- 'leveldb/util/cache.cc', +- 'leveldb/util/coding.cc', +- 'leveldb/util/comparator.cc', +- 'leveldb/util/crc32c.cc', +- 'leveldb/util/env.cc', +- 'leveldb/util/env_posix.cc', +- 'leveldb/util/filter_policy.cc', +- 'leveldb/util/hash.cc', +- 'leveldb/util/histogram.cc', +- 'leveldb/util/logging.cc', +- 'leveldb/util/options.cc', +- 'leveldb/util/status.cc', +- 'leveldb/port/port_posix.cc', - - # python stuff - 'leveldb_ext.cc', - 'leveldb_object.cc', + # python stuff + 'leveldb_ext.cc', + 'leveldb_object.cc', + ], +- libraries = ['stdc++'], ++ libraries = ['snappy', 'leveldb', 'stdc++', 'c++_shared'], + extra_compile_args = extra_compile_args, + ) + ] diff --git a/pythonforandroid/recipes/snappy/__init__.py b/pythonforandroid/recipes/snappy/__init__.py index 4ca61a219a..c57f797af9 100644 --- a/pythonforandroid/recipes/snappy/__init__.py +++ b/pythonforandroid/recipes/snappy/__init__.py @@ -1,13 +1,28 @@ -from pythonforandroid.toolchain import Recipe +from pythonforandroid.recipe import Recipe +from pythonforandroid.logger import shprint +from pythonforandroid.util import current_directory +from os.path import join +import sh class SnappyRecipe(Recipe): - version = '1.1.3' - url = 'https://github.com/google/snappy/releases/download/{version}/snappy-{version}.tar.gz' + version = '1.1.7' + url = 'https://github.com/google/snappy/archive/{version}.tar.gz' + built_libraries = {'libsnappy.so': '.'} - def should_build(self, arch): - # Only download to use in leveldb recipe - return False + def build_arch(self, arch): + env = self.get_recipe_env(arch) + source_dir = self.get_build_dir(arch.arch) + with current_directory(source_dir): + shprint(sh.cmake, source_dir, + '-DANDROID_ABI={}'.format(arch.arch), + '-DANDROID_NATIVE_API_LEVEL={}'.format(self.ctx.ndk_api), + '-DCMAKE_TOOLCHAIN_FILE={}'.format( + join(self.ctx.ndk_dir, 'build', 'cmake', + 'android.toolchain.cmake')), + '-DBUILD_SHARED_LIBS=1', + _env=env) + shprint(sh.make, _env=env) recipe = SnappyRecipe() From 9e39fbea705cc7c1263665139c5776e71045323a Mon Sep 17 00:00:00 2001 From: Pol Canelles Date: Thu, 29 Aug 2019 22:51:35 +0200 Subject: [PATCH 13/72] [WIP][LIBS - PART IV] Rework of shapely and libgeos (#1967) * [recipe-stl] Make libgeos a library recipe * [recipe-stl] Fix shapely for reworked libgeos --- pythonforandroid/recipes/libgeos/__init__.py | 74 +++++++++++--------- pythonforandroid/recipes/shapely/__init__.py | 37 +++++++--- pythonforandroid/recipes/shapely/setup.patch | 56 +++++++++++---- 3 files changed, 112 insertions(+), 55 deletions(-) diff --git a/pythonforandroid/recipes/libgeos/__init__.py b/pythonforandroid/recipes/libgeos/__init__.py index 30786f8ea4..cff9fe0f5e 100644 --- a/pythonforandroid/recipes/libgeos/__init__.py +++ b/pythonforandroid/recipes/libgeos/__init__.py @@ -1,44 +1,52 @@ -from pythonforandroid.toolchain import Recipe, shprint, shutil, current_directory -from os.path import exists, join -import sh +from pythonforandroid.util import current_directory, ensure_dir +from pythonforandroid.toolchain import shprint +from pythonforandroid.recipe import Recipe from multiprocessing import cpu_count +from os.path import join +import sh class LibgeosRecipe(Recipe): - version = '3.5' - # url = 'http://download.osgeo.org/geos/geos-{version}.tar.bz2' - url = 'https://github.com/libgeos/libgeos/archive/svn-{version}.zip' + version = '3.7.1' + url = 'https://github.com/libgeos/libgeos/archive/{version}.zip' depends = [] - - def should_build(self, arch): - super(LibgeosRecipe, self).should_build(arch) - return not exists(join(self.ctx.get_libs_dir(arch.arch), 'libgeos_c.so')) + built_libraries = { + 'libgeos.so': 'install_target/lib', + 'libgeos_c.so': 'install_target/lib' + } + need_stl_shared = True def build_arch(self, arch): - super(LibgeosRecipe, self).build_arch(arch) - env = self.get_recipe_env(arch) - - with current_directory(self.get_build_dir(arch.arch)): - dst_dir = join(self.get_build_dir(arch.arch), 'dist') - bash = sh.Command('bash') - print("If this fails make sure you have autoconf and libtool installed") - shprint(bash, 'autogen.sh') # Requires autoconf and libtool - shprint(bash, 'configure', '--host=arm-linux-androideabi', '--enable-shared', '--prefix={}'.format(dst_dir), _env=env) - shprint(sh.make, '-j', str(cpu_count()), _env=env) + source_dir = self.get_build_dir(arch.arch) + build_target = join(source_dir, 'build_target') + install_target = join(source_dir, 'install_target') + + ensure_dir(build_target) + with current_directory(build_target): + env = self.get_recipe_env(arch) + shprint(sh.cmake, source_dir, + '-DANDROID_ABI={}'.format(arch.arch), + '-DANDROID_NATIVE_API_LEVEL={}'.format(self.ctx.ndk_api), + '-DANDROID_STL=' + self.stl_lib_name, + + '-DCMAKE_TOOLCHAIN_FILE={}'.format( + join(self.ctx.ndk_dir, 'build', 'cmake', + 'android.toolchain.cmake')), + '-DCMAKE_INSTALL_PREFIX={}'.format(install_target), + '-DCMAKE_BUILD_TYPE=Release', + + '-DGEOS_ENABLE_TESTS=OFF', + + '-DBUILD_SHARED_LIBS=1', + + _env=env) + shprint(sh.make, '-j' + str(cpu_count()), _env=env) + + # We make the install because this way we will have all the + # includes in one place (mostly we are interested in `geos_c.h`, + # which is not in the include folder, so this way we make easier to + # link with this library...case of shapely's recipe) shprint(sh.make, 'install', _env=env) - shutil.copyfile('{}/lib/libgeos_c.so'.format(dst_dir), join(self.ctx.get_libs_dir(arch.arch), 'libgeos_c.so')) - - def get_recipe_env(self, arch): - env = super(LibgeosRecipe, self).get_recipe_env(arch) - env['CXXFLAGS'] += ' -I{}/sources/cxx-stl/gnu-libstdc++/4.8/include'.format(self.ctx.ndk_dir) - env['CXXFLAGS'] += ' -I{}/sources/cxx-stl/gnu-libstdc++/4.8/libs/{}/include'.format( - self.ctx.ndk_dir, arch) - env['CXXFLAGS'] += ' -L{}/sources/cxx-stl/gnu-libstdc++/4.8/libs/{}'.format( - self.ctx.ndk_dir, arch) - env['CXXFLAGS'] += ' -lgnustl_shared' - env['LDFLAGS'] += ' -L{}/sources/cxx-stl/gnu-libstdc++/4.8/libs/{}'.format( - self.ctx.ndk_dir, arch) - return env recipe = LibgeosRecipe() diff --git a/pythonforandroid/recipes/shapely/__init__.py b/pythonforandroid/recipes/shapely/__init__.py index e0b093766b..f70876132e 100644 --- a/pythonforandroid/recipes/shapely/__init__.py +++ b/pythonforandroid/recipes/shapely/__init__.py @@ -1,21 +1,38 @@ -from pythonforandroid.recipe import Recipe, CythonRecipe +from pythonforandroid.recipe import CythonRecipe +from os.path import join class ShapelyRecipe(CythonRecipe): - version = '1.5' - url = 'https://github.com/Toblerity/Shapely/archive/master.zip' + version = '1.7a1' + url = 'https://github.com/Toblerity/Shapely/archive/{version}.tar.gz' depends = ['setuptools', 'libgeos'] + + # Actually, this recipe seems to compile/install fine for python2, but it + # fails at runtime when importing module with: + # `[Errno 2] No such file or directory` + conflicts = ['python2'] + call_hostpython_via_targetpython = False - patches = ['setup.patch'] # Patch to force setup to fail when C extention fails to build + # Patch to avoid libgeos check (because it fails), insert environment + # variables for our libgeos build (includes, lib paths...) and force + # the cython's compilation to raise an error in case that it fails + patches = ['setup.patch'] + + # Don't Force Cython + # setup_extra_args = ['sdist'] + + def get_recipe_env(self, arch=None, with_flags_in_cc=True): + env = super(ShapelyRecipe, self).get_recipe_env(arch) - # setup_extra_args = ['sdist'] # DontForce Cython + libgeos_install = join(self.get_recipe( + 'libgeos', self.ctx).get_build_dir(arch.arch), 'install_target') + # All this `GEOS_X` variables should be string types, separated + # by commas in case that we need to pass more than one value + env['GEOS_INCLUDE_DIRS'] = join(libgeos_install, 'include') + env['GEOS_LIBRARY_DIRS'] = join(libgeos_install, 'lib') + env['GEOS_LIBRARIES'] = 'geos_c,geos' - def get_recipe_env(self, arch, with_flags_in_cc=True): - """ Add libgeos headers to path """ - env = super(ShapelyRecipe, self).get_recipe_env(arch, with_flags_in_cc) - libgeos_dir = Recipe.get_recipe('libgeos', self.ctx).get_build_dir(arch.arch) - env['CFLAGS'] += " -I{}/dist/include".format(libgeos_dir) return env diff --git a/pythonforandroid/recipes/shapely/setup.patch b/pythonforandroid/recipes/shapely/setup.patch index 9523f357bc..7fd1ca9149 100644 --- a/pythonforandroid/recipes/shapely/setup.patch +++ b/pythonforandroid/recipes/shapely/setup.patch @@ -1,12 +1,44 @@ -*** shapely/setup.py 2016-06-29 11:29:49.000000000 -0400 ---- b/setup.py 2016-07-09 01:51:37.759670990 -0400 -*************** -*** 359,364 **** ---- 359,365 ---- - construct_build_ext(existing_build_ext) - setup(ext_modules=ext_modules, **setup_args) - except BuildFailed as ex: -+ raise # Force python only build to fail - BUILD_EXT_WARNING = "The C extension could not be compiled, " \ - "speedups are not enabled." - log.warn(ex) +This patch does three things: + - disable the libgeos check, because, even setting the proper env variables, + it fails to load our libgeos library, so we skip that because it's not + mandatory for the cythonizing. + - sets some environment variables into the setup.py file, so we can pass + our libgeos information (includes, lib path and libraries) + - force to raise an error when cython file to compile (our current build + system relies on this failure to do the proper `cythonizing`, if we don't + raise the error, we will end up with the package installed without the + speed optimizations. +--- Shapely-1.7a1/setup.py.orig 2018-07-29 22:53:13.000000000 +0200 ++++ Shapely-1.7a1/setup.py 2019-02-24 14:26:19.178610660 +0100 +@@ -82,8 +82,8 @@ if not (py_version == (2, 7) or py_versi + + # Get geos_version from GEOS dynamic library, which depends on + # GEOS_LIBRARY_PATH and/or GEOS_CONFIG environment variables +-from shapely._buildcfg import geos_version_string, geos_version, \ +- geos_config, get_geos_config ++# from shapely._buildcfg import geos_version_string, geos_version, \ ++# geos_config, get_geos_config + + logging.basicConfig() + log = logging.getLogger(__file__) +@@ -248,9 +248,9 @@ if sys.platform == 'win32': + setup_args['package_data']['shapely'].append('shapely/DLLs/*.dll') + + # Prepare build opts and args for the speedups extension module. +-include_dirs = [] +-library_dirs = [] +-libraries = [] ++include_dirs = os.environ.get('GEOS_INCLUDE_DIRS', '').split(',') ++library_dirs = os.environ.get('GEOS_LIBRARY_DIRS', '').split(',') ++libraries = os.environ.get('GEOS_LIBRARIES', '').split(',') + extra_link_args = [] + + # If NO_GEOS_CONFIG is set in the environment, geos-config will not +@@ -375,6 +375,7 @@ try: + construct_build_ext(existing_build_ext) + setup(ext_modules=ext_modules, **setup_args) + except BuildFailed as ex: ++ raise # Force python only build to fail + BUILD_EXT_WARNING = "The C extension could not be compiled, " \ + "speedups are not enabled." + log.warn(ex) From dfad9c0b581b9b76b7fc60842df8ce3d30558036 Mon Sep 17 00:00:00 2001 From: Pol Canelles Date: Thu, 29 Aug 2019 22:52:14 +0200 Subject: [PATCH 14/72] [recipe-stl] Make libzmq a library recipe (#1968) --- pythonforandroid/recipes/libzmq/__init__.py | 55 +++------------------ 1 file changed, 8 insertions(+), 47 deletions(-) diff --git a/pythonforandroid/recipes/libzmq/__init__.py b/pythonforandroid/recipes/libzmq/__init__.py index 7bf6c2b762..243517bc96 100644 --- a/pythonforandroid/recipes/libzmq/__init__.py +++ b/pythonforandroid/recipes/libzmq/__init__.py @@ -1,21 +1,18 @@ -from pythonforandroid.toolchain import Recipe, shprint, shutil, current_directory -from pythonforandroid.util import ensure_dir -from os.path import exists, join +from pythonforandroid.recipe import Recipe +from pythonforandroid.logger import shprint +from pythonforandroid.util import current_directory +from os.path import join import sh class LibZMQRecipe(Recipe): - version = '4.3.1' + version = '4.3.2' url = 'https://github.com/zeromq/libzmq/releases/download/v{version}/zeromq-{version}.zip' depends = [] - - def should_build(self, arch): - super(LibZMQRecipe, self).should_build(arch) - return True - return not exists(join(self.ctx.get_libs_dir(arch.arch), 'libzmq.so')) + built_libraries = {'libzmq.so': 'src/.libs'} + need_stl_shared = True def build_arch(self, arch): - super(LibZMQRecipe, self).build_arch(arch) env = self.get_recipe_env(arch) # # libsodium_recipe = Recipe.get_recipe('libsodium', self.ctx) @@ -27,6 +24,7 @@ def build_arch(self, arch): curdir = self.get_build_dir(arch.arch) prefix = join(curdir, "install") + with current_directory(curdir): bash = sh.Command('sh') shprint( @@ -39,43 +37,6 @@ def build_arch(self, arch): _env=env) shprint(sh.make, _env=env) shprint(sh.make, 'install', _env=env) - shutil.copyfile('src/.libs/libzmq.so', join( - self.ctx.get_libs_dir(arch.arch), 'libzmq.so')) - - bootstrap_obj_dir = join(self.ctx.bootstrap.build_dir, 'obj', 'local', arch.arch) - ensure_dir(bootstrap_obj_dir) - shutil.copyfile( - '{}/sources/cxx-stl/gnu-libstdc++/{}/libs/{}/libgnustl_shared.so'.format( - self.ctx.ndk_dir, self.ctx.toolchain_version, arch), - join(bootstrap_obj_dir, 'libgnustl_shared.so')) - - # Copy libgnustl_shared.so - with current_directory(self.get_build_dir(arch.arch)): - sh.cp( - "{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}/libgnustl_shared.so".format(ctx=self.ctx, arch=arch), - self.ctx.get_libs_dir(arch.arch) - ) - - def get_include_dirs(self, arch): - return [join(self.get_build_dir(arch.arch), 'include')] - - def get_recipe_env(self, arch): - # XXX should stl be configuration for the toolchain itself? - env = super(LibZMQRecipe, self).get_recipe_env(arch) - env['CFLAGS'] += ' -Os' - env['CXXFLAGS'] += ' -Os -fPIC -fvisibility=default' - env['CXXFLAGS'] += ' -I{}/sources/cxx-stl/gnu-libstdc++/{}/include'.format( - self.ctx.ndk_dir, self.ctx.toolchain_version) - env['CXXFLAGS'] += ' -I{}/sources/cxx-stl/gnu-libstdc++/{}/libs/{}/include'.format( - self.ctx.ndk_dir, self.ctx.toolchain_version, arch) - env['CXXFLAGS'] += ' -L{}/sources/cxx-stl/gnu-libstdc++/{}/libs/{}'.format( - self.ctx.ndk_dir, self.ctx.toolchain_version, arch) - env['CXXFLAGS'] += ' -lgnustl_shared' - env['LDFLAGS'] += ' -L{}/sources/cxx-stl/gnu-libstdc++/{}/libs/{}'.format( - self.ctx.ndk_dir, self.ctx.toolchain_version, arch) - env['CXXFLAGS'] += ' --sysroot={}/platforms/android-{}/{}'.format( - self.ctx.ndk_dir, self.ctx.ndk_api, arch.platform_dir) - return env recipe = LibZMQRecipe() From 9e93257f576d8b3b6d02365c965307641e9e7b40 Mon Sep 17 00:00:00 2001 From: Pol Canelles Date: Thu, 29 Aug 2019 22:52:47 +0200 Subject: [PATCH 15/72] [recipe-stl] Rework of libtorrent and boost (#1971) which: - fix build for android's NDK r19+ - allow us to build `boost+libtorrent` for any arch - Update boost to version `1.69.0` - update libtorrent to version `1.2.1` The build method needs to be changed because one of the scripts used in `boost+libtorrent` build (make-standalone-toolchain.sh), has been removed from android's NDK. [This is because](https://developer.android.com/ndk/guides/standalone_toolchain): `As of r19, the NDK's default toolchains are standalone toolchains, which renders this process unnecessary.` Note: since 3887d2b, `python-for-android` uses android's NDK r19 as the minimum supported and also changed the build method used by p4a as [described in here](https://developer.android.com/ndk/guides/other_build_systems) --- pythonforandroid/recipes/boost/__init__.py | 89 ++++++++++--------- .../recipes/boost/fix-android-issues.patch | 88 ++++++++++-------- .../recipes/boost/user-config.jam | 47 +++------- .../recipes/libtorrent/__init__.py | 26 ++++-- .../recipes/libtorrent/setup-lib-name.patch | 2 +- 5 files changed, 135 insertions(+), 117 deletions(-) diff --git a/pythonforandroid/recipes/boost/__init__.py b/pythonforandroid/recipes/boost/__init__.py index 53d9388877..978ab6f5b0 100644 --- a/pythonforandroid/recipes/boost/__init__.py +++ b/pythonforandroid/recipes/boost/__init__.py @@ -1,10 +1,13 @@ -from pythonforandroid.toolchain import Recipe, shprint, shutil, current_directory +from pythonforandroid.util import current_directory, build_platform +from pythonforandroid.recipe import Recipe +from pythonforandroid.logger import shprint from os.path import join, exists from os import environ +import shutil import sh """ -This recipe creates a custom toolchain and bootstraps Boost from source to build Boost.Build +This recipe bootstraps Boost from source to build Boost.Build including python bindings """ @@ -12,7 +15,8 @@ class BoostRecipe(Recipe): # Todo: make recipe compatible with all p4a architectures ''' - .. note:: This recipe can be built only against API 21+ and arch armeabi-v7a + .. note:: This recipe can be built only against API 21+ and an android + ndk >= r19 .. versionchanged:: 0.6.0 Rewrote recipe to support clang's build. The following changes has @@ -24,14 +28,24 @@ class BoostRecipe(Recipe): - Default compiler for ndk's toolchain set to clang - Python version will be detected via user-config.jam - Changed stl's lib from ``gnustl_shared`` to ``c++_shared`` + + .. versionchanged:: 2019.08.09.1.dev0 + + - Bumped version number to 1.68.0 + - Adapted to work with ndk-r19+ ''' - version = '1.68.0' - url = 'http://downloads.sourceforge.net/project/boost/' \ - 'boost/{version}/boost_{version_underscore}.tar.bz2' + version = '1.69.0' + url = ( + 'http://downloads.sourceforge.net/project/boost/' + 'boost/{version}/boost_{version_underscore}.tar.bz2' + ) depends = [('python2', 'python3')] - patches = ['disable-so-version.patch', - 'use-android-libs.patch', - 'fix-android-issues.patch'] + patches = [ + 'disable-so-version.patch', + 'use-android-libs.patch', + 'fix-android-issues.patch', + ] + need_stl_shared = True @property def versioned_url(self): @@ -39,7 +53,8 @@ def versioned_url(self): return None return self.url.format( version=self.version, - version_underscore=self.version.replace('.', '_')) + version_underscore=self.version.replace('.', '_'), + ) def should_build(self, arch): return not exists(join(self.get_build_dir(arch.arch), 'b2')) @@ -48,56 +63,50 @@ def prebuild_arch(self, arch): super(BoostRecipe, self).prebuild_arch(arch) env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): - if not exists(env['CROSSHOME']): - # Make custom toolchain - bash = sh.Command('bash') - shprint(bash, join(self.ctx.ndk_dir, 'build/tools/make-standalone-toolchain.sh'), - '--arch=' + env['ARCH'], - '--platform=android-' + str(self.ctx.android_api), - '--toolchain=' + env['CROSSHOST'] + '-' + self.ctx.toolchain_version + ':-llvm', - '--use-llvm', - '--stl=libc++', - '--install-dir=' + env['CROSSHOME'] - ) # Set custom configuration - shutil.copyfile(join(self.get_recipe_dir(), 'user-config.jam'), - join(env['BOOST_BUILD_PATH'], 'user-config.jam')) + shutil.copyfile( + join(self.get_recipe_dir(), 'user-config.jam'), + join(env['BOOST_BUILD_PATH'], 'user-config.jam'), + ) def build_arch(self, arch): super(BoostRecipe, self).build_arch(arch) env = self.get_recipe_env(arch) env['PYTHON_HOST'] = self.ctx.hostpython with current_directory(self.get_build_dir(arch.arch)): - # Compile Boost.Build engine with this custom toolchain - bash = sh.Command('bash') - shprint(bash, 'bootstrap.sh') # Do not pass env - # Install app stl - shutil.copyfile( - join(self.ctx.ndk_dir, 'sources/cxx-stl/llvm-libc++/libs/' - 'armeabi-v7a/libc++_shared.so'), - join(self.ctx.get_libs_dir(arch.arch), 'libc++_shared.so')) - - def select_build_arch(self, arch): - return arch.arch.replace('eabi-v7a', '').replace('eabi', '') + if not exists('b2'): + # Compile Boost.Build engine with this custom toolchain + bash = sh.Command('bash') + shprint(bash, 'bootstrap.sh') # Do not pass env def get_recipe_env(self, arch): # We don't use the normal env because we # are building with a standalone toolchain env = environ.copy() - env['BOOST_BUILD_PATH'] = self.get_build_dir(arch.arch) # find user-config.jam - env['BOOST_ROOT'] = env['BOOST_BUILD_PATH'] # find boost source + # find user-config.jam + env['BOOST_BUILD_PATH'] = self.get_build_dir(arch.arch) + # find boost source + env['BOOST_ROOT'] = env['BOOST_BUILD_PATH'] env['PYTHON_ROOT'] = self.ctx.python_recipe.link_root(arch.arch) env['PYTHON_INCLUDE'] = self.ctx.python_recipe.include_root(arch.arch) env['PYTHON_MAJOR_MINOR'] = self.ctx.python_recipe.version[:3] - env['PYTHON_LINK_VERSION'] = self.ctx.python_recipe.major_minor_version_string + env[ + 'PYTHON_LINK_VERSION' + ] = self.ctx.python_recipe.major_minor_version_string if 'python3' in self.ctx.python_recipe.name: env['PYTHON_LINK_VERSION'] += 'm' - env['ARCH'] = self.select_build_arch(arch) - env['CROSSHOST'] = env['ARCH'] + '-linux-androideabi' - env['CROSSHOME'] = join(env['BOOST_ROOT'], 'standalone-' + env['ARCH'] + '-toolchain') + env['ARCH'] = arch.arch.replace('-', '') + env['TARGET_TRIPLET'] = arch.target + env['CROSSHOST'] = arch.command_prefix + env['CROSSHOME'] = join( + self.ctx.ndk_dir, + 'toolchains/llvm/prebuilt/{build_platform}'.format( + build_platform=build_platform + ), + ) return env diff --git a/pythonforandroid/recipes/boost/fix-android-issues.patch b/pythonforandroid/recipes/boost/fix-android-issues.patch index 54134800a1..40bdea42dc 100644 --- a/pythonforandroid/recipes/boost/fix-android-issues.patch +++ b/pythonforandroid/recipes/boost/fix-android-issues.patch @@ -1,10 +1,26 @@ -diff -u -r boost_1_68_0.orig/boost/config/user.hpp boost_1_68_0/boost/config/user.hpp ---- boost_1_68_0.orig/boost/config/user.hpp 2018-08-01 22:50:46.000000000 +0200 -+++ boost_1_68_0/boost/config/user.hpp 2018-08-27 15:43:38.000000000 +0200 +diff -u -r boost_1_69_0.orig/boost/asio/detail/config.hpp boost_1_69_0/boost/asio/detail/config.hpp +--- boost_1_69_0.orig/boost/asio/detail/config.hpp 2018-12-05 20:58:15.000000000 +0100 ++++ boost_1_69_0/boost/asio/detail/config.hpp 2018-12-13 14:52:06.000000000 +0100 +@@ -815,7 +815,11 @@ + # if (_LIBCPP_VERSION < 7000) + # if (__cplusplus >= 201402) + # if __has_include() +-# define BOOST_ASIO_HAS_STD_EXPERIMENTAL_STRING_VIEW 1 ++# if __clang_major__ >= 7 ++# undef BOOST_ASIO_HAS_STD_EXPERIMENTAL_STRING_VIEW ++# else ++# define BOOST_ASIO_HAS_STD_EXPERIMENTAL_STRING_VIEW 1 ++# endif // __clang_major__ >= 7 + # endif // __has_include() + # endif // (__cplusplus >= 201402) + # endif // (_LIBCPP_VERSION < 7000) +diff -u -r boost_1_69_0.orig/boost/config/user.hpp boost_1_69_0/boost/config/user.hpp +--- boost_1_69_0.orig/boost/config/user.hpp 2018-12-05 20:58:16.000000000 +0100 ++++ boost_1_69_0/boost/config/user.hpp 2018-12-13 14:35:29.000000000 +0100 @@ -13,6 +13,12 @@ // configuration policy: // - + +// Android defines +// There is problem with std::atomic on android (and some other platforms). +// See this link for more info: @@ -13,41 +29,25 @@ diff -u -r boost_1_68_0.orig/boost/config/user.hpp boost_1_68_0/boost/config/use + // define this to locate a compiler config file: // #define BOOST_COMPILER_CONFIG - -diff -u -r boost_1_68_0.orig/boost/asio/detail/config.hpp boost_1_68_0/boost/asio/detail/config.hpp ---- boost_1_68_0.orig/boost/asio/detail/config.hpp 2018-08-01 22:50:46.000000000 +0200 -+++ boost_1_68_0/boost/asio/detail/config.hpp 2018-09-19 12:39:56.000000000 +0200 -@@ -804,7 +804,11 @@ - # if defined(__clang__) - # if (__cplusplus >= 201402) - # if __has_include() --# define BOOST_ASIO_HAS_STD_EXPERIMENTAL_STRING_VIEW 1 -+# if __clang_major__ >= 7 -+# undef BOOST_ASIO_HAS_STD_EXPERIMENTAL_STRING_VIEW -+# else -+# define BOOST_ASIO_HAS_STD_EXPERIMENTAL_STRING_VIEW 1 -+# endif // __clang_major__ >= 7 - # endif // __has_include() - # endif // (__cplusplus >= 201402) - # endif // defined(__clang__) -diff -u -r boost_1_68_0.orig/boost/system/error_code.hpp boost_1_68_0/boost/system/error_code.hpp ---- boost_1_68_0.orig/boost/system/error_code.hpp 2018-08-01 22:50:53.000000000 +0200 -+++ boost_1_68_0/boost/system/error_code.hpp 2018-08-27 15:44:29.000000000 +0200 -@@ -17,6 +17,7 @@ - #include - #include - #include + +diff -u -r boost_1_69_0.orig/boost/system/error_code.hpp boost_1_69_0/boost/system/error_code.hpp +--- boost_1_69_0.orig/boost/system/error_code.hpp 2018-12-05 20:58:23.000000000 +0100 ++++ boost_1_69_0/boost/system/error_code.hpp 2018-12-13 14:53:33.000000000 +0100 +@@ -14,6 +14,7 @@ + #include + #include + #include +#include #include #include - #include -diff -u -r boost_1_68_0.orig/libs/filesystem/src/operations.cpp boost_1_68_0/libs/filesystem/src/operations.cpp ---- boost_1_68_0.orig/libs/filesystem/src/operations.cpp 2018-08-01 22:50:47.000000000 +0200 -+++ boost_1_68_0/libs/filesystem/src/operations.cpp 2018-08-27 15:47:15.000000000 +0200 + #include +diff -u -r boost_1_69_0.orig/libs/filesystem/src/operations.cpp boost_1_69_0/libs/filesystem/src/operations.cpp +--- boost_1_69_0.orig/libs/filesystem/src/operations.cpp 2018-12-05 20:58:17.000000000 +0100 ++++ boost_1_69_0/libs/filesystem/src/operations.cpp 2018-12-13 14:55:41.000000000 +0100 @@ -232,6 +232,21 @@ - + # if defined(BOOST_POSIX_API) - + +# if defined(__ANDROID__) +# define truncate libboost_truncate_wrapper +// truncate() is present in Android libc only starting from ABI 21, so here's a simple wrapper @@ -64,5 +64,23 @@ diff -u -r boost_1_68_0.orig/libs/filesystem/src/operations.cpp boost_1_68_0/lib +# endif + typedef int err_t; - + // POSIX uses a 0 return to indicate success +diff -u -r boost_1_69_0.orig/tools/build/src/tools/common.jam boost_1_69_0/tools/build/src/tools/common.jam +--- boost_1_69_0.orig/tools/build/src/tools/common.jam 2019-01-25 23:18:34.544755629 +0200 ++++ boost_1_69_0/tools/build/src/tools/common.jam 2019-01-25 23:20:42.309047754 +0200 +@@ -976,10 +976,10 @@ + } + + # Ditto, from Clang 4 +- if $(tag) in clang clangw && [ numbers.less 3 $(version[1]) ] +- { +- version = $(version[1]) ; +- } ++ #if $(tag) in clang clangw && [ numbers.less 3 $(version[1]) ] ++ #{ ++ # version = $(version[1]) ; ++ #} + + # On intel, version is not added, because it does not matter and it is the + # version of vc used as backend that matters. Ideally, we should encode the diff --git a/pythonforandroid/recipes/boost/user-config.jam b/pythonforandroid/recipes/boost/user-config.jam index e50b50afea..fa1eef1337 100644 --- a/pythonforandroid/recipes/boost/user-config.jam +++ b/pythonforandroid/recipes/boost/user-config.jam @@ -1,6 +1,7 @@ import os ; local ARCH = [ os.environ ARCH ] ; +local TARGET_TRIPLET = [ os.environ TARGET_TRIPLET ] ; local CROSSHOME = [ os.environ CROSSHOME ] ; local PYTHON_HOST = [ os.environ PYTHON_HOST ] ; local PYTHON_ROOT = [ os.environ PYTHON_ROOT ] ; @@ -8,42 +9,22 @@ local PYTHON_INCLUDE = [ os.environ PYTHON_INCLUDE ] ; local PYTHON_LINK_VERSION = [ os.environ PYTHON_LINK_VERSION ] ; local PYTHON_MAJOR_MINOR = [ os.environ PYTHON_MAJOR_MINOR ] ; -using clang : $(ARCH) : $(CROSSHOME)/bin/arm-linux-androideabi-clang++ : -$(CROSSHOME)/bin/arm-linux-androideabi-ar -$(CROSSHOME)/sysroot -$(ARCH) --fexceptions --frtti --fpic +using clang : $(ARCH) : $(CROSSHOME)/bin/$(TARGET_TRIPLET)-clang++ : +$(CROSSHOME)/bin/llvm-ar +-fPIC -ffunction-sections +-fdata-sections -funwind-tables --march=armv7-a --msoft-float --mfpu=neon --mthumb --march=armv7-a --Wl,--fix-cortex-a8 --Os --fomit-frame-pointer --fno-strict-aliasing --DANDROID --D__ANDROID__ --DANDROID_TOOLCHAIN=clang --DANDROID_ABI=armv7-a --DANDROID_STL=c++_shared --DBOOST_ALL_NO_LIB -#-DNDEBUG --O2 +-fstack-protector-strong +-no-canonical-prefixes +-Wformat +-Werror=format-security +-frtti +-fexceptions +-DNDEBUG -g --fvisibility=hidden --fvisibility-inlines-hidden --fdata-sections --D__arm__ --D_REENTRANT --D_GLIBCXX__PTHREADS --Wno-long-long --Wno-missing-field-initializers --Wno-unused-variable +-Oz +-mthumb -Wl,-z,relro -Wl,-z,now -lc++_shared diff --git a/pythonforandroid/recipes/libtorrent/__init__.py b/pythonforandroid/recipes/libtorrent/__init__.py index c73bb02962..0eb50672b7 100644 --- a/pythonforandroid/recipes/libtorrent/__init__.py +++ b/pythonforandroid/recipes/libtorrent/__init__.py @@ -5,8 +5,8 @@ import sh # This recipe builds libtorrent with Python bindings -# It depends on Boost.Build and the source of several Boost libraries present in BOOST_ROOT, -# which is all provided by the boost recipe +# It depends on Boost.Build and the source of several Boost libraries present +# in BOOST_ROOT, which is all provided by the boost recipe def get_lib_from(search_directory, lib_extension='.so'): @@ -24,7 +24,8 @@ def get_lib_from(search_directory, lib_extension='.so'): class LibtorrentRecipe(Recipe): # Todo: make recipe compatible with all p4a architectures ''' - .. note:: This recipe can be built only against API 21+ and arch armeabi-v7a + .. note:: This recipe can be built only against API 21+ and an android + ndk >= r19 .. versionchanged:: 0.6.0 Rewrote recipe to support clang's build and boost 1.68. The following @@ -33,9 +34,14 @@ class LibtorrentRecipe(Recipe): - Bumped version number to 1.2.0 - added python 3 compatibility - new system to detect/copy generated libraries + + .. versionchanged:: 2019.08.09.1.dev0 + + - Bumped version number to 1.2.1 + - Adapted to work with ndk-r19+ ''' - version = '1_2_0' - url = 'https://github.com/arvidn/libtorrent/archive/libtorrent_{version}.tar.gz' + version = '1_2_1' + url = 'https://github.com/arvidn/libtorrent/archive/libtorrent-{version}.tar.gz' depends = ['boost'] opt_depends = ['openssl'] @@ -76,7 +82,7 @@ def build_arch(self, arch): '-j' + str(cpu_count()), '--debug-configuration', # so we know if our python is detected # '--deprecated-functions=off', - 'toolset=clang-arm', + 'toolset=clang-{arch}'.format(arch=env['ARCH']), 'abi=aapcs', 'binary-format=elf', 'cxxflags=-std=c++11', @@ -105,8 +111,12 @@ def build_arch(self, arch): # Copy only the boost shared libraries into the libs folder. Because # boost build two boost_python libraries, we force to search the lib # into the corresponding build path. - b2_build_dir = 'build/clang-linux-arm/release/{encryption}/' \ - 'lt-visibility-hidden/'.format(encryption=crypto_folder) + b2_build_dir = ( + 'build/clang-linux-{arch}/release/{encryption}/' + 'lt-visibility-hidden/'.format( + arch=env['ARCH'], encryption=crypto_folder + ) + ) boost_libs_dir = join(env['BOOST_BUILD_PATH'], 'bin.v2/libs') for boost_lib in listdir(boost_libs_dir): lib_path = get_lib_from(join(boost_libs_dir, boost_lib, b2_build_dir)) diff --git a/pythonforandroid/recipes/libtorrent/setup-lib-name.patch b/pythonforandroid/recipes/libtorrent/setup-lib-name.patch index 183705c839..4b688be35b 100644 --- a/pythonforandroid/recipes/libtorrent/setup-lib-name.patch +++ b/pythonforandroid/recipes/libtorrent/setup-lib-name.patch @@ -15,6 +15,6 @@ setup( - name='python-libtorrent', + name='libtorrent', - version='1.2.0', + version='1.2.1', author='Arvid Norberg', author_email='arvid@libtorrent.org', From e76aad9e7e79b1d021c09a78898963ea66d03fe4 Mon Sep 17 00:00:00 2001 From: jtc0de Date: Thu, 29 Aug 2019 22:57:23 +0200 Subject: [PATCH 16/72] Also copy the service/main.py when building with setup.py (#1936) --- .../bootstraps/common/build/build.py | 36 ++++++++++++++----- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/pythonforandroid/bootstraps/common/build/build.py b/pythonforandroid/bootstraps/common/build/build.py index 8068c6f53a..4a53ae04ec 100644 --- a/pythonforandroid/bootstraps/common/build/build.py +++ b/pythonforandroid/bootstraps/common/build/build.py @@ -321,16 +321,36 @@ def make_package(args): 'full private data into .apk.') tar_dirs.append(args.private) else: - print('Copying main.py ONLY, since other app data is ' - 'expected in site-packages.') + print("Copying main.py's ONLY, since other app data is " + "expected in site-packages.") main_py_only_dir = tempfile.mkdtemp() _temp_dirs_to_clean.append(main_py_only_dir) - if exists(join(args.private, "main.pyo")): - shutil.copyfile(join(args.private, "main.pyo"), - join(main_py_only_dir, "main.pyo")) - elif exists(join(args.private, "main.py")): - shutil.copyfile(join(args.private, "main.py"), - join(main_py_only_dir, "main.py")) + + # Check all main.py files we need to copy: + copy_paths = ["main.py", join("service", "main.py")] + for copy_path in copy_paths: + variants = [ + copy_path, + copy_path.partition(".")[0] + ".pyc", + copy_path.partition(".")[0] + ".pyo", + ] + # Check in all variants with all possible endings: + for variant in variants: + if exists(join(args.private, variant)): + # Make sure surrounding directly exists: + dir_path = os.path.dirname(variant) + if (len(dir_path) > 0 and + not exists( + join(main_py_only_dir, dir_path) + )): + os.mkdir(join(main_py_only_dir, dir_path)) + # Copy actual file: + shutil.copyfile( + join(args.private, variant), + join(main_py_only_dir, variant), + ) + + # Append directory with all main.py's to result apk paths: tar_dirs.append(main_py_only_dir) for python_bundle_dir in ('private', '_python_bundle'): if exists(python_bundle_dir): From e4ed9462cc6d346c0ee4846e8667bece20fca874 Mon Sep 17 00:00:00 2001 From: Pol Canelles Date: Fri, 30 Aug 2019 15:59:15 +0200 Subject: [PATCH 17/72] [recipe-stl] Rework of protobuf_cpp recipe (#1969) In here we do: - inherit from CppCompiledComponentsPythonRecipe Because depends on android's STL library - make use of the base class methods for library recipes - Split build_arch into proper methods - Shorten some long lines (to be PEP8 friendly) - make generated library shared - remove recipe from CI/constants --- ci/constants.py | 1 - .../recipes/protobuf_cpp/__init__.py | 72 +++++++++---------- 2 files changed, 33 insertions(+), 40 deletions(-) diff --git a/ci/constants.py b/ci/constants.py index 6699d24cd3..f5ab1dad59 100644 --- a/ci/constants.py +++ b/ci/constants.py @@ -67,7 +67,6 @@ class TargetPython(Enum): 'ffpyplayer', # requires `libpq-dev` system dependency e.g. for `pg_config` binary 'psycopg2', - 'protobuf_cpp', # most likely some setup in the Docker container, because it works in host 'pyjnius', 'pyopenal', # SyntaxError: invalid syntax (Python2) diff --git a/pythonforandroid/recipes/protobuf_cpp/__init__.py b/pythonforandroid/recipes/protobuf_cpp/__init__.py index 30ca030a2a..84a5e3c398 100644 --- a/pythonforandroid/recipes/protobuf_cpp/__init__.py +++ b/pythonforandroid/recipes/protobuf_cpp/__init__.py @@ -1,6 +1,6 @@ -from pythonforandroid.recipe import PythonRecipe +from pythonforandroid.recipe import CppCompiledComponentsPythonRecipe from pythonforandroid.logger import shprint, info_notify -from pythonforandroid.util import current_directory, shutil +from pythonforandroid.util import current_directory from os.path import exists, join import sh from multiprocessing import cpu_count @@ -9,13 +9,19 @@ import os -class ProtobufCppRecipe(PythonRecipe): +class ProtobufCppRecipe(CppCompiledComponentsPythonRecipe): + """This is a two-in-one recipe: + - build labraru `libprotobuf.so` + - build and install python binding for protobuf_cpp + """ name = 'protobuf_cpp' version = '3.6.1' url = 'https://github.com/google/protobuf/releases/download/v{version}/protobuf-python-{version}.tar.gz' call_hostpython_via_targetpython = False depends = ['cffi', 'setuptools'] site_packages_name = 'google/protobuf/pyext' + setup_extra_args = ['--cpp_implementation'] + built_libraries = {'libprotobuf.so': 'src/.libs'} protoc_dir = None def prebuild_arch(self, arch): @@ -65,42 +71,37 @@ def prebuild_arch(self, arch): def build_arch(self, arch): env = self.get_recipe_env(arch) - # Build libproto.a + # Build libproto.so with current_directory(self.get_build_dir(arch.arch)): - env['HOSTARCH'] = 'arm-eabi' - env['BUILDARCH'] = shprint(sh.gcc, '-dumpmachine').stdout.decode('utf-8').split('\n')[0] + build_arch = ( + shprint(sh.gcc, '-dumpmachine') + .stdout.decode('utf-8') + .split('\n')[0] + ) if not exists('configure'): shprint(sh.Command('./autogen.sh'), _env=env) shprint(sh.Command('./configure'), - '--host={}'.format(env['HOSTARCH']), + '--build={}'.format(build_arch), + '--host={}'.format(arch.command_prefix), + '--target={}'.format(arch.command_prefix), + '--disable-static', '--enable-shared', _env=env) with current_directory(join(self.get_build_dir(arch.arch), 'src')): shprint(sh.make, 'libprotobuf.la', '-j'+str(cpu_count()), _env=env) - shprint(sh.cp, '.libs/libprotobuf.a', join(self.ctx.get_libs_dir(arch.arch), 'libprotobuf.a')) - - # Copy stl library - shutil.copyfile( - self.ctx.ndk_dir + '/sources/cxx-stl/gnu-libstdc++/' + self.ctx.toolchain_version + '/libs/' + arch.arch + '/libgnustl_shared.so', - join(self.ctx.get_libs_dir(arch.arch), 'libgnustl_shared.so')) + def build_compiled_components(self, arch): # Build python bindings and _message.so + env = self.get_recipe_env(arch) with current_directory(join(self.get_build_dir(arch.arch), 'python')): hostpython = sh.Command(self.hostpython_location) shprint(hostpython, 'setup.py', 'build_ext', - '--cpp_implementation', _env=env) - - # Install python bindings - self.install_python_package(arch) - - # Create __init__.py which is missing (cf. https://github.com/protocolbuffers/protobuf/issues/1296 - # and https://stackoverflow.com/questions/13862562/google-protocol-buffers-not-found-when-trying-to-freeze-python-app) - open(join(self.ctx.get_site_packages_dir(), 'google', '__init__.py'), 'a').close() + _env=env, *self.setup_extra_args) def install_python_package(self, arch): env = self.get_recipe_env(arch) @@ -114,32 +115,25 @@ def install_python_package(self, arch): shprint(hostpython, 'setup.py', 'install', '-O2', '--root={}'.format(self.ctx.get_python_install_dir()), '--install-lib=.', - '--cpp_implementation', _env=hpenv, *self.setup_extra_args) + # Create __init__.py which is missing, see also: + # - https://github.com/protocolbuffers/protobuf/issues/1296 + # - https://stackoverflow.com/questions/13862562/ + # google-protocol-buffers-not-found-when-trying-to-freeze-python-app + open( + join(self.ctx.get_site_packages_dir(), 'google', '__init__.py'), + 'a', + ).close() + def get_recipe_env(self, arch): env = super(ProtobufCppRecipe, self).get_recipe_env(arch) if self.protoc_dir is not None: # we need protoc with binary for host platform env['PROTOC'] = join(self.protoc_dir, 'bin', 'protoc') env['TARGET_OS'] = 'OS_ANDROID_CROSSCOMPILE' - env['CFLAGS'] += ( - ' -I' + self.ctx.ndk_dir + '/platforms/android-' + - str(self.ctx.android_api) + - '/arch-' + arch.arch.replace('eabi', '') + '/usr/include' + - ' -I' + self.ctx.ndk_dir + '/sources/cxx-stl/gnu-libstdc++/' + - self.ctx.toolchain_version + '/include' + - ' -I' + self.ctx.ndk_dir + '/sources/cxx-stl/gnu-libstdc++/' + - self.ctx.toolchain_version + '/libs/' + arch.arch + '/include') - env['CFLAGS'] += ' -std=gnu++11' - env['CXXFLAGS'] = env['CFLAGS'] - env['CXXFLAGS'] += ' -frtti' - env['CXXFLAGS'] += ' -fexceptions' - env['LDFLAGS'] += ( - ' -lgnustl_shared -landroid -llog' + - ' -L' + self.ctx.ndk_dir + - '/sources/cxx-stl/gnu-libstdc++/' + self.ctx.toolchain_version + - '/libs/' + arch.arch) + env['CXXFLAGS'] += ' -std=c++11' + env['LDFLAGS'] += ' -lm -landroid -llog' return env From 57be583240fa78541041c5b7655e1e968b52683b Mon Sep 17 00:00:00 2001 From: Pol Canelles Date: Sun, 1 Sep 2019 21:12:28 +0200 Subject: [PATCH 18/72] [CI] Fix CI's test for `arm64-v8a` (#1977) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 317ad33283..cc5dd0e170 100644 --- a/.travis.yml +++ b/.travis.yml @@ -75,7 +75,7 @@ jobs: # overrides requirements to skip `peewee` pure python module, see: # https://github.com/kivy/python-for-android/issues/1263#issuecomment-390421054 env: - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools' --arch=arm64-v8a + COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools --arch=arm64-v8a' - <<: *testing name: Python 3 armeabi-v7a os: osx From 7f18efa2cb6ae8c1b08a033fa1f9e5907f2bf6b3 Mon Sep 17 00:00:00 2001 From: Richard Larkin Date: Thu, 12 Sep 2019 18:19:56 +0200 Subject: [PATCH 19/72] Updated README.md to clarify NDK versions (#1981) * :books: Updated README.dm to clarify NDK versions * :wrench: Change back to r19b --- README.md | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 82f7c1d9d8..bf108c3442 100644 --- a/README.md +++ b/README.md @@ -92,33 +92,33 @@ Please refer to the LICENSE file. ## History -In 2015 these tools were rewritten to provide a new, easier to use and -extend interface. If you'd like to browse the old toolchain, its +In 2015 these tools were rewritten to provide a new, easier-to-use and +easier-to-extend interface. If you'd like to browse the old toolchain, its status is recorded for posterity at at https://github.com/kivy/python-for-android/tree/old_toolchain. -In the last quarter of 2018 the python recipes has been changed, the -new recipe for python3 (3.7.1) has a new build system which has been +In the last quarter of 2018 the python recipes were changed. The +new recipe for python3 (3.7.1) had a new build system which was applied to the ancient python recipe, allowing us to bump the python2 -version number to 2.7.15. This change, unifies the build process for -both python recipes, and probably solve some issues detected over the -years. Also should be mentioned that this **unified python recipes** -require to target to a **minimum api level of 21**, -*Android 5.0 - Lollipop*, so in case the you need to build targeting an -api level below 21, you must use an old version of python-for-android -(<=0.7.1). All this work has been done using android ndk version r17c, -and your build should success with that version...but be aware that the -project is in constant development so, as per time of writing, -``we recommend to use android's NDK r19b`` because the toolchains installed by -default with the NDK may be used *in-place* and the python-for-android project -has been adapted to that feature, so, it's mandatory to use, at least, ndk -version r19 (be aware that more modern versions of the -android's ndk may not work). +version number to 2.7.15. This change unified the build process for +both python recipes, and probably solved various issues detected over the +years. It should also be mentioned that these **unified python recipes** +require a **minimum target api level of 21**, +*Android 5.0 - Lollipop*, so in the case that you need to build targeting an +api level below 21, you should use an older version of python-for-android +(<=0.7.1). + +Be aware that this project is in constant development so, as per time of writing, +you should use a minimum on Android's NDK r19, and ``we recommend using NDK r19b``. +This is because the toolchains installed by +default with the NDK can be used *in-place* and the python-for-android project +has been adapted for that feature. Also be aware that more recent versions of the +Android's NDK may not work. Those mentioned changes has been done this way to make easier the transition between python3 and python2. We will slowly phase out python2 support towards 2020...so...if you are using python2 in your projects you should -consider to migrate it into python3. +consider migrating it into python3. ## Contributors From be7abe910f110c577d24437e6ffac16e79b65192 Mon Sep 17 00:00:00 2001 From: Pol Canelles Date: Sat, 14 Sep 2019 20:43:43 +0200 Subject: [PATCH 20/72] A bunch of tests for library recipes (#1982) The main idea is to test as many recipes as we can with the less code possible and without creating any file/directory so our tests can be performed as fast as possible (all this tests will only add between 2 and 3 seconds to our CI tests and will cover almost 100% of the code for each tested recipe) To do so, we create a couple of modules: tests.recipe_ctx: allow us to create a proper Context to test our recipes tests.recipe_lib_test: which holds some base classes to be used to test a recipe depending on the build method used. For starters we introduce two kind of base classes: BaseTestForMakeRecipe: To test an standard library build (this will iinclude the recipes which requires the classical build commandsconfigure/make) BaseTestForCmakeRecipe: To test an library recipe which is compiled with cmake We also refactor the existing recipes tests, so we can remove some lines in there...the ones that creates a Context. * [test] Add module `tests.recipe_ctx` A helper module to test recipes. Here we will initialize a `Context` to test recipes. * [test] Refactor `setUp/tearDown` for `test_icu` * [test] Refactor `setUp/tearDown` for `test_pyicu` * [test] Refactor `setUp` for `test_reportlab` * [test] Refactor `setUp` for `test_gevent` * [test] Add module `tests.recipe_lib_test` A helper module to test recipes which will allow to test any recipe using `configure/make` commands. * [test] Add test for `libffi` * [test] Add test for `libexpat` * [test] Add test for `libcurl` * [test] Add test for `libiconv` * [test] Add test for `libogg` * [test] Add test for `libpq` * [test] Add test for `libsecp256k1` * [test] Add test for `libshine` * [test] Add test for `libvorbis` * [test] Add test for `libx264` * [test] Add test for `libxml2` * [test] Add test for `libxslt` * [test] Add test for `png` * [test] Add test for `freetype` * [test] Add test for `harfbuzz` * [test] Add test for `openssl` * [test] Add `BaseTestForCmakeRecipe` * [test] Add test for `jpeg` and clean code We can remove the `get_recipe_env` because the environment that we use is already using a clang as default compiler (not the case when we migrated the jpeg recipe to use `cmake`)...so we can do a little refactor :) * [test] Add test for `snappy` * [test] Add test for `leveldb` * [test] Add test for `libgeos` * [test] Add test for `libmysqlclient` * [test] Add test for `openal` * [test] Make the `super` calls Python3 style * [test] Move mock tests outside context manager... Because there is no need to do it there. Also rewrote the inline comments. --- pythonforandroid/recipes/jpeg/__init__.py | 19 +-- tests/recipes/recipe_ctx.py | 52 +++++++ tests/recipes/recipe_lib_test.py | 157 ++++++++++++++++++++++ tests/recipes/test_freetype.py | 10 ++ tests/recipes/test_gevent.py | 16 +-- tests/recipes/test_harfbuzz.py | 10 ++ tests/recipes/test_icu.py | 78 +++-------- tests/recipes/test_jpeg.py | 9 ++ tests/recipes/test_leveldb.py | 9 ++ tests/recipes/test_libcurl.py | 10 ++ tests/recipes/test_libexpat.py | 10 ++ tests/recipes/test_libffi.py | 15 +++ tests/recipes/test_libgeos.py | 29 ++++ tests/recipes/test_libiconv.py | 10 ++ tests/recipes/test_libmysqlclient.py | 32 +++++ tests/recipes/test_libogg.py | 10 ++ tests/recipes/test_libpq.py | 30 +++++ tests/recipes/test_libsecp256k1.py | 10 ++ tests/recipes/test_libshine.py | 10 ++ tests/recipes/test_libvorbis.py | 31 +++++ tests/recipes/test_libx264.py | 10 ++ tests/recipes/test_libxml2.py | 14 ++ tests/recipes/test_libxslt.py | 15 +++ tests/recipes/test_openal.py | 62 +++++++++ tests/recipes/test_openssl.py | 65 +++++++++ tests/recipes/test_png.py | 10 ++ tests/recipes/test_pyicu.py | 43 ++---- tests/recipes/test_reportlab.py | 24 +--- tests/recipes/test_snappy.py | 9 ++ 29 files changed, 665 insertions(+), 144 deletions(-) create mode 100644 tests/recipes/recipe_ctx.py create mode 100644 tests/recipes/recipe_lib_test.py create mode 100644 tests/recipes/test_freetype.py create mode 100644 tests/recipes/test_harfbuzz.py create mode 100644 tests/recipes/test_jpeg.py create mode 100644 tests/recipes/test_leveldb.py create mode 100644 tests/recipes/test_libcurl.py create mode 100644 tests/recipes/test_libexpat.py create mode 100644 tests/recipes/test_libffi.py create mode 100644 tests/recipes/test_libgeos.py create mode 100644 tests/recipes/test_libiconv.py create mode 100644 tests/recipes/test_libmysqlclient.py create mode 100644 tests/recipes/test_libogg.py create mode 100644 tests/recipes/test_libpq.py create mode 100644 tests/recipes/test_libsecp256k1.py create mode 100644 tests/recipes/test_libshine.py create mode 100644 tests/recipes/test_libvorbis.py create mode 100644 tests/recipes/test_libx264.py create mode 100644 tests/recipes/test_libxml2.py create mode 100644 tests/recipes/test_libxslt.py create mode 100644 tests/recipes/test_openal.py create mode 100644 tests/recipes/test_openssl.py create mode 100644 tests/recipes/test_png.py create mode 100644 tests/recipes/test_snappy.py diff --git a/pythonforandroid/recipes/jpeg/__init__.py b/pythonforandroid/recipes/jpeg/__init__.py index 1937ecb1d8..5e5b447a24 100644 --- a/pythonforandroid/recipes/jpeg/__init__.py +++ b/pythonforandroid/recipes/jpeg/__init__.py @@ -2,7 +2,6 @@ from pythonforandroid.logger import shprint from pythonforandroid.util import current_directory from os.path import join -from os import environ, uname import sh @@ -35,10 +34,9 @@ def build_arch(self, arch): '-DCMAKE_POSITION_INDEPENDENT_CODE=1', '-DCMAKE_ANDROID_ARCH_ABI={arch}'.format(arch=arch.arch), '-DCMAKE_ANDROID_NDK=' + self.ctx.ndk_dir, - '-DCMAKE_C_COMPILER={toolchain}/bin/clang'.format( - toolchain=env['TOOLCHAIN']), - '-DCMAKE_CXX_COMPILER={toolchain}/bin/clang++'.format( - toolchain=env['TOOLCHAIN']), + '-DCMAKE_C_COMPILER={cc}'.format(cc=arch.get_clang_exe()), + '-DCMAKE_CXX_COMPILER={cc_plus}'.format( + cc_plus=arch.get_clang_exe(plus_plus=True)), '-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_INSTALL_PREFIX=./install', '-DCMAKE_TOOLCHAIN_FILE=' + toolchain_file, @@ -54,16 +52,5 @@ def build_arch(self, arch): _env=env) shprint(sh.make, _env=env) - def get_recipe_env(self, arch=None, with_flags_in_cc=False): - env = environ.copy() - - build_platform = '{system}-{machine}'.format( - system=uname()[0], machine=uname()[-1]).lower() - env['TOOLCHAIN'] = join(self.ctx.ndk_dir, 'toolchains/llvm/' - 'prebuilt/{build_platform}'.format( - build_platform=build_platform)) - - return env - recipe = JpegRecipe() diff --git a/tests/recipes/recipe_ctx.py b/tests/recipes/recipe_ctx.py new file mode 100644 index 0000000000..938b19e471 --- /dev/null +++ b/tests/recipes/recipe_ctx.py @@ -0,0 +1,52 @@ +import os + +from pythonforandroid.bootstrap import Bootstrap +from pythonforandroid.distribution import Distribution +from pythonforandroid.recipe import Recipe +from pythonforandroid.build import Context +from pythonforandroid.archs import ArchAarch_64 + + +class RecipeCtx: + """ + An base class for unit testing a recipe. This will create a context so we + can test any recipe using this context. Implement `setUp` and `tearDown` + methods used by unit testing. + """ + + ctx = None + arch = None + recipe = None + + recipe_name = "" + "The name of the recipe to test." + + recipes = [] + """A List of recipes to pass to `Distribution.get_distribution`. Should + contain the target recipe to test as well as a python recipe.""" + recipe_build_order = [] + """A recipe_build_order which should take into account the recipe we want + to test as well as the possible dependant recipes""" + + def setUp(self): + self.ctx = Context() + self.ctx.ndk_api = 21 + self.ctx.android_api = 27 + self.ctx._sdk_dir = "/opt/android/android-sdk" + self.ctx._ndk_dir = "/opt/android/android-ndk" + self.ctx.setup_dirs(os.getcwd()) + self.ctx.bootstrap = Bootstrap().get_bootstrap("sdl2", self.ctx) + self.ctx.bootstrap.distribution = Distribution.get_distribution( + self.ctx, name="sdl2", recipes=self.recipes + ) + self.ctx.recipe_build_order = self.recipe_build_order + self.ctx.python_recipe = Recipe.get_recipe("python3", self.ctx) + self.arch = ArchAarch_64(self.ctx) + self.ctx.ndk_platform = ( + f"{self.ctx._ndk_dir}/platforms/" + f"android-{self.ctx.ndk_api}/{self.arch.platform_dir}" + ) + self.recipe = Recipe.get_recipe(self.recipe_name, self.ctx) + + def tearDown(self): + self.ctx = None diff --git a/tests/recipes/recipe_lib_test.py b/tests/recipes/recipe_lib_test.py new file mode 100644 index 0000000000..dbb476028b --- /dev/null +++ b/tests/recipes/recipe_lib_test.py @@ -0,0 +1,157 @@ +from unittest import mock +from tests.recipes.recipe_ctx import RecipeCtx + + +class BaseTestForMakeRecipe(RecipeCtx): + """ + An unittest for testing any recipe using the standard build commands + (`configure/make`). + + .. note:: Note that Some cmake recipe may need some more specific testing + ...but this should cover the basics. + """ + + recipe_name = None + recipes = ["python3", "kivy"] + recipe_build_order = ["hostpython3", "python3", "sdl2", "kivy"] + expected_compiler = ( + "{android_ndk}/toolchains/llvm/prebuilt/linux-x86_64/bin/clang" + ) + + sh_command_calls = ["./configure"] + """The expected commands that the recipe runs via `sh.command`.""" + + extra_env_flags = {} + """ + This must be a dictionary containing pairs of key (env var) and value. + """ + + def __new__(cls, *args): + obj = super().__new__(cls) + if obj.recipe_name is not None: + print(f"We are testing recipe: {obj.recipe_name}") + obj.recipes.append(obj.recipe_name) + obj.recipe_build_order.insert(1, obj.recipe_name) + return obj + + @mock.patch("pythonforandroid.recipe.Recipe.check_recipe_choices") + @mock.patch("pythonforandroid.build.ensure_dir") + @mock.patch("pythonforandroid.archs.glob") + @mock.patch("pythonforandroid.archs.find_executable") + def test_get_recipe_env( + self, + mock_find_executable, + mock_glob, + mock_ensure_dir, + mock_check_recipe_choices, + ): + """ + Test that get_recipe_env contains some expected arch flags and that + some internal methods has been called. + """ + mock_find_executable.return_value = self.expected_compiler.format( + android_ndk=self.ctx._ndk_dir + ) + mock_glob.return_value = ["llvm"] + mock_check_recipe_choices.return_value = sorted( + self.ctx.recipe_build_order + ) + + # make sure the arch flags are in env + env = self.recipe.get_recipe_env(self.arch) + for flag in self.arch.arch_cflags: + self.assertIn(flag, env["CFLAGS"]) + self.assertIn( + f"-target {self.arch.target}", + env["CFLAGS"], + ) + + for flag, value in self.extra_env_flags.items(): + self.assertIn(value, env[flag]) + + # make sure that the mocked methods are actually called + mock_glob.assert_called() + mock_ensure_dir.assert_called() + mock_find_executable.assert_called() + mock_check_recipe_choices.assert_called() + + @mock.patch("pythonforandroid.util.chdir") + @mock.patch("pythonforandroid.build.ensure_dir") + @mock.patch("pythonforandroid.archs.glob") + @mock.patch("pythonforandroid.archs.find_executable") + def test_build_arch( + self, + mock_find_executable, + mock_glob, + mock_ensure_dir, + mock_current_directory, + ): + mock_find_executable.return_value = self.expected_compiler.format( + android_ndk=self.ctx._ndk_dir + ) + mock_glob.return_value = ["llvm"] + + # Since the following mocks are dynamic, + # we mock it inside a Context Manager + with mock.patch( + f"pythonforandroid.recipes.{self.recipe_name}.sh.Command" + ) as mock_sh_command, mock.patch( + f"pythonforandroid.recipes.{self.recipe_name}.sh.make" + ) as mock_make: + self.recipe.build_arch(self.arch) + + # make sure that the mocked methods are actually called + for command in self.sh_command_calls: + self.assertIn( + mock.call(command), + mock_sh_command.mock_calls, + ) + mock_make.assert_called() + mock_glob.assert_called() + mock_ensure_dir.assert_called() + mock_current_directory.assert_called() + mock_find_executable.assert_called() + + +class BaseTestForCmakeRecipe(BaseTestForMakeRecipe): + """ + An unittest for testing any recipe using `cmake`. It inherits from + `BaseTestForMakeRecipe` but we override the build method to match the cmake + build method. + + .. note:: Note that Some cmake recipe may need some more specific testing + ...but this should cover the basics. + """ + + @mock.patch("pythonforandroid.util.chdir") + @mock.patch("pythonforandroid.build.ensure_dir") + @mock.patch("pythonforandroid.archs.glob") + @mock.patch("pythonforandroid.archs.find_executable") + def test_build_arch( + self, + mock_find_executable, + mock_glob, + mock_ensure_dir, + mock_current_directory, + ): + mock_find_executable.return_value = self.expected_compiler.format( + android_ndk=self.ctx._ndk_dir + ) + mock_glob.return_value = ["llvm"] + + # Since the following mocks are dynamic, + # we mock it inside a Context Manager + with mock.patch( + f"pythonforandroid.recipes.{self.recipe_name}.sh.make" + ) as mock_make, mock.patch( + f"pythonforandroid.recipes.{self.recipe_name}.sh.cmake" + ) as mock_cmake: + self.recipe.build_arch(self.arch) + + # make sure that the mocked methods are actually called + mock_cmake.assert_called() + mock_make.assert_called() + mock_glob.assert_called() + mock_ensure_dir.assert_called() + mock_current_directory.assert_called() + mock_find_executable.assert_called() diff --git a/tests/recipes/test_freetype.py b/tests/recipes/test_freetype.py new file mode 100644 index 0000000000..981ae631b7 --- /dev/null +++ b/tests/recipes/test_freetype.py @@ -0,0 +1,10 @@ +import unittest +from tests.recipes.recipe_lib_test import BaseTestForMakeRecipe + + +class TestFreetypeRecipe(BaseTestForMakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.freetype` + """ + recipe_name = "freetype" + sh_command_calls = ["./configure"] diff --git a/tests/recipes/test_gevent.py b/tests/recipes/test_gevent.py index 8548ffa64a..3b1ce62842 100644 --- a/tests/recipes/test_gevent.py +++ b/tests/recipes/test_gevent.py @@ -1,21 +1,11 @@ import unittest from mock import patch -from pythonforandroid.archs import ArchARMv7_a -from pythonforandroid.build import Context -from pythonforandroid.recipe import Recipe +from tests.recipes.recipe_ctx import RecipeCtx -class TestGeventRecipe(unittest.TestCase): +class TestGeventRecipe(RecipeCtx, unittest.TestCase): - def setUp(self): - """ - Setups recipe and context. - """ - self.context = Context() - self.context.ndk_api = 21 - self.context.android_api = 27 - self.arch = ArchARMv7_a(self.context) - self.recipe = Recipe.get_recipe('gevent', self.context) + recipe_name = "gevent" def test_get_recipe_env(self): """ diff --git a/tests/recipes/test_harfbuzz.py b/tests/recipes/test_harfbuzz.py new file mode 100644 index 0000000000..6006fb94c7 --- /dev/null +++ b/tests/recipes/test_harfbuzz.py @@ -0,0 +1,10 @@ +import unittest +from tests.recipes.recipe_lib_test import BaseTestForMakeRecipe + + +class TestHarfbuzzRecipe(BaseTestForMakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.harfbuzz` + """ + recipe_name = "harfbuzz" + sh_command_calls = ["./configure"] diff --git a/tests/recipes/test_icu.py b/tests/recipes/test_icu.py index 17c207c8be..22de4b3dd3 100644 --- a/tests/recipes/test_icu.py +++ b/tests/recipes/test_icu.py @@ -1,72 +1,34 @@ import os - import unittest +from unittest import mock -try: - from unittest import mock -except ImportError: - # `Python 2` or lower than `Python 3.3` does not - # have the `unittest.mock` module built-in - import mock -from pythonforandroid.bootstrap import Bootstrap -from pythonforandroid.distribution import Distribution - -from pythonforandroid.build import Context -from pythonforandroid.archs import ArchARMv7_a +from tests.recipes.recipe_ctx import RecipeCtx from pythonforandroid.recipes.icu import ICURecipe -from pythonforandroid.recipe import Recipe -class TestIcuRecipe(unittest.TestCase): +class TestIcuRecipe(RecipeCtx, unittest.TestCase): """ An unittest for recipe :mod:`~pythonforandroid.recipes.icu` """ - ctx = None - - def setUp(self): - self.ctx = Context() - self.ctx.ndk_api = 21 - self.ctx.android_api = 27 - self.ctx._sdk_dir = "/opt/android/android-sdk" - self.ctx._ndk_dir = "/opt/android/android-ndk" - self.ctx.setup_dirs(os.getcwd()) - self.ctx.bootstrap = Bootstrap().get_bootstrap("sdl2", self.ctx) - self.ctx.bootstrap.distribution = Distribution.get_distribution( - self.ctx, name="sdl2", recipes=["python3", "kivy", "icu"] - ) - self.ctx.python_recipe = ICURecipe.get_recipe("python3", self.ctx) - self.ctx.recipe_build_order = [ - "hostpython3", - "icu", - "python3", - "sdl2", - "kivy", - ] - - def tearDown(self): - self.ctx = None + recipe_name = "icu" + recipes = ["python3", "kivy", "icu"] + recipe_build_order = ["hostpython3", "icu", "python3", "sdl2", "kivy"] def test_url(self): - recipe = ICURecipe() - recipe.ctx = self.ctx - self.assertTrue(recipe.versioned_url.startswith("http")) - self.assertIn(recipe.version, recipe.versioned_url) + self.assertTrue(self.recipe.versioned_url.startswith("http")) + self.assertIn(self.recipe.version, self.recipe.versioned_url) @mock.patch( "pythonforandroid.recipe.Recipe.url", new_callable=mock.PropertyMock ) def test_url_none(self, mock_url): mock_url.return_value = None - recipe = ICURecipe() - recipe.ctx = self.ctx - self.assertIsNone(recipe.versioned_url) + self.assertIsNone(self.recipe.versioned_url) def test_get_recipe_dir(self): - recipe = ICURecipe() - recipe.ctx = self.ctx expected_dir = os.path.join(self.ctx.root_dir, "recipes", "icu") - self.assertEqual(recipe.get_recipe_dir(), expected_dir) + self.assertEqual(self.recipe.get_recipe_dir(), expected_dir) @mock.patch("pythonforandroid.util.makedirs") @mock.patch("pythonforandroid.util.chdir") @@ -85,8 +47,6 @@ def test_build_arch( mock_chdir, mock_makedirs, ): - recipe = ICURecipe() - recipe.ctx = self.ctx mock_find_executable.return_value = os.path.join( self.ctx._ndk_dir, "toolchains/llvm/prebuilt/linux-x86_64/bin/clang", @@ -94,13 +54,12 @@ def test_build_arch( mock_archs_glob.return_value = [ os.path.join(self.ctx._ndk_dir, "toolchains", "llvm") ] - arch = ArchARMv7_a(self.ctx) - self.ctx.toolchain_prefix = arch.toolchain_prefix + self.ctx.toolchain_prefix = self.arch.toolchain_prefix self.ctx.toolchain_version = "4.9" - recipe.build_arch(arch) + self.recipe.build_arch(self.arch) - # We expect to calls to `sh.Command` - build_root = recipe.get_build_dir(arch.arch) + # We expect some calls to `sh.Command` + build_root = self.recipe.get_build_dir(self.arch.arch) mock_sh_command.has_calls( [ mock.call( @@ -111,7 +70,7 @@ def test_build_arch( ) mock_ensure_dir.assert_called() mock_chdir.assert_called() - # we expect for calls to sh.make command + # we expect multiple calls to sh.make command expected_host_cppflags = ( "-O3 -fno-short-wchar -DU_USING_ICU_NAMESPACE=1 -fno-short-enums " "-DU_HAVE_NL_LANGINFO_CODESET=0 -D__STDC_INT64__ -DU_TIMEZONE=0 " @@ -147,10 +106,7 @@ def test_build_arch( @mock.patch("pythonforandroid.recipes.icu.sh.cp") @mock.patch("pythonforandroid.util.makedirs") def test_install_libraries(self, mock_makedirs, mock_sh_cp): - arch = ArchARMv7_a(self.ctx) - recipe = Recipe.get_recipe("icu", self.ctx) - recipe.ctx = self.ctx - recipe.install_libraries(arch) + self.recipe.install_libraries(self.arch) mock_makedirs.assert_called() mock_sh_cp.assert_called() @@ -158,6 +114,8 @@ def test_install_libraries(self, mock_makedirs, mock_sh_cp): def test_get_recipe_dir_with_local_recipes(self, mock_exists): self.ctx.local_recipes = "/home/user/p4a_local_recipes" + # we don't use `self.recipe` because, somehow, the modified variable + # above is not updated in the `ctx` and makes the test fail... recipe = ICURecipe() recipe.ctx = self.ctx recipe_dir = recipe.get_recipe_dir() diff --git a/tests/recipes/test_jpeg.py b/tests/recipes/test_jpeg.py new file mode 100644 index 0000000000..2d43562061 --- /dev/null +++ b/tests/recipes/test_jpeg.py @@ -0,0 +1,9 @@ +import unittest +from tests.recipes.recipe_lib_test import BaseTestForCmakeRecipe + + +class TestJpegRecipe(BaseTestForCmakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.jpeg` + """ + recipe_name = "jpeg" diff --git a/tests/recipes/test_leveldb.py b/tests/recipes/test_leveldb.py new file mode 100644 index 0000000000..f501398c63 --- /dev/null +++ b/tests/recipes/test_leveldb.py @@ -0,0 +1,9 @@ +import unittest +from tests.recipes.recipe_lib_test import BaseTestForCmakeRecipe + + +class TestLeveldbRecipe(BaseTestForCmakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.leveldb` + """ + recipe_name = "leveldb" diff --git a/tests/recipes/test_libcurl.py b/tests/recipes/test_libcurl.py new file mode 100644 index 0000000000..d5e2fa45d4 --- /dev/null +++ b/tests/recipes/test_libcurl.py @@ -0,0 +1,10 @@ +import unittest +from tests.recipes.recipe_lib_test import BaseTestForMakeRecipe + + +class TestLibcurlRecipe(BaseTestForMakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.libcurl` + """ + recipe_name = "libcurl" + sh_command_calls = ["./configure"] diff --git a/tests/recipes/test_libexpat.py b/tests/recipes/test_libexpat.py new file mode 100644 index 0000000000..c9e0ed69ff --- /dev/null +++ b/tests/recipes/test_libexpat.py @@ -0,0 +1,10 @@ +import unittest +from tests.recipes.recipe_lib_test import BaseTestForMakeRecipe + + +class TestLibexpatRecipe(BaseTestForMakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.libexpat` + """ + recipe_name = "libexpat" + sh_command_calls = ["./buildconf.sh", "./configure"] diff --git a/tests/recipes/test_libffi.py b/tests/recipes/test_libffi.py new file mode 100644 index 0000000000..68d4ce3db1 --- /dev/null +++ b/tests/recipes/test_libffi.py @@ -0,0 +1,15 @@ +import unittest +from tests.recipes.recipe_lib_test import BaseTestForMakeRecipe + + +class TestLibffiRecipe(BaseTestForMakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.libffi` + """ + recipe_name = "libffi" + sh_command_calls = ["./autogen.sh", "autoreconf", "./configure"] + + def test_get_include_dirs(self): + list_of_includes = self.recipe.get_include_dirs(self.arch) + self.assertIsInstance(list_of_includes, list) + self.assertTrue(list_of_includes[0].endswith("include")) diff --git a/tests/recipes/test_libgeos.py b/tests/recipes/test_libgeos.py new file mode 100644 index 0000000000..6914faf17a --- /dev/null +++ b/tests/recipes/test_libgeos.py @@ -0,0 +1,29 @@ +import unittest +from unittest import mock +from tests.recipes.recipe_lib_test import BaseTestForCmakeRecipe + + +class TestLibgeosRecipe(BaseTestForCmakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.libgeos` + """ + recipe_name = "libgeos" + + @mock.patch("pythonforandroid.util.makedirs") + @mock.patch("pythonforandroid.util.chdir") + @mock.patch("pythonforandroid.build.ensure_dir") + @mock.patch("pythonforandroid.archs.glob") + @mock.patch("pythonforandroid.archs.find_executable") + def test_build_arch( + self, + mock_find_executable, + mock_glob, + mock_ensure_dir, + mock_current_directory, + mock_makedirs, + ): + # We overwrite the base test method because we + # want to avoid any file/directory creation + super().test_build_arch() + # make sure that the mocked methods are actually called + mock_makedirs.assert_called() diff --git a/tests/recipes/test_libiconv.py b/tests/recipes/test_libiconv.py new file mode 100644 index 0000000000..d81649fd58 --- /dev/null +++ b/tests/recipes/test_libiconv.py @@ -0,0 +1,10 @@ +import unittest +from tests.recipes.recipe_lib_test import BaseTestForMakeRecipe + + +class TestLibiconvRecipe(BaseTestForMakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.libiconv` + """ + recipe_name = "libiconv" + sh_command_calls = ["./configure"] diff --git a/tests/recipes/test_libmysqlclient.py b/tests/recipes/test_libmysqlclient.py new file mode 100644 index 0000000000..8acadb8645 --- /dev/null +++ b/tests/recipes/test_libmysqlclient.py @@ -0,0 +1,32 @@ +import unittest +from unittest import mock +from tests.recipes.recipe_lib_test import BaseTestForCmakeRecipe + + +class TestLibmysqlclientRecipe(BaseTestForCmakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.libmysqlclient` + """ + recipe_name = "libmysqlclient" + + @mock.patch("pythonforandroid.recipes.libmysqlclient.sh.rm") + @mock.patch("pythonforandroid.recipes.libmysqlclient.sh.cp") + @mock.patch("pythonforandroid.util.chdir") + @mock.patch("pythonforandroid.build.ensure_dir") + @mock.patch("pythonforandroid.archs.glob") + @mock.patch("pythonforandroid.archs.find_executable") + def test_build_arch( + self, + mock_find_executable, + mock_glob, + mock_ensure_dir, + mock_current_directory, + mock_sh_cp, + mock_sh_rm, + ): + # We overwrite the base test method because we need + # to mock a little more (`sh.cp` and `sh.rm`) + super().test_build_arch() + # make sure that the mocked methods are actually called + mock_sh_cp.assert_called() + mock_sh_rm.assert_called() diff --git a/tests/recipes/test_libogg.py b/tests/recipes/test_libogg.py new file mode 100644 index 0000000000..883cfa491b --- /dev/null +++ b/tests/recipes/test_libogg.py @@ -0,0 +1,10 @@ +import unittest +from tests.recipes.recipe_lib_test import BaseTestForMakeRecipe + + +class TestLiboggRecipe(BaseTestForMakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.libogg` + """ + recipe_name = "libogg" + sh_command_calls = ["./configure"] diff --git a/tests/recipes/test_libpq.py b/tests/recipes/test_libpq.py new file mode 100644 index 0000000000..2c3e145775 --- /dev/null +++ b/tests/recipes/test_libpq.py @@ -0,0 +1,30 @@ +import unittest +from unittest import mock +from tests.recipes.recipe_lib_test import BaseTestForMakeRecipe + + +class TestLibpqRecipe(BaseTestForMakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.libpq` + """ + recipe_name = "libpq" + sh_command_calls = ["./configure"] + + @mock.patch("pythonforandroid.recipes.libpq.sh.cp") + @mock.patch("pythonforandroid.util.chdir") + @mock.patch("pythonforandroid.build.ensure_dir") + @mock.patch("pythonforandroid.archs.glob") + @mock.patch("pythonforandroid.archs.find_executable") + def test_build_arch( + self, + mock_find_executable, + mock_glob, + mock_ensure_dir, + mock_current_directory, + mock_sh_cp, + ): + # We overwrite the base test method because we need to mock a little + # more with this recipe (`sh.cp`) + super().test_build_arch() + # make sure that the mocked methods are actually called + mock_sh_cp.assert_called() diff --git a/tests/recipes/test_libsecp256k1.py b/tests/recipes/test_libsecp256k1.py new file mode 100644 index 0000000000..983b0bf3de --- /dev/null +++ b/tests/recipes/test_libsecp256k1.py @@ -0,0 +1,10 @@ +import unittest +from tests.recipes.recipe_lib_test import BaseTestForMakeRecipe + + +class TestLibsecp256k1Recipe(BaseTestForMakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.libsecp256k1` + """ + recipe_name = "libsecp256k1" + sh_command_calls = ["./autogen.sh", "./configure"] diff --git a/tests/recipes/test_libshine.py b/tests/recipes/test_libshine.py new file mode 100644 index 0000000000..66f8ecb354 --- /dev/null +++ b/tests/recipes/test_libshine.py @@ -0,0 +1,10 @@ +import unittest +from tests.recipes.recipe_lib_test import BaseTestForMakeRecipe + + +class TestLibshineRecipe(BaseTestForMakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.libshine` + """ + recipe_name = "libshine" + sh_command_calls = ["./bootstrap", "./configure"] diff --git a/tests/recipes/test_libvorbis.py b/tests/recipes/test_libvorbis.py new file mode 100644 index 0000000000..95a4c3cd85 --- /dev/null +++ b/tests/recipes/test_libvorbis.py @@ -0,0 +1,31 @@ +import unittest +from unittest import mock +from tests.recipes.recipe_lib_test import BaseTestForMakeRecipe + + +class TestLibvorbisRecipe(BaseTestForMakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.libvorbis` + """ + recipe_name = "libvorbis" + sh_command_calls = ["./configure"] + extra_env_flags = {'CFLAGS': 'libogg/include'} + + @mock.patch("pythonforandroid.recipes.libvorbis.sh.cp") + @mock.patch("pythonforandroid.util.chdir") + @mock.patch("pythonforandroid.build.ensure_dir") + @mock.patch("pythonforandroid.archs.glob") + @mock.patch("pythonforandroid.archs.find_executable") + def test_build_arch( + self, + mock_find_executable, + mock_glob, + mock_ensure_dir, + mock_current_directory, + mock_sh_cp, + ): + # We overwrite the base test method because we need to mock a little + # more with this recipe (`sh.cp`) + super().test_build_arch() + # make sure that the mocked methods are actually called + mock_sh_cp.assert_called() diff --git a/tests/recipes/test_libx264.py b/tests/recipes/test_libx264.py new file mode 100644 index 0000000000..d928b476a8 --- /dev/null +++ b/tests/recipes/test_libx264.py @@ -0,0 +1,10 @@ +import unittest +from tests.recipes.recipe_lib_test import BaseTestForMakeRecipe + + +class TestLibx264Recipe(BaseTestForMakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.libx264` + """ + recipe_name = "libx264" + sh_command_calls = ["./configure"] diff --git a/tests/recipes/test_libxml2.py b/tests/recipes/test_libxml2.py new file mode 100644 index 0000000000..d55909cc28 --- /dev/null +++ b/tests/recipes/test_libxml2.py @@ -0,0 +1,14 @@ +import unittest +from tests.recipes.recipe_lib_test import BaseTestForMakeRecipe + + +class TestLibxml2Recipe(BaseTestForMakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.libxml2` + """ + recipe_name = "libxml2" + sh_command_calls = ["./autogen.sh", "autoreconf", "./configure"] + extra_env_flags = { + "CONFIG_SHELL": "/bin/bash", + "SHELL": "/bin/bash", + } diff --git a/tests/recipes/test_libxslt.py b/tests/recipes/test_libxslt.py new file mode 100644 index 0000000000..24d6566999 --- /dev/null +++ b/tests/recipes/test_libxslt.py @@ -0,0 +1,15 @@ +import unittest +from tests.recipes.recipe_lib_test import BaseTestForMakeRecipe + + +class TestLibxsltRecipe(BaseTestForMakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.libxslt` + """ + recipe_name = "libxslt" + sh_command_calls = ["./autogen.sh", "autoreconf", "./configure"] + extra_env_flags = { + "CONFIG_SHELL": "/bin/bash", + "SHELL": "/bin/bash", + "LIBS": "-lxml2 -lz -lm", + } diff --git a/tests/recipes/test_openal.py b/tests/recipes/test_openal.py new file mode 100644 index 0000000000..9f3a6cf4ba --- /dev/null +++ b/tests/recipes/test_openal.py @@ -0,0 +1,62 @@ +import unittest +from unittest import mock +from tests.recipes.recipe_lib_test import BaseTestForCmakeRecipe + + +class TestOpenalRecipe(BaseTestForCmakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.openal` + """ + recipe_name = "openal" + + @mock.patch("pythonforandroid.recipes.openal.sh.cmake") + @mock.patch("pythonforandroid.recipes.openal.sh.make") + @mock.patch("pythonforandroid.recipes.openal.sh.cp") + @mock.patch("pythonforandroid.util.chdir") + @mock.patch("pythonforandroid.build.ensure_dir") + @mock.patch("pythonforandroid.archs.glob") + @mock.patch("pythonforandroid.archs.find_executable") + def test_prebuild_arch( + self, + mock_find_executable, + mock_glob, + mock_ensure_dir, + mock_current_directory, + mock_sh_cp, + mock_sh_make, + mock_sh_cmake, + ): + mock_find_executable.return_value = ( + "/opt/android/android-ndk/toolchains/" + "llvm/prebuilt/linux-x86_64/bin/clang" + ) + mock_glob.return_value = ["llvm"] + self.recipe.build_arch(self.arch) + + # make sure that the mocked methods are actually called + mock_glob.assert_called() + mock_ensure_dir.assert_called() + mock_current_directory.assert_called() + mock_find_executable.assert_called() + mock_sh_cp.assert_called() + mock_sh_make.assert_called() + mock_sh_cmake.assert_called() + + @mock.patch("pythonforandroid.recipes.openal.sh.cp") + @mock.patch("pythonforandroid.util.chdir") + @mock.patch("pythonforandroid.build.ensure_dir") + @mock.patch("pythonforandroid.archs.glob") + @mock.patch("pythonforandroid.archs.find_executable") + def test_build_arch( + self, + mock_find_executable, + mock_glob, + mock_ensure_dir, + mock_current_directory, + mock_sh_cp, + ): + # We overwrite the base test method because we need to mock a little + # more with this recipe (`sh.cp` and `sh.rm`) + super().test_build_arch() + # make sure that the mocked methods are actually called + mock_sh_cp.assert_called() diff --git a/tests/recipes/test_openssl.py b/tests/recipes/test_openssl.py new file mode 100644 index 0000000000..51fd2da1d0 --- /dev/null +++ b/tests/recipes/test_openssl.py @@ -0,0 +1,65 @@ +import unittest +from unittest import mock +from tests.recipes.recipe_lib_test import BaseTestForMakeRecipe + + +class TestOpensslRecipe(BaseTestForMakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.openssl` + """ + + recipe_name = "openssl" + sh_command_calls = ["perl"] + + @mock.patch("pythonforandroid.recipes.openssl.sh.patch") + @mock.patch("pythonforandroid.util.chdir") + @mock.patch("pythonforandroid.build.ensure_dir") + @mock.patch("pythonforandroid.archs.glob") + @mock.patch("pythonforandroid.archs.find_executable") + def test_build_arch( + self, + mock_find_executable, + mock_glob, + mock_ensure_dir, + mock_current_directory, + mock_sh_patch, + ): + # We overwrite the base test method because we need to mock a little + # more with this recipe (`sh.cp` and `sh.rm`) + super().test_build_arch() + # make sure that the mocked methods are actually called + mock_sh_patch.assert_called() + + def test_versioned_url(self): + self.assertEqual( + self.recipe.url.format(url_version=self.recipe.url_version), + self.recipe.versioned_url, + ) + + def test_include_flags(self): + inc = self.recipe.include_flags(self.arch) + build_dir = self.recipe.get_build_dir(self.arch) + for i in {"include/internal", "include/openssl"}: + self.assertIn(f"-I{build_dir}/{i}", inc) + + def test_link_flags(self): + build_dir = self.recipe.get_build_dir(self.arch) + openssl_version = self.recipe.version + self.assertEqual( + f" -L{build_dir} -lcrypto{openssl_version} -lssl{openssl_version}", + self.recipe.link_flags(self.arch), + ) + + def test_select_build_arch(self): + expected_build_archs = { + "armeabi": "android", + "armeabi-v7a": "android-arm", + "arm64-v8a": "android-arm64", + "x86": "android-x86", + "x86_64": "android-x86_64", + } + for arch in self.ctx.archs: + self.assertEqual( + expected_build_archs[arch.arch], + self.recipe.select_build_arch(arch), + ) diff --git a/tests/recipes/test_png.py b/tests/recipes/test_png.py new file mode 100644 index 0000000000..ac734bfefe --- /dev/null +++ b/tests/recipes/test_png.py @@ -0,0 +1,10 @@ +import unittest +from tests.recipes.recipe_lib_test import BaseTestForMakeRecipe + + +class TestPngRecipe(BaseTestForMakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.png` + """ + recipe_name = "png" + sh_command_calls = ["./configure"] diff --git a/tests/recipes/test_pyicu.py b/tests/recipes/test_pyicu.py index ba346bc2ad..2bcf1feeaf 100644 --- a/tests/recipes/test_pyicu.py +++ b/tests/recipes/test_pyicu.py @@ -1,47 +1,23 @@ -import os - import unittest - -try: - from unittest import mock -except ImportError: - # `Python 2` or lower than `Python 3.3` does not - # have the `unittest.mock` module built-in - import mock -from pythonforandroid.bootstrap import Bootstrap -from pythonforandroid.distribution import Distribution +from unittest import mock +from tests.recipes.recipe_ctx import RecipeCtx from pythonforandroid.recipe import Recipe -from pythonforandroid.build import Context -from pythonforandroid.archs import ArchARMv7_a -class TestPyIcuRecipe(unittest.TestCase): +class TestPyIcuRecipe(RecipeCtx, unittest.TestCase): """ An unittest for recipe :mod:`~pythonforandroid.recipes.pyicu` """ - - ctx = None - - def setUp(self): - self.ctx = Context() - self.ctx.ndk_api = 21 - self.ctx.android_api = 27 - self.ctx._sdk_dir = "/opt/android/android-sdk" - self.ctx._ndk_dir = "/opt/android/android-ndk" - self.ctx.setup_dirs(os.getcwd()) - self.ctx.bootstrap = Bootstrap().get_bootstrap("sdl2", self.ctx) - self.ctx.bootstrap.distribution = Distribution.get_distribution( - self.ctx, name="sdl2", recipes=["python3", "kivy", "pyicu"] - ) - self.ctx.recipe_build_order = [ + recipe_name = "pyicu" + recipes = ["python3", "kivy", "pyicu"] + recipe_build_order = [ "hostpython3", "icu", "python3", "sdl2", "pyicu", "kivy", - ] - self.ctx.python_recipe = Recipe.get_recipe("python3", self.ctx) + ] @mock.patch("pythonforandroid.recipe.Recipe.check_recipe_choices") @mock.patch("pythonforandroid.build.ensure_dir") @@ -59,9 +35,6 @@ def test_get_recipe_env( :meth:`~pythonforandroid.recipes.pyicu.PyICURecipe.get_recipe_env` returns the expected flags """ - arch = ArchARMv7_a(self.ctx) - recipe = Recipe.get_recipe("pyicu", self.ctx) - icu_recipe = Recipe.get_recipe("icu", self.ctx) mock_find_executable.return_value = ( @@ -76,7 +49,7 @@ def test_get_recipe_env( expected_pyicu_libs = [ lib[3:-3] for lib in icu_recipe.built_libraries.keys() ] - env = recipe.get_recipe_env(arch) + env = self.recipe.get_recipe_env(self.arch) self.assertEqual(":".join(expected_pyicu_libs), env["PYICU_LIBRARIES"]) self.assertIn("include/icu", env["CPPFLAGS"]) self.assertIn("icu4c/icu_build/lib", env["LDFLAGS"]) diff --git a/tests/recipes/test_reportlab.py b/tests/recipes/test_reportlab.py index 41667a6932..24934efdaf 100644 --- a/tests/recipes/test_reportlab.py +++ b/tests/recipes/test_reportlab.py @@ -1,34 +1,18 @@ import os -import tempfile import unittest from mock import patch -from pythonforandroid.archs import ArchARMv7_a -from pythonforandroid.build import Context -from pythonforandroid.graph import get_recipe_order_and_bootstrap -from pythonforandroid.recipe import Recipe +from tests.recipes.recipe_ctx import RecipeCtx from pythonforandroid.util import ensure_dir -class TestReportLabRecipe(unittest.TestCase): +class TestReportLabRecipe(RecipeCtx, unittest.TestCase): + recipe_name = "reportlab" def setUp(self): """ Setups recipe and context. """ - self.context = Context() - self.context.ndk_api = 21 - self.context.android_api = 27 - self.arch = ArchARMv7_a(self.context) - self.recipe = Recipe.get_recipe('reportlab', self.context) - self.recipe.ctx = self.context - self.bootstrap = None - recipe_build_order, python_modules, bootstrap = \ - get_recipe_order_and_bootstrap( - self.context, [self.recipe.name], self.bootstrap) - self.context.recipe_build_order = recipe_build_order - self.context.python_modules = python_modules - self.context.setup_dirs(tempfile.gettempdir()) - self.bootstrap = bootstrap + super().setUp() self.recipe_dir = self.recipe.get_build_dir(self.arch.arch) ensure_dir(self.recipe_dir) diff --git a/tests/recipes/test_snappy.py b/tests/recipes/test_snappy.py new file mode 100644 index 0000000000..6439454e29 --- /dev/null +++ b/tests/recipes/test_snappy.py @@ -0,0 +1,9 @@ +import unittest +from tests.recipes.recipe_lib_test import BaseTestForCmakeRecipe + + +class TestSnappyRecipe(BaseTestForCmakeRecipe, unittest.TestCase): + """ + An unittest for recipe :mod:`~pythonforandroid.recipes.snappy` + """ + recipe_name = "snappy" From 2f98b818334fc0556472d1f6c3e878b7d6306e4b Mon Sep 17 00:00:00 2001 From: Pol Canelles Date: Sun, 15 Sep 2019 17:05:35 +0200 Subject: [PATCH 21/72] Recipes tests enhancements (#1984) This is a follow up of #1982 In here we do: - Move from `__new__` to `__init__` for `BaseTestForMakeRecipe` ([discussed at here](https://github.com/kivy/python-for-android/pull/1982#discussion_r324419892)) - Convert cls attributes to instance attributes ([discussed at here](https://github.com/kivy/python-for-android/pull/1982#discussion_r324419973)) - Remove `mock` dependency for py3 tests (Because, as per Python 3.3+, `mock` it's a built-in-module). (Unfortunately we still will have a couple of `import mock` entries that we cannot remove until we completely remove python2 from our tests) * [test] From `__new__` to `__init__` for `BaseTestForMakeRecipe` * [test] Remove mock dependency for py3 tests Because, as per Python 3.3+, `mock` it's a built-in-module * [test] Convert cls attributes to instance attributes Because we may have some side effects without willing, considering that cls attributes are shared between instances. --- tests/recipes/recipe_lib_test.py | 16 +++++++--------- tests/recipes/test_gevent.py | 2 +- tests/recipes/test_icu.py | 2 -- tests/recipes/test_pyicu.py | 9 --------- tests/recipes/test_reportlab.py | 2 +- tests/test_archs.py | 8 +------- tests/test_bootstrap.py | 7 +------ tests/test_build.py | 7 +------ tests/test_distribution.py | 7 +------ tests/test_graph.py | 2 +- tests/test_logger.py | 2 +- tests/test_pythonpackage_basic.py | 2 +- tests/test_recipe.py | 2 +- tests/test_toolchain.py | 2 +- tests/test_util.py | 7 +------ tox.ini | 2 +- 16 files changed, 20 insertions(+), 59 deletions(-) diff --git a/tests/recipes/recipe_lib_test.py b/tests/recipes/recipe_lib_test.py index dbb476028b..5622397aa5 100644 --- a/tests/recipes/recipe_lib_test.py +++ b/tests/recipes/recipe_lib_test.py @@ -12,8 +12,6 @@ class BaseTestForMakeRecipe(RecipeCtx): """ recipe_name = None - recipes = ["python3", "kivy"] - recipe_build_order = ["hostpython3", "python3", "sdl2", "kivy"] expected_compiler = ( "{android_ndk}/toolchains/llvm/prebuilt/linux-x86_64/bin/clang" ) @@ -26,13 +24,13 @@ class BaseTestForMakeRecipe(RecipeCtx): This must be a dictionary containing pairs of key (env var) and value. """ - def __new__(cls, *args): - obj = super().__new__(cls) - if obj.recipe_name is not None: - print(f"We are testing recipe: {obj.recipe_name}") - obj.recipes.append(obj.recipe_name) - obj.recipe_build_order.insert(1, obj.recipe_name) - return obj + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.recipes = ["python3", "kivy", self.recipe_name] + self.recipe_build_order = [ + "hostpython3", self.recipe_name, "python3", "sdl2", "kivy" + ] + print(f"We are testing recipe: {self.recipe_name}") @mock.patch("pythonforandroid.recipe.Recipe.check_recipe_choices") @mock.patch("pythonforandroid.build.ensure_dir") diff --git a/tests/recipes/test_gevent.py b/tests/recipes/test_gevent.py index 3b1ce62842..fa56fe6a11 100644 --- a/tests/recipes/test_gevent.py +++ b/tests/recipes/test_gevent.py @@ -1,5 +1,5 @@ import unittest -from mock import patch +from unittest.mock import patch from tests.recipes.recipe_ctx import RecipeCtx diff --git a/tests/recipes/test_icu.py b/tests/recipes/test_icu.py index 22de4b3dd3..506cdb74b2 100644 --- a/tests/recipes/test_icu.py +++ b/tests/recipes/test_icu.py @@ -12,8 +12,6 @@ class TestIcuRecipe(RecipeCtx, unittest.TestCase): """ recipe_name = "icu" - recipes = ["python3", "kivy", "icu"] - recipe_build_order = ["hostpython3", "icu", "python3", "sdl2", "kivy"] def test_url(self): self.assertTrue(self.recipe.versioned_url.startswith("http")) diff --git a/tests/recipes/test_pyicu.py b/tests/recipes/test_pyicu.py index 2bcf1feeaf..ac70c4ea18 100644 --- a/tests/recipes/test_pyicu.py +++ b/tests/recipes/test_pyicu.py @@ -9,15 +9,6 @@ class TestPyIcuRecipe(RecipeCtx, unittest.TestCase): An unittest for recipe :mod:`~pythonforandroid.recipes.pyicu` """ recipe_name = "pyicu" - recipes = ["python3", "kivy", "pyicu"] - recipe_build_order = [ - "hostpython3", - "icu", - "python3", - "sdl2", - "pyicu", - "kivy", - ] @mock.patch("pythonforandroid.recipe.Recipe.check_recipe_choices") @mock.patch("pythonforandroid.build.ensure_dir") diff --git a/tests/recipes/test_reportlab.py b/tests/recipes/test_reportlab.py index 24934efdaf..83191e5286 100644 --- a/tests/recipes/test_reportlab.py +++ b/tests/recipes/test_reportlab.py @@ -1,6 +1,6 @@ import os import unittest -from mock import patch +from unittest.mock import patch from tests.recipes.recipe_ctx import RecipeCtx from pythonforandroid.util import ensure_dir diff --git a/tests/test_archs.py b/tests/test_archs.py index 7cbe5421ad..24c118eb40 100644 --- a/tests/test_archs.py +++ b/tests/test_archs.py @@ -1,13 +1,7 @@ import os import unittest from os import environ - -try: - from unittest import mock -except ImportError: - # `Python 2` or lower than `Python 3.3` does not - # have the `unittest.mock` module built-in - import mock +from unittest import mock from pythonforandroid.bootstrap import Bootstrap from pythonforandroid.distribution import Distribution diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index e3e75b945f..6297eef551 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -3,12 +3,7 @@ import sh import unittest -try: - from unittest import mock -except ImportError: - # `Python 2` or lower than `Python 3.3` does not - # have the `unittest.mock` module built-in - import mock +from unittest import mock from pythonforandroid.bootstrap import ( _cmp_bootstraps_by_priority, Bootstrap, expand_dependencies, ) diff --git a/tests/test_build.py b/tests/test_build.py index 784e24c23f..6e659a5837 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -1,11 +1,6 @@ import unittest +from unittest import mock -try: - from unittest import mock -except ImportError: - # `Python 2` or lower than `Python 3.3` does not - # have the `unittest.mock` module built-in - import mock from pythonforandroid.build import run_pymodules_install diff --git a/tests/test_distribution.py b/tests/test_distribution.py index dece511074..e533a5af5e 100644 --- a/tests/test_distribution.py +++ b/tests/test_distribution.py @@ -1,13 +1,8 @@ import os import json import unittest +from unittest import mock -try: - from unittest import mock -except ImportError: - # `Python 2` or lower than `Python 3.3` does not - # have the `unittest.mock` module built-in - import mock from pythonforandroid.bootstrap import Bootstrap from pythonforandroid.distribution import Distribution from pythonforandroid.recipe import Recipe diff --git a/tests/test_graph.py b/tests/test_graph.py index ccade98561..ebe9fb400d 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -8,7 +8,7 @@ from pythonforandroid.util import BuildInterruptingException from itertools import product -import mock +from unittest import mock import pytest ctx = Context() diff --git a/tests/test_logger.py b/tests/test_logger.py index 8212b4596f..773e7e54a0 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -1,5 +1,5 @@ import unittest -from mock import MagicMock +from unittest.mock import MagicMock from pythonforandroid import logger diff --git a/tests/test_pythonpackage_basic.py b/tests/test_pythonpackage_basic.py index 121f0e946b..7b92f79721 100644 --- a/tests/test_pythonpackage_basic.py +++ b/tests/test_pythonpackage_basic.py @@ -5,7 +5,6 @@ while the other additional ones aren't (for build time reasons). """ -import mock import os import pytest import shutil @@ -13,6 +12,7 @@ import subprocess import tempfile import textwrap +from unittest import mock from pythonforandroid.pythonpackage import ( _extract_info_from_package, diff --git a/tests/test_recipe.py b/tests/test_recipe.py index c3ff2d6b6e..c01ab83507 100644 --- a/tests/test_recipe.py +++ b/tests/test_recipe.py @@ -3,7 +3,7 @@ import types import unittest import warnings -import mock +from unittest import mock from backports import tempfile from pythonforandroid.build import Context from pythonforandroid.recipe import Recipe, import_recipe diff --git a/tests/test_toolchain.py b/tests/test_toolchain.py index 11e35ff989..7e3078133b 100644 --- a/tests/test_toolchain.py +++ b/tests/test_toolchain.py @@ -1,7 +1,7 @@ import io import sys import pytest -import mock +from unittest import mock from pythonforandroid.recipe import Recipe from pythonforandroid.toolchain import ToolchainCL from pythonforandroid.util import BuildInterruptingException diff --git a/tests/test_util.py b/tests/test_util.py index 9568514890..0653991639 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,13 +1,8 @@ import os import types import unittest +from unittest import mock -try: - from unittest import mock -except ImportError: - # `Python 2` or lower than `Python 3.3` does not - # have the `unittest.mock` module built-in - import mock from pythonforandroid import util diff --git a/tox.ini b/tox.ini index 2ca84ae803..6a90ef435d 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ basepython = python3 [testenv] deps = - mock + py27: mock pytest virtualenv py3: coveralls From 9f6d6fcce748be96ed606815aca9294644ac90cc Mon Sep 17 00:00:00 2001 From: Andre Miras Date: Sun, 29 Sep 2019 16:15:15 +0200 Subject: [PATCH 22/72] Fixes test_virtualenv and test_venv failing, closes #1994 (#1995) Recent pep517 release seems to break tests. --- setup.py | 2 +- tests/test_pythonpackage_basic.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 52dc745763..3c89586416 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ install_reqs = [ 'appdirs', 'colorama>=0.3.3', 'jinja2', 'six', 'enum34; python_version<"3.4"', 'sh>=1.10; sys_platform!="nt"', - 'pep517', 'pytoml', 'virtualenv' + 'pep517<0.7.0"', 'pytoml', 'virtualenv' ] # (pep517, pytoml and virtualenv are used by pythonpackage.py) diff --git a/tests/test_pythonpackage_basic.py b/tests/test_pythonpackage_basic.py index 7b92f79721..3d1a156df6 100644 --- a/tests/test_pythonpackage_basic.py +++ b/tests/test_pythonpackage_basic.py @@ -303,7 +303,7 @@ def test_virtualenv(self): ]) subprocess.check_output([ os.path.join(test_dir, "virtualenv", "bin", "pip"), - "install", "-U", "pep517" + "install", "-U", "pep517<0.7.0" ]) sys_python_path = self.run__get_system_python_executable( os.path.join(test_dir, "virtualenv", "bin", "python") @@ -336,7 +336,7 @@ def test_venv(self): ]) subprocess.check_output([ os.path.join(test_dir, "venv", "bin", "pip"), - "install", "-U", "pep517" + "install", "-U", "pep517<0.7.0" ]) sys_python_path = self.run__get_system_python_executable( os.path.join(test_dir, "venv", "bin", "python") From 01c060704f4852d60a3889ccc273c43ed6e3354c Mon Sep 17 00:00:00 2001 From: Alexander Taylor Date: Sun, 29 Sep 2019 15:15:49 +0100 Subject: [PATCH 23/72] Made p4a use per-arch dist build dirs (#1986) * Made p4a use per-arch dist build dirs * Fixed up check on existing folder when overwriting distributions * Improved Distribution.get_distribution api * Fixed test_archs.py * Fixed test_bootstrap.py * Fixed test_distribution.py * Fixed recipes tests * Fixed formatting * Made sure whole build process accesses dist_dir correctly * Fixed apk output name formatting by moving version to end of string * Further test fixes following dist_dir changes * Removed unnecessary variable --- pythonforandroid/bootstrap.py | 16 +++--- pythonforandroid/build.py | 11 ++-- pythonforandroid/distribution.py | 95 ++++++++++++++++++++++++-------- pythonforandroid/python.py | 7 ++- pythonforandroid/toolchain.py | 16 ++++-- tests/recipes/recipe_ctx.py | 6 +- tests/test_archs.py | 7 ++- tests/test_bootstrap.py | 22 +++++--- tests/test_distribution.py | 20 +++++-- 9 files changed, 140 insertions(+), 60 deletions(-) diff --git a/pythonforandroid/bootstrap.py b/pythonforandroid/bootstrap.py index cd11c6e1a9..03925a3e01 100755 --- a/pythonforandroid/bootstrap.py +++ b/pythonforandroid/bootstrap.py @@ -8,10 +8,9 @@ import shlex import shutil -from pythonforandroid.logger import (warning, shprint, info, logger, - debug) -from pythonforandroid.util import (current_directory, ensure_dir, - temp_directory) +from pythonforandroid.logger import (shprint, info, logger, debug) +from pythonforandroid.util import ( + current_directory, ensure_dir, temp_directory, BuildInterruptingException) from pythonforandroid.recipe import Recipe @@ -75,7 +74,6 @@ class Bootstrap(object): bootstrap_dir = None build_dir = None - dist_dir = None dist_name = None distribution = None @@ -97,9 +95,9 @@ class Bootstrap(object): def dist_dir(self): '''The dist dir at which to place the finished distribution.''' if self.distribution is None: - warning('Tried to access {}.dist_dir, but {}.distribution ' - 'is None'.format(self, self)) - exit(1) + raise BuildInterruptingException( + 'Internal error: tried to access {}.dist_dir, but {}.distribution ' + 'is None'.format(self, self)) return self.distribution.dist_dir @property @@ -158,7 +156,7 @@ def prepare_build_dir(self): with open('project.properties', 'w') as fileh: fileh.write('target=android-{}'.format(self.ctx.android_api)) - def prepare_dist_dir(self, name): + def prepare_dist_dir(self): ensure_dir(self.dist_dir) def run_distribute(self): diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index cc6cea1406..8f0ac40a82 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -92,8 +92,13 @@ class Context(object): # in which bootstraps are copied for building # and recipes are built build_dir = None + + distribution = None + """The Distribution object representing the current build target location.""" + # the Android project folder where everything ends up dist_dir = None + # where Android libs are cached after build # but before being placed in dists libs_dir = None @@ -106,7 +111,6 @@ class Context(object): ndk_platform = None # the ndk platform directory - dist_name = None # should be deprecated in favour of self.dist.dist_name bootstrap = None bootstrap_build_dir = None @@ -485,9 +489,8 @@ def prepare_bootstrap(self, bs): self.bootstrap.prepare_build_dir() self.bootstrap_build_dir = self.bootstrap.build_dir - def prepare_dist(self, name): - self.dist_name = name - self.bootstrap.prepare_dist_dir(self.dist_name) + def prepare_dist(self): + self.bootstrap.prepare_dist_dir() def get_site_packages_dir(self, arch=None): '''Returns the location of site-packages in the python-install build diff --git a/pythonforandroid/distribution.py b/pythonforandroid/distribution.py index f088ac01e6..379cd90852 100644 --- a/pythonforandroid/distribution.py +++ b/pythonforandroid/distribution.py @@ -24,7 +24,7 @@ class Distribution(object): ndk_api = None archs = [] - '''The arch targets that the dist is built for.''' + '''The names of the arch targets that the dist is built for.''' recipes = [] @@ -42,12 +42,19 @@ def __repr__(self): return str(self) @classmethod - def get_distribution(cls, ctx, name=None, recipes=[], - ndk_api=None, - force_build=False, - extra_dist_dirs=[], - require_perfect_match=False, - allow_replace_dist=True): + def get_distribution( + cls, + ctx, + *, + arch_name, # required keyword argument: there is no sensible default + name=None, + recipes=[], + ndk_api=None, + force_build=False, + extra_dist_dirs=[], + require_perfect_match=False, + allow_replace_dist=True + ): '''Takes information about the distribution, and decides what kind of distribution it will be. @@ -60,6 +67,12 @@ def get_distribution(cls, ctx, name=None, recipes=[], name : str The name of the distribution. If a dist with this name already ' exists, it will be used. + ndk_api : int + The NDK API to compile against, included in the dist because it cannot + be changed later during APK packaging. + arch_name : str + The target architecture name to compile against, included in the dist because + it cannot be changed later during APK packaging. recipes : list The recipes that the distribution must contain. force_download: bool @@ -77,17 +90,24 @@ def get_distribution(cls, ctx, name=None, recipes=[], a new one with the current requirements. ''' - existing_dists = Distribution.get_distributions(ctx) + possible_dists = Distribution.get_distributions(ctx) - possible_dists = existing_dists + # Will hold dists that would be built in the same folder as an existing dist + folder_match_dist = None - name_match_dist = None - - # 0) Check if a dist with that name already exists + # 0) Check if a dist with that name and architecture already exists if name is not None and name: - possible_dists = [d for d in possible_dists if d.name == name] + possible_dists = [ + d for d in possible_dists if + (d.name == name) and (arch_name in d.archs)] + if possible_dists: - name_match_dist = possible_dists[0] + # There should only be one folder with a given dist name *and* arch. + # We could check that here, but for compatibility let's let it slide + # and just record the details of one of them. We only use this data to + # possibly fail the build later, so it doesn't really matter if there + # was more than one clash. + folder_match_dist = possible_dists[0] # 1) Check if any existing dists meet the requirements _possible_dists = [] @@ -110,12 +130,14 @@ def get_distribution(cls, ctx, name=None, recipes=[], else: info('No existing dists meet the given requirements!') - # If any dist has perfect recipes and ndk API, return it + # If any dist has perfect recipes, arch and NDK API, return it for dist in possible_dists: if force_build: continue if ndk_api is not None and dist.ndk_api != ndk_api: continue + if arch_name is not None and arch_name not in dist.archs: + continue if (set(dist.recipes) == set(recipes) or (set(recipes).issubset(set(dist.recipes)) and not require_perfect_match)): @@ -123,12 +145,10 @@ def get_distribution(cls, ctx, name=None, recipes=[], .format(dist.name)) return dist - assert len(possible_dists) < 2 - # If there was a name match but we didn't already choose it, # then the existing dist is incompatible with the requested # configuration and the build cannot continue - if name_match_dist is not None and not allow_replace_dist: + if folder_match_dist is not None and not allow_replace_dist: raise BuildInterruptingException( 'Asked for dist with name {name} with recipes ({req_recipes}) and ' 'NDK API {req_ndk_api}, but a dist ' @@ -136,9 +156,11 @@ def get_distribution(cls, ctx, name=None, recipes=[], '({dist_recipes}) or NDK API {dist_ndk_api}'.format( name=name, req_ndk_api=ndk_api, - dist_ndk_api=name_match_dist.ndk_api, + dist_ndk_api=folder_match_dist.ndk_api, req_recipes=', '.join(recipes), - dist_recipes=', '.join(name_match_dist.recipes))) + dist_recipes=', '.join(folder_match_dist.recipes))) + + assert len(possible_dists) < 2 # If we got this far, we need to build a new dist dist = Distribution(ctx) @@ -152,9 +174,16 @@ def get_distribution(cls, ctx, name=None, recipes=[], name = filen.format(i) dist.name = name - dist.dist_dir = join(ctx.dist_dir, dist.name) + dist.dist_dir = join( + ctx.dist_dir, + generate_dist_folder_name( + name, + [arch_name] if arch_name is not None else None, + ) + ) dist.recipes = recipes dist.ndk_api = ctx.ndk_api + dist.archs = [arch_name] return dist @@ -182,7 +211,7 @@ def get_distributions(cls, ctx, extra_dist_dirs=[]): with open(join(folder, 'dist_info.json')) as fileh: dist_info = json.load(fileh) dist = cls(ctx) - dist.name = folder.split('/')[-1] + dist.name = dist_info['dist_name'] dist.dist_dir = folder dist.needs_build = False dist.recipes = dist_info['recipes'] @@ -210,7 +239,7 @@ def save_info(self, dirn): with current_directory(dirn): info('Saving distribution info') with open('dist_info.json', 'w') as fileh: - json.dump({'dist_name': self.ctx.dist_name, + json.dump({'dist_name': self.name, 'bootstrap': self.ctx.bootstrap.name, 'archs': [arch.arch for arch in self.ctx.archs], 'ndk_api': self.ctx.ndk_api, @@ -236,3 +265,23 @@ def pretty_log_dists(dists, log_func=info): for line in infos: log_func('\t' + line) + + +def generate_dist_folder_name(base_dist_name, arch_names=None): + """Generate the distribution folder name to use, based on a + combination of the input arguments. + + Parameters + ---------- + base_dist_name : str + The core distribution identifier string + arch_names : list of str + The architecture compile targets + """ + if arch_names is None: + arch_names = ["no_arch_specified"] + + return '{}__{}'.format( + base_dist_name, + '_'.join(arch_names) + ) diff --git a/pythonforandroid/python.py b/pythonforandroid/python.py index c4c4ebcc19..a46b602c60 100755 --- a/pythonforandroid/python.py +++ b/pythonforandroid/python.py @@ -341,8 +341,11 @@ def create_python_bundle(self, dirn, arch): python_lib_name = 'libpython' + self.major_minor_version_string if self.major_minor_version_string[0] == '3': python_lib_name += 'm' - shprint(sh.cp, join(python_build_dir, python_lib_name + '.so'), - join(self.ctx.dist_dir, self.ctx.dist_name, 'libs', arch.arch)) + shprint( + sh.cp, + join(python_build_dir, python_lib_name + '.so'), + join(self.ctx.bootstrap.dist_dir, 'libs', arch.arch) + ) info('Renaming .so files to reflect cross-compile') self.reduce_object_file_names(join(dirn, 'site-packages')) diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py index cb15ca0392..04cebfc673 100644 --- a/pythonforandroid/toolchain.py +++ b/pythonforandroid/toolchain.py @@ -101,6 +101,8 @@ def check_python_dependencies(): toolchain_dir = dirname(__file__) sys.path.insert(0, join(toolchain_dir, "tools", "external")) +APK_SUFFIX = '.apk' + def add_boolean_option(parser, names, no_names=None, default=True, dest=None, description=None): @@ -163,6 +165,7 @@ def dist_from_args(ctx, args): ctx, name=args.dist_name, recipes=split_argument_list(args.requirements), + arch_name=args.arch, ndk_api=args.ndk_api, force_build=args.force_build, require_perfect_match=args.require_perfect_match, @@ -195,10 +198,10 @@ def build_dist_from_args(ctx, dist, args): info('Dist will also contain modules ({}) installed from pip'.format( ', '.join(ctx.python_modules))) - ctx.dist_name = bs.distribution.name + ctx.distribution = dist ctx.prepare_bootstrap(bs) if dist.needs_build: - ctx.prepare_dist(ctx.dist_name) + ctx.prepare_dist() build_recipes(build_order, python_modules, ctx, getattr(args, "private", None), @@ -211,7 +214,7 @@ def build_dist_from_args(ctx, dist, args): info_main('# Your distribution was created successfully, exiting.') info('Dist can be found at (for now) {}' - .format(join(ctx.dist_dir, ctx.dist_name))) + .format(join(ctx.dist_dir, ctx.distribution.dist_dir))) def split_argument_list(l): @@ -304,7 +307,7 @@ def __init__(self): '(default: {})'.format(default_storage_dir))) generic_parser.add_argument( - '--arch', help='The archs to build for, separated by commas.', + '--arch', help='The arch to build for.', default='armeabi-v7a') # Options for specifying the Distribution @@ -912,6 +915,7 @@ def export_dist(self, args): def _dist(self): ctx = self.ctx dist = dist_from_args(ctx, self.args) + ctx.distribution = dist return dist @require_prebuilt_dist @@ -1062,9 +1066,9 @@ def apk(self, args): info_main('# Found APK file: {}'.format(apk_file)) if apk_add_version: info('# Add version number to APK') - apk_name, apk_suffix = basename(apk_file).split("-", 1) + apk_name = basename(apk_file)[:-len(APK_SUFFIX)] apk_file_dest = "{}-{}-{}".format( - apk_name, build_args.version, apk_suffix) + apk_name, build_args.version, APK_SUFFIX) info('# APK renamed to {}'.format(apk_file_dest)) shprint(sh.cp, apk_file, apk_file_dest) else: diff --git a/tests/recipes/recipe_ctx.py b/tests/recipes/recipe_ctx.py index 938b19e471..a162e8f0dc 100644 --- a/tests/recipes/recipe_ctx.py +++ b/tests/recipes/recipe_ctx.py @@ -26,7 +26,9 @@ class RecipeCtx: contain the target recipe to test as well as a python recipe.""" recipe_build_order = [] """A recipe_build_order which should take into account the recipe we want - to test as well as the possible dependant recipes""" + to test as well as the possible dependent recipes""" + + TEST_ARCH = 'arm64-v8a' def setUp(self): self.ctx = Context() @@ -37,7 +39,7 @@ def setUp(self): self.ctx.setup_dirs(os.getcwd()) self.ctx.bootstrap = Bootstrap().get_bootstrap("sdl2", self.ctx) self.ctx.bootstrap.distribution = Distribution.get_distribution( - self.ctx, name="sdl2", recipes=self.recipes + self.ctx, name="sdl2", recipes=self.recipes, arch_name=self.TEST_ARCH, ) self.ctx.recipe_build_order = self.recipe_build_order self.ctx.python_recipe = Recipe.get_recipe("python3", self.ctx) diff --git a/tests/test_archs.py b/tests/test_archs.py index 24c118eb40..ddb18b80e2 100644 --- a/tests/test_archs.py +++ b/tests/test_archs.py @@ -50,6 +50,8 @@ class ArchSetUpBaseClass(object): ctx = None expected_compiler = "" + TEST_ARCH = 'armeabi-v7a' + def setUp(self): self.ctx = Context() self.ctx.ndk_api = 21 @@ -59,7 +61,10 @@ def setUp(self): self.ctx.setup_dirs(os.getcwd()) self.ctx.bootstrap = Bootstrap().get_bootstrap("sdl2", self.ctx) self.ctx.bootstrap.distribution = Distribution.get_distribution( - self.ctx, name="sdl2", recipes=["python3", "kivy"] + self.ctx, + name="sdl2", + recipes=["python3", "kivy"], + arch_name=self.TEST_ARCH, ) self.ctx.python_recipe = Recipe.get_recipe("python3", self.ctx) # Here we define the expected compiler, which, as per ndk >= r19, diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index 6297eef551..02d54f0d9e 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -7,10 +7,11 @@ from pythonforandroid.bootstrap import ( _cmp_bootstraps_by_priority, Bootstrap, expand_dependencies, ) -from pythonforandroid.distribution import Distribution +from pythonforandroid.distribution import Distribution, generate_dist_folder_name from pythonforandroid.recipe import Recipe from pythonforandroid.archs import ArchARMv7_a from pythonforandroid.build import Context +from pythonforandroid.util import BuildInterruptingException from test_graph import get_fake_recipe @@ -22,6 +23,8 @@ class BaseClassSetupBootstrap(object): `setUp` and `tearDown` methods. """ + TEST_ARCH = 'armeabi-v7a' + def setUp(self): self.ctx = Context() self.ctx.ndk_api = 21 @@ -43,7 +46,9 @@ def setUp_distribution_with_bootstrap(self, bs): """ self.ctx.bootstrap = bs self.ctx.bootstrap.distribution = Distribution.get_distribution( - self.ctx, name="test_prj", recipes=["python3", "kivy"] + self.ctx, name="test_prj", + recipes=["python3", "kivy"], + arch_name=self.TEST_ARCH, ) def tearDown(self): @@ -71,15 +76,16 @@ def test_attributes(self): self.assertEqual(bs.jni_dir, "sdl2/jni") self.assertEqual(bs.get_build_dir_name(), "sdl2-python3") - # test dist_dir error + # bs.dist_dir should raise an error if there is no distribution to query bs.distribution = None - with self.assertRaises(SystemExit) as e: + with self.assertRaises(BuildInterruptingException): bs.dist_dir - self.assertEqual(e.exception.args[0], 1) # test dist_dir success self.setUp_distribution_with_bootstrap(bs) - self.assertTrue(bs.dist_dir.endswith("dists/test_prj")) + expected_folder_name = generate_dist_folder_name('test_prj', [self.TEST_ARCH]) + self.assertTrue( + bs.dist_dir.endswith(f"dists/{expected_folder_name}")) def test_build_dist_dirs(self): """A test which will initialize a bootstrap and will check if the @@ -249,8 +255,8 @@ def test_prepare_dist_dir(self, mock_ensure_dir): """ bs = Bootstrap().get_bootstrap("sdl2", self.ctx) - bs.prepare_dist_dir("fake_name") - mock_ensure_dir.assert_called_once_with(bs.dist_dir) + bs.prepare_dist_dir() + mock_ensure_dir.assert_called_once() @mock.patch("pythonforandroid.bootstrap.open", create=True) @mock.patch("pythonforandroid.util.chdir") diff --git a/tests/test_distribution.py b/tests/test_distribution.py index e533a5af5e..3342b5f143 100644 --- a/tests/test_distribution.py +++ b/tests/test_distribution.py @@ -10,7 +10,7 @@ from pythonforandroid.build import Context dist_info_data = { - "dist_name": None, + "dist_name": "sdl2_dist", "bootstrap": "sdl2", "archs": ["armeabi", "armeabi-v7a", "x86", "x86_64", "arm64-v8a"], "ndk_api": 21, @@ -27,6 +27,8 @@ class TestDistribution(unittest.TestCase): :mod:`~pythonforandroid.distribution`. """ + TEST_ARCH = 'armeabi-v7a' + def setUp(self): """Configure a :class:`~pythonforandroid.build.Context` so we can perform our unittests""" @@ -51,6 +53,7 @@ def setUp_distribution_with_bootstrap(self, bs, **kwargs): self.ctx, name=kwargs.pop("name", "test_prj"), recipes=kwargs.pop("recipes", ["python3", "kivy"]), + arch_name=self.TEST_ARCH, **kwargs ) @@ -108,7 +111,7 @@ def test_get_distribution_no_name(self, mock_exists): returns the proper result which should `unnamed_dist_1`.""" mock_exists.return_value = False self.ctx.bootstrap = Bootstrap().get_bootstrap("sdl2", self.ctx) - dist = Distribution.get_distribution(self.ctx) + dist = Distribution.get_distribution(self.ctx, arch_name=self.TEST_ARCH) self.assertEqual(dist.name, "unnamed_dist_1") @mock.patch("pythonforandroid.util.chdir") @@ -157,7 +160,8 @@ def test_get_distributions( self.assertIsInstance(dists, list) self.assertEqual(len(dists), 1) self.assertIsInstance(dists[0], Distribution) - self.assertEqual(dists[0].name, "sdl2-python3") + self.assertEqual(dists[0].name, "sdl2_dist") + self.assertEqual(dists[0].dist_dir, "sdl2-python3") self.assertEqual(dists[0].ndk_api, 21) self.assertEqual( dists[0].recipes, @@ -206,7 +210,10 @@ def test_get_distributions_error_ndk_api_mismatch( distribution with the same `name` but different `ndk_api`. """ expected_dist = Distribution.get_distribution( - self.ctx, name="test_prj", recipes=["python3", "kivy"] + self.ctx, + name="test_prj", + recipes=["python3", "kivy"], + arch_name=self.TEST_ARCH, ) mock_get_dists.return_value = [expected_dist] mock_glob.return_value = ["sdl2-python3"] @@ -254,7 +261,10 @@ def test_get_distributions_possible_dists(self, mock_get_dists): `:class:`~pythonforandroid.distribution.Distribution`. """ expected_dist = Distribution.get_distribution( - self.ctx, name="test_prj", recipes=["python3", "kivy"] + self.ctx, + name="test_prj", + recipes=["python3", "kivy"], + arch_name=self.TEST_ARCH, ) mock_get_dists.return_value = [expected_dist] self.setUp_distribution_with_bootstrap( From a095a553c677ea4ab9f4898a9904750d466c6777 Mon Sep 17 00:00:00 2001 From: Andre Miras Date: Sun, 29 Sep 2019 14:15:32 +0200 Subject: [PATCH 24/72] Fixes libiconv & libzbar configure host Having `--host=x86` was making the configure script fail at enabling shared library build on `x86`. This was resulting in failing to copy `libiconv.so` and `libzbar.so` on `x86` arch. --- pythonforandroid/recipes/libiconv/__init__.py | 2 +- pythonforandroid/recipes/libzbar/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pythonforandroid/recipes/libiconv/__init__.py b/pythonforandroid/recipes/libiconv/__init__.py index 6bde196e56..530497a2ed 100644 --- a/pythonforandroid/recipes/libiconv/__init__.py +++ b/pythonforandroid/recipes/libiconv/__init__.py @@ -20,7 +20,7 @@ def build_arch(self, arch): with current_directory(self.get_build_dir(arch.arch)): shprint( sh.Command('./configure'), - '--host=' + arch.toolchain_prefix, + '--host=' + arch.command_prefix, '--prefix=' + self.ctx.get_python_install_dir(), _env=env) shprint(sh.make, '-j' + str(cpu_count()), _env=env) diff --git a/pythonforandroid/recipes/libzbar/__init__.py b/pythonforandroid/recipes/libzbar/__init__.py index 8d7d48e2b7..5b1b62d8c1 100644 --- a/pythonforandroid/recipes/libzbar/__init__.py +++ b/pythonforandroid/recipes/libzbar/__init__.py @@ -32,7 +32,7 @@ def build_arch(self, arch): shprint(sh.Command('autoreconf'), '-vif', _env=env) shprint( sh.Command('./configure'), - '--host=' + arch.toolchain_prefix, + '--host=' + arch.command_prefix, '--target=' + arch.toolchain_prefix, '--prefix=' + self.ctx.get_python_install_dir(), # Python bindings are compiled in a separated recipe From e40260ec2edcc829d7446a75f31ff343854c8003 Mon Sep 17 00:00:00 2001 From: Andre Miras Date: Mon, 30 Sep 2019 13:56:10 +0200 Subject: [PATCH 25/72] Updates Java version troubleshooting (#1991) Adds the NoClassDefFoundError XmlSchema case and groups related cases together. Gives instructions for both Ubuntu and macOS to fix. --- doc/source/troubleshooting.rst | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/doc/source/troubleshooting.rst b/doc/source/troubleshooting.rst index f213e8d9eb..4277d51bbf 100644 --- a/doc/source/troubleshooting.rst +++ b/doc/source/troubleshooting.rst @@ -145,19 +145,26 @@ the build (e.g. if buildozer was previously used). Removing this directory should fix the problem, and is desirable anyway since you don't want it in the APK. -Exception in thread "main" java.lang.UnsupportedClassVersionError: com/android/dx/command/Main : Unsupported major.minor version 52.0 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Errors related to Java version +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This occurs due to a java version mismatch, it should be fixed by -installing Java 8 (e.g. the openjdk-8-jdk package on Ubuntu). +The errors listed below are related to Java version mismatch, it should be +fixed by installing Java 8. -java.lang.NoClassDefFoundError: sun/misc/BASE64Encoder -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +- :code:`java.lang.UnsupportedClassVersionError: com/android/dx/command/Main` +- :code:`java.lang.NoClassDefFoundError: sun/misc/BASE64Encoder` +- :code:`java.lang.NoClassDefFoundError: javax/xml/bind/annotation/XmlSchema` -Also make sure you're running Java 8, on OS X:: +On Ubuntu fix it my making sure only the :code:`openjdk-8-jdk` package is installed:: + + apt remove --purge openjdk-*-jdk + apt install openjdk-8-jdk + +In the similar fashion for macOS you need to install the :code:`java8` package:: brew cask install java8 + JNI DETECTED ERROR IN APPLICATION: static jfieldID 0x0000000 not valid for class java.lang.Class ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From cb403c9c74c56ceb1a8d9ab063adb475e3ea6ae2 Mon Sep 17 00:00:00 2001 From: Alexander Taylor Date: Sun, 6 Oct 2019 12:16:26 +0100 Subject: [PATCH 26/72] Made on-device unit tests app use the develop branch by default (#1985) --- testapps/on_device_unit_tests/buildozer.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testapps/on_device_unit_tests/buildozer.spec b/testapps/on_device_unit_tests/buildozer.spec index b313bad059..a2b55302c8 100644 --- a/testapps/on_device_unit_tests/buildozer.spec +++ b/testapps/on_device_unit_tests/buildozer.spec @@ -153,7 +153,7 @@ android.whitelist = unittest/* #android.add_activites = com.example.ExampleActivity # (str) python-for-android branch to use, defaults to master -p4a.branch = master +p4a.branch = develop # (str) OUYA Console category. Should be one of GAME or APP # If you leave this blank, OUYA support will not be enabled From 1ad1539fc0ac6ac621821968cbf1b0c975745521 Mon Sep 17 00:00:00 2001 From: opacam Date: Mon, 14 Oct 2019 22:51:08 +0200 Subject: [PATCH 27/72] :bar_chart: Fix error on py2 when `import numpy` Also: - make use of python3 `super()` for numpy recipe --- pythonforandroid/recipes/numpy/__init__.py | 9 +++++++-- .../numpy/patches/fix-py2-numpy-import.patch | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 pythonforandroid/recipes/numpy/patches/fix-py2-numpy-import.patch diff --git a/pythonforandroid/recipes/numpy/__init__.py b/pythonforandroid/recipes/numpy/__init__.py index 027b5465fe..a36c6fdd7c 100644 --- a/pythonforandroid/recipes/numpy/__init__.py +++ b/pythonforandroid/recipes/numpy/__init__.py @@ -18,14 +18,19 @@ class NumpyRecipe(CompiledComponentsPythonRecipe): call_hostpython_via_targetpython = False + def apply_patches(self, arch, build_dir=None): + if 'python2' in self.ctx.recipe_build_order: + self.patches.append(join('patches', 'fix-py2-numpy-import.patch')) + super().apply_patches(arch, build_dir=build_dir) + def build_compiled_components(self, arch): self.setup_extra_args = ['-j', str(cpu_count())] - super(NumpyRecipe, self).build_compiled_components(arch) + super().build_compiled_components(arch) self.setup_extra_args = [] def rebuild_compiled_components(self, arch, env): self.setup_extra_args = ['-j', str(cpu_count())] - super(NumpyRecipe, self).rebuild_compiled_components(arch, env) + super().rebuild_compiled_components(arch, env) self.setup_extra_args = [] diff --git a/pythonforandroid/recipes/numpy/patches/fix-py2-numpy-import.patch b/pythonforandroid/recipes/numpy/patches/fix-py2-numpy-import.patch new file mode 100644 index 0000000000..e75bd37dfd --- /dev/null +++ b/pythonforandroid/recipes/numpy/patches/fix-py2-numpy-import.patch @@ -0,0 +1,15 @@ +There is an open issue at numpy talking about the import error with numpy: + https://github.com/numpy/numpy/issues/13248 +The solution applied here is mentioned there and it seems to only affect our +python2 version. +--- numpy-1.16.4/numpy/core/overrides.py.orig 2019-05-27 12:41:06.000000000 +0200 ++++ numpy-1.16.4/numpy/core/overrides.py 2019-10-13 18:51:06.387037239 +0200 +@@ -140,6 +140,8 @@ def array_function_dispatch(dispatcher, + ------- + Function suitable for decorating the implementation of a NumPy function. + """ ++ if dispatcher.__doc__ is None: ++ dispatcher.__doc__ = "" + + if not ENABLE_ARRAY_FUNCTION: + # __array_function__ requires an explicit opt-in for now From 9666980c2a4be9b033db69afef51331aff71ab96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Poisson?= Date: Sun, 20 Oct 2019 16:58:51 +0200 Subject: [PATCH 28/72] twisted: updated to 19.7.0 Last version is building fine with p4a. --- pythonforandroid/recipes/twisted/__init__.py | 2 +- .../recipes/twisted/incremental.patch | 22 ++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/pythonforandroid/recipes/twisted/__init__.py b/pythonforandroid/recipes/twisted/__init__.py index ca222794ac..11da3094d5 100644 --- a/pythonforandroid/recipes/twisted/__init__.py +++ b/pythonforandroid/recipes/twisted/__init__.py @@ -2,7 +2,7 @@ class TwistedRecipe(CythonRecipe): - version = '17.9.0' + version = '19.7.0' url = 'https://github.com/twisted/twisted/archive/twisted-{version}.tar.gz' depends = ['setuptools', 'zope_interface', 'incremental', 'constantly'] diff --git a/pythonforandroid/recipes/twisted/incremental.patch b/pythonforandroid/recipes/twisted/incremental.patch index 85e53077d4..61656fc22b 100644 --- a/pythonforandroid/recipes/twisted/incremental.patch +++ b/pythonforandroid/recipes/twisted/incremental.patch @@ -1,13 +1,15 @@ -diff -Naur twisted-twisted-17.9.0/src/twisted/python/_setup.py twisted-twisted-17.9.0_patched/src/twisted/python/_setup.py ---- twisted-twisted-17.9.0/src/twisted/python/_setup.py 2017-09-23 07:56:08.000000000 +0200 -+++ twisted-twisted-17.9.0_patched/src/twisted/python/_setup.py 2018-10-05 11:06:23.305860722 +0200 -@@ -227,14 +227,11 @@ - requirements = ["zope.interface >= 3.6.0"] - - requirements.append("constantly >= 15.1") -- requirements.append("incremental >= 16.10.1") - requirements.append("Automat >= 0.3.0") - requirements.append("hyperlink >= 17.1.1") +diff -Naur twisted-twisted-19.7.0/src/twisted/python/_setup.py twisted-twisted-19.7.0_patched/src/twisted/python/_setup.py +--- twisted-twisted-19.7.0/src/twisted/python/_setup.py 2019-07-28 11:17:29.000000000 +0200 ++++ twisted-twisted-19.7.0_patched/src/twisted/python/_setup.py 2019-10-21 22:10:03.643068863 +0200 +@@ -282,7 +282,6 @@ + requirements = [ + "zope.interface >= 4.4.2", + "constantly >= 15.1", +- "incremental >= 16.10.1", + "Automat >= 0.3.0", + "hyperlink >= 17.1.1", + "PyHamcrest >= 1.9.0", +@@ -291,8 +290,6 @@ arguments.update(dict( packages=find_packages("src"), From c7d76e42c6f08fc6e288ea056e395fbe23cf7282 Mon Sep 17 00:00:00 2001 From: Andre Miras Date: Sun, 27 Oct 2019 00:15:16 +0200 Subject: [PATCH 29/72] Travis CI revamp part 1, refs #2008 Revamping Travis and Docker setup introducing a `Makefile`. The idea is to move the CI complexity from .travis.yml to `Makefile`. That makes a single entry point via `make` command and reproducible builds via Docker. It makes it easy to run some commands outside docker, such as: ```sh make testapps/python3/armeabi-v7a ``` Or the same command inside docker: ```sh make docker/run/make/testapps/python3/armeabi-v7a ``` This pull request also starts introducing some docker layer cache optimization as needed by #2009 to speed up docker pull/push and rebuilds from cache. It also introduces other Docker images good practices like ordering dependencies alphabetically or always enforcing `apt update` prior install, refs: https://docs.docker.com/develop/develop-images/dockerfile_best-practices/ Subsequent pull requests would simplify the process furthermore and leverage the cache to speed up builds. --- .env | 7 ++++ .travis.yml | 67 +++++++++------------------------- Dockerfile.py3 | 97 +++++++++++++++++++++++++++++++------------------- Makefile | 78 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 162 insertions(+), 87 deletions(-) create mode 100644 .env create mode 100644 Makefile diff --git a/.env b/.env new file mode 100644 index 0000000000..618f1bf109 --- /dev/null +++ b/.env @@ -0,0 +1,7 @@ +# used by coveralls.io, refs: +# https://coveralls-python.readthedocs.io/en/latest/usage/tox.html#travisci +CI +TRAVIS +TRAVIS_BRANCH +TRAVIS_JOB_ID +TRAVIS_PULL_REQUEST diff --git a/.travis.yml b/.travis.yml index cc5dd0e170..601f387007 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,10 @@ sudo: required -dist: xenial # needed for more recent python 3 and python3-venv - language: generic stages: - - lint - - test + - unit tests + - build testapps services: - docker @@ -18,8 +16,8 @@ before_install: jobs: include: - - &linting - stage: lint + - &unittests + stage: unit tests language: python python: 3.7 before_script: @@ -36,47 +34,24 @@ jobs: - pip3.7 install pyOpenSSL - pip3.7 install coveralls script: - # we want to fail fast on tox errors without having to `docker build` first + # ignores test_pythonpackage.py since it runs for too long - tox -- tests/ --ignore tests/test_pythonpackage.py - # (we ignore test_pythonpackage.py since these run way too long!! - # test_pythonpackage_basic.py will still be run.) name: "Tox Pep8" env: TOXENV=pep8 - - <<: *linting + - <<: *unittests name: "Tox Python 2" env: TOXENV=py27 - - <<: *linting + - <<: *unittests name: "Tox Python 3 & Coverage" env: TOXENV=py3 after_success: - coveralls - - - &testing - stage: test - before_script: - # build docker image - - docker build --tag=p4a --file Dockerfile.py3 . - # Run a background process to make sure that travis will not kill our tests in - # case that the travis log doesn't produce any output for more than 10 minutes - - while sleep 540; do echo "==== Still running (travis, don't kill me) ===="; done & - script: - - > - docker run - -e CI - -e TRAVIS_JOB_ID - -e TRAVIS_BRANCH - -e ANDROID_SDK_HOME="/home/user/.android/android-sdk" - -e ANDROID_NDK_HOME="/home/user/.android/android-ndk" - p4a /bin/sh -c "$COMMAND" - after_script: - # kill the background process started before run docker - - kill %1 + - &testapps name: Python 3 arm64-v8a - # overrides requirements to skip `peewee` pure python module, see: - # https://github.com/kivy/python-for-android/issues/1263#issuecomment-390421054 - env: - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools --arch=arm64-v8a' - - <<: *testing + stage: build testapps + before_script: make docker/pull + script: make docker/run/make/testapps/python3/arm64-v8a + - <<: *testapps name: Python 3 armeabi-v7a os: osx osx_image: xcode11 # since xcode1.3, python3 is the default interpreter @@ -84,18 +59,10 @@ jobs: # installs java 1.8, android's SDK/NDK and p4a - make -f ci/makefiles/osx.mk - export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home - # Run a background process (like we do with linux tests) - - while sleep 540; do echo "==== Still running (travis, don't kill me) ===="; done & - script: - - > - cd testapps && python3 setup_testapp_python3_sqlite_openssl.py apk - --sdk-dir $HOME/.android/android-sdk - --ndk-dir $HOME/.android/android-ndk - --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools - --arch=armeabi-v7a - - <<: *testing + script: make testapps/python3/armeabi-v7a PYTHON_WITH_VERSION=python3 + - <<: *testapps name: Python 2 armeabi-v7a (with numpy) - env: COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements sdl2,pyjnius,kivy,python2,openssl,requests,sqlite3,setuptools,numpy' - - <<: *testing + script: make docker/run/make/testapps/python2/armeabi-v7a + - <<: *testapps name: Rebuild updated recipes - env: COMMAND='. venv/bin/activate && ./ci/rebuild_updated_recipes.py' + script: make docker/run/make/rebuild_updated_recipes diff --git a/Dockerfile.py3 b/Dockerfile.py3 index 70c9417180..cd6e9f8405 100644 --- a/Dockerfile.py3 +++ b/Dockerfile.py3 @@ -25,9 +25,12 @@ ENV LANG="en_US.UTF-8" \ LANGUAGE="en_US.UTF-8" \ LC_ALL="en_US.UTF-8" -RUN apt -y update -qq \ - && apt -y install -qq --no-install-recommends curl unzip ca-certificates \ - && apt -y autoremove +RUN apt -y update -qq > /dev/null && apt -y install -qq --no-install-recommends \ + ca-certificates \ + curl \ + && apt -y autoremove \ + && apt -y clean \ + && rm -rf /var/lib/apt/lists/* # retry helper script, refs: # https://github.com/kivy/python-for-android/issues/1306 @@ -37,37 +40,52 @@ RUN curl https://raw.githubusercontent.com/kadwanev/retry/1.0.1/retry \ ENV USER="user" ENV HOME_DIR="/home/${USER}" -ENV ANDROID_HOME="${HOME_DIR}/.android" -ENV WORK_DIR="${HOME_DIR}" \ - PATH="${HOME_DIR}/.local/bin:${PATH}" +ENV WORK_DIR="${HOME_DIR}/app" \ + PATH="${HOME_DIR}/.local/bin:${PATH}" \ + ANDROID_HOME="${HOME_DIR}/.android" \ + JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64" -# install system dependencies -RUN ${RETRY} apt -y install -qq --no-install-recommends \ - python3 virtualenv python3-pip python3-venv \ - wget lbzip2 patch sudo python python-pip \ - && apt -y autoremove -# build dependencies -# https://buildozer.readthedocs.io/en/latest/installation.html#android-on-ubuntu-16-04-64bit +# install system dependencies RUN dpkg --add-architecture i386 \ - && ${RETRY} apt -y update -qq \ + && ${RETRY} apt -y update -qq > /dev/null \ && ${RETRY} apt -y install -qq --no-install-recommends \ - build-essential ccache git python3 python3-dev \ - libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 \ - libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 \ - zip zlib1g-dev zlib1g:i386 \ - && apt -y autoremove - -# specific recipes dependencies (e.g. libffi requires autoreconf binary) -RUN ${RETRY} apt -y install -qq --no-install-recommends \ - libffi-dev autoconf automake cmake gettext libltdl-dev libtool pkg-config \ + autoconf \ + automake \ + build-essential \ + ccache \ + cmake \ + gettext \ + git \ + lbzip2 \ + libffi-dev \ + libgtk2.0-0:i386 \ + libidn11:i386 \ + libltdl-dev \ + libncurses5:i386 \ + libpangox-1.0-0:i386 \ + libpangoxft-1.0-0:i386 \ + libstdc++6:i386 \ + libtool \ + openjdk-8-jdk \ + patch \ + pkg-config \ + python \ + python-pip \ + python3 \ + python3-dev \ + python3-pip \ + python3-venv \ + sudo \ + unzip \ + virtualenv \ + wget \ + zip \ + zlib1g-dev \ + zlib1g:i386 \ && apt -y autoremove \ - && apt -y clean - -# Install Java and set JAVA_HOME (to accept android's SDK licenses) -RUN ${RETRY} apt -y install -qq --no-install-recommends openjdk-8-jdk \ - && apt -y autoremove && apt -y clean -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 + && apt -y clean \ + && rm -rf /var/lib/apt/lists/* # prepare non root env RUN useradd --create-home --shell /bin/bash ${USER} @@ -77,18 +95,23 @@ RUN usermod -append --groups sudo ${USER} RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers # install cython for python 2 (for python 3 it's inside the venv) -RUN pip2 install --upgrade Cython==0.28.6 +RUN pip2 install --upgrade Cython==0.28.6 \ + && rm -rf ~/.cache/ WORKDIR ${WORK_DIR} -COPY --chown=user:user . ${WORK_DIR} -RUN mkdir ${ANDROID_HOME} && chown --recursive ${USER} ${ANDROID_HOME} +RUN mkdir ${ANDROID_HOME} && chown --recursive ${USER} ${HOME_DIR} ${ANDROID_HOME} USER ${USER} # Download and install android's NDK/SDK -RUN make -f ci/makefiles/android.mk target_os=linux +COPY ci/makefiles/android.mk /tmp/android.mk +RUN make --file /tmp/android.mk target_os=linux \ + && sudo rm /tmp/android.mk # install python-for-android from current branch -RUN virtualenv --python=python3 venv \ - && . venv/bin/activate \ - && pip3 install --upgrade Cython==0.28.6 \ - && pip3 install -e . +COPY --chown=user:user Makefile README.md setup.py pythonforandroid/__init__.py ${WORK_DIR}/ +RUN mkdir pythonforandroid \ + && mv __init__.py pythonforandroid/ \ + && make virtualenv \ + && rm -rf ~/.cache/ + +COPY --chown=user:user . ${WORK_DIR} diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..05687dffd6 --- /dev/null +++ b/Makefile @@ -0,0 +1,78 @@ +VIRTUAL_ENV ?= venv +PIP=$(VIRTUAL_ENV)/bin/pip +TOX=`which tox` +ACTIVATE=$(VIRTUAL_ENV)/bin/activate +PYTHON=$(VIRTUAL_ENV)/bin/python +FLAKE8=$(VIRTUAL_ENV)/bin/flake8 +PYTEST=$(VIRTUAL_ENV)/bin/pytest +SOURCES=src/ tests/ +PYTHON_MAJOR_VERSION=3 +PYTHON_MINOR_VERSION=6 +PYTHON_VERSION=$(PYTHON_MAJOR_VERSION).$(PYTHON_MINOR_VERSION) +PYTHON_MAJOR_MINOR=$(PYTHON_MAJOR_VERSION)$(PYTHON_MINOR_VERSION) +PYTHON_WITH_VERSION=python$(PYTHON_VERSION) +DOCKER_IMAGE=kivy/python-for-android +ANDROID_SDK_HOME ?= $(HOME)/.android/android-sdk +ANDROID_NDK_HOME ?= $(HOME)/.android/android-ndk + + +all: virtualenv + +$(VIRTUAL_ENV): + virtualenv --python=$(PYTHON_WITH_VERSION) $(VIRTUAL_ENV) + $(PIP) install Cython==0.28.6 + $(PIP) install -e . + +virtualenv: $(VIRTUAL_ENV) + +# ignores test_pythonpackage.py since it runs for too long +test: + $(TOX) -- tests/ --ignore tests/test_pythonpackage.py + @if test -n "$$CI"; then .tox/py$(PYTHON_MAJOR_MINOR)/bin/coveralls; fi; \ + +rebuild_updated_recipes: virtualenv + $(PYTHON) ci/rebuild_updated_recipes.py + +testapps/python2/armeabi-v7a: virtualenv + . $(ACTIVATE) && cd testapps/ && \ + python setup_testapp_python2_sqlite_openssl.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ + --requirements sdl2,pyjnius,kivy,python2,openssl,requests,sqlite3,setuptools,numpy + +testapps/python3/arm64-v8a: virtualenv + . $(ACTIVATE) && cd testapps/ && \ + python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ + --arch=arm64-v8a + +testapps/python3/armeabi-v7a: virtualenv + . $(ACTIVATE) && cd testapps/ && \ + python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ + --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools \ + --arch=armeabi-v7a + +clean: + find . -type d -name "__pycache__" -exec rm -r {} + + find . -type d -name "*.egg-info" -exec rm -r {} + + +clean/all: clean + rm -rf $(VIRTUAL_ENV) .tox/ + +docker/pull: + docker pull $(DOCKER_IMAGE):latest || true + +docker/build: + docker build --cache-from=$(DOCKER_IMAGE) --tag=$(DOCKER_IMAGE) --file=Dockerfile.py3 . + +docker/push: + docker push $(DOCKER_IMAGE) + +docker/run/test: docker/build + docker run --rm --env-file=.env $(DOCKER_IMAGE) 'make test' + +docker/run/command: docker/build + docker run --rm --env-file=.env $(DOCKER_IMAGE) /bin/sh -c "$(COMMAND)" + +docker/run/make/%: docker/build + docker run --rm --env-file=.env $(DOCKER_IMAGE) make $* + +docker/run/shell: docker/build + docker run --rm --env-file=.env -it $(DOCKER_IMAGE) From 1a079d64c49e365dd990c22ebab44758d0a79bea Mon Sep 17 00:00:00 2001 From: Aditya Bhawsingka Date: Wed, 30 Oct 2019 18:36:13 +0530 Subject: [PATCH 30/72] Documentation: android hide loading screen Correct sample code for hiding the splash screen in android. --- doc/source/apis.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/apis.rst b/doc/source/apis.rst index beae347625..975d5dfc7f 100644 --- a/doc/source/apis.rst +++ b/doc/source/apis.rst @@ -88,11 +88,11 @@ longer than necessary (with your app already being loaded) due to a limitation with the way we check if the app has properly started. In this case, the splash screen overlaps the app gui for a short time. -To dismiss the loading screen explicitely in your code, use the `android` +To dismiss the loading screen explicitly in your code, use the `android` module:: - from android import hide_loading_screen - hide_loading_screen() + from android import loadingscreen + loadingscreen.hide_loading_screen() You can call it e.g. using ``kivy.clock.Clock.schedule_once`` to run it in the first active frame of your app, or use the app build method. From 18926d828aa7b5baa65c400f38c2a8d17c67f570 Mon Sep 17 00:00:00 2001 From: Andre Miras Date: Sun, 10 Nov 2019 01:22:43 +0100 Subject: [PATCH 31/72] Exposes ANDROID_SDK_HOME to rebuild_updated_recipes The error was: ``` Traceback (most recent call last): File "ci/rebuild_updated_recipes.py", line 112, in main() File "ci/rebuild_updated_recipes.py", line 108, in main build(target_python, recipes) File "ci/rebuild_updated_recipes.py", line 59, in build android_sdk_home = os.environ['ANDROID_SDK_HOME'] File "/home/user/app/venv/lib/python3.6/os.py", line 669, in __getitem__ raise KeyError(key) from None KeyError: 'ANDROID_SDK_HOME' ``` https://travis-ci.org/kivy/python-for-android/jobs/609719811 --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 05687dffd6..0b492a65b7 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,7 @@ test: @if test -n "$$CI"; then .tox/py$(PYTHON_MAJOR_MINOR)/bin/coveralls; fi; \ rebuild_updated_recipes: virtualenv + ANDROID_SDK_HOME=$(ANDROID_SDK_HOME) ANDROID_NDK_HOME=$(ANDROID_NDK_HOME) \ $(PYTHON) ci/rebuild_updated_recipes.py testapps/python2/armeabi-v7a: virtualenv From 139c7786ba31351e372751985fc1b8de661ffa99 Mon Sep 17 00:00:00 2001 From: joergbrech Date: Sat, 9 Nov 2019 17:21:49 +0100 Subject: [PATCH 32/72] update pyproj recipe --- pythonforandroid/recipes/pyproj/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pythonforandroid/recipes/pyproj/__init__.py b/pythonforandroid/recipes/pyproj/__init__.py index 71b272d136..0c47b29514 100644 --- a/pythonforandroid/recipes/pyproj/__init__.py +++ b/pythonforandroid/recipes/pyproj/__init__.py @@ -2,8 +2,8 @@ class PyProjRecipe(CythonRecipe): - version = '1.9.5.1' - url = 'https://github.com/jswhit/pyproj/archive/master.zip' + version = '1.9.6' + url = 'https://github.com/pyproj4/pyproj/archive/v{version}rel.zip' depends = ['setuptools'] call_hostpython_via_targetpython = False From b27ff0b11fe76b2aafaf944c5bce067498fe483b Mon Sep 17 00:00:00 2001 From: Pol Canelles Date: Sun, 17 Nov 2019 15:14:40 +0100 Subject: [PATCH 33/72] Fix rebuild updated recipes, refs #2011 (#2024) * [bug] Fix rebuild_updated_recipes * [pep8] Shorten long line for `zope_interface` recipe To test `rebuild_updated_recipes` --- Makefile | 1 + pythonforandroid/recipes/zope_interface/__init__.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0b492a65b7..9483eb71af 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,7 @@ test: @if test -n "$$CI"; then .tox/py$(PYTHON_MAJOR_MINOR)/bin/coveralls; fi; \ rebuild_updated_recipes: virtualenv + . $(ACTIVATE) && \ ANDROID_SDK_HOME=$(ANDROID_SDK_HOME) ANDROID_NDK_HOME=$(ANDROID_NDK_HOME) \ $(PYTHON) ci/rebuild_updated_recipes.py diff --git a/pythonforandroid/recipes/zope_interface/__init__.py b/pythonforandroid/recipes/zope_interface/__init__.py index b1fb0bd122..245ebc8082 100644 --- a/pythonforandroid/recipes/zope_interface/__init__.py +++ b/pythonforandroid/recipes/zope_interface/__init__.py @@ -25,7 +25,11 @@ def build_arch(self, arch): def prebuild_arch(self, arch): super(ZopeInterfaceRecipe, self).prebuild_arch(arch) with current_directory(self.get_build_dir(arch.arch)): - sh.rm('-rf', 'src/zope/interface/tests', 'src/zope/interface/common/tests') + sh.rm( + '-rf', + 'src/zope/interface/tests', + 'src/zope/interface/common/tests', + ) recipe = ZopeInterfaceRecipe() From 7cdf79bad8326be7495ea5dad167bb961835be30 Mon Sep 17 00:00:00 2001 From: Mathieu Virbel Date: Wed, 9 Oct 2019 22:34:54 +0200 Subject: [PATCH 34/72] fixes libcurl with openssl --- pythonforandroid/recipes/libcurl/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pythonforandroid/recipes/libcurl/__init__.py b/pythonforandroid/recipes/libcurl/__init__.py index 27b142e071..83e4cf0858 100644 --- a/pythonforandroid/recipes/libcurl/__init__.py +++ b/pythonforandroid/recipes/libcurl/__init__.py @@ -15,8 +15,11 @@ class LibcurlRecipe(Recipe): def build_arch(self, arch): env = self.get_recipe_env(arch) - r = self.get_recipe('openssl', self.ctx) - openssl_dir = r.get_build_dir(arch.arch) + openssl_recipe = self.get_recipe('openssl', self.ctx) + openssl_dir = openssl_recipe.get_build_dir(arch.arch) + + env['LDFLAGS'] += openssl_recipe.link_dirs_flags(arch) + env['LIBS'] = env.get('LIBS', '') + openssl_recipe.link_libs_flags() with current_directory(self.get_build_dir(arch.arch)): dst_dir = join(self.get_build_dir(arch.arch), 'dist') From e2851d9b48ffbaeb83b59311a86aba165b013ee2 Mon Sep 17 00:00:00 2001 From: Mirko Date: Sat, 23 Nov 2019 20:59:50 +0100 Subject: [PATCH 35/72] Fixes ffmpeg build on ndk 19 (#1997) * ffmpeg builds w/ libx264 and openssl via ndk 19 fixes #1996 * target a specific hash * update travis_wait to 30 --- .travis.yml | 2 +- pythonforandroid/recipes/ffmpeg/__init__.py | 33 ++++++++---- .../recipes/ffmpeg/patches/configure.patch | 52 ++++++------------- .../recipes/ffpyplayer_codecs/__init__.py | 3 +- 4 files changed, 41 insertions(+), 49 deletions(-) diff --git a/.travis.yml b/.travis.yml index 601f387007..a94cc418d5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,4 +65,4 @@ jobs: script: make docker/run/make/testapps/python2/armeabi-v7a - <<: *testapps name: Rebuild updated recipes - script: make docker/run/make/rebuild_updated_recipes + script: travis_wait 30 make docker/run/make/rebuild_updated_recipes diff --git a/pythonforandroid/recipes/ffmpeg/__init__.py b/pythonforandroid/recipes/ffmpeg/__init__.py index 4a04f4844f..519fbe991d 100644 --- a/pythonforandroid/recipes/ffmpeg/__init__.py +++ b/pythonforandroid/recipes/ffmpeg/__init__.py @@ -4,7 +4,7 @@ class FFMpegRecipe(Recipe): - version = 'n3.4.5' + version = '007e03348dbd8d3de3eb09022d72c734a8608144' # Moved to github.com instead of ffmpeg.org to improve download speed url = 'https://github.com/FFmpeg/FFmpeg/archive/{version}.zip' depends = ['sdl2'] # Need this to build correct recipe order @@ -37,22 +37,31 @@ def build_arch(self, arch): '--enable-nonfree', '--enable-protocol=https,tls_openssl', ] - build_dir = Recipe.get_recipe('openssl', self.ctx).get_build_dir(arch.arch) - cflags += ['-I' + build_dir + '/include/'] + build_dir = Recipe.get_recipe( + 'openssl', self.ctx).get_build_dir(arch.arch) + cflags += ['-I' + build_dir + '/include/', + '-DOPENSSL_API_COMPAT=0x10002000L'] ldflags += ['-L' + build_dir] if 'ffpyplayer_codecs' in self.ctx.recipe_build_order: # libx264 flags += ['--enable-libx264'] - build_dir = Recipe.get_recipe('libx264', self.ctx).get_build_dir(arch.arch) + build_dir = Recipe.get_recipe( + 'libx264', self.ctx).get_build_dir(arch.arch) cflags += ['-I' + build_dir + '/include/'] ldflags += ['-lx264', '-L' + build_dir + '/lib/'] + """ + WARNING: DISABLED during migration to ndk19, cause We're + getting a runtime error for a missing symbol. + # libshine flags += ['--enable-libshine'] build_dir = Recipe.get_recipe('libshine', self.ctx).get_build_dir(arch.arch) cflags += ['-I' + build_dir + '/include/'] ldflags += ['-lshine', '-L' + build_dir + '/lib/'] + ldflags += ['-lm'] + """ # Enable all codecs: flags += [ @@ -79,22 +88,20 @@ def build_arch(self, arch): # disable binaries / doc flags += [ - '--disable-ffmpeg', - '--disable-ffplay', - '--disable-ffprobe', - '--disable-ffserver', + '--disable-programs', '--disable-doc', ] # other flags: flags += [ '--enable-filter=aresample,resample,crop,adelay,volume,scale', - '--enable-protocol=file,http', + '--enable-protocol=file,http,hls', '--enable-small', '--enable-hwaccels', '--enable-gpl', '--enable-pic', '--disable-static', + '--disable-debug', '--enable-shared', ] @@ -108,9 +115,13 @@ def build_arch(self, arch): # android: flags += [ '--target-os=android', - '--cross-prefix={}'.format(cross_prefix), + '--enable-cross-compile', + '--cross-prefix={}-'.format(arch.target), '--arch={}'.format(arch_flag), - '--sysroot=' + self.ctx.ndk_platform, + '--strip={}strip'.format(cross_prefix), + '--sysroot={}'.format(join(self.ctx.ndk_dir, 'toolchains', + 'llvm', 'prebuilt', 'linux-x86_64', + 'sysroot')), '--enable-neon', '--prefix={}'.format(realpath('.')), ] diff --git a/pythonforandroid/recipes/ffmpeg/patches/configure.patch b/pythonforandroid/recipes/ffmpeg/patches/configure.patch index b898c7f526..5746918e34 100644 --- a/pythonforandroid/recipes/ffmpeg/patches/configure.patch +++ b/pythonforandroid/recipes/ffmpeg/patches/configure.patch @@ -1,40 +1,20 @@ ---- ./configure.orig 2017-12-11 00:35:18.000000000 +0300 -+++ ./configure 2017-12-19 09:47:54.104914600 +0300 -@@ -4841,9 +4841,6 @@ - add_cflags -std=c11 || - check_cflags -std=c99 - --check_cppflags -D_FILE_OFFSET_BITS=64 --check_cppflags -D_LARGEFILE_SOURCE -- - add_host_cppflags -D_ISOC99_SOURCE - check_host_cflags -std=c99 - check_host_cflags -Wall -@@ -5979,7 +5976,7 @@ +--- ./configure 2019-07-21 18:36:31.000000000 +0200 ++++ ./configure_new 2019-10-04 12:43:41.798448200 +0200 +@@ -6222,7 +6222,7 @@ enabled librsvg && require_pkg_config librsvg librsvg-2.0 librsvg-2.0/librsvg/rsvg.h rsvg_handle_render_cairo enabled librtmp && require_pkg_config librtmp librtmp librtmp/rtmp.h RTMP_Socket - enabled librubberband && require_pkg_config librubberband "rubberband >= 1.8.1" rubberband/rubberband-c.h rubberband_new + enabled librubberband && require_pkg_config librubberband "rubberband >= 1.8.1" rubberband/rubberband-c.h rubberband_new -lstdc++ && append librubberband_extralibs "-lstdc++" -enabled libshine && require_pkg_config libshine shine shine/layer3.h shine_encode_buffer -+enabled libshine && require "shine" shine/layer3.h shine_encode_buffer -lshine - enabled libsmbclient && { use_pkg_config libsmbclient smbclient libsmbclient.h smbc_init || - require smbclient libsmbclient.h smbc_init -lsmbclient; } - enabled libsnappy && require libsnappy snappy-c.h snappy_compress -lsnappy - -diff -Naur ffmpeg/configure ffmpeg-1/configure ---- ffmpeg/configure 2019-01-11 09:30:02.824961600 +0100 -+++ ffmpeg-1/configure 2019-01-11 09:29:54.976149600 +0100 -@@ -6068,11 +6068,11 @@ - { ! enabled cross_compile && add_cflags -isystem/opt/vc/include/IL && check_header OMX_Core.h ; } || - die "ERROR: OpenMAX IL headers not found"; } - enabled omx && require_header OMX_Core.h --enabled openssl && { use_pkg_config openssl openssl openssl/ssl.h OPENSSL_init_ssl || -+enabled openssl && { use_pkg_config openssl openssl openssl/ssl.h OPENSSL_init_ssl || - use_pkg_config openssl openssl openssl/ssl.h SSL_library_init || -- check_lib openssl openssl/ssl.h OPENSSL_init_ssl -lssl -lcrypto || -- check_lib openssl openssl/ssl.h OPENSSL_init_ssl -lssl32 -leay32 || -- check_lib openssl openssl/ssl.h OPENSSL_init_ssl -lssl -lcrypto -lws2_32 -lgdi32 || -+ check_lib openssl openssl/ssl.h SSL_library_init -lssl -lcrypto || -+ check_lib openssl openssl/ssl.h SSL_library_init -lssl32 -leay32 || -+ check_lib openssl openssl/ssl.h SSL_library_init -lssl -lcrypto -lws2_32 -lgdi32 || ++enabled libshine && require "shine" shine/layer3.h shine_encode_buffer -lshine -lm + enabled libsmbclient && { check_pkg_config libsmbclient smbclient libsmbclient.h smbc_init || + require libsmbclient libsmbclient.h smbc_init -lsmbclient; } + enabled libsnappy && require libsnappy snappy-c.h snappy_compress -lsnappy -lstdc++ +@@ -6322,7 +6322,7 @@ + die "ERROR: OpenMAX IL headers not found"; } && enable omx + enabled openssl && { check_pkg_config openssl openssl openssl/ssl.h OPENSSL_init_ssl || + check_pkg_config openssl openssl openssl/ssl.h SSL_library_init || +- check_lib openssl openssl/ssl.h SSL_library_init -lssl -lcrypto || ++ check_lib openssl openssl/ssl.h OPENSSL_init_ssl -lssl -lcrypto || + check_lib openssl openssl/ssl.h SSL_library_init -lssl32 -leay32 || + check_lib openssl openssl/ssl.h SSL_library_init -lssl -lcrypto -lws2_32 -lgdi32 || die "ERROR: openssl not found"; } - enabled rkmpp && { { require_pkg_config rockchip_mpp rockchip_mpp rockchip/rk_mpi.h mpp_create || diff --git a/pythonforandroid/recipes/ffpyplayer_codecs/__init__.py b/pythonforandroid/recipes/ffpyplayer_codecs/__init__.py index b32419470d..da67d928f6 100644 --- a/pythonforandroid/recipes/ffpyplayer_codecs/__init__.py +++ b/pythonforandroid/recipes/ffpyplayer_codecs/__init__.py @@ -2,7 +2,8 @@ class FFPyPlayerCodecsRecipe(Recipe): - depends = ['libshine', 'libx264'] + depends = ['libx264'] + # disabled libshine due a missing symbol error (see ffmpeg recipe) def build_arch(self, arch): pass From 11a9eb0ac0fc7815e50aced2312d37e5d93e7aa4 Mon Sep 17 00:00:00 2001 From: Pol Canelles Date: Sat, 23 Nov 2019 21:05:14 +0100 Subject: [PATCH 36/72] Introduce github-actions (push & pull_requests) (#2025) * [actions] Add push workflow (linting) * [actions] Fix `expected compiler` for tests Because the compiler path depends on the platform where the tests are run. * [actions] Mock environ variables for `test_create_no_sdk_dir` * [actions] Fix `test_toolchain.test_create` for macos * [actions] Fix venv/virtualenv tests * [actions] Add build stage: pull docker image & build an testapp * [actions] Make flake8 tests via tox * [actions] Run pytest via tox We also remove the env var `COVERALLS_REPO_TOKEN`, so we test if coveralls is working with `gh-actions` Note: Since Python 2 it's almost at the end of his life we only perform the Python3 tests, but we still have the travis tests for Python 2 * [actions] Reintroduce `COVERALLS_REPO_TOKEN` * [actions] Make tox tests via Makefile The coveralls report will only be send with our travis tests, since the `CI` environment variable it will only exist for travis. * [actions] Add `rebuild updated recipes` test * [actions] Shorten long line below 79 characters for libtorrent The purpose of this commit is to test the rebuild updated recipes behaviour with a long time consuming recipe * [actions] Revert "[actions] Shorten long line ..." This reverts commit c6c3ea8c, to keep travis happy --- .github/workflows/push.yml | 70 +++++++++++++++++++++++++++++++ tests/recipes/recipe_lib_test.py | 9 ++-- tests/recipes/test_icu.py | 3 +- tests/test_archs.py | 5 ++- tests/test_bootstrap.py | 4 +- tests/test_pythonpackage_basic.py | 8 +++- tests/test_recipe.py | 6 ++- tests/test_toolchain.py | 16 +++++-- 8 files changed, 105 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/push.yml diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000000..254832ceaf --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,70 @@ +name: Unit tests & Build Testapp + +on: ['push', 'pull_request'] + +jobs: + + flake8: + name: Flake8 tests + runs-on: ubuntu-latest + steps: + - name: Checkout python-for-android + uses: actions/checkout@master + - name: Set up Python 3.7 + uses: actions/setup-python@v1.1.0 + with: + python-version: 3.7 + - name: Run flake8 + run: | + python -m pip install --upgrade pip + pip install tox>=2.0 + tox -e pep8 + + test: + name: Pytest [Python ${{ matrix.python-version }} | ${{ matrix.os }}] + needs: flake8 + runs-on: ${{ matrix.os }} + strategy: + matrix: + python-version: [3.6, 3.7] + os: [ubuntu-latest, macOs-latest] + steps: + - name: Checkout python-for-android + uses: actions/checkout@master + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1.1.0 + with: + python-version: ${{ matrix.python-version }} + - name: Tox tests + run: | + python -m pip install --upgrade pip + pip install tox>=2.0 + make test + + build: + name: Build testapp + needs: [flake8] + runs-on: ubuntu-latest + steps: + - name: Checkout python-for-android + uses: actions/checkout@master + - name: Pull docker image + run: | + make docker/pull + - name: Build apk for Python 3 arm64-v8a + run: | + make docker/run/make/testapps/python3/arm64-v8a + + rebuild_updated_recipes: + name: Test updated recipes + needs: [flake8] + runs-on: ubuntu-latest + steps: + - name: Checkout python-for-android + uses: actions/checkout@master + - name: Pull docker image + run: | + make docker/pull + - name: Rebuild updated recipes + run: | + make docker/run/make/rebuild_updated_recipes diff --git a/tests/recipes/recipe_lib_test.py b/tests/recipes/recipe_lib_test.py index 5622397aa5..6c48474384 100644 --- a/tests/recipes/recipe_lib_test.py +++ b/tests/recipes/recipe_lib_test.py @@ -1,4 +1,5 @@ from unittest import mock +from platform import system from tests.recipes.recipe_ctx import RecipeCtx @@ -13,7 +14,7 @@ class BaseTestForMakeRecipe(RecipeCtx): recipe_name = None expected_compiler = ( - "{android_ndk}/toolchains/llvm/prebuilt/linux-x86_64/bin/clang" + "{android_ndk}/toolchains/llvm/prebuilt/{system}-x86_64/bin/clang" ) sh_command_calls = ["./configure"] @@ -48,7 +49,7 @@ def test_get_recipe_env( some internal methods has been called. """ mock_find_executable.return_value = self.expected_compiler.format( - android_ndk=self.ctx._ndk_dir + android_ndk=self.ctx._ndk_dir, system=system().lower() ) mock_glob.return_value = ["llvm"] mock_check_recipe_choices.return_value = sorted( @@ -85,7 +86,7 @@ def test_build_arch( mock_current_directory, ): mock_find_executable.return_value = self.expected_compiler.format( - android_ndk=self.ctx._ndk_dir + android_ndk=self.ctx._ndk_dir, system=system().lower() ) mock_glob.return_value = ["llvm"] @@ -133,7 +134,7 @@ def test_build_arch( mock_current_directory, ): mock_find_executable.return_value = self.expected_compiler.format( - android_ndk=self.ctx._ndk_dir + android_ndk=self.ctx._ndk_dir, system=system().lower() ) mock_glob.return_value = ["llvm"] diff --git a/tests/recipes/test_icu.py b/tests/recipes/test_icu.py index 506cdb74b2..00cfbdbbe7 100644 --- a/tests/recipes/test_icu.py +++ b/tests/recipes/test_icu.py @@ -1,6 +1,7 @@ import os import unittest from unittest import mock +from platform import system from tests.recipes.recipe_ctx import RecipeCtx from pythonforandroid.recipes.icu import ICURecipe @@ -47,7 +48,7 @@ def test_build_arch( ): mock_find_executable.return_value = os.path.join( self.ctx._ndk_dir, - "toolchains/llvm/prebuilt/linux-x86_64/bin/clang", + f"toolchains/llvm/prebuilt/{system().lower()}-x86_64/bin/clang", ) mock_archs_glob.return_value = [ os.path.join(self.ctx._ndk_dir, "toolchains", "llvm") diff --git a/tests/test_archs.py b/tests/test_archs.py index ddb18b80e2..61887e0e51 100644 --- a/tests/test_archs.py +++ b/tests/test_archs.py @@ -2,6 +2,7 @@ import unittest from os import environ from unittest import mock +from platform import system from pythonforandroid.bootstrap import Bootstrap from pythonforandroid.distribution import Distribution @@ -70,8 +71,8 @@ def setUp(self): # Here we define the expected compiler, which, as per ndk >= r19, # should be the same for all the tests (no more gcc compiler) self.expected_compiler = ( - "/opt/android/android-ndk/toolchains/" - "llvm/prebuilt/linux-x86_64/bin/clang" + f"/opt/android/android-ndk/toolchains/" + f"llvm/prebuilt/{system().lower()}-x86_64/bin/clang" ) diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index 02d54f0d9e..b164985e38 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -4,6 +4,8 @@ import unittest from unittest import mock +from platform import system + from pythonforandroid.bootstrap import ( _cmp_bootstraps_by_priority, Bootstrap, expand_dependencies, ) @@ -546,7 +548,7 @@ def test_bootstrap_strip( ): mock_find_executable.return_value = os.path.join( self.ctx._ndk_dir, - "toolchains/llvm/prebuilt/linux-x86_64/bin/clang", + f"toolchains/llvm/prebuilt/{system().lower()}-x86_64/bin/clang", ) mock_glob.return_value = [ os.path.join(self.ctx._ndk_dir, "toolchains", "llvm") diff --git a/tests/test_pythonpackage_basic.py b/tests/test_pythonpackage_basic.py index 3d1a156df6..a39d8a44e3 100644 --- a/tests/test_pythonpackage_basic.py +++ b/tests/test_pythonpackage_basic.py @@ -308,7 +308,9 @@ def test_virtualenv(self): sys_python_path = self.run__get_system_python_executable( os.path.join(test_dir, "virtualenv", "bin", "python") ) - assert os.path.normpath(sys_python_path) == os.path.normpath(pybin) + assert os.path.normpath(sys_python_path).startswith( + os.path.normpath(pybin) + ) finally: shutil.rmtree(test_dir) @@ -341,6 +343,8 @@ def test_venv(self): sys_python_path = self.run__get_system_python_executable( os.path.join(test_dir, "venv", "bin", "python") ) - assert os.path.normpath(sys_python_path) == os.path.normpath(pybin) + assert os.path.normpath(sys_python_path).startswith( + os.path.normpath(pybin) + ) finally: shutil.rmtree(test_dir) diff --git a/tests/test_recipe.py b/tests/test_recipe.py index c01ab83507..9ff1713c95 100644 --- a/tests/test_recipe.py +++ b/tests/test_recipe.py @@ -5,6 +5,8 @@ import warnings from unittest import mock from backports import tempfile +from platform import system + from pythonforandroid.build import Context from pythonforandroid.recipe import Recipe, import_recipe from pythonforandroid.archs import ArchAarch_64 @@ -278,8 +280,8 @@ def test_get_recipe_env_with( should be tested in the proper test. """ expected_compiler = ( - "/opt/android/android-ndk/toolchains/" - "llvm/prebuilt/linux-x86_64/bin/clang" + f"/opt/android/android-ndk/toolchains/" + f"llvm/prebuilt/{system().lower()}-x86_64/bin/clang" ) mock_find_executable.return_value = expected_compiler mock_glob.return_value = ["llvm"] diff --git a/tests/test_toolchain.py b/tests/test_toolchain.py index 7e3078133b..e299794ebb 100644 --- a/tests/test_toolchain.py +++ b/tests/test_toolchain.py @@ -78,10 +78,14 @@ def test_create(self): m_get_ndk_platform_dir.return_value = ( '/tmp/android-ndk/platforms/android-21/arch-arm', True) ToolchainCL() - assert m_get_available_apis.call_args_list == [ - mock.call('/tmp/android-sdk')] - assert m_get_toolchain_versions.call_args_list == [ - mock.call('/tmp/android-ndk', mock.ANY)] + assert m_get_available_apis.call_args_list in [ + [mock.call('/tmp/android-sdk')], # linux case + [mock.call('/private/tmp/android-sdk')] # macos case + ] + assert m_get_toolchain_versions.call_args_list in [ + [mock.call('/tmp/android-ndk', mock.ANY)], # linux case + [mock.call('/private/tmp/android-ndk', mock.ANY)], # macos case + ] build_order = [ 'hostpython3', 'libffi', 'openssl', 'sqlite3', 'python3', 'genericndkbuild', 'setuptools', 'six', 'pyjnius', 'android', @@ -100,6 +104,10 @@ def test_create(self): ] assert m_run_distribute.call_args_list == [mock.call()] + @mock.patch( + 'pythonforandroid.build.environ', + # Make sure that no environ variable modifies `sdk_dir` + {'ANDROIDSDK': None, 'ANDROID_HOME': None}) def test_create_no_sdk_dir(self): """ The `--sdk-dir` is mandatory to `create` a distribution. From 0c6acfc4254ecf9d770009b4c365867bdbb6a6ec Mon Sep 17 00:00:00 2001 From: Pol Canelles Date: Sun, 24 Nov 2019 18:43:53 +0100 Subject: [PATCH 37/72] Fix libshine and re-enable it for ffmpeg & ffpyplayer_codecs (#2027) * [libs] Rework of libshine to be an shared library - make it build as shared - move to python3's f-strings - fix linking when building the shared library * [libs] Re-enable libshine for ffmpeg & ffpyplayer_codecs --- pythonforandroid/recipes/ffmpeg/__init__.py | 5 ----- .../recipes/ffpyplayer_codecs/__init__.py | 3 +-- pythonforandroid/recipes/libshine/__init__.py | 18 +++++++++++++----- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/pythonforandroid/recipes/ffmpeg/__init__.py b/pythonforandroid/recipes/ffmpeg/__init__.py index 519fbe991d..3418ed1a90 100644 --- a/pythonforandroid/recipes/ffmpeg/__init__.py +++ b/pythonforandroid/recipes/ffmpeg/__init__.py @@ -51,17 +51,12 @@ def build_arch(self, arch): cflags += ['-I' + build_dir + '/include/'] ldflags += ['-lx264', '-L' + build_dir + '/lib/'] - """ - WARNING: DISABLED during migration to ndk19, cause We're - getting a runtime error for a missing symbol. - # libshine flags += ['--enable-libshine'] build_dir = Recipe.get_recipe('libshine', self.ctx).get_build_dir(arch.arch) cflags += ['-I' + build_dir + '/include/'] ldflags += ['-lshine', '-L' + build_dir + '/lib/'] ldflags += ['-lm'] - """ # Enable all codecs: flags += [ diff --git a/pythonforandroid/recipes/ffpyplayer_codecs/__init__.py b/pythonforandroid/recipes/ffpyplayer_codecs/__init__.py index da67d928f6..558c0c223d 100644 --- a/pythonforandroid/recipes/ffpyplayer_codecs/__init__.py +++ b/pythonforandroid/recipes/ffpyplayer_codecs/__init__.py @@ -2,8 +2,7 @@ class FFPyPlayerCodecsRecipe(Recipe): - depends = ['libx264'] - # disabled libshine due a missing symbol error (see ffmpeg recipe) + depends = ['libx264', 'libshine'] def build_arch(self, arch): pass diff --git a/pythonforandroid/recipes/libshine/__init__.py b/pythonforandroid/recipes/libshine/__init__.py index 7d114d25a1..32fa9e175d 100644 --- a/pythonforandroid/recipes/libshine/__init__.py +++ b/pythonforandroid/recipes/libshine/__init__.py @@ -10,7 +10,15 @@ class LibShineRecipe(Recipe): version = 'c72aba9031bde18a0995e7c01c9b53f2e08a0e46' url = 'https://github.com/toots/shine/archive/{version}.zip' - built_libraries = {'libshine.a': 'lib'} + built_libraries = {'libshine.so': 'lib'} + + def get_recipe_env(self, arch=None, with_flags_in_cc=True): + env = super().get_recipe_env(arch, with_flags_in_cc) + # technically, libraries should go to `LDLIBS`, but it seems + # that libshine doesn't like so, and it will fail on linking stage + env['LDLIBS'] = env['LDLIBS'].replace(' -lm', '') + env['LDFLAGS'] += ' -lm' + return env def build_arch(self, arch): with current_directory(self.get_build_dir(arch.arch)): @@ -18,11 +26,11 @@ def build_arch(self, arch): shprint(sh.Command('./bootstrap')) configure = sh.Command('./configure') shprint(configure, - '--host=arm-linux', + f'--host={arch.command_prefix}', '--enable-pic', - '--disable-shared', - '--enable-static', - '--prefix={}'.format(realpath('.')), + '--disable-static', + '--enable-shared', + f'--prefix={realpath(".")}', _env=env) shprint(sh.make, '-j', str(cpu_count()), _env=env) shprint(sh.make, 'install', _env=env) From 8bd79b829796dbdabc8caf16f7205bf976823e01 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sun, 24 Nov 2019 22:57:50 +0100 Subject: [PATCH 38/72] from kivy.base import runTouchApp for line 75 --- testapps/testapp_matplotlib/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testapps/testapp_matplotlib/main.py b/testapps/testapp_matplotlib/main.py index fa798204b9..2c2d9d00e7 100644 --- a/testapps/testapp_matplotlib/main.py +++ b/testapps/testapp_matplotlib/main.py @@ -31,6 +31,7 @@ print('saved fig') from kivy.app import App +from kivy.base import runTouchApp from kivy.uix.image import Image from kivy.lang import Builder From 2610330cfca7f43d2fa69e175f4b9c3d09338680 Mon Sep 17 00:00:00 2001 From: Pol Canelles Date: Thu, 5 Dec 2019 13:31:37 +0100 Subject: [PATCH 39/72] [gh-actions] Add new testapp and upload artifacts... (#2033) * [gh-actions] Add testapp for `armeabi-v7a` and create artifacts * [gh-actions] Pin actions/checkout to v1 --- .github/workflows/push.yml | 20 ++++++++++++++------ Makefile | 11 +++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 254832ceaf..48ef9133ca 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout python-for-android - uses: actions/checkout@master + uses: actions/checkout@v1 - name: Set up Python 3.7 uses: actions/setup-python@v1.1.0 with: @@ -30,7 +30,7 @@ jobs: os: [ubuntu-latest, macOs-latest] steps: - name: Checkout python-for-android - uses: actions/checkout@master + uses: actions/checkout@v1 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1.1.0 with: @@ -45,15 +45,23 @@ jobs: name: Build testapp needs: [flake8] runs-on: ubuntu-latest + strategy: + matrix: + build-arch: ['arm64-v8a', 'armeabi-v7a'] steps: - name: Checkout python-for-android - uses: actions/checkout@master + uses: actions/checkout@v1 - name: Pull docker image run: | make docker/pull - - name: Build apk for Python 3 arm64-v8a + - name: Build apk for Python 3 ${{ matrix.build-arch }} run: | - make docker/run/make/testapps/python3/arm64-v8a + mkdir -p apks + make docker/run/make/with-artifact/testapps/python3/${{ matrix.build-arch }} + - uses: actions/upload-artifact@v1 + with: + name: bdisttest_python3_sqlite_openssl_googlendk__${{ matrix.build-arch }}-debug-1.1.apk + path: apks rebuild_updated_recipes: name: Test updated recipes @@ -61,7 +69,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout python-for-android - uses: actions/checkout@master + uses: actions/checkout@v1 - name: Pull docker image run: | make docker/pull diff --git a/Makefile b/Makefile index 9483eb71af..aac0e51c71 100644 --- a/Makefile +++ b/Makefile @@ -76,5 +76,16 @@ docker/run/command: docker/build docker/run/make/%: docker/build docker run --rm --env-file=.env $(DOCKER_IMAGE) make $* +docker/run/make/with-artifact/%: docker/build +ifeq (,$(findstring python3,$($*))) + $(eval $@_APP_NAME := bdisttest_python3_sqlite_openssl_googlendk) +else + $(eval $@_APP_NAME := bdisttest_python2_sqlite_openssl) +endif + $(eval $@_APP_ARCH := $(shell basename $*)) + docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $* + docker cp p4a-latest:/home/user/app/testapps/$($@_APP_NAME)__$($@_APP_ARCH)-debug-1.1-.apk ./apks + docker rm -fv p4a-latest + docker/run/shell: docker/build docker run --rm --env-file=.env -it $(DOCKER_IMAGE) From 74fb8e5fc756c52bd192d2e2f57e02d6cd2686a5 Mon Sep 17 00:00:00 2001 From: Tom Goldberg <30830488+tomgold182@users.noreply.github.com> Date: Sat, 7 Dec 2019 19:18:59 +0200 Subject: [PATCH 40/72] added a recipe for bcrypt library (#2035) closes #1910 --- pythonforandroid/recipes/bcrypt/__init__.py | 22 +++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 pythonforandroid/recipes/bcrypt/__init__.py diff --git a/pythonforandroid/recipes/bcrypt/__init__.py b/pythonforandroid/recipes/bcrypt/__init__.py new file mode 100644 index 0000000000..e7a5659b8f --- /dev/null +++ b/pythonforandroid/recipes/bcrypt/__init__.py @@ -0,0 +1,22 @@ +from pythonforandroid.recipe import CompiledComponentsPythonRecipe, Recipe + + +class BCryptRecipe(CompiledComponentsPythonRecipe): + name = 'bcrypt' + version = '3.1.7' + url = 'https://github.com/pyca/bcrypt/archive/{version}.tar.gz' + depends = ['openssl', 'cffi'] + call_hostpython_via_targetpython = False + + def get_recipe_env(self, arch): + env = super(BCryptRecipe, self).get_recipe_env(arch) + + openssl_recipe = Recipe.get_recipe('openssl', self.ctx) + env['CFLAGS'] += openssl_recipe.include_flags(arch) + env['LDFLAGS'] += openssl_recipe.link_dirs_flags(arch) + env['LIBS'] = openssl_recipe.link_libs_flags() + + return env + + +recipe = BCryptRecipe() From 3a155e3c709db718f8d9136a43ab7119bb805972 Mon Sep 17 00:00:00 2001 From: Pol Canelles Date: Mon, 9 Dec 2019 11:11:51 +0100 Subject: [PATCH 41/72] [testapp] Fix vibration for testapps (#2034) Method `vibrate(milliseconds)` was deprecated in API level 26, now we should use ` vibrate(android.os.VibrationEffect)`. See also: https://developer.android.com/reference/android/os/Vibrator#vibrate(long) --- testapps/testapp/main.py | 29 ++++++++++++----- testapps/testapp_encryption/main.py | 33 ++++++++++++++------ testapps/testapp_flask/main.py | 23 ++++++++++++-- testapps/testapp_keyboard/main.py | 29 ++++++++++++----- testapps/testapp_pillow/main.py | 41 +++++++++++++++---------- testapps/testapp_sqlite_openssl/main.py | 31 +++++++++++++------ 6 files changed, 133 insertions(+), 53 deletions(-) diff --git a/testapps/testapp/main.py b/testapps/testapp/main.py index 5baa420f3c..2148e7c68a 100644 --- a/testapps/testapp/main.py +++ b/testapps/testapp/main.py @@ -123,21 +123,34 @@ def on_pause(self): def test_pyjnius(self, *args): try: - from jnius import autoclass + from jnius import autoclass, cast except ImportError: raise_error('Could not import pyjnius') return - print('Attempting to vibrate with pyjnius') - # PythonActivity = autoclass('org.renpy.android.PythonActivity') - # activity = PythonActivity.mActivity + ANDROID_VERSION = autoclass('android.os.Build$VERSION') + SDK_INT = ANDROID_VERSION.SDK_INT + + Context = autoclass("android.content.Context") PythonActivity = autoclass('org.kivy.android.PythonActivity') activity = PythonActivity.mActivity - Intent = autoclass('android.content.Intent') - Context = autoclass('android.content.Context') - vibrator = activity.getSystemService(Context.VIBRATOR_SERVICE) - vibrator.vibrate(1000) + vibrator_service = activity.getSystemService(Context.VIBRATOR_SERVICE) + vibrator = cast("android.os.Vibrator", vibrator_service) + + if vibrator and SDK_INT >= 26: + print("Using android's `VibrationEffect` (SDK >= 26)") + VibrationEffect = autoclass("android.os.VibrationEffect") + vibrator.vibrate( + VibrationEffect.createOneShot( + 1000, VibrationEffect.DEFAULT_AMPLITUDE, + ), + ) + elif vibrator: + print("Using deprecated android's vibrate (SDK < 26)") + vibrator.vibrate(1000) + else: + print('Something happened...vibrator service disabled?') def test_ctypes(self, *args): import ctypes diff --git a/testapps/testapp_encryption/main.py b/testapps/testapp_encryption/main.py index b3289e857a..3b4db91020 100644 --- a/testapps/testapp_encryption/main.py +++ b/testapps/testapp_encryption/main.py @@ -324,19 +324,34 @@ def on_pause(self): def test_pyjnius(self, *args): try: - from jnius import autoclass + from jnius import autoclass, cast except ImportError: raise_error('Could not import pyjnius') return - print('Attempting to vibrate with pyjnius') - python_activity = autoclass('org.kivy.android.PythonActivity') - activity = python_activity.mActivity - intent = autoclass('android.content.Intent') - context = autoclass('android.content.Context') - vibrator = activity.getSystemService(context.VIBRATOR_SERVICE) - - vibrator.vibrate(1000) + ANDROID_VERSION = autoclass('android.os.Build$VERSION') + SDK_INT = ANDROID_VERSION.SDK_INT + + Context = autoclass("android.content.Context") + PythonActivity = autoclass('org.kivy.android.PythonActivity') + activity = PythonActivity.mActivity + + vibrator_service = activity.getSystemService(Context.VIBRATOR_SERVICE) + vibrator = cast("android.os.Vibrator", vibrator_service) + + if vibrator and SDK_INT >= 26: + print("Using android's `VibrationEffect` (SDK >= 26)") + VibrationEffect = autoclass("android.os.VibrationEffect") + vibrator.vibrate( + VibrationEffect.createOneShot( + 1000, VibrationEffect.DEFAULT_AMPLITUDE, + ), + ) + elif vibrator: + print("Using deprecated android's vibrate (SDK < 26)") + vibrator.vibrate(1000) + else: + print('Something happened...vibrator service disabled?') def test_ctypes(self, *args): import ctypes diff --git a/testapps/testapp_flask/main.py b/testapps/testapp_flask/main.py index fcd0f232e3..1f74d1e9e6 100644 --- a/testapps/testapp_flask/main.py +++ b/testapps/testapp_flask/main.py @@ -26,12 +26,16 @@ print('imported flask etc') print('importing pyjnius') -from jnius import autoclass +from jnius import autoclass, cast + +ANDROID_VERSION = autoclass('android.os.Build$VERSION') +SDK_INT = ANDROID_VERSION.SDK_INT Context = autoclass('android.content.Context') PythonActivity = autoclass('org.kivy.android.PythonActivity') activity = PythonActivity.mActivity -vibrator = activity.getSystemService(Context.VIBRATOR_SERVICE) +vibrator_service = activity.getSystemService(Context.VIBRATOR_SERVICE) +vibrator = cast("android.os.Vibrator", vibrator_service) ActivityInfo = autoclass('android.content.pm.ActivityInfo') @@ -50,7 +54,20 @@ def vibrate(): print('ERROR: asked to vibrate but without time argument') print('asked to vibrate', args['time']) - vibrator.vibrate(float(args['time']) * 1000) + if vibrator and SDK_INT >= 26: + print("Using android's `VibrationEffect` (SDK >= 26)") + VibrationEffect = autoclass("android.os.VibrationEffect") + vibrator.vibrate( + VibrationEffect.createOneShot( + int(float(args['time']) * 1000), + VibrationEffect.DEFAULT_AMPLITUDE, + ), + ) + elif vibrator: + print("Using deprecated android's vibrate (SDK < 26)") + vibrator.vibrate(int(float(args['time']) * 1000)) + else: + print('Something happened...vibrator service disabled?') print('vibrated') @app.route('/loadUrl') diff --git a/testapps/testapp_keyboard/main.py b/testapps/testapp_keyboard/main.py index cb76b7af99..27cea40d18 100644 --- a/testapps/testapp_keyboard/main.py +++ b/testapps/testapp_keyboard/main.py @@ -121,21 +121,34 @@ def on_pause(self): def test_pyjnius(self, *args): try: - from jnius import autoclass + from jnius import autoclass, cast except ImportError: raise_error('Could not import pyjnius') return - print('Attempting to vibrate with pyjnius') - # PythonActivity = autoclass('org.renpy.android.PythonActivity') - # activity = PythonActivity.mActivity + ANDROID_VERSION = autoclass('android.os.Build$VERSION') + SDK_INT = ANDROID_VERSION.SDK_INT + + Context = autoclass("android.content.Context") PythonActivity = autoclass('org.kivy.android.PythonActivity') activity = PythonActivity.mActivity - Intent = autoclass('android.content.Intent') - Context = autoclass('android.content.Context') - vibrator = activity.getSystemService(Context.VIBRATOR_SERVICE) - vibrator.vibrate(1000) + vibrator_service = activity.getSystemService(Context.VIBRATOR_SERVICE) + vibrator = cast("android.os.Vibrator", vibrator_service) + + if vibrator and SDK_INT >= 26: + print("Using android's `VibrationEffect` (SDK >= 26)") + VibrationEffect = autoclass("android.os.VibrationEffect") + vibrator.vibrate( + VibrationEffect.createOneShot( + 1000, VibrationEffect.DEFAULT_AMPLITUDE, + ), + ) + elif vibrator: + print("Using deprecated android's vibrate (SDK < 26)") + vibrator.vibrate(1000) + else: + print('Something happened...vibrator service disabled?') def test_ctypes(self, *args): import ctypes diff --git a/testapps/testapp_pillow/main.py b/testapps/testapp_pillow/main.py index ec0aa4add5..fa0c1255df 100644 --- a/testapps/testapp_pillow/main.py +++ b/testapps/testapp_pillow/main.py @@ -337,25 +337,34 @@ def on_stop(self): def test_pyjnius(self, *args): try: - from jnius import autoclass + from jnius import autoclass, cast except ImportError: - raise_error("Could not import pyjnius") + raise_error('Could not import pyjnius') return - - print("Attempting to vibrate with pyjnius") - # Todo: fix vibrate with Api level >= 26 - # vibrate was deprecated in API level 26: - # https://developer.android.com/reference/android/os/Vibrator - try: - PythonActivity = autoclass("org.kivy.android.PythonActivity") - activity = PythonActivity.mActivity - Intent = autoclass("android.content.Intent") - Context = autoclass("android.content.Context") - vibrator = activity.getSystemService(Context.VIBRATOR_SERVICE) - + print('Attempting to vibrate with pyjnius') + ANDROID_VERSION = autoclass('android.os.Build$VERSION') + SDK_INT = ANDROID_VERSION.SDK_INT + + Context = autoclass("android.content.Context") + PythonActivity = autoclass('org.kivy.android.PythonActivity') + activity = PythonActivity.mActivity + + vibrator_service = activity.getSystemService(Context.VIBRATOR_SERVICE) + vibrator = cast("android.os.Vibrator", vibrator_service) + + if vibrator and SDK_INT >= 26: + print("Using android's `VibrationEffect` (SDK >= 26)") + VibrationEffect = autoclass("android.os.VibrationEffect") + vibrator.vibrate( + VibrationEffect.createOneShot( + 1000, VibrationEffect.DEFAULT_AMPLITUDE, + ), + ) + elif vibrator: + print("Using deprecated android's vibrate (SDK < 26)") vibrator.vibrate(1000) - except Exception as e: - raise_error("Error when trying to vibrate: {}".format(e)) + else: + print('Something happened...vibrator service disabled?') def test_ctypes(self, *args): try: diff --git a/testapps/testapp_sqlite_openssl/main.py b/testapps/testapp_sqlite_openssl/main.py index a8083ab961..9b75c755ef 100644 --- a/testapps/testapp_sqlite_openssl/main.py +++ b/testapps/testapp_sqlite_openssl/main.py @@ -192,24 +192,37 @@ def on_pause(self): def test_pyjnius(self, *args): try: - from jnius import autoclass + from jnius import autoclass, cast except ImportError: raise_error('Could not import pyjnius') return - print('Attempting to vibrate with pyjnius') - # PythonActivity = autoclass('org.renpy.android.PythonActivity') - # activity = PythonActivity.mActivity + ANDROID_VERSION = autoclass('android.os.Build$VERSION') + SDK_INT = ANDROID_VERSION.SDK_INT + + Context = autoclass("android.content.Context") PythonActivity = autoclass('org.kivy.android.PythonActivity') activity = PythonActivity.mActivity - Intent = autoclass('android.content.Intent') - Context = autoclass('android.content.Context') - vibrator = activity.getSystemService(Context.VIBRATOR_SERVICE) - vibrator.vibrate(1000) + vibrator_service = activity.getSystemService(Context.VIBRATOR_SERVICE) + vibrator = cast("android.os.Vibrator", vibrator_service) + + if vibrator and SDK_INT >= 26: + print("Using android's `VibrationEffect` (SDK >= 26)") + VibrationEffect = autoclass("android.os.VibrationEffect") + vibrator.vibrate( + VibrationEffect.createOneShot( + 1000, VibrationEffect.DEFAULT_AMPLITUDE, + ), + ) + elif vibrator: + print("Using deprecated android's vibrate (SDK < 26)") + vibrator.vibrate(1000) + else: + print('Something happened...vibrator service disabled?') def test_ctypes(self, *args): - import ctypes + pass def test_numpy(self, *args): import numpy From 5304fea377d60374aaed798cb19d9cf3726622ea Mon Sep 17 00:00:00 2001 From: Gabriel Pettier Date: Sun, 15 Dec 2019 12:12:08 +0100 Subject: [PATCH 42/72] update pyjnius recipe (#2036) --- pythonforandroid/recipes/pyjnius/__init__.py | 2 +- .../genericndkbuild_jnienv_getter.patch | 25 ++++++++++--------- .../recipes/pyjnius/sdl2_jnienv_getter.patch | 24 ++++++++++-------- 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/pythonforandroid/recipes/pyjnius/__init__.py b/pythonforandroid/recipes/pyjnius/__init__.py index b5afdca665..12f3717872 100644 --- a/pythonforandroid/recipes/pyjnius/__init__.py +++ b/pythonforandroid/recipes/pyjnius/__init__.py @@ -8,7 +8,7 @@ class PyjniusRecipe(CythonRecipe): # "6553ad4" is one commit after last release (1.2.0) # it fixes method resolution, required for resolving requestPermissions() - version = '6553ad4' + version = '1.2.1' url = 'https://github.com/kivy/pyjnius/archive/{version}.zip' name = 'pyjnius' depends = [('genericndkbuild', 'sdl2'), 'six'] diff --git a/pythonforandroid/recipes/pyjnius/genericndkbuild_jnienv_getter.patch b/pythonforandroid/recipes/pyjnius/genericndkbuild_jnienv_getter.patch index ff2699415b..3d41dbb64c 100644 --- a/pythonforandroid/recipes/pyjnius/genericndkbuild_jnienv_getter.patch +++ b/pythonforandroid/recipes/pyjnius/genericndkbuild_jnienv_getter.patch @@ -10,16 +10,17 @@ index ac89fec..71daa43 100644 cdef JNIEnv *get_platform_jnienv(): - return SDL_ANDROID_GetJNIEnv() + return WebView_AndroidGetJNIEnv() -diff --git a/setup.py b/setup.py -index 740510f..0c8e55f 100644 ---- a/setup.py -+++ b/setup.py -@@ -53,7 +53,7 @@ except ImportError: +diff --git a/jnius/env.py b/jnius/env.py +--- a/jnius/env.py ++++ b/jnius/env.py +@@ -185,10 +185,10 @@ except ImportError: - if PLATFORM == 'android': - # for android, we use SDL... -- LIBRARIES = ['sdl', 'log'] -+ LIBRARIES = ['main', 'log'] - LIBRARY_DIRS = ['libs/' + getenv('ARCH')] - elif PLATFORM == 'darwin': - import subprocess + def get_libraries(platform): + if platform == 'android': + # for android, we use SDL... +- return ['sdl', 'log'] ++ return ['main', 'log'] + + elif platform == 'win32': + return ['jvm'] + diff --git a/pythonforandroid/recipes/pyjnius/sdl2_jnienv_getter.patch b/pythonforandroid/recipes/pyjnius/sdl2_jnienv_getter.patch index d208e5f69c..3fc63a2935 100644 --- a/pythonforandroid/recipes/pyjnius/sdl2_jnienv_getter.patch +++ b/pythonforandroid/recipes/pyjnius/sdl2_jnienv_getter.patch @@ -10,16 +10,18 @@ index ac89fec..71daa43 100644 cdef JNIEnv *get_platform_jnienv(): - return SDL_ANDROID_GetJNIEnv() + return SDL_AndroidGetJNIEnv() -diff --git a/setup.py b/setup.py +diff --git a/env.py b/env.py index 740510f..0c8e55f 100644 ---- a/setup.py -+++ b/setup.py -@@ -53,7 +53,7 @@ except ImportError: +--- a/jnius/env.py ++++ b/jnius/env.py +@@ -185,10 +185,10 @@ except ImportError: - if PLATFORM == 'android': - # for android, we use SDL... -- LIBRARIES = ['sdl', 'log'] -+ LIBRARIES = ['SDL2', 'log'] - LIBRARY_DIRS = ['libs/' + getenv('ARCH')] - elif PLATFORM == 'darwin': - import subprocess + def get_libraries(platform): + if platform == 'android': + # for android, we use SDL... +- return ['sdl', 'log'] ++ return ['SDL2', 'log'] + + elif platform == 'win32': + return ['jvm'] + From 32e380544b9eb31b615e944edc0a78a879ef9c06 Mon Sep 17 00:00:00 2001 From: Alexander Taylor Date: Sun, 22 Dec 2019 18:44:12 +0000 Subject: [PATCH 43/72] Updated version to 2019.10.06 (#2042) --- pythonforandroid/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonforandroid/__init__.py b/pythonforandroid/__init__.py index db365eba80..dc3923732e 100644 --- a/pythonforandroid/__init__.py +++ b/pythonforandroid/__init__.py @@ -1 +1 @@ -__version__ = '2019.08.09.1.dev0' +__version__ = '2019.10.06' From 777f09b5d42898d29ce0c43f73c8c1213f37a35f Mon Sep 17 00:00:00 2001 From: Alexander Taylor Date: Sun, 22 Dec 2019 18:45:43 +0000 Subject: [PATCH 44/72] Updated version after 2019.10.06 release (#2043) --- pythonforandroid/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonforandroid/__init__.py b/pythonforandroid/__init__.py index dc3923732e..17472f4540 100644 --- a/pythonforandroid/__init__.py +++ b/pythonforandroid/__init__.py @@ -1 +1 @@ -__version__ = '2019.10.06' +__version__ = '2019.10.06.post0' From 2acddd5a8db7008024244be6511ecfabdfce336c Mon Sep 17 00:00:00 2001 From: Alexander Taylor Date: Tue, 24 Dec 2019 00:07:20 +0000 Subject: [PATCH 45/72] Added an extra-manifest-xml option to build.py (#2040) --- pythonforandroid/bootstraps/common/build/build.py | 3 +++ .../bootstraps/sdl2/build/templates/AndroidManifest.tmpl.xml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/pythonforandroid/bootstraps/common/build/build.py b/pythonforandroid/bootstraps/common/build/build.py index 4a53ae04ec..07a5be01d8 100644 --- a/pythonforandroid/bootstraps/common/build/build.py +++ b/pythonforandroid/bootstraps/common/build/build.py @@ -767,6 +767,9 @@ def parse_args(args=None): action='store_false', default=True, help=('Whether to compile to optimised .pyo files, using -OO ' '(strips docstrings and asserts)')) + ap.add_argument('--extra-manifest-xml', default='', + help=('Extra xml to write directly inside the element of' + 'AndroidManifest.xml')) # Put together arguments, and add those from .p4a config file: if args is None: diff --git a/pythonforandroid/bootstraps/sdl2/build/templates/AndroidManifest.tmpl.xml b/pythonforandroid/bootstraps/sdl2/build/templates/AndroidManifest.tmpl.xml index 6f9b03903d..86e69371db 100644 --- a/pythonforandroid/bootstraps/sdl2/build/templates/AndroidManifest.tmpl.xml +++ b/pythonforandroid/bootstraps/sdl2/build/templates/AndroidManifest.tmpl.xml @@ -42,6 +42,9 @@ {% endif %} + {{ args.extra_manifest_xml }} + +