diff --git a/CMakeLists.txt b/CMakeLists.txt index 5d3410f251..cb84c0e6df 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -86,9 +86,6 @@ if (NOT FIREBASE_ANDROID_STL STREQUAL "") set(ANDROID_STL ${FIREBASE_ANDROID_STL}) endif() -set(FIREBASE_PYTHON_EXECUTABLE "python" CACHE FILEPATH - "The Python interpreter to use, such as one from a venv") - set(FIREBASE_XCODE_TARGET_FORMAT "frameworks" CACHE STRING "Format to output, 'frameworks' or 'libraries'") diff --git a/analytics/CMakeLists.txt b/analytics/CMakeLists.txt index 6bc41fbc51..e511c7edb0 100644 --- a/analytics/CMakeLists.txt +++ b/analytics/CMakeLists.txt @@ -14,6 +14,13 @@ # CMake file for the firebase_analytics library +include(python_setup) +FirebaseSetupPythonInterpreter( + OUTVAR MY_PYTHON_EXECUTABLE + KEY Analytics + REQUIREMENTS absl-py +) + # Analytics generates header files for default events, parameters, and # properties based on the iOS SDK, that are used across all platforms. set(analytics_generated_headers_dir @@ -29,7 +36,7 @@ file(MAKE_DIRECTORY ${analytics_generated_headers_dir}) function(generate_analytics_header OBJC_FILE CPP_FILE) add_custom_command( OUTPUT ${CPP_FILE} - COMMAND ${FIREBASE_PYTHON_EXECUTABLE} "${CMAKE_CURRENT_LIST_DIR}/generate_constants.py" + COMMAND ${MY_PYTHON_EXECUTABLE} "${CMAKE_CURRENT_LIST_DIR}/generate_constants.py" "--objc_header=${OBJC_FILE}" "--cpp_header=${CPP_FILE}" DEPENDS ${OBJC_FILE} diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt index f2d1eba528..113a1d32b1 100644 --- a/app/CMakeLists.txt +++ b/app/CMakeLists.txt @@ -14,6 +14,13 @@ # CMake file for the firebase_app library +include(python_setup) +FirebaseSetupPythonInterpreter( + OUTVAR MY_PYTHON_EXECUTABLE + KEY App + REQUIREMENTS absl-py +) + # Define how to generate google_services_resource_(source/header) binary_to_array("google_services_resource" "${CMAKE_CURRENT_LIST_DIR}/google_services.fbs" @@ -47,7 +54,7 @@ add_custom_target(FIREBASE_APP_GENERATED_HEADERS DEPENDS "${version_header}") file(MAKE_DIRECTORY ${version_header_dir}) add_custom_command( OUTPUT ${version_header} - COMMAND ${FIREBASE_PYTHON_EXECUTABLE} "${FIREBASE_SCRIPT_DIR}/version_header.py" + COMMAND ${MY_PYTHON_EXECUTABLE} "${FIREBASE_SCRIPT_DIR}/version_header.py" "--input_file=${FIREBASE_SCRIPT_DIR}/cpp_sdk_version.json" "--output_file=${version_header}" "--build_type=released" @@ -430,7 +437,7 @@ if (IOS) function(generate_analytics_header OBJC_FILE CPP_FILE) add_custom_command( OUTPUT ${CPP_FILE} - COMMAND ${FIREBASE_PYTHON_EXECUTABLE} "${FIREBASE_SOURCE_DIR}/analytics/generate_constants.py" + COMMAND ${MY_PYTHON_EXECUTABLE} "${FIREBASE_SOURCE_DIR}/analytics/generate_constants.py" "--objc_header=${OBJC_FILE}" "--cpp_header=${CPP_FILE}" DEPENDS ${OBJC_FILE} diff --git a/build_scripts/android/build.sh b/build_scripts/android/build.sh index f2373e3ea9..64a2cbc84c 100755 --- a/build_scripts/android/build.sh +++ b/build_scripts/android/build.sh @@ -59,9 +59,9 @@ set +e # network connectivity issues that cause the download to fail. gradleparams="-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false\ -Dmaven.wagon.httpconnectionManager.ttlSeconds=120" -for retry in {1..10} error; do +for retry in {1..1} error; do if [[ $retry == "error" ]]; then exit 5; fi - ./gradlew assembleRelease "${gradleparams}" && break + ./gradlew --stacktrace assembleRelease "${gradleparams}" && break sleep 300 done set -e diff --git a/build_scripts/android/cmake_gradle_setup.py b/build_scripts/android/cmake_gradle_setup.py new file mode 100644 index 0000000000..a6c8af25f4 --- /dev/null +++ b/build_scripts/android/cmake_gradle_setup.py @@ -0,0 +1,151 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Updates all build.gradle files to explicitly specify the version of cmake to use. + +This avoids gradle builds using the default cmake version 3.6 that is bundled with the +Android SDK, since our minimum-supported cmake version is 3.7. +""" + +from __future__ import annotations + +import io +import logging +import os +import pathlib +import platform +import re +import shutil +import sys +import subprocess +import tempfile +from typing import Iterable, Tuple +import venv + + +def main() -> None: + if len(sys.argv) > 1: + raise Exception(f"unexpected argument: {sys.argv[1]}") + + logging.basicConfig( + level=logging.INFO, + format=os.path.basename(__file__) + " %(message)s", + ) + + run() + + +def run() -> None: + logging.info("Python Interpreter: %s", sys.executable) + logging.info("Python Version: %s", sys.version) + + (cmake_version, cmake_dir) = install_cmake() + + build_gradle_cmake_files = tuple(sorted(iter_build_gradle_cmake_files())) + + expr = re.compile(r"(\s*)path\s+'.*CMakeLists.txt'(\s*)") + for build_gradle_file in build_gradle_cmake_files: + logging.info("Setting cmake version to %s in %s", cmake_version, build_gradle_file) + with build_gradle_file.open("rt", encoding="utf8") as f: + lines = list(f) + + version_line_added = False + arguments_modified = False + for i in range(len(lines)): + line = lines[i] + match = expr.fullmatch(line) + if match: + indent = match.group(1) + eol = match.group(2) + lines.insert(i+1, indent + "version '" + cmake_version + "'" + eol) + version_line_added = True + elif line.strip().startswith("arguments "): + eol = line[len(line.rstrip()):] + lines[i] = ( + line.rstrip() + + (" " if line.rstrip().endswith(",") else ", ") + + "'-DFIREBASE_PYTHON_HOST_EXECUTABLE:FILEPATH=" + + sys.executable.replace("\\", "\\\\") + + "'" + + ("," if line.rstrip().endswith(",") else "") + + eol + ) + arguments_modified = True + + if not version_line_added: + raise Exception(f"Unable to find place to insert cmake version in {build_gradle_file}") + elif not arguments_modified: + raise Exception(f"Unable to find place to edit cmake arguments in {build_gradle_file}") + + with build_gradle_file.open("wt", encoding="utf8") as f: + f.writelines(lines) + + cmake_dir_local_properties_line = "cmake.dir=" + re.sub(r"([:\\])", r"\\\1", str(cmake_dir.resolve())) + for build_gradle_file in (pathlib.Path.cwd() / "build.gradle",) + build_gradle_cmake_files: + local_properties_file = build_gradle_file.parent / "local.properties" + logging.info("Setting %s in %s", cmake_dir_local_properties_line, local_properties_file) + with local_properties_file.open("at", encoding="utf8") as f: + print("", file=f) + print(cmake_dir_local_properties_line, file=f) + + +def iter_build_gradle_cmake_files() -> Iterable[pathlib.Path]: + for build_gradle_path in pathlib.Path.cwd().glob("**/build.gradle"): + with build_gradle_path.open("rt", encoding="utf8", errors="ignore") as f: + build_gradle_text = f.read() + if "externalNativeBuild" in build_gradle_text and "cmake" in build_gradle_text: + yield build_gradle_path + + +def install_cmake() -> Tuple[str, pathlib.Path]: + cmake_version = "3.18.0" + venv_path_str = tempfile.mkdtemp(prefix=f"cmake-{cmake_version}_", dir=pathlib.Path.cwd()) + venv_path = pathlib.Path(venv_path_str) + logging.info("Creating a Python virtualenv for cmake %s in %s", cmake_version, venv_path) + + venv.create(venv_path, with_pip=True) + + if platform.system() == "Windows": + pip_exe = venv_path / "Scripts" / "pip.exe" + cmake_exe = venv_path / "Scripts" / "cmake.exe" + ninja_exe = venv_path / "Scripts" / "ninja.exe" + else: + pip_exe = venv_path / "bin" / "pip" + cmake_exe = venv_path / "bin" / "cmake" + ninja_exe = venv_path / "bin" / "ninja" + + logging.info("Installing cmake %s in %s using %s", cmake_version, venv_path, pip_exe) + subprocess.check_output([str(pip_exe), "install", f"cmake=={cmake_version}", "ninja"]) + + if not cmake_exe.exists(): + raise Exception(f"File not found: {cmake_exe}") + if not ninja_exe.exists(): + raise Exception(f"File not found: {ninja_exe}") + + if platform.system() == "Windows": + bin_dir = venv_path / "bin" + bin_dir.mkdir(exist_ok=True) + cmake_bin_exe = bin_dir / cmake_exe.name + ninja_bin_exe = bin_dir / ninja_exe.name + logging.info("Copying %s to %s", cmake_exe, cmake_bin_exe) + shutil.copy2(cmake_exe, cmake_bin_exe) + logging.info("Copying %s to %s", ninja_exe, ninja_bin_exe) + shutil.copy2(ninja_exe, ninja_bin_exe) + + return (cmake_version, venv_path) + + +if __name__ == "__main__": + main() diff --git a/build_scripts/android/install_prereqs.sh b/build_scripts/android/install_prereqs.sh index ffc601da2d..1f1c0b9c0a 100755 --- a/build_scripts/android/install_prereqs.sh +++ b/build_scripts/android/install_prereqs.sh @@ -60,6 +60,8 @@ if [[ -z "${ANDROID_HOME}" ]]; then exit 1 fi +python build_scripts/android/cmake_gradle_setup.py + if [[ -z "${NDK_ROOT}" || -z $(grep "Pkg\.Revision = 16\." "${NDK_ROOT}/source.properties") ]]; then if [[ -d /tmp/android-ndk-r16b && \ -n $(grep "Pkg\.Revision = 16\." "/tmp/android-ndk-r16b/source.properties") ]]; then diff --git a/cmake/binary_to_array.cmake b/cmake/binary_to_array.cmake index d95bb61c6d..12e685518a 100644 --- a/cmake/binary_to_array.cmake +++ b/cmake/binary_to_array.cmake @@ -26,6 +26,13 @@ # CPP_NAMESPACE: The namespace to use in the generated files. # OUTPUT_DIRECTORY: Where the generated files should be written to. function(binary_to_array NAME INPUT CPP_NAMESPACE OUTPUT_DIRECTORY) + include(python_setup) + FirebaseSetupPythonInterpreter( + OUTVAR MY_PYTHON_EXECUTABLE + KEY BinaryToArray + REQUIREMENTS absl-py + ) + # Guarantee the output directory exists file(MAKE_DIRECTORY ${OUTPUT_DIRECTORY}) @@ -39,7 +46,7 @@ function(binary_to_array NAME INPUT CPP_NAMESPACE OUTPUT_DIRECTORY) OUTPUT ${output_source} ${output_header} DEPENDS ${INPUT} - COMMAND ${FIREBASE_PYTHON_EXECUTABLE} "${FIREBASE_SCRIPT_DIR}/binary_to_array.py" + COMMAND ${MY_PYTHON_EXECUTABLE} "${FIREBASE_SCRIPT_DIR}/binary_to_array.py" "--input=${INPUT}" "--output_header=${output_header}" "--output_source=${output_source}" diff --git a/cmake/python_setup.cmake b/cmake/python_setup.cmake new file mode 100644 index 0000000000..bdb7b9f6ae --- /dev/null +++ b/cmake/python_setup.cmake @@ -0,0 +1,183 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Sets up an isolated Python interpreter, installing required dependencies. +# +# This function does the following: +# 1. Finds a Python interpreter using the best-available built-in cmake +# mechanism do do so. This is referred to as the "host" interpreter. +# 2. Creates a Python virtualenv in the cmake binary directory using the +# host Python interpreter found in the previous step. +# 3. Locates the Python interpreter in the virtualenv and sets its path in +# the specified OUTVAR variable. +# 4. Runs `pip install` to install the specified required dependencies, if any, +# in the virtualenv. +# +# This function also writes "stamp files" into the virtualenv. These files +# are used to determine if the virtualenv is up-to-date from a previous cmake +# run or if it needs to be recreated from scratch. It will simply be re-used if +# possible. +# +# If any errors occur (e.g. cannot install one of the given requirements) then a +# fatal error is logged, causing the cmake processing to terminate. +# +# See https://docs.python.org/3/library/venv.html for details about virtualenv. +# +# Arguments: +# OUTVAR - The name of the variable into which to store the path of the +# Python executable from the virtualenv. +# KEY - A unique key to ensure isolation from other Python virtualenv +# environments created by this function. This value will be incorporated +# into the path of the virtualenv and incorporated into the name of the +# cmake cache variable that stores its path. +# REQUIREMENTS - (Optional) A list of Python packages to install in the +# virtualenv. These will be given as arguments to `pip install`. +# +# Example: +# include(python_setup) +# FirebaseSetupPythonInterpreter( +# OUTVAR MY_PYTHON_EXECUTABLE +# KEY ScanStuff +# REQUIREMENTS six absl-py +# ) +# execute_process(COMMAND "${MY_PYTHON_EXECUTABLE}" scan_stuff.py) +function(FirebaseSetupPythonInterpreter) + cmake_parse_arguments( + PARSE_ARGV 0 + ARG + "" # zero-value arguments + "OUTVAR;KEY" # single-value arguments + "REQUIREMENTS" # multi-value arguments + ) + + # Validate this function's arguments. + if("${ARG_OUTVAR}" STREQUAL "") + message(FATAL_ERROR "OUTVAR must be specified to ${CMAKE_CURRENT_FUNCTION}") + elseif("${ARG_KEY}" STREQUAL "") + message(FATAL_ERROR "KEY must be specified to ${CMAKE_CURRENT_FUNCTION}") + endif() + + # Calculate the name of the cmake *cache* variable into which to store the + # path of the Python interpreter from the virtualenv. + set(CACHEVAR "FIREBASE_PYTHON_EXECUTABLE_${ARG_KEY}") + + set(LOG_PREFIX "${CMAKE_CURRENT_FUNCTION}(${ARG_KEY})") + + # Find a "host" Python interpreter using the best available mechanism. + if(${CMAKE_VERSION} VERSION_LESS "3.12") + include(FindPythonInterp) + set(DEFAULT_PYTHON_HOST_EXECUTABLE "${PYTHON_EXECUTABLE}") + else() + find_package(Python3 COMPONENTS Interpreter REQUIRED) + set(DEFAULT_PYTHON_HOST_EXECUTABLE "${Python3_EXECUTABLE}") + endif() + + # Get the host Python interpreter on the host system to use. + set( + FIREBASE_PYTHON_HOST_EXECUTABLE + "${DEFAULT_PYTHON_HOST_EXECUTABLE}" + CACHE FILEPATH + "The Python interpreter on the host system to use" + ) + + # Check if the virtualenv is already up-to-date by examining the contents of + # its stamp files. The stamp files store the path of the host Python + # interpreter and the dependencies that were installed by pip. If both of + # these files exist and contain the same Python interpreter and dependencies + # then just re-use the virtualenv; otherwise, re-create it. + set(PYVENV_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/pyvenv/${ARG_KEY}") + set(STAMP_FILE1 "${PYVENV_DIRECTORY}/cmake_firebase_python_stamp1.txt") + set(STAMP_FILE2 "${PYVENV_DIRECTORY}/cmake_firebase_python_stamp2.txt") + + if(EXISTS "${STAMP_FILE1}" AND EXISTS "${STAMP_FILE2}") + file(READ "${STAMP_FILE1}" STAMP_FILE1_CONTENTS) + file(READ "${STAMP_FILE2}" STAMP_FILE2_CONTENTS) + if( + ("${STAMP_FILE1_CONTENTS}" STREQUAL "${FIREBASE_PYTHON_HOST_EXECUTABLE}") + AND + ("${STAMP_FILE2_CONTENTS}" STREQUAL "${ARG_REQUIREMENTS}") + ) + set("${ARG_OUTVAR}" "$CACHE{${CACHEVAR}}" PARENT_SCOPE) + message(STATUS "${LOG_PREFIX}: Using Python interpreter: $CACHE{${CACHEVAR}}") + return() + endif() + endif() + + # Create the virtualenv. + message(STATUS + "${LOG_PREFIX}: Creating Python virtualenv in ${PYVENV_DIRECTORY} " + "using ${FIREBASE_PYTHON_HOST_EXECUTABLE}" + ) + file(REMOVE_RECURSE "${PYVENV_DIRECTORY}") + execute_process( + COMMAND + "${FIREBASE_PYTHON_HOST_EXECUTABLE}" + -m + venv + "${PYVENV_DIRECTORY}" + RESULT_VARIABLE + FIREBASE_PYVENV_CREATE_RESULT + ) + if(NOT FIREBASE_PYVENV_CREATE_RESULT EQUAL 0) + message(FATAL_ERROR + "Failed to create a Python virtualenv in ${PYVENV_DIRECTORY} " + "using ${FIREBASE_PYTHON_HOST_EXECUTABLE}") + endif() + + # Find the Python interpreter in the virtualenv. + find_program( + "${CACHEVAR}" + DOC "The Python interpreter to use for ${ARG_KEY}" + NAMES python3 python + PATHS "${PYVENV_DIRECTORY}" + PATH_SUFFIXES bin Scripts + NO_DEFAULT_PATH + ) + if(NOT ${CACHEVAR}) + message(FATAL_ERROR "Unable to find Python executable in ${PYVENV_DIRECTORY}") + else() + set(PYTHON_EXECUTABLE "$CACHE{${CACHEVAR}}") + message(STATUS "${LOG_PREFIX}: Found Python executable in virtualenv: ${PYTHON_EXECUTABLE}") + endif() + + # Install the dependencies in the virtualenv, if any are requested. + if(NOT ("${ARG_REQUIREMENTS}" STREQUAL "")) + message(STATUS + "${LOG_PREFIX}: Installing Python dependencies into " + "${PYVENV_DIRECTORY}: ${ARG_REQUIREMENTS}" + ) + execute_process( + COMMAND + "${PYTHON_EXECUTABLE}" + -m + pip + install + ${ARG_REQUIREMENTS} + RESULT_VARIABLE + PIP_INSTALL_RESULT + ) + if(NOT PIP_INSTALL_RESULT EQUAL 0) + message(FATAL_ERROR + "Failed to install Python dependencies into " + "${PYVENV_DIRECTORY}: ${ARG_REQUIREMENTS}" + ) + endif() + endif() + + # Write the stamp files. + file(WRITE "${STAMP_FILE1}" "${FIREBASE_PYTHON_HOST_EXECUTABLE}") + file(WRITE "${STAMP_FILE2}" "${ARG_REQUIREMENTS}") + + set("${ARG_OUTVAR}" "${PYTHON_EXECUTABLE}" PARENT_SCOPE) +endfunction(FirebaseSetupPythonInterpreter) diff --git a/external/vcpkg_custom_data/toolchains/linux_32.cmake b/external/vcpkg_custom_data/toolchains/linux_32.cmake index 778e986303..c7f57d3d38 100644 --- a/external/vcpkg_custom_data/toolchains/linux_32.cmake +++ b/external/vcpkg_custom_data/toolchains/linux_32.cmake @@ -12,6 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +include(python_setup) +FirebaseSetupPythonInterpreter( + OUTVAR MY_PYTHON_EXECUTABLE + KEY VcpkgLinux32 +) + # Toolchain settings for building 32-bit Linux libraries set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32") @@ -436,7 +442,7 @@ function(add_executable name) elseif(_VCPKG_TARGET_TRIPLET_PLAT MATCHES "osx") if (NOT MACOSX_BUNDLE_IDX EQUAL -1) add_custom_command(TARGET ${name} POST_BUILD - COMMAND ${FIREBASE_PYTHON_EXECUTABLE} ${_VCPKG_TOOLCHAIN_DIR}/osx/applocal.py + COMMAND ${MY_PYTHON_EXECUTABLE} ${_VCPKG_TOOLCHAIN_DIR}/osx/applocal.py $ "${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}$<$:/debug>" ) diff --git a/external/vcpkg_custom_data/toolchains/macos_arm64.cmake b/external/vcpkg_custom_data/toolchains/macos_arm64.cmake index c590c582ce..ca66f5ecd4 100644 --- a/external/vcpkg_custom_data/toolchains/macos_arm64.cmake +++ b/external/vcpkg_custom_data/toolchains/macos_arm64.cmake @@ -12,6 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +include(python_setup) +FirebaseSetupPythonInterpreter( + OUTVAR MY_PYTHON_EXECUTABLE + KEY VcpkgMacOSArm64 +) + # Toolchain settings for building ARM64 MacOS libraries set(VCPKG_TARGET_ARCHITECTURE arm64) set(VCPKG_CRT_LINKAGE dynamic) @@ -438,7 +444,7 @@ function(add_executable name) elseif(_VCPKG_TARGET_TRIPLET_PLAT MATCHES "osx") if (NOT MACOSX_BUNDLE_IDX EQUAL -1) add_custom_command(TARGET ${name} POST_BUILD - COMMAND ${FIREBASE_PYTHON_EXECUTABLE} ${_VCPKG_TOOLCHAIN_DIR}/osx/applocal.py + COMMAND ${MY_PYTHON_EXECUTABLE} ${_VCPKG_TOOLCHAIN_DIR}/osx/applocal.py $ "${_VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}$<$:/debug>" ) diff --git a/scripts/gha/build_testapps.py b/scripts/gha/build_testapps.py index 6a3d71785d..44e78b70f2 100644 --- a/scripts/gha/build_testapps.py +++ b/scripts/gha/build_testapps.py @@ -76,6 +76,7 @@ import datetime import os import platform +import re import shutil import subprocess import sys @@ -477,7 +478,8 @@ def _build_android(project_dir, sdk_dir): """Builds an Android binary (apk).""" if platform.system() == "Windows": gradlew = "gradlew.bat" - sdk_dir = sdk_dir.replace("\\", "/") # Gradle misinterprets backslashes. + # Gradle misinterprets backslashes and colons. + sdk_dir = re.sub(r"([:\\])", r"\\\1", sdk_dir) else: gradlew = "./gradlew" logging.info("Patching gradle properties with path to SDK")