Create individual Python virtualenv's in cmake builds

CMakeLists.txt

@@ -86,9 +86,6 @@ if (NOT FIREBASE_ANDROID_STL STREQUAL "")
   set(ANDROID_STL ${FIREBASE_ANDROID_STL})
 endif()
 
-set(FIREBASE_PYTHON_EXECUTABLE "python" CACHE FILEPATH
-  "The Python interpreter to use, such as one from a venv")
-
 set(FIREBASE_XCODE_TARGET_FORMAT "frameworks" CACHE STRING
                                  "Format to output, 'frameworks' or 'libraries'")
 

analytics/CMakeLists.txt

@@ -14,6 +14,13 @@
 
 # CMake file for the firebase_analytics library
 
+include(python_setup)
+FirebaseSetupPythonInterpreter(
+  OUTVAR MY_PYTHON_EXECUTABLE
+  KEY Analytics
+  REQUIREMENTS absl-py
+)
+
 # Analytics generates header files for default events, parameters, and
 # properties based on the iOS SDK, that are used across all platforms.
 set(analytics_generated_headers_dir
@@ -29,7 +36,7 @@ file(MAKE_DIRECTORY ${analytics_generated_headers_dir})
 function(generate_analytics_header OBJC_FILE CPP_FILE)
   add_custom_command(
     OUTPUT ${CPP_FILE}
-    COMMAND ${FIREBASE_PYTHON_EXECUTABLE} "${CMAKE_CURRENT_LIST_DIR}/generate_constants.py"
+    COMMAND ${MY_PYTHON_EXECUTABLE} "${CMAKE_CURRENT_LIST_DIR}/generate_constants.py"
       "--objc_header=${OBJC_FILE}"
       "--cpp_header=${CPP_FILE}"
     DEPENDS ${OBJC_FILE}

app/CMakeLists.txt

@@ -14,6 +14,13 @@
 
 # CMake file for the firebase_app library
 
+include(python_setup)
+FirebaseSetupPythonInterpreter(
+  OUTVAR MY_PYTHON_EXECUTABLE
+  KEY App
+  REQUIREMENTS absl-py
+)
+
 # Define how to generate google_services_resource_(source/header)
 binary_to_array("google_services_resource"
     "${CMAKE_CURRENT_LIST_DIR}/google_services.fbs"
@@ -47,7 +54,7 @@ add_custom_target(FIREBASE_APP_GENERATED_HEADERS DEPENDS "${version_header}")
 file(MAKE_DIRECTORY ${version_header_dir})
 add_custom_command(
     OUTPUT ${version_header}
-    COMMAND ${FIREBASE_PYTHON_EXECUTABLE} "${FIREBASE_SCRIPT_DIR}/version_header.py"
+    COMMAND ${MY_PYTHON_EXECUTABLE} "${FIREBASE_SCRIPT_DIR}/version_header.py"
         "--input_file=${FIREBASE_SCRIPT_DIR}/cpp_sdk_version.json"
         "--output_file=${version_header}"
         "--build_type=released"
@@ -430,7 +437,7 @@ if (IOS)
   function(generate_analytics_header OBJC_FILE CPP_FILE)
     add_custom_command(
       OUTPUT ${CPP_FILE}
-      COMMAND ${FIREBASE_PYTHON_EXECUTABLE} "${FIREBASE_SOURCE_DIR}/analytics/generate_constants.py"
+      COMMAND ${MY_PYTHON_EXECUTABLE} "${FIREBASE_SOURCE_DIR}/analytics/generate_constants.py"
         "--objc_header=${OBJC_FILE}"
         "--cpp_header=${CPP_FILE}"
       DEPENDS ${OBJC_FILE}

build_scripts/android/build.sh

@@ -59,9 +59,9 @@ set +e
 # network connectivity issues that cause the download to fail.
 gradleparams="-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false\
  -Dmaven.wagon.httpconnectionManager.ttlSeconds=120"
-for retry in {1..10} error; do
+for retry in {1..1} error; do
     if [[ $retry == "error" ]]; then exit 5; fi
-    ./gradlew assembleRelease "${gradleparams}" && break
+    ./gradlew --stacktrace assembleRelease "${gradleparams}" && break
     sleep 300
 done
 set -e

build_scripts/android/cmake_gradle_setup.py

@@ -0,0 +1,151 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Updates all build.gradle files to explicitly specify the version of cmake to use.
+
+This avoids gradle builds using the default cmake version 3.6 that is bundled with the
+Android SDK, since our minimum-supported cmake version is 3.7.
+"""
+
+from __future__ import annotations
+
+import io
+import logging
+import os
+import pathlib
+import platform
+import re
+import shutil
+import sys
+import subprocess
+import tempfile
+from typing import Iterable, Tuple
+import venv
+
+
+def main() -> None:
+  if len(sys.argv) > 1:
+    raise Exception(f"unexpected argument: {sys.argv[1]}")
+
+  logging.basicConfig(
+    level=logging.INFO,
+    format=os.path.basename(__file__) + " %(message)s",
+  )
+
+  run()
+
+
+def run() -> None:
+  logging.info("Python Interpreter: %s", sys.executable)
+  logging.info("Python Version: %s", sys.version)
+
+  (cmake_version, cmake_dir) = install_cmake()
+
+  build_gradle_cmake_files = tuple(sorted(iter_build_gradle_cmake_files()))
+
+  expr = re.compile(r"(\s*)path\s+'.*CMakeLists.txt'(\s*)")
+  for build_gradle_file in build_gradle_cmake_files:
+    logging.info("Setting cmake version to %s in %s", cmake_version, build_gradle_file)
+    with build_gradle_file.open("rt", encoding="utf8") as f:
+      lines = list(f)
+
+    version_line_added = False
+    arguments_modified = False
+    for i in range(len(lines)):
+      line = lines[i]
+      match = expr.fullmatch(line)
+      if match:
+        indent = match.group(1)
+        eol = match.group(2)
+        lines.insert(i+1, indent + "version '" + cmake_version + "'" + eol)
+        version_line_added = True
+      elif line.strip().startswith("arguments "):
+        eol = line[len(line.rstrip()):]
+        lines[i] = (
+          line.rstrip()
+          + (" " if line.rstrip().endswith(",") else ", ")
+          + "'-DFIREBASE_PYTHON_HOST_EXECUTABLE:FILEPATH="
+          + sys.executable.replace("\\", "\\\\")
+          + "'"
+          + ("," if line.rstrip().endswith(",") else "")
+          + eol
+        )
+        arguments_modified = True
+
+    if not version_line_added:
+      raise Exception(f"Unable to find place to insert cmake version in {build_gradle_file}")
+    elif not arguments_modified:
+      raise Exception(f"Unable to find place to edit cmake arguments in {build_gradle_file}")
+
+    with build_gradle_file.open("wt", encoding="utf8") as f:
+      f.writelines(lines)
+
+  cmake_dir_local_properties_line = "cmake.dir=" + re.sub(r"([:\\])", r"\\\1", str(cmake_dir.resolve()))
+  for build_gradle_file in (pathlib.Path.cwd() / "build.gradle",) + build_gradle_cmake_files:
+    local_properties_file = build_gradle_file.parent / "local.properties"
+    logging.info("Setting %s in %s", cmake_dir_local_properties_line, local_properties_file)
+    with local_properties_file.open("at", encoding="utf8") as f:
+      print("", file=f)
+      print(cmake_dir_local_properties_line, file=f)
+
+
+def iter_build_gradle_cmake_files() -> Iterable[pathlib.Path]:
+  for build_gradle_path in pathlib.Path.cwd().glob("**/build.gradle"):
+    with build_gradle_path.open("rt", encoding="utf8", errors="ignore") as f:
+      build_gradle_text = f.read()
+    if "externalNativeBuild" in build_gradle_text and "cmake" in build_gradle_text:
+      yield build_gradle_path
+
+
+def install_cmake() -> Tuple[str, pathlib.Path]:
+  cmake_version = "3.18.0"
+  venv_path_str = tempfile.mkdtemp(prefix=f"cmake-{cmake_version}_", dir=pathlib.Path.cwd())
+  venv_path = pathlib.Path(venv_path_str)
+  logging.info("Creating a Python virtualenv for cmake %s in %s", cmake_version, venv_path)
+
+  venv.create(venv_path, with_pip=True)
+
+  if platform.system() == "Windows":
+    pip_exe = venv_path / "Scripts" / "pip.exe"
+    cmake_exe = venv_path / "Scripts" / "cmake.exe"
+    ninja_exe = venv_path / "Scripts" / "ninja.exe"
+  else:
+    pip_exe = venv_path / "bin" / "pip"
+    cmake_exe = venv_path / "bin" / "cmake"
+    ninja_exe = venv_path / "bin" / "ninja"
+
+  logging.info("Installing cmake %s in %s using %s", cmake_version, venv_path, pip_exe)
+  subprocess.check_output([str(pip_exe), "install", f"cmake=={cmake_version}", "ninja"])
+
+  if not cmake_exe.exists():
+    raise Exception(f"File not found: {cmake_exe}")
+  if not ninja_exe.exists():
+    raise Exception(f"File not found: {ninja_exe}")
+
+  if platform.system() == "Windows":
+    bin_dir = venv_path / "bin"
+    bin_dir.mkdir(exist_ok=True)
+    cmake_bin_exe = bin_dir / cmake_exe.name
+    ninja_bin_exe = bin_dir / ninja_exe.name
+    logging.info("Copying %s to %s", cmake_exe, cmake_bin_exe)
+    shutil.copy2(cmake_exe, cmake_bin_exe)
+    logging.info("Copying %s to %s", ninja_exe, ninja_bin_exe)
+    shutil.copy2(ninja_exe, ninja_bin_exe)
+
+  return (cmake_version, venv_path)
+
+
+if __name__ == "__main__":
+  main()

build_scripts/android/install_prereqs.sh

@@ -60,6 +60,8 @@ if [[ -z "${ANDROID_HOME}" ]]; then
     exit 1
 fi
 
+python build_scripts/android/cmake_gradle_setup.py
+
 if [[ -z "${NDK_ROOT}" || -z $(grep "Pkg\.Revision = 16\." "${NDK_ROOT}/source.properties") ]]; then
     if [[ -d /tmp/android-ndk-r16b && \
 	      -n $(grep "Pkg\.Revision = 16\." "/tmp/android-ndk-r16b/source.properties") ]]; then

cmake/binary_to_array.cmake

@@ -26,6 +26,13 @@
 #   CPP_NAMESPACE: The namespace to use in the generated files.
 #   OUTPUT_DIRECTORY: Where the generated files should be written to.
 function(binary_to_array NAME INPUT CPP_NAMESPACE OUTPUT_DIRECTORY)
+  include(python_setup)
+  FirebaseSetupPythonInterpreter(
+    OUTVAR MY_PYTHON_EXECUTABLE
+    KEY BinaryToArray
+    REQUIREMENTS absl-py
+  )
+
   # Guarantee the output directory exists
   file(MAKE_DIRECTORY ${OUTPUT_DIRECTORY})
 
@@ -39,7 +46,7 @@ function(binary_to_array NAME INPUT CPP_NAMESPACE OUTPUT_DIRECTORY)
     OUTPUT ${output_source}
       ${output_header}
     DEPENDS ${INPUT}
-    COMMAND ${FIREBASE_PYTHON_EXECUTABLE} "${FIREBASE_SCRIPT_DIR}/binary_to_array.py"
+    COMMAND ${MY_PYTHON_EXECUTABLE} "${FIREBASE_SCRIPT_DIR}/binary_to_array.py"
       "--input=${INPUT}"
       "--output_header=${output_header}"
       "--output_source=${output_source}"