From 8e3e81033c55695de179f52359b7b91a853a5d19 Mon Sep 17 00:00:00 2001 From: Alan de Freitas Date: Wed, 30 Jul 2025 17:33:52 -0500 Subject: [PATCH] build: installation workflow uses Ninja for all projects --- CMakeUserPresets.json.example | 126 +++++++++------------------------- bootstrap.py | 108 ++++++++++++++++++++++++++--- 2 files changed, 131 insertions(+), 103 deletions(-) diff --git a/CMakeUserPresets.json.example b/CMakeUserPresets.json.example index 449dc055c7..1beab242ec 100644 --- a/CMakeUserPresets.json.example +++ b/CMakeUserPresets.json.example @@ -14,8 +14,8 @@ "generator": "Ninja", "binaryDir": "${sourceDir}/build/${presetName}", "cacheVariables": { - "LLVM_ROOT": "C:\\Users\\$env{USERNAME}\\Libraries\\llvm\\install\\MSVC\\DebWithOpt", - "Clang_ROOT": "C:\\Users\\$env{USERNAME}\\Libraries\\llvm\\install\\MSVC\\DebWithOpt", + "LLVM_ROOT": "C:\\Users\\$env{USERNAME}\\Libraries\\llvm\\install\\MSVC\\Debug", + "Clang_ROOT": "C:\\Users\\$env{USERNAME}\\Libraries\\llvm\\install\\MSVC\\Debug", "CMAKE_EXPORT_COMPILE_COMMANDS": "ON", "CMAKE_CXX_FLAGS": "/W4", "CMAKE_C_FLAGS": "/W4" @@ -34,17 +34,6 @@ } } }, - { - "name": "debwithopt-msvc", - "displayName": "Debug with Optimizations MSVC", - "description": "Build on Windows + MSVC natively (Debug with optimizations). This is the preset typically used for development. CI tests it and generates LLVM binaries for it.", - "inherits": "debug-msvc", - "binaryDir": "${sourceDir}/build/${presetName}", - "cacheVariables": { - "LLVM_ROOT": "C:\\Users\\$env{USERNAME}\\Libraries\\llvm\\DebWithOpt", - "Clang_ROOT": "C:\\Users\\$env{USERNAME}\\Libraries\\llvm\\DebWithOpt" - } - }, { "name": "release-msvc", "displayName": "Release MSVC", @@ -57,29 +46,6 @@ "Clang_ROOT": "C:\\Users\\$env{USERNAME}\\Libraries\\llvm\\Release" } }, - { - "name": "relwithdebinfo-msvc", - "displayName": "RelWithDebInfo MSVC", - "description": "Build on Windows + MSVC natively (RelWithDebInfo). This is the preset used for the release builds.", - "inherits": "debug-msvc", - "binaryDir": "${sourceDir}/build/${presetName}", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "RelWithDebInfo", - "LLVM_ROOT": "C:\\Users\\$env{USERNAME}\\Libraries\\llvm\\Release", - "Clang_ROOT": "C:\\Users\\$env{USERNAME}\\Libraries\\llvm\\Release" - } - }, - { - "name": "debwithopt-clang-cl", - "displayName": "Debug with Optimizations Clang-CL", - "description": "Build on Windows + Clang natively. Windows developers can use this preset to catch Clang-specific issues natively.", - "inherits": "debwithopt-msvc", - "binaryDir": "${sourceDir}/build/${presetName}", - "cacheVariables": { - "CMAKE_C_COMPILER": "clang-cl.exe", - "CMAKE_CXX_COMPILER": "clang-cl.exe" - } - }, { "name": "debug-wsl-gcc", "displayName": "Debug WSL GCC", @@ -117,18 +83,6 @@ "CMAKE_CXX_FLAGS": "-fsanitize=address -fno-omit-frame-pointer -g -O0 -fno-inline-functions" } }, - { - "name": "debwithopt-wsl-gcc", - "displayName": "Debug with Optimizations WSL GCC", - "description": "Build on WSL + GCC (Debug with optimizations). Windows developers can use this preset to catch GCC-specific issues with WSL. WSL tends to be very slow, so this is not recommended for interactive development.", - "inherits": "debug-wsl-gcc", - "binaryDir": "${sourceDir}/build/${presetName}", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "LLVM_ROOT": "/home/$env{USER}/libraries/llvm-project/llvm/install/Linux/Release", - "Clang_ROOT": "/home/$env{USER}/libraries/llvm-project/llvm/install/Linux/Release" - } - }, { "name": "release-wsl-gcc", "displayName": "Release WSL GCC", @@ -171,7 +125,7 @@ }, { "name": "debug-macos", - "displayName": "Debug macOS", + "displayName": "Debug (macOS)", "description": "Preset for building MrDocs in Debug mode with the default compiler in macOS.", "inherits": "debug", "binaryDir": "${sourceDir}/build/${presetName}", @@ -186,7 +140,8 @@ "MRDOCS_BUILD_TESTS": true, "MRDOCS_BUILD_DOCS": false, "MRDOCS_GENERATE_REFERENCE": false, - "MRDOCS_GENERATE_ANTORA_REFERENCE": false + "MRDOCS_GENERATE_ANTORA_REFERENCE": false, + "CMAKE_MAKE_PROGRAM": "$env{HOME}/Developer/cpp-libs/ninja/ninja" }, "condition": { "type": "equals", @@ -195,39 +150,42 @@ }, "warnings": { "unusedCli": false - } + }, + "generator": "Ninja" }, { - "name": "debwithopt-macos", - "displayName": "DebWithOpt macOS", - "description": "Preset for building MrDocs in DebWithOpt mode with the default compiler in macOS.", + "name": "debug-macos-fast", + "displayName": "Debug with Optimized Dependencies (macOS)", + "description": "Preset for building MrDocs in Debug mode with the default compiler in macOS.", "inherits": "debug", "binaryDir": "${sourceDir}/build/${presetName}", "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", - "LLVM_ROOT": "$env{HOME}/Developer/cpp-libs/llvm-project/install/debwithopt", - "Clang_ROOT": "$env{HOME}/Developer/cpp-libs/llvm-project/install/debwithopt", - "duktape_ROOT": "$env{HOME}/Developer/cpp-libs/duktape/install/debwithopt", - "Duktape_ROOT": "$env{HOME}/Developer/cpp-libs/duktape/install/debwithopt", + "LLVM_ROOT": "$env{HOME}/Developer/cpp-libs/llvm-project/install/release", + "Clang_ROOT": "$env{HOME}/Developer/cpp-libs/llvm-project/install/release", + "duktape_ROOT": "$env{HOME}/Developer/cpp-libs/duktape/install/release", + "Duktape_ROOT": "$env{HOME}/Developer/cpp-libs/duktape/install/release", "libxml2_ROOT": "$env{HOME}/Developer/cpp-libs/libxml2/install/release", "LibXml2_ROOT": "$env{HOME}/Developer/cpp-libs/libxml2/install/release", "MRDOCS_BUILD_TESTS": true, "MRDOCS_BUILD_DOCS": false, "MRDOCS_GENERATE_REFERENCE": false, - "MRDOCS_GENERATE_ANTORA_REFERENCE": false + "MRDOCS_GENERATE_ANTORA_REFERENCE": false, + "CMAKE_MAKE_PROGRAM": "$env{HOME}/Developer/cpp-libs/ninja/ninja" + }, + "warnings": { + "unusedCli": false }, "condition": { "type": "equals", "lhs": "${hostSystemName}", "rhs": "Darwin" }, - "warnings": { - "unusedCli": false - } + "generator": "Ninja" }, { "name": "release-macos", - "displayName": "Release macOS", + "displayName": "Release (macOS)", "description": "Preset for building MrDocs in Release mode with the default compiler in macOS.", "inherits": "release", "binaryDir": "${sourceDir}/build/${presetName}", @@ -242,7 +200,8 @@ "MRDOCS_BUILD_TESTS": true, "MRDOCS_BUILD_DOCS": false, "MRDOCS_GENERATE_REFERENCE": false, - "MRDOCS_GENERATE_ANTORA_REFERENCE": false + "MRDOCS_GENERATE_ANTORA_REFERENCE": false, + "CMAKE_MAKE_PROGRAM": "$env{HOME}/Developer/cpp-libs/ninja/ninja" }, "condition": { "type": "equals", @@ -251,11 +210,12 @@ }, "warnings": { "unusedCli": false - } + }, + "generator": "Ninja" }, { "name": "release-macos-gcc", - "displayName": "Release macOS (gcc)", + "displayName": "Release (macOS) (gcc)", "description": "Preset for building MrDocs in Release mode with the gcc compiler in macOS.", "inherits": "release", "binaryDir": "${sourceDir}/build/${presetName}", @@ -270,7 +230,10 @@ "MRDOCS_BUILD_TESTS": true, "MRDOCS_BUILD_DOCS": false, "MRDOCS_GENERATE_REFERENCE": false, - "MRDOCS_GENERATE_ANTORA_REFERENCE": false + "MRDOCS_GENERATE_ANTORA_REFERENCE": false, + "CMAKE_C_COMPILER": "/usr/bin/gcc", + "CMAKE_CXX_COMPILER": "/usr/bin/g++", + "CMAKE_MAKE_PROGRAM": "$env{HOME}/Developer/cpp-libs/ninja/ninja" }, "warnings": { "unusedCli": false @@ -279,35 +242,8 @@ "type": "equals", "lhs": "${hostSystemName}", "rhs": "Darwin" - } - }, - { - "name": "debug-macos-fast", - "displayName": "Debug with Optimized Dependencies (macOS)", - "description": "Preset for building MrDocs in Debug mode with the default compiler in macOS.", - "inherits": "debug", - "binaryDir": "${sourceDir}/build/${presetName}", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "LLVM_ROOT": "$env{HOME}/Developer/cpp-libs/llvm-project/install/release", - "Clang_ROOT": "$env{HOME}/Developer/cpp-libs/llvm-project/install/release", - "duktape_ROOT": "$env{HOME}/Developer/cpp-libs/duktape/install/release", - "Duktape_ROOT": "$env{HOME}/Developer/cpp-libs/duktape/install/release", - "libxml2_ROOT": "$env{HOME}/Developer/cpp-libs/libxml2/install/release", - "LibXml2_ROOT": "$env{HOME}/Developer/cpp-libs/libxml2/install/release", - "MRDOCS_BUILD_TESTS": true, - "MRDOCS_BUILD_DOCS": false, - "MRDOCS_GENERATE_REFERENCE": false, - "MRDOCS_GENERATE_ANTORA_REFERENCE": false }, - "warnings": { - "unusedCli": false - }, - "condition": { - "type": "equals", - "lhs": "${hostSystemName}", - "rhs": "Darwin" - } + "generator": "Ninja" } ] } \ No newline at end of file diff --git a/bootstrap.py b/bootstrap.py index b23ed3cbe4..76f2580371 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -12,6 +12,7 @@ import subprocess import os import sys +import platform import shutil from dataclasses import dataclass, field import dataclasses @@ -20,6 +21,7 @@ import json import shlex import re +import zipfile from functools import lru_cache @@ -51,11 +53,14 @@ class InstallOptions: cc: str = '' cxx: str = '' - # Tools + # Required tools git_path: str = '' cmake_path: str = '' java_path: str = '' + # Optional tools + ninja_path: str = '' + # MrDocs mrdocs_src_dir: str = field( default_factory=lambda: os.getcwd() if running_from_mrdocs_source_dir() else os.path.join(os.getcwd(), @@ -114,6 +119,7 @@ class InstallOptions: "git_path": "Path to the git executable, if not in system PATH.", "cmake_path": "Path to the cmake executable, if not in system PATH.", "java_path": "Path to the java executable, if not in system PATH.", + "ninja_path": "Path to the ninja executable. Leave empty to download it automatically.", "mrdocs_src_dir": "MrDocs source directory.", "mrdocs_repo": "URL of the MrDocs repository to clone.", "mrdocs_branch": "Branch or tag of the MrDocs repository to use.", @@ -448,6 +454,8 @@ def cmake_workflow(self, src_dir, build_type, build_dir, install_dir, extra_args cmake_build_type = build_type if not build_type_is_optimizeddebug else "Debug" if build_dir: config_args.extend(["-B", build_dir]) + if self.options.ninja_path: + config_args.extend(["-G", "Ninja", f"-DCMAKE_MAKE_PROGRAM={self.options.ninja_path}"]) if build_type: config_args.extend([f"-DCMAKE_BUILD_TYPE={cmake_build_type}"]) if build_type_is_optimizeddebug: @@ -582,6 +590,82 @@ def setup_third_party_dir(self): self.prompt_dependency_path_option("third_party_src_dir") os.makedirs(self.options.third_party_src_dir, exist_ok=True) + def install_ninja(self): + # 1. Check if the user has set a ninja_path option + if self.prompt_option("ninja_path"): + if not os.path.isabs(self.options.ninja_path): + self.options.ninja_path = shutil.which(self.options.ninja_path) + if not self.is_executable(self.options.ninja_path): + raise FileNotFoundError(f"Ninja executable not found at {self.options.ninja_path}.") + return + + # 2. If ninja_path is not set, but does the user have it available in PATH? + ninja_path = shutil.which("ninja") + if ninja_path: + print(f"Ninja found in PATH at {ninja_path}. Using it.") + self.options.ninja_path = ninja_path + return + + # 3. Ninja path isn't set and not available in PATH, so we download it + destination_dir = self.options.third_party_src_dir + ninja_dir = os.path.join(destination_dir, "ninja") + exe_name = 'ninja.exe' if platform.system().lower() == 'windows' else 'ninja' + ninja_path = os.path.join(ninja_dir, exe_name) + if os.path.exists(ninja_path) and self.is_executable(ninja_path): + print(f"Ninja already exists at {ninja_path}. Using it.") + self.options.ninja_path = ninja_path + return + + # 3a. Determine the ninja asset name based on the platform and architecture + system = platform.system().lower() + arch = platform.machine().lower() + if system == 'linux': + if arch in ('aarch64', 'arm64'): + asset_name = 'ninja-linux-aarch64.zip' + else: + asset_name = 'ninja-linux.zip' + elif system == 'darwin': + asset_name = 'ninja-mac.zip' + elif system == 'windows': + if arch in ('arm64', 'aarch64'): + asset_name = 'ninja-winarm64.zip' + else: + asset_name = 'ninja-win.zip' + else: + return + + # 3b. Find the download URL for the latest Ninja release asset + api_url = 'https://api.github.com/repos/ninja-build/ninja/releases/latest' + with urllib.request.urlopen(api_url) as resp: + data = json.load(resp) + release_assets = data.get('assets', []) + download_url = None + for a in release_assets: + if a.get('name') == asset_name: + download_url = a.get('browser_download_url') + break + if not download_url: + # Could not find release asset named asset_name + return + + # 3c. Download the asset to the third-party source directory + tmpzip = os.path.join(destination_dir, asset_name) + os.makedirs(destination_dir, exist_ok=True) + print(f'Downloading {asset_name} …') + urllib.request.urlretrieve(download_url, tmpzip) + + # 3d. Extract the downloaded zip file into the ninja dir + print('Extracting…') + os.makedirs(ninja_dir, exist_ok=True) + with zipfile.ZipFile(tmpzip, 'r') as z: + z.extractall(ninja_dir) + os.remove(tmpzip) + + # 3e. Set the ninja_path option to the extracted ninja executable + if platform.system().lower() != 'windows': + os.chmod(ninja_path, 0o755) + self.options.ninja_path = ninja_path + def is_abi_compatible(self, build_type_a, build_type_b): if not self.is_windows(): return True @@ -606,6 +690,7 @@ def install_duktape(self): continue member.name = member_path tar.extract(member, path=self.options.duktape_src_dir) + os.remove(archive_path) duktape_patches = os.path.join(self.options.mrdocs_src_dir, 'third-party', 'duktape') if os.path.exists(duktape_patches): for patch_file in os.listdir(duktape_patches): @@ -800,14 +885,19 @@ def create_cmake_presets(self): } } - # Iterate the cacheVariables and, - # 1) If any starts with the value of the parent of mrdocs-src-dir, we replace it with ${sourceParentDir} to make it relative - # 2) If any starts with the value of mrdocs-src-dir, we replace it with ${sourceDir} to make it relative - # 3) if any starts with the value of $HOME, we replace it with $env{HOME} to make it relative + if self.options.cc: + new_preset["cacheVariables"]["CMAKE_C_COMPILER"] = self.options.cc + if self.options.cxx: + new_preset["cacheVariables"]["CMAKE_CXX_COMPILER"] = self.options.cxx + if self.options.ninja_path: + new_preset["generator"] = "Ninja" + new_preset["cacheVariables"]["CMAKE_MAKE_PROGRAM"] = self.options.ninja_path + + # Update cache variables path prefixes with their relative equivalents mrdocs_src_dir_parent = os.path.dirname(self.options.mrdocs_src_dir) if mrdocs_src_dir_parent == self.options.mrdocs_src_dir: mrdocs_src_dir_parent = '' - mrdocs_home_dir = os.path.expanduser("~") + home_dir = os.path.expanduser("~") for key, value in new_preset["cacheVariables"].items(): if not isinstance(value, str): continue @@ -820,10 +910,11 @@ def create_cmake_presets(self): new_value = "${sourceDir}" + value[len(self.options.mrdocs_src_dir):] new_preset["cacheVariables"][key] = new_value # Replace $HOME with $env{HOME} - elif mrdocs_home_dir and value.startswith(mrdocs_home_dir): - new_value = "$env{HOME}" + value[len(mrdocs_home_dir):] + elif home_dir and value.startswith(home_dir): + new_value = "$env{HOME}" + value[len(home_dir):] new_preset["cacheVariables"][key] = new_value + # Upsert preset preset_exists = False for preset in user_presets.get("configurePresets", []): if preset.get("name") == self.options.mrdocs_preset_name: @@ -1382,6 +1473,7 @@ def install_all(self): self.check_tools() self.setup_mrdocs_src_dir() self.setup_third_party_dir() + self.install_ninja() self.install_duktape() self.install_llvm() if self.prompt_option("mrdocs_build_tests"):