From 173404454c874943f0eb946210b78e9c7e230d91 Mon Sep 17 00:00:00 2001 From: Jakub Piasecki Date: Mon, 23 Mar 2026 06:11:07 -0700 Subject: [PATCH] Generate snapshots in parallel (#56184) Summary: Changelog: [Internal] Updates the snapshot generator script to run generation steps for all snapshots in parallel instead of in sequence, significantly improving the generation time. Differential Revision: D97714334 --- .../react-native/.doxygen.config.template | 2 +- .../manual_test/.doxygen.config.template | 2 +- scripts/cxx-api/parser/__main__.py | 135 ++++++++++++------ scripts/cxx-api/parser/doxygen.py | 30 ++-- 4 files changed, 115 insertions(+), 54 deletions(-) diff --git a/packages/react-native/.doxygen.config.template b/packages/react-native/.doxygen.config.template index 4f982c734bc2..f06eeddd9b57 100644 --- a/packages/react-native/.doxygen.config.template +++ b/packages/react-native/.doxygen.config.template @@ -74,7 +74,7 @@ PROJECT_ICON = # entered, it will be relative to the location where Doxygen was started. If # left blank the current directory will be used. -OUTPUT_DIRECTORY = api +OUTPUT_DIRECTORY = ${OUTPUT_DIR} # If the CREATE_SUBDIRS tag is set to YES then Doxygen will create up to 4096 # sub-directories (in 2 levels) under the output directory of each output format diff --git a/scripts/cxx-api/manual_test/.doxygen.config.template b/scripts/cxx-api/manual_test/.doxygen.config.template index cac0975942a6..c8e1f2a82f02 100644 --- a/scripts/cxx-api/manual_test/.doxygen.config.template +++ b/scripts/cxx-api/manual_test/.doxygen.config.template @@ -74,7 +74,7 @@ PROJECT_ICON = # entered, it will be relative to the location where Doxygen was started. If # left blank the current directory will be used. -OUTPUT_DIRECTORY = api +OUTPUT_DIRECTORY = ${OUTPUT_DIR} # If the CREATE_SUBDIRS tag is set to YES then Doxygen will create up to 4096 # sub-directories (in 2 levels) under the output directory of each output format diff --git a/scripts/cxx-api/parser/__main__.py b/scripts/cxx-api/parser/__main__.py index cc317a25ca58..84dbb9a57cbd 100644 --- a/scripts/cxx-api/parser/__main__.py +++ b/scripts/cxx-api/parser/__main__.py @@ -12,11 +12,12 @@ """ import argparse +import concurrent.futures import os -import shutil import subprocess import sys import tempfile +import traceback from .config import ApiViewSnapshotConfig, parse_config_file from .doxygen import get_doxygen_bin, run_doxygen @@ -36,13 +37,20 @@ def run_command( if result.returncode != 0: if verbose: print(f"{label} finished with error: {result.stderr}") - sys.exit(1) + raise RuntimeError( + f"{label} finished with error (exit code {result.returncode})" + ) elif verbose: print(f"{label} finished successfully") return result -def build_codegen(platform: str, verbose: bool = False) -> str: +def build_codegen( + platform: str, + verbose: bool = False, + output_path: str = "./api/codegen", + label: str = "", +) -> str: react_native_dir = os.path.join(get_react_native_dir(), "packages", "react-native") run_command( @@ -52,17 +60,21 @@ def build_codegen(platform: str, verbose: bool = False) -> str: "--path", "./", "--outputPath", - "./api/codegen", + output_path, "--targetPlatform", platform, "--forceOutputPath", ], - label="Codegen", + label=f"[{label}] Codegen" if label else "Codegen", verbose=verbose, cwd=react_native_dir, + capture_output=True, + text=True, ) - return os.path.join(react_native_dir, "api", "codegen") + if os.path.isabs(output_path): + return output_path + return os.path.join(react_native_dir, output_path) def build_snapshot_for_view( @@ -75,21 +87,29 @@ def build_snapshot_for_view( codegen_platform: str | None = None, verbose: bool = True, input_filter: str = None, + work_dir: str | None = None, ) -> str: if verbose: - print(f"Generating API view: {api_view}") + print(f"[{api_view}] Generating API view") - api_dir = os.path.join(react_native_dir, "api") - if os.path.exists(api_dir): - if verbose: - print(" Deleting existing output directory") - shutil.rmtree(api_dir) + include_directories = list(include_directories) + + if work_dir is None: + work_dir = os.path.join(react_native_dir, "api") if codegen_platform is not None: - codegen_dir = build_codegen(codegen_platform, verbose=verbose) + codegen_output = os.path.join(work_dir, "codegen") + codegen_dir = build_codegen( + codegen_platform, + verbose=verbose, + output_path=codegen_output, + label=api_view, + ) include_directories.append(codegen_dir) elif verbose: - print(" Skipping codegen") + print(f"[{api_view}] Skipping codegen") + + config_file = f".doxygen.config.{api_view}.generated" run_doxygen( working_dir=react_native_dir, @@ -98,12 +118,15 @@ def build_snapshot_for_view( definitions=definitions, input_filter=input_filter, verbose=verbose, + output_dir=work_dir, + config_file=config_file, + label=api_view, ) if verbose: - print(" Building snapshot") + print(f"[{api_view}] Building snapshot") - snapshot = build_snapshot(os.path.join(api_dir, "xml")) + snapshot = build_snapshot(os.path.join(work_dir, "xml")) snapshot_string = snapshot.to_string() output_file = os.path.join(output_dir, f"{api_view}Cxx.api") @@ -126,36 +149,66 @@ def build_snapshots( is_test: bool = False, ) -> None: if not is_test: - for config in snapshot_configs: - if view_filter and config.snapshot_name != view_filter: - continue - - build_snapshot_for_view( - api_view=config.snapshot_name, + configs_to_build = [ + config + for config in snapshot_configs + if not view_filter or config.snapshot_name == view_filter + ] + + with tempfile.TemporaryDirectory(prefix="cxx-api-") as parent_tmp: + with concurrent.futures.ThreadPoolExecutor() as executor: + futures = {} + for config in configs_to_build: + work_dir = os.path.join(parent_tmp, config.snapshot_name) + os.makedirs(work_dir, exist_ok=True) + future = executor.submit( + build_snapshot_for_view, + api_view=config.snapshot_name, + react_native_dir=react_native_dir, + include_directories=config.inputs, + exclude_patterns=config.exclude_patterns, + definitions=config.definitions, + output_dir=output_dir, + codegen_platform=config.codegen_platform, + verbose=verbose, + input_filter=input_filter if config.input_filter else None, + work_dir=work_dir, + ) + futures[future] = config.snapshot_name + + errors = [] + for future in concurrent.futures.as_completed(futures): + view_name = futures[future] + try: + future.result() + except Exception as e: + errors.append((view_name, e)) + if verbose: + print( + f"[{view_name}] Error generating:\n" + f"{traceback.format_exc()}" + ) + + if errors: + failed_views = ", ".join(name for name, _ in errors) + raise RuntimeError(f"Failed to generate snapshots: {failed_views}") + else: + with tempfile.TemporaryDirectory(prefix="cxx-api-test-") as work_dir: + snapshot = build_snapshot_for_view( + api_view="Test", react_native_dir=react_native_dir, - include_directories=config.inputs, - exclude_patterns=config.exclude_patterns, - definitions=config.definitions, + include_directories=[], + exclude_patterns=[], + definitions={}, output_dir=output_dir, - codegen_platform=config.codegen_platform, + codegen_platform=None, verbose=verbose, - input_filter=input_filter if config.input_filter else None, + input_filter=input_filter, + work_dir=work_dir, ) - else: - snapshot = build_snapshot_for_view( - api_view="Test", - react_native_dir=react_native_dir, - include_directories=[], - exclude_patterns=[], - definitions={}, - output_dir=output_dir, - codegen_platform=None, - verbose=verbose, - input_filter=input_filter, - ) - if verbose: - print(snapshot) + if verbose: + print(snapshot) def get_default_snapshot_dir() -> str: diff --git a/scripts/cxx-api/parser/doxygen.py b/scripts/cxx-api/parser/doxygen.py index bb0f1914060b..108a7ca80d40 100644 --- a/scripts/cxx-api/parser/doxygen.py +++ b/scripts/cxx-api/parser/doxygen.py @@ -9,7 +9,6 @@ import os import subprocess -import sys _DOXYGEN_CONFIG_FILE = ".doxygen.config.generated" @@ -24,6 +23,8 @@ def build_doxygen_config( exclude_patterns: list[str] = None, definitions: dict[str, str | int] = None, input_filter: str = None, + output_dir: str = "api", + config_file: str = _DOXYGEN_CONFIG_FILE, ) -> None: if include_directories is None: include_directories = [] @@ -54,9 +55,10 @@ def build_doxygen_config( .replace("${EXCLUDE_PATTERNS}", exclude_patterns_str) .replace("${PREDEFINED}", definitions_str) .replace("${DOXYGEN_INPUT_FILTER}", input_filter_str) + .replace("${OUTPUT_DIR}", output_dir) ) - with open(os.path.join(directory, _DOXYGEN_CONFIG_FILE), "w") as f: + with open(os.path.join(directory, config_file), "w") as f: f.write(config) @@ -67,10 +69,14 @@ def run_doxygen( definitions: dict[str, str | int], input_filter: str = None, verbose: bool = True, + output_dir: str = "api", + config_file: str = _DOXYGEN_CONFIG_FILE, + label: str = "", ) -> None: """Generate Doxygen config, run Doxygen, and clean up the config file.""" + prefix = f"[{label}] " if label else "" if verbose: - print(" Generating Doxygen config file") + print(f"{prefix}Generating Doxygen config file") build_doxygen_config( working_dir, @@ -78,17 +84,19 @@ def run_doxygen( exclude_patterns=exclude_patterns, definitions=definitions, input_filter=input_filter, + output_dir=output_dir, + config_file=config_file, ) if verbose: - print(" Running Doxygen") + print(f"{prefix}Running Doxygen") if input_filter: - print(f" Using input filter: {input_filter}") + print(f"{prefix}Using input filter: {input_filter}") doxygen_bin = get_doxygen_bin() result = subprocess.run( - [doxygen_bin, _DOXYGEN_CONFIG_FILE], + [doxygen_bin, config_file], cwd=working_dir, capture_output=True, text=True, @@ -96,11 +104,11 @@ def run_doxygen( if result.returncode != 0: if verbose: - print(f" Doxygen finished with error: {result.stderr}") - sys.exit(1) + print(f"{prefix}Doxygen finished with error: {result.stderr}") + raise RuntimeError(f"Doxygen finished with error: {result.stderr}") elif verbose: - print(" Doxygen finished successfully") + print(f"{prefix}Doxygen finished successfully") if verbose: - print(" Deleting Doxygen config file") - os.remove(os.path.join(working_dir, _DOXYGEN_CONFIG_FILE)) + print(f"{prefix}Deleting Doxygen config file") + os.remove(os.path.join(working_dir, config_file))