From 4ec2494d218655d8770a9497c9de48c7d838b6ae Mon Sep 17 00:00:00 2001 From: Eloston Date: Tue, 19 Jul 2016 22:27:28 -0700 Subject: [PATCH] Implement GYP configuration. See #16 Add WIP GN args --- building/debian.py | 30 ++++++ building/generic.py | 146 +++++++++++++++++++------- building/resources/debian/gn_args.ini | 63 +++++++++++ building/resources/debian/gyp_flags | 31 ++++++ gn_args.ini | 55 ++++++++++ gyp_flags | 34 ++++++ 6 files changed, 322 insertions(+), 37 deletions(-) create mode 100644 building/resources/debian/gn_args.ini create mode 100644 building/resources/debian/gyp_flags create mode 100644 gn_args.ini create mode 100644 gyp_flags diff --git a/building/debian.py b/building/debian.py index 9304f7fb37..86e600cfd8 100644 --- a/building/debian.py +++ b/building/debian.py @@ -78,3 +78,33 @@ def apply_patches(self): result = subprocess.run(["quilt", "push", "-a"], env=new_env, cwd=str(self.sandbox_root)) if not result.returncode == 0: raise Exception("Quilt returned non-zero exit code: {}".format(result.returncode)) + + #def generate_build_configuration(self, gn_args=pathlib.Path("gn_args.ini"), build_output=pathlib.Path("out", "Default"), debian_gn_args=(self.PLATFORM_RESOURCES / pathlib.Path("gn_args.ini")): + # (self.sandbox_root / build_output).mkdir(parents=True, exist_ok=True) + # common_config = configparser.ConfigParser() + # common_config.read(str(gn_args)) + # debian_config = configparser.ConfigParser() + # debian_config.read(str(debian_gn_args)) + # combined_dict = dict() + # for section in common_config: + # if not section == "DEFAULT": + # combined_dict[section] = dict() + # for config_key in common_config[section]: + # combined_dict[section][config_key] = common_config[section][config_key] + # for section in debian_config: + # if not section == "DEFAULT": + # if not section in combined_dict: + # combined_dict[section] = dict() + # for config_key in debian_config[section]: + # combined_dict[section][config_key] = debian_config[section][config_key] + # self._gn_write_args(combined_dict, build_output) + # self._gn_generate_ninja(build_output) + + def generate_build_configuration(self, gyp_flags=pathlib.Path("gyp_flags"), build_output=pathlib.Path("out", "Release"), python2_command=None, debian_gyp_flags=(PLATFORM_RESOURCES / pathlib.Path("gyp_flags"))): + self.logger.info("Running gyp command with additional Debian gyp flags...") + gyp_list = list() + with gyp_flags.open() as f: + gyp_list = f.read().splitlines() + with debian_gyp_flags.open() as f: + gyp_list += f.read().splitlines() + self._gyp_generate_ninja(gyp_list, build_output, python2_command) diff --git a/building/generic.py b/building/generic.py index 2b03f93e64..7519a6eb17 100644 --- a/building/generic.py +++ b/building/generic.py @@ -26,6 +26,7 @@ import re import subprocess import logging +import configparser class GenericPlatform: def __init__(self, version, revision, logger=None, sandbox_root=pathlib.Path("build_sandbox")): @@ -61,9 +62,11 @@ def __init__(self, version, revision, logger=None, sandbox_root=pathlib.Path("bu self.logger.info("ungoogled_dir does not exist. Creating...") self.ungoogled_dir.mkdir() - self.gn_command = None self.sourcearchive = None self.sourcearchive_hashes = None + self.gn_command = None + self.ninja_command = None + self.build_output = None def _check_source_archive(self): ''' @@ -151,20 +154,76 @@ def _domain_substitute(self, regex_defs, file_list, log_warnings=True): self.logger.error("Exception thrown for path {}".format(path)) raise e - def _build_gn(self, python2_command): + def _gyp_generate_ninja(self, args_list, build_output, python2_command): + command_list = list() + if not python2_command is None: + command_list.append(python2_command) + command_list.append(str(pathlib.Path("build", "gyp_chromium"))) + command_list += ["--depth=.", "--check"] + for i in args_list: + command_list.append("-D{}".format(i)) + self.logger.debug("GYP command: {}".format(" ".join(command_list))) + result = subprocess.run(command_list, cwd=str(self.sandbox_root)) + if not result.returncode == 0: + raise Exception("GYP command returned non-zero exit code: {}".format(result.returncode)) + + def _gn_write_args(self, args_map, build_output): ''' - Build the GN tool to out/gn-tool in the build sandbox + `args_map` can be any object supporting the mapping interface ''' - command_list = [str(pathlib.Path("tools", "gn", "bootstrap", "bootstrap.py")), "-v", "--no-clean", "-o", str(pathlib.Path("out", "gn_tool")), "--gn-gen-args= use_sysroot=false"] - if not python2_command is None: - command_list.insert(0, python2_command) - subprocess.run(command_list, cwd=str(self.sandbox_root)) - - def _run_gn(self): # Run GN configuration - pass + gn_imports = list() + gn_flags = list() + for gn_path in args_map: + if not gn_path == "DEFAULT" and not gn_path == "global": # Checking against DEFAULT for configparser mapping interface + if not gn_path.lower().endswith(".gn"): + gn_imports.append('import("{}")'.format(gn_path)) + for flag in args_map[gn_path]: + gn_flags.append("{}={}".format(flag, args_map[gn_path][flag])) + with (self.sandbox_root / build_output / pathlib.Path("args.gn")).open("w") as f: + f.write("\n".join(gn_imports)) + f.write("\n") + f.write("\n".join(gn_flags)) + + def _gn_generate_ninja(self, build_output, gn_override=None): + command_list = list() + if gn_override is None: + command_list.append(self.gn_command) + else: + command_list.append(gn_override) + command_list.append("gen") + command_list.append(str(build_output)) + result = subprocess.run(command_list, cwd=str(self.sandbox_root)) + if not result.returncode == 0: + raise Exception("gn gen returned non-zero exit code: {}".format(result.returncode)) + + def _run_ninja(self, build_output, targets): + result = subprocess.run([self.ninja_command, "-C", str(build_output), *targets], cwd=str(self.sandbox_root)) + if not result.returncode == 0: + raise Exception("ninja returned non-zero exit code: {}".format(result.returncode)) - def _run_ninja(self): # Run build command - pass + def _build_gn(self, python2_command): + ''' + Build the GN tool to out/gn_tool in the build sandbox. Returns the gn command string. Only works on Linux or Mac. + ''' + self.logger.info("Building gn...") + temp_gn_executable = pathlib.Path("out", "temp_gn") + if (self.sandbox_root / temp_gn_executable).exists(): + self.logger.info("Bootstrap gn already exists") + else: + self.logger.info("Building bootstrap gn") + command_list = [str(pathlib.Path("tools", "gn", "bootstrap", "bootstrap.py")), "-v", "-s", "-o", str(temp_gn_executable), "--gn-gen-args= use_sysroot=false"] + if not python2_command is None: + command_list.insert(0, python2_command) + result = subprocess.run(command_list, cwd=str(self.sandbox_root)) + if not result.returncode == 0: + raise Exception("GN bootstrap command returned non-zero exit code: {}".format(result.returncode)) + self.logger.info("Building gn using bootstrap gn...") + build_output = pathlib.Path("out", "gn_release") + (self.sandbox_root / build_output).mkdir(parents=True, exist_ok=True) + self._gn_write_args({"global": {"use_sysroot": "false", "is_debug": "false"}}, build_output) + self._gn_generate_ninja(build_output, gn_override=str(temp_gn_executable)) + self._run_ninja(build_output, ["gn"]) + return str(build_output / pathlib.Path("gn")) def setup_chromium_source(self, check_if_exists=True, force_download=False, check_integrity=True, extract_archive=True, destination_dir=pathlib.Path("."), cleaning_list=pathlib.Path("cleaning_list"), archive_path=None, hashes_path=None): ''' @@ -237,30 +296,43 @@ def apply_patches(self): # TODO: Use Python to apply patches defined in `patch_order` pass - def setup_build_utilities(self, build_gn=True, gn_command=None, python2_command=None): - ''' - Sets up the utilities required for building. For now, this is just the "gn" tool. - - If `build_gn` is True, then the `tools/gn/bootstrap/bootstrap.py` script is invoked in the build directory to build gn. - If `python2_command` is set, it must be a string of a command to invoke Python 2 for running bootstrap.py. Otherwise, the bootstrap.py path will be the executable path. - - If `gn_command` is set, it must be a string of a command to invoke gn. - - `build_gn` and `gn_command` are mutually exclusive. - ''' - if build_gn and not gn_command is None: - raise Exception("Conflicting arguments: build_gn and gn_path") - - if build_gn: - self._build_gn(python2_command) - else: - self.gn_command = gn_command - - def generate_build_configuration(self): - pass - - def pre_build_finalization(self): - pass + #def setup_build_utilities(self, build_gn=True, gn_command=None, python2_command=None, ninja_command="ninja"): + # ''' + # Sets up the utilities required for building. For now, this is just the "gn" tool. + # + # If `build_gn` is True, then the `tools/gn/bootstrap/bootstrap.py` script is invoked in the build directory to build gn. + # If `python2_command` is set, it must be a string of a command to invoke Python 2 for running bootstrap.py. Otherwise, the bootstrap.py path will be the executable path. + # + # If `gn_command` is set, it must be a string of a command to invoke gn. + # + # `build_gn` and `gn_command` are mutually exclusive. + # ''' + # if build_gn and not gn_command is None: + # raise Exception("Conflicting arguments: build_gn and gn_path") + # self.ninja_command = ninja_command + # if build_gn: + # self.gn_command = self._build_gn(python2_command) + # else: + # self.gn_command = gn_command + + def setup_build_utilities(self, ninja_command="ninja"): + self.ninja_command = ninja_command + + #def generate_build_configuration(self, gn_args=pathlib.Path("gn_args.ini"), build_output=pathlib.Path("out", "Default")): + # (self.sandbox_root / build_output).mkdir(parents=True, exist_ok=True) + # config = configparser.ConfigParser() + # config.read(str(gn_args)) + # self._gn_write_args(config, build_output) + # self._gn_generate_ninja(build_output) + + def generate_build_configuration(self, gyp_flags=pathlib.Path("gyp_flags"), build_output=pathlib.Path("out", "Release"), python2_command=None): + self.logger.info("Running gyp command...") + gyp_list = list() + with gyp_flags.open() as f: + gyp_list = f.read().splitlines() + self._gyp_generate_ninja(gyp_list, build_output, python2_command) + self.build_output = build_output def build(self): - pass + self.logger.info("Running build command...") + self._run_ninja(self.build_output, ["chrome"]) diff --git a/building/resources/debian/gn_args.ini b/building/resources/debian/gn_args.ini new file mode 100644 index 0000000000..23a43aaca9 --- /dev/null +++ b/building/resources/debian/gn_args.ini @@ -0,0 +1,63 @@ +# NOTE: THIS FILE IS NOT USED YET + +# Debian-specific GN args +# Can override values set by the cross-platform gn_args.ini + +# Values from GYP that don't have a GN equivalent: +# linux_breakpad=0 +# linux_use_libgps=0 +# linux_use_bundled_gold=0 +# linux_link_gsettings=0 +# linux_link_libpci=1 +# linux_link_libspeechd=1 +# linux_strip_binary=1 +# use_system_re2=1 +# use_system_yasm=1 +# use_system_opus=1 (rtc_build_opus?) +# use_system_zlib=1 +# use_system_speex=1 +# use_system_expat=1 (rtc_build_expat?) +# use_system_libpng=1 +# use_system_libxml=1 +# use_system_libwebp=1 +# use_system_libxslt=1 +# use_system_libsrtp=1 +# use_system_jsoncpp=1 (rtc_build_json?) +# use_system_libevent=1 +# use_system_xdg_utils=1 + +[global] +is_clang=true +use_sysroot=false + +[//build/config/clang/BUILD.gn] +clang_use_chrome_plugins=false + +[//build/config/features.gni] +use_gconf=false +use_gio=true + +[//build/config/allocator.gni] +use_allocator="none" + +[//build/config/compiler/BUILD.gn] +linux_use_bundled_binutils=false + +[//build/config/compiler/compiler.gni] +symbol_level=1 + +[//media/media_options.gni] +link_pulseaudio=true + +[//third_party/BUILD.gn] +use_system_libjpeg=true + +[//third_party/harfbuzz-ng/BUILD.gn] +use_system_harfbuzz=true + +[//third_party/webrtc/build/webrtc.gni] +rtc_build_opus=false +rtc_build_expat=false +rtc_build_libjpeg=false +rtc_build_json=false +rtc_jsoncpp_root="/usr/include/jsoncpp/json" diff --git a/building/resources/debian/gyp_flags b/building/resources/debian/gyp_flags new file mode 100644 index 0000000000..08a142401b --- /dev/null +++ b/building/resources/debian/gyp_flags @@ -0,0 +1,31 @@ +use_ozone=0 +use_gconf=0 +use_sysroot=0 +use_allocator=none +linux_breakpad=0 +linux_use_libgps=0 +linux_use_bundled_gold=0 +linux_use_bundled_binutils=0 +use_gnome_keyring=0 +linux_link_gsettings=0 +linux_link_gnome_keyring= +use_gio=1 +use_pulseaudio=1 +linux_link_libpci=1 +linux_link_libspeechd=1 +use_system_re2=1 +use_system_yasm=1 +use_system_opus=1 +use_system_zlib=1 +use_system_speex=1 +use_system_expat=1 +use_system_libpng=1 +use_system_libxml=1 +use_system_libjpeg=1 +use_system_libwebp=1 +use_system_libxslt=1 +use_system_libsrtp=1 +use_system_jsoncpp=1 +use_system_libevent=1 +use_system_harfbuzz=1 +use_system_xdg_utils=1 diff --git a/gn_args.ini b/gn_args.ini new file mode 100644 index 0000000000..d445184e51 --- /dev/null +++ b/gn_args.ini @@ -0,0 +1,55 @@ +# NOTE: THIS FILE IS NOT USED YET + +# Cross-platform GN arguments + +# Values from GYP that don't have a GN equivalent: +# enable_automation=0 +# enable_wifi_bootstrapping=0 +# enable_pre_sync_backup=0 +# enable_prod_wallet_service=0 +# disable_newlib=1 + +[//build/config/BUILDCONFIG.gn] +is_debug=false + +[//build/config/compiler/BUILD.gn] +treat_warnings_as_errors=false + +[//build/config/ui.gni] +use_ozone=false + +[//build/config/features.gni] +enable_remoting=false +enable_nacl=false +safe_browsing_mode=0 +enable_webrtc=false +enable_hangout_services_extension=false +enable_supervised_users=false +fieldtrial_testing_like_official_build=true +proprietary_codecs=true + +[//chrome/common/features.gni] +enable_google_now=false +enable_one_click_signin=false + +[//chrome/browser/BUILD.gn] +enable_hotwording=false + +[//google_apis/BUILD.gn] +google_api_key="" +google_default_client_id="" +google_default_client_secret="" +use_official_google_api_keys=false + +[//media/media_options.gni] +enable_hevc_demuxing=true +enable_mse_mpeg2ts_stream_parser=true + +[//third_party/ffmpeg/ffmpeg_options.gni] +ffmpeg_branding="ChromeOS" + +[//third_party/icu/config.gni] +icu_use_data_file=false + +[//third_party/WebKit/Source/config.gni] +remove_webcore_debug_symbols=true diff --git a/gyp_flags b/gyp_flags new file mode 100644 index 0000000000..6e649d1bde --- /dev/null +++ b/gyp_flags @@ -0,0 +1,34 @@ +werror= +default_apps_list=[] +default_apps_list_linux_dest=[] +remoting=0 +disable_nacl=1 +disable_pnacl=1 +enable_remoting_host=0 +enable_automation=0 +enable_google_now=0 +safe_browsing=0 +enable_webrtc=0 +remoting=0 +enable_hotwording=0 +enable_hangout_services_extension=0 +enable_wifi_bootstrapping=0 +use_official_google_api_keys=0 +enable_rlz=0 +enable_pre_sync_backup=0 +enable_prod_wallet_service=0 +enable_supervised_users=0 +enable_one_click_signin=0 +disable_newlib=1 +icu_use_data_file_flag=0 +enable_hidpi=1 +fastbuild=1 +disable_fatal_linker_warnings=1 +tracing_like_official_build=1 +fieldtrial_testing_like_official_build=1 +linux_strip_binary=1 +proprietary_codecs=1 +ffmpeg_branding=ChromeOS +enable_mpeg2ts_stream_parser=1 +enable_hevc_demuxing=1 +remove_webcore_debug_symbols=1