Implement GYP configuration. See #16

Add WIP GN args

building/debian.py

@@ -78,3 +78,33 @@ def apply_patches(self):
         result = subprocess.run(["quilt", "push", "-a"], env=new_env, cwd=str(self.sandbox_root))
         if not result.returncode == 0:
             raise Exception("Quilt returned non-zero exit code: {}".format(result.returncode))
+
+    #def generate_build_configuration(self, gn_args=pathlib.Path("gn_args.ini"), build_output=pathlib.Path("out", "Default"), debian_gn_args=(self.PLATFORM_RESOURCES / pathlib.Path("gn_args.ini")):
+    #    (self.sandbox_root / build_output).mkdir(parents=True, exist_ok=True)
+    #    common_config = configparser.ConfigParser()
+    #    common_config.read(str(gn_args))
+    #    debian_config = configparser.ConfigParser()
+    #    debian_config.read(str(debian_gn_args))
+    #    combined_dict = dict()
+    #    for section in common_config:
+    #        if not section == "DEFAULT":
+    #            combined_dict[section] = dict()
+    #            for config_key in common_config[section]:
+    #                combined_dict[section][config_key] = common_config[section][config_key]
+    #    for section in debian_config:
+    #        if not section == "DEFAULT":
+    #            if not section in combined_dict:
+    #                combined_dict[section] = dict()
+    #            for config_key in debian_config[section]:
+    #                combined_dict[section][config_key] = debian_config[section][config_key]
+    #    self._gn_write_args(combined_dict, build_output)
+    #    self._gn_generate_ninja(build_output)
+
+    def generate_build_configuration(self, gyp_flags=pathlib.Path("gyp_flags"), build_output=pathlib.Path("out", "Release"), python2_command=None, debian_gyp_flags=(PLATFORM_RESOURCES / pathlib.Path("gyp_flags"))):
+        self.logger.info("Running gyp command with additional Debian gyp flags...")
+        gyp_list = list()
+        with gyp_flags.open() as f:
+            gyp_list = f.read().splitlines()
+        with debian_gyp_flags.open() as f:
+            gyp_list += f.read().splitlines()
+        self._gyp_generate_ninja(gyp_list, build_output, python2_command)

building/generic.py

@@ -26,6 +26,7 @@
 import re
 import subprocess
 import logging
+import configparser
 
 class GenericPlatform:
     def __init__(self, version, revision, logger=None, sandbox_root=pathlib.Path("build_sandbox")):
@@ -61,9 +62,11 @@ def __init__(self, version, revision, logger=None, sandbox_root=pathlib.Path("bu
             self.logger.info("ungoogled_dir does not exist. Creating...")
             self.ungoogled_dir.mkdir()
 
-        self.gn_command = None
         self.sourcearchive = None
         self.sourcearchive_hashes = None
+        self.gn_command = None
+        self.ninja_command = None
+        self.build_output = None
 
     def _check_source_archive(self):
         '''
@@ -151,20 +154,76 @@ def _domain_substitute(self, regex_defs, file_list, log_warnings=True):
                 self.logger.error("Exception thrown for path {}".format(path))
                 raise e
 
-    def _build_gn(self, python2_command):
+    def _gyp_generate_ninja(self, args_list, build_output, python2_command):
+        command_list = list()
+        if not python2_command is None:
+            command_list.append(python2_command)
+        command_list.append(str(pathlib.Path("build", "gyp_chromium")))
+        command_list += ["--depth=.", "--check"]
+        for i in args_list:
+            command_list.append("-D{}".format(i))
+        self.logger.debug("GYP command: {}".format(" ".join(command_list)))
+        result = subprocess.run(command_list, cwd=str(self.sandbox_root))
+        if not result.returncode == 0:
+            raise Exception("GYP command returned non-zero exit code: {}".format(result.returncode))
+
+    def _gn_write_args(self, args_map, build_output):
         '''
-        Build the GN tool to out/gn-tool in the build sandbox
+        `args_map` can be any object supporting the mapping interface
         '''
-        command_list = [str(pathlib.Path("tools", "gn", "bootstrap", "bootstrap.py")), "-v", "--no-clean", "-o", str(pathlib.Path("out", "gn_tool")), "--gn-gen-args= use_sysroot=false"]
-        if not python2_command is None:
-            command_list.insert(0, python2_command)
-        subprocess.run(command_list, cwd=str(self.sandbox_root))
-
-    def _run_gn(self): # Run GN configuration
-        pass
+        gn_imports = list()
+        gn_flags = list()
+        for gn_path in args_map:
+            if not gn_path == "DEFAULT" and not gn_path == "global": # Checking against DEFAULT for configparser mapping interface
+                if not gn_path.lower().endswith(".gn"):
+                    gn_imports.append('import("{}")'.format(gn_path))
+            for flag in args_map[gn_path]:
+                gn_flags.append("{}={}".format(flag, args_map[gn_path][flag]))
+        with (self.sandbox_root / build_output / pathlib.Path("args.gn")).open("w") as f:
+            f.write("\n".join(gn_imports))
+            f.write("\n")
+            f.write("\n".join(gn_flags))
+
+    def _gn_generate_ninja(self, build_output, gn_override=None):
+        command_list = list()
+        if gn_override is None:
+            command_list.append(self.gn_command)
+        else:
+            command_list.append(gn_override)
+        command_list.append("gen")
+        command_list.append(str(build_output))
+        result = subprocess.run(command_list, cwd=str(self.sandbox_root))
+        if not result.returncode == 0:
+            raise Exception("gn gen returned non-zero exit code: {}".format(result.returncode))
+
+    def _run_ninja(self, build_output, targets):
+        result = subprocess.run([self.ninja_command, "-C", str(build_output), *targets], cwd=str(self.sandbox_root))
+        if not result.returncode == 0:
+            raise Exception("ninja returned non-zero exit code: {}".format(result.returncode))
 
-    def _run_ninja(self): # Run build command
-        pass
+    def _build_gn(self, python2_command):
+        '''
+        Build the GN tool to out/gn_tool in the build sandbox. Returns the gn command string. Only works on Linux or Mac.
+        '''
+        self.logger.info("Building gn...")
+        temp_gn_executable = pathlib.Path("out", "temp_gn")
+        if (self.sandbox_root / temp_gn_executable).exists():
+            self.logger.info("Bootstrap gn already exists")
+        else:
+            self.logger.info("Building bootstrap gn")
+            command_list = [str(pathlib.Path("tools", "gn", "bootstrap", "bootstrap.py")), "-v", "-s", "-o", str(temp_gn_executable), "--gn-gen-args= use_sysroot=false"]
+            if not python2_command is None:
+                command_list.insert(0, python2_command)
+            result = subprocess.run(command_list, cwd=str(self.sandbox_root))
+            if not result.returncode == 0:
+                raise Exception("GN bootstrap command returned non-zero exit code: {}".format(result.returncode))
+        self.logger.info("Building gn using bootstrap gn...")
+        build_output = pathlib.Path("out", "gn_release")
+        (self.sandbox_root / build_output).mkdir(parents=True, exist_ok=True)
+        self._gn_write_args({"global": {"use_sysroot": "false", "is_debug": "false"}}, build_output)
+        self._gn_generate_ninja(build_output, gn_override=str(temp_gn_executable))
+        self._run_ninja(build_output, ["gn"])
+        return str(build_output / pathlib.Path("gn"))
 
     def setup_chromium_source(self, check_if_exists=True, force_download=False, check_integrity=True, extract_archive=True, destination_dir=pathlib.Path("."), cleaning_list=pathlib.Path("cleaning_list"), archive_path=None, hashes_path=None):
         '''
@@ -237,30 +296,43 @@ def apply_patches(self):
         # TODO: Use Python to apply patches defined in `patch_order`
         pass
 
-    def setup_build_utilities(self, build_gn=True, gn_command=None, python2_command=None):
-        '''
-        Sets up the utilities required for building. For now, this is just the "gn" tool.
-
-        If `build_gn` is True, then the `tools/gn/bootstrap/bootstrap.py` script is invoked in the build directory to build gn.
-        If `python2_command` is set, it must be a string of a command to invoke Python 2 for running bootstrap.py. Otherwise, the bootstrap.py path will be the executable path.
-
-        If `gn_command` is set, it must be a string of a command to invoke gn.
-
-        `build_gn` and `gn_command` are mutually exclusive.
-        '''
-        if build_gn and not gn_command is None:
-            raise Exception("Conflicting arguments: build_gn and gn_path")
-
-        if build_gn:
-            self._build_gn(python2_command)
-        else:
-            self.gn_command = gn_command
-
-    def generate_build_configuration(self):
-        pass
-
-    def pre_build_finalization(self):
-        pass
+    #def setup_build_utilities(self, build_gn=True, gn_command=None, python2_command=None, ninja_command="ninja"):
+    #    '''
+    #    Sets up the utilities required for building. For now, this is just the "gn" tool.
+    #
+    #    If `build_gn` is True, then the `tools/gn/bootstrap/bootstrap.py` script is invoked in the build directory to build gn.
+    #    If `python2_command` is set, it must be a string of a command to invoke Python 2 for running bootstrap.py. Otherwise, the bootstrap.py path will be the executable path.
+    #
+    #    If `gn_command` is set, it must be a string of a command to invoke gn.
+    #
+    #    `build_gn` and `gn_command` are mutually exclusive.
+    #    '''
+    #    if build_gn and not gn_command is None:
+    #        raise Exception("Conflicting arguments: build_gn and gn_path")
+    #    self.ninja_command = ninja_command
+    #    if build_gn:
+    #        self.gn_command = self._build_gn(python2_command)
+    #    else:
+    #        self.gn_command = gn_command
+
+    def setup_build_utilities(self, ninja_command="ninja"):
+        self.ninja_command = ninja_command
+
+    #def generate_build_configuration(self, gn_args=pathlib.Path("gn_args.ini"), build_output=pathlib.Path("out", "Default")):
+    #    (self.sandbox_root / build_output).mkdir(parents=True, exist_ok=True)
+    #    config = configparser.ConfigParser()
+    #    config.read(str(gn_args))
+    #    self._gn_write_args(config, build_output)
+    #    self._gn_generate_ninja(build_output)
+
+    def generate_build_configuration(self, gyp_flags=pathlib.Path("gyp_flags"), build_output=pathlib.Path("out", "Release"), python2_command=None):
+        self.logger.info("Running gyp command...")
+        gyp_list = list()
+        with gyp_flags.open() as f:
+            gyp_list = f.read().splitlines()
+        self._gyp_generate_ninja(gyp_list, build_output, python2_command)
+        self.build_output = build_output
 
     def build(self):
-        pass
+        self.logger.info("Running build command...")
+        self._run_ninja(self.build_output, ["chrome"])

building/resources/debian/gn_args.ini

@@ -0,0 +1,63 @@
+# NOTE: THIS FILE IS NOT USED YET
+
+# Debian-specific GN args
+# Can override values set by the cross-platform gn_args.ini
+
+# Values from GYP that don't have a GN equivalent:
+# linux_breakpad=0
+# linux_use_libgps=0
+# linux_use_bundled_gold=0
+# linux_link_gsettings=0
+# linux_link_libpci=1
+# linux_link_libspeechd=1
+# linux_strip_binary=1
+# use_system_re2=1
+# use_system_yasm=1
+# use_system_opus=1 (rtc_build_opus?)
+# use_system_zlib=1
+# use_system_speex=1
+# use_system_expat=1 (rtc_build_expat?)
+# use_system_libpng=1
+# use_system_libxml=1
+# use_system_libwebp=1
+# use_system_libxslt=1
+# use_system_libsrtp=1
+# use_system_jsoncpp=1 (rtc_build_json?)
+# use_system_libevent=1
+# use_system_xdg_utils=1
+
+[global]
+is_clang=true
+use_sysroot=false
+
+[//build/config/clang/BUILD.gn]
+clang_use_chrome_plugins=false
+
+[//build/config/features.gni]
+use_gconf=false
+use_gio=true
+
+[//build/config/allocator.gni]
+use_allocator="none"
+
+[//build/config/compiler/BUILD.gn]
+linux_use_bundled_binutils=false
+
+[//build/config/compiler/compiler.gni]
+symbol_level=1
+
+[//media/media_options.gni]
+link_pulseaudio=true
+
+[//third_party/BUILD.gn]
+use_system_libjpeg=true
+
+[//third_party/harfbuzz-ng/BUILD.gn]
+use_system_harfbuzz=true
+
+[//third_party/webrtc/build/webrtc.gni]
+rtc_build_opus=false
+rtc_build_expat=false
+rtc_build_libjpeg=false
+rtc_build_json=false
+rtc_jsoncpp_root="/usr/include/jsoncpp/json"

building/resources/debian/gyp_flags

@@ -0,0 +1,31 @@
+use_ozone=0
+use_gconf=0
+use_sysroot=0
+use_allocator=none
+linux_breakpad=0
+linux_use_libgps=0
+linux_use_bundled_gold=0
+linux_use_bundled_binutils=0
+use_gnome_keyring=0
+linux_link_gsettings=0
+linux_link_gnome_keyring=
+use_gio=1
+use_pulseaudio=1
+linux_link_libpci=1
+linux_link_libspeechd=1
+use_system_re2=1
+use_system_yasm=1
+use_system_opus=1
+use_system_zlib=1
+use_system_speex=1
+use_system_expat=1
+use_system_libpng=1
+use_system_libxml=1
+use_system_libjpeg=1
+use_system_libwebp=1
+use_system_libxslt=1
+use_system_libsrtp=1
+use_system_jsoncpp=1
+use_system_libevent=1
+use_system_harfbuzz=1
+use_system_xdg_utils=1

gn_args.ini

@@ -0,0 +1,55 @@
+# NOTE: THIS FILE IS NOT USED YET
+
+# Cross-platform GN arguments
+
+# Values from GYP that don't have a GN equivalent:
+# enable_automation=0
+# enable_wifi_bootstrapping=0
+# enable_pre_sync_backup=0
+# enable_prod_wallet_service=0
+# disable_newlib=1
+
+[//build/config/BUILDCONFIG.gn]
+is_debug=false
+
+[//build/config/compiler/BUILD.gn]
+treat_warnings_as_errors=false
+
+[//build/config/ui.gni]
+use_ozone=false
+
+[//build/config/features.gni]
+enable_remoting=false
+enable_nacl=false
+safe_browsing_mode=0
+enable_webrtc=false
+enable_hangout_services_extension=false
+enable_supervised_users=false
+fieldtrial_testing_like_official_build=true
+proprietary_codecs=true
+
+[//chrome/common/features.gni]
+enable_google_now=false
+enable_one_click_signin=false
+
+[//chrome/browser/BUILD.gn]
+enable_hotwording=false
+
+[//google_apis/BUILD.gn]
+google_api_key=""
+google_default_client_id=""
+google_default_client_secret=""
+use_official_google_api_keys=false
+
+[//media/media_options.gni]
+enable_hevc_demuxing=true
+enable_mse_mpeg2ts_stream_parser=true
+
+[//third_party/ffmpeg/ffmpeg_options.gni]
+ffmpeg_branding="ChromeOS"
+
+[//third_party/icu/config.gni]
+icu_use_data_file=false
+
+[//third_party/WebKit/Source/config.gni]
+remove_webcore_debug_symbols=true

gyp_flags

@@ -0,0 +1,34 @@
+werror=
+default_apps_list=[]
+default_apps_list_linux_dest=[]
+remoting=0
+disable_nacl=1
+disable_pnacl=1
+enable_remoting_host=0
+enable_automation=0
+enable_google_now=0
+safe_browsing=0
+enable_webrtc=0
+remoting=0
+enable_hotwording=0
+enable_hangout_services_extension=0
+enable_wifi_bootstrapping=0
+use_official_google_api_keys=0
+enable_rlz=0
+enable_pre_sync_backup=0
+enable_prod_wallet_service=0
+enable_supervised_users=0
+enable_one_click_signin=0
+disable_newlib=1
+icu_use_data_file_flag=0
+enable_hidpi=1
+fastbuild=1
+disable_fatal_linker_warnings=1
+tracing_like_official_build=1
+fieldtrial_testing_like_official_build=1
+linux_strip_binary=1
+proprietary_codecs=1
+ffmpeg_branding=ChromeOS
+enable_mpeg2ts_stream_parser=1
+enable_hevc_demuxing=1
+remove_webcore_debug_symbols=1