chromium: build with upstream llvm & PGO

pkgs/applications/networking/browsers/chromium/common.nix

@@ -225,6 +225,13 @@ let
       is_official_build = true;
       is_debug = false;
 
+      chrome_pgo_phase = 2; # TODO Remove after updating stable to M87+
+      pgo_data_path =
+        builtins.toString (fetchurl {
+          url = "https://commondatastorage.googleapis.com/chromium-optimization-profiles/pgo_profiles/${upstream-info.pgo-profile.name}";
+          sha256 = upstream-info.pgo-profile.sha256;
+        });
+
       proprietary_codecs = false;
       use_sysroot = false;
       use_gnome_keyring = gnomeKeyringSupport;

pkgs/applications/networking/browsers/chromium/default.nix

@@ -22,15 +22,20 @@
 }:
 
 let
-  llvmPackages = llvmPackages_11;
+  upstream-info = (lib.importJSON ./upstream-info.json).${channel};
+
+  llvmPackages = llvmPackages_11.override {
+    buildLlvmTools = llvmPackages.tools;
+    targetLlvmLibraries = llvmPackages.libraries;
+    buildOverride = upstream-info.deps.clang;
+  };
+
   stdenv = llvmPackages.stdenv;
 
   callPackage = newScope chromium;
 
   chromium = rec {
-    inherit stdenv llvmPackages;
-
-    upstream-info = (lib.importJSON ./upstream-info.json).${channel};
+    inherit upstream-info stdenv llvmPackages;
 
     mkChromiumDerivation = callPackage ./common.nix ({
       inherit channel gnome gnomeSupport gnomeKeyringSupport proprietaryCodecs

pkgs/applications/networking/browsers/chromium/update.py

@@ -16,6 +16,7 @@
 HISTORY_URL = 'https://omahaproxy.appspot.com/history?os=linux'
 DEB_URL = 'https://dl.google.com/linux/chrome/deb/pool/main/g'
 BUCKET_URL = 'https://commondatastorage.googleapis.com/chromium-browser-official'
+PGO_URL = 'https://commondatastorage.googleapis.com/chromium-optimization-profiles/pgo_profiles'
 
 JSON_PATH = dirname(abspath(__file__)) + '/upstream-info.json'
 
@@ -39,11 +40,29 @@ def get_file_revision(revision, file_path):
         return http_response.read()
 
 def get_channel_dependencies(channel):
+    clang_script = get_file_revision(channel['version'], 'tools/clang/scripts/update.py')
+    clang_commit_pattern = b"CLANG_REVISION = '.+([0-9a-f]{8})'"
+    clang_release_pattern = b"RELEASE_VERSION = '(\d+\.?\d+\.?\d+)'"
+    clang_commit = re.search(clang_commit_pattern, clang_script).group(1).decode()
+    clang_release = re.search(clang_release_pattern, clang_script).group(1).decode()
+    clang_github_owner = 'llvm'
+    clang_github_repo = 'llvm-project'
+    clang = nix_prefetch_git(f'https://github.com/{clang_github_owner}/{clang_github_repo}.git', clang_commit)
+
     deps = get_file_revision(channel['version'], 'DEPS')
     gn_pattern = b"'gn_version': 'git_revision:([0-9a-f]{40})'"
     gn_commit = re.search(gn_pattern, deps).group(1).decode()
     gn = nix_prefetch_git('https://gn.googlesource.com/gn', gn_commit)
+
     return {
+        'clang': {
+            'version': datetime.fromisoformat(clang['date']).date().isoformat(),
+            'release': clang_release,
+            'owner': clang_github_owner,
+            'repo': clang_github_repo,
+            'rev': clang['rev'],
+            'sha256': clang['sha256']
+        },
         'gn': {
             'version': datetime.fromisoformat(gn['date']).date().isoformat(),
             'url': gn['url'],
@@ -79,6 +98,13 @@ def get_channel_dependencies(channel):
         try:
             channel['sha256'] = nix_prefetch_url(f'{BUCKET_URL}/chromium-{build["version"]}.tar.xz')
             channel['sha256bin64'] = nix_prefetch_url(f'{DEB_URL}/google-chrome-{suffix}/google-chrome-{suffix}_{build["version"]}-1_amd64.deb')
+
+            pgo_filename = get_file_revision(build['version'], 'chrome/build/linux.pgo.txt').decode("utf-8").rstrip()
+            channel['pgo-profile'] = {
+                'name': pgo_filename,
+                'sha256': nix_prefetch_url(f'{PGO_URL}/{pgo_filename}')
+            }
+
         except subprocess.CalledProcessError:
             # This build isn't actually available yet.  Continue to
             # the next one.

pkgs/applications/networking/browsers/chromium/upstream-info.json

@@ -3,7 +3,19 @@
     "version": "86.0.4240.193",
     "sha256": "0d55xkw3fygqpa3a5bvz7vqmzb0d9w1kis72h54cnwsqgw4xag90",
     "sha256bin64": "1x1f8fv0fi965apk2a5v6jmhz8j5m3rz0k5x5vmkwqprvlblfjpw",
+    "pgo-profile": {
+      "name": "chrome-linux-master-1597082402-c5bd486642405d828e9e780a3fe4cf0c960a0b58.profdata",
+      "sha256": "0ifxvii6fzx5aq1wlksz49qrljvv58779y472mlf5y0vxjk5knkd"
+    },
     "deps": {
+      "clang": {
+        "version": "2020-08-18",
+        "release": "12.0.0",
+        "owner": "llvm",
+        "repo": "llvm-project",
+        "rev": "a1caa302970de86b15d360212b526be8f1d59641",
+        "sha256": "1j3z3ra9dp5jlfdfncq6y279dkb68xfr5mf5286db69i57zfdxjc"
+      },
       "gn": {
         "version": "2020-08-07",
         "url": "https://gn.googlesource.com/gn",
@@ -16,7 +28,19 @@
     "version": "87.0.4280.47",
     "sha256": "0a3c4kz85857wrbcxqknbqmai250d36nq332l66a83r8sidhbxfq",
     "sha256bin64": "0bjls2airr4p448rx7im1a7gff68yfnfb5vymr4bin4kz0s38mpw",
+    "pgo-profile": {
+      "name": "chrome-linux-4280-1604356843-75758dc7691325d08848840a6660d8bbac79840f.profdata",
+      "sha256": "1lajj3jizfrf8zzq39714g7w6n7b1758542x6927i9s0d4y1qirx"
+    },
     "deps": {
+      "clang": {
+        "version": "2020-09-05",
+        "release": "12.0.0",
+        "owner": "llvm",
+        "repo": "llvm-project",
+        "rev": "d0abc757495349fd053beeaea81cd954c2e457e7",
+        "sha256": "0w2y8xxhpsnfm3wpnrky5hfnnjxmzxq88gkmr6wgnpbssl8z0q0g"
+      },
       "gn": {
         "version": "2020-09-09",
         "url": "https://gn.googlesource.com/gn",
@@ -29,7 +53,19 @@
     "version": "88.0.4315.5",
     "sha256": "0k6man2xq8vmnk73dqv3fcc9929agqq8j6s6nvc9z653q4jdphkw",
     "sha256bin64": "0m139w8lwaawridk8in24ss1g1i6ah5298xky226k3vhk5i4mn3j",
+    "pgo-profile": {
+      "name": "chrome-linux-master-1604426269-c37b60c52a129039000efce9374c5d976d3c4e8a.profdata",
+      "sha256": "10mzlb5qqm2qz8s7f15hyjnpl7sbx0j1d6sr3gpm60icymdhqps3"
+    },
     "deps": {
+      "clang": {
+        "version": "2020-09-12",
+        "release": "12.0.0",
+        "owner": "llvm",
+        "repo": "llvm-project",
+        "rev": "f086e85eea94a51eb42115496ac5d24f07bc8791",
+        "sha256": "0s6k2x0knkx3r28pkzz2b3l4ixgxlckl08b7lhqq242hicp83jdb"
+      },
       "gn": {
         "version": "2020-10-21",
         "url": "https://gn.googlesource.com/gn",

pkgs/development/compilers/llvm/11/clang/default.nix

@@ -1,4 +1,5 @@
-{ stdenv, fetch, cmake, libxml2, llvm, version, clang-tools-extra_src, python3, lld
+{ stdenv, fetch, cmake, libxml2, llvm, version, buildOverride
+, clang-tools-extra_src, python3, lld
 , fixDarwinDylibNames
 , enableManpages ? false
 }:
@@ -11,13 +12,21 @@ let
     src = fetch "clang" "02ajkij85966vd150iy246mv16dsaph1kfi0y8wnncp8w6nar5hg";
     inherit clang-tools-extra_src;
 
-    unpackPhase = ''
-      unpackFile $src
-      mv clang-${version}* clang
-      sourceRoot=$PWD/clang
-      unpackFile ${clang-tools-extra_src}
-      mv clang-tools-extra-* $sourceRoot/tools/extra
-    '';
+    unpackPhase =
+      if buildOverride == null then ''
+        unpackFile $src
+        mv clang-${version}* clang
+        sourceRoot=$PWD/clang
+        unpackFile ${clang-tools-extra_src}
+        mv clang-tools-extra-* $sourceRoot/tools/extra
+      ''
+      else ''
+        sourceRoot=$PWD/clang
+        mkdir -p $sourceRoot/tools
+        cp -r $src/clang .
+        cp -r $src/clang-tools-extra $sourceRoot/tools/extra
+        chmod -R u+w .
+      '';
 
     nativeBuildInputs = [ cmake python3 lld ]
       ++ stdenv.lib.optional enableManpages python3.pkgs.sphinx

pkgs/development/compilers/llvm/11/compiler-rt.nix

@@ -1,4 +1,4 @@
-{ stdenv, version, fetch, cmake, python3, llvm, libcxxabi }:
+{ stdenv, version, buildOverride, fetch, cmake, python3, llvm, libcxxabi }:
 
 let
 
@@ -13,6 +13,13 @@ stdenv.mkDerivation rec {
   inherit version;
   src = fetch pname "0d5j5l8phwqjjscmk8rmqn0i2i0abl537gdbkagl8fjpzy1gyjip";
 
+  unpackPhase =
+    if buildOverride == null then null
+    else ''
+      cp -r $src/compiler-rt/. .
+      chmod -R u+w .
+    '';
+
   nativeBuildInputs = [ cmake python3 llvm ];
   buildInputs = stdenv.lib.optional stdenv.hostPlatform.isDarwin libcxxabi;
 

pkgs/development/compilers/llvm/11/default.nix

@@ -1,24 +1,38 @@
 { lowPrio, newScope, pkgs, stdenv, cmake, gccForLibs
-, libxml2, python3, isl, fetchurl, overrideCC, wrapCCWith, wrapBintoolsWith
+, libxml2, python3, isl, fetchurl, fetchFromGitHub
+, overrideCC, wrapCCWith, wrapBintoolsWith
 , buildPackages
 , buildLlvmTools # tools, but from the previous stage, for cross
 , targetLlvmLibraries # libraries, but from the next stage, for cross
+, buildOverride ? null
 }:
 
 let
-  release_version = "11.0.0";
-  version = release_version; # differentiating these (variables) is important for RCs
+  release_version =
+    if buildOverride == null then "11.0.0"
+    else buildOverride.release;
+  # differentiating these (variables) is important for RCs
+  version =
+    if buildOverride == null then release_version
+    else "unstable-${buildOverride.version}";
   targetConfig = stdenv.targetPlatform.config;
 
-  fetch = name: sha256: fetchurl {
-    url = "https://github.com/llvm/llvm-project/releases/download/llvmorg-${release_version}/${name}-${version}.src.tar.xz";
-    inherit sha256;
-  };
+  fetch =
+    if buildOverride == null
+    then
+      name: sha256: fetchurl {
+        url = "https://github.com/llvm/llvm-project/releases/download/llvmorg-${release_version}/${name}-${version}.src.tar.xz";
+        inherit sha256;
+      }
+    else
+      _: _: fetchFromGitHub {
+        inherit (buildOverride) owner repo rev sha256;
+      };
 
   clang-tools-extra_src = fetch "clang-tools-extra" "02bcwwn54661madhq4nxc069s7p7pj5gpqi8ww50w3anbpviilzy";
 
   tools = stdenv.lib.makeExtensible (tools: let
-    callPackage = newScope (tools // { inherit stdenv cmake libxml2 python3 isl release_version version fetch; });
+    callPackage = newScope (tools // { inherit stdenv cmake libxml2 python3 isl release_version version buildOverride fetch; });
     mkExtraBuildCommands = cc: ''
       rsrc="$out/resource-root"
       mkdir "$rsrc"
@@ -162,7 +176,7 @@ let
   });
 
   libraries = stdenv.lib.makeExtensible (libraries: let
-    callPackage = newScope (libraries // buildLlvmTools // { inherit stdenv cmake libxml2 python3 isl release_version version fetch; });
+    callPackage = newScope (libraries // buildLlvmTools // { inherit stdenv cmake libxml2 python3 isl release_version version buildOverride fetch; });
   in {
 
     compiler-rt = callPackage ./compiler-rt.nix ({} //

pkgs/development/compilers/llvm/11/fix-out-of-tree-build.patch

@@ -0,0 +1,53 @@
+From 5e4409f30817798377689ef6c229f1c82815e6e8 Mon Sep 17 00:00:00 2001
+From: Dave Airlie <airlied@redhat.com>
+Date: Tue, 6 Oct 2020 18:13:44 -0400
+Subject: [PATCH] Fix out-of-tree clang build due to sysexits change
+
+The sysexists change broke clang building out of tree against llvm.
+
+https://reviews.llvm.org/D88467
+---
+ llvm/include/llvm/Config/config.h.cmake      | 3 ---
+ llvm/include/llvm/Config/llvm-config.h.cmake | 3 +++
+ llvm/include/llvm/Support/ExitCodes.h        | 2 +-
+ 3 files changed, 4 insertions(+), 4 deletions(-)
+
+diff --git a/llvm/include/llvm/Config/config.h.cmake b/llvm/include/llvm/Config/config.h.cmake
+index 9ad0d827dfd8..fc3f94570e98 100644
+--- a/llvm/include/llvm/Config/config.h.cmake
++++ b/llvm/include/llvm/Config/config.h.cmake
+@@ -208,9 +208,6 @@
+ /* Define to 1 if you have the <sys/types.h> header file. */
+ #cmakedefine HAVE_SYS_TYPES_H ${HAVE_SYS_TYPES_H}
+
+-/* Define to 1 if you have the <sysexits.h> header file. */
+-#cmakedefine HAVE_SYSEXITS_H ${HAVE_SYSEXITS_H}
+-
+ /* Define if the setupterm() function is supported this platform. */
+ #cmakedefine LLVM_ENABLE_TERMINFO ${LLVM_ENABLE_TERMINFO}
+
+diff --git a/llvm/include/llvm/Config/llvm-config.h.cmake b/llvm/include/llvm/Config/llvm-config.h.cmake
+index ee299876825e..452c56e408de 100644
+--- a/llvm/include/llvm/Config/llvm-config.h.cmake
++++ b/llvm/include/llvm/Config/llvm-config.h.cmake
+@@ -88,4 +88,7 @@
+ /* Define if LLVM was built with a dependency to the tensorflow compiler */
+ #cmakedefine LLVM_HAVE_TF_AOT
+
++/* Define to 1 if you have the <sysexits.h> header file. */
++#cmakedefine HAVE_SYSEXITS_H ${HAVE_SYSEXITS_H}
++
+ #endif
+diff --git a/llvm/include/llvm/Support/ExitCodes.h b/llvm/include/llvm/Support/ExitCodes.h
+index 2715c5ca4128..b9041f5557d5 100644
+--- a/llvm/include/llvm/Support/ExitCodes.h
++++ b/llvm/include/llvm/Support/ExitCodes.h
+@@ -16,7 +16,7 @@
+ #ifndef LLVM_SUPPORT_EXITCODES_H
+ #define LLVM_SUPPORT_EXITCODES_H
+
+-#include "llvm/Config/config.h"
++#include "llvm/Config/llvm-config.h"
+
+ #if HAVE_SYSEXITS_H
+ #include <sysexits.h>

pkgs/development/compilers/llvm/11/lld.nix

@@ -4,6 +4,7 @@
 , libxml2
 , llvm
 , version
+, buildOverride
 }:
 
 stdenv.mkDerivation rec {
@@ -12,6 +13,13 @@ stdenv.mkDerivation rec {
 
   src = fetch pname "077xyh7sij6mhp4dc4kdcmp9whrpz332fa12rwxnzp3wgd5bxrzg";
 
+  unpackPhase =
+    if buildOverride == null then null
+    else ''
+      cp -r $src/lld/. .
+      chmod -R u+w .
+    '';
+
   nativeBuildInputs = [ cmake ];
   buildInputs = [ llvm libxml2 ];
 

pkgs/development/compilers/llvm/11/llvm.nix

@@ -9,6 +9,9 @@
 , ncurses
 , version
 , release_version
+, buildOverride
+, chromiumBeta
+, chromiumDev
 , zlib
 , buildPackages
 , debugVersion ? false
@@ -35,14 +38,23 @@ in stdenv.mkDerivation (rec {
   src = fetch pname "0s94lwil98w7zb7cjrbnxli0z7gklb312pkw74xs1d6zk346hgwi";
   polly_src = fetch "polly" "0h442ivcslr3dv3q3g1nw5avh77f8cxsp6zild1hgspj266xpynw";
 
-  unpackPhase = ''
-    unpackFile $src
-    mv llvm-${version}* llvm
-    sourceRoot=$PWD/llvm
-  '' + optionalString enablePolly ''
-    unpackFile $polly_src
-    mv polly-* $sourceRoot/tools/polly
-  '';
+  unpackPhase =
+    if buildOverride == null then ''
+      unpackFile $src
+      mv llvm-${version}* llvm
+      sourceRoot=$PWD/llvm
+    '' + optionalString enablePolly ''
+      unpackFile $polly_src
+      mv polly-* $sourceRoot/tools/polly
+    ''
+    else ''
+      cp -r $src/. .
+      chmod -R u+w .
+      sourceRoot=$PWD/llvm
+    '' + optionalString enablePolly ''
+      cp -r $src/polly $sourceRoot/tools
+      chmod -R u+w $sourceRoot/tools
+    '';
 
   outputs = [ "out" "python" ]
     ++ optional enableSharedLibraries "lib";
@@ -87,6 +99,13 @@ in stdenv.mkDerivation (rec {
     rm test/ExecutionEngine/frem.ll
   '' + ''
     patchShebangs test/BugPoint/compile-custom.ll.py
+  '' + optionalString (
+    (buildOverride == chromiumBeta.upstream-info.deps.clang) ||
+    (buildOverride == chromiumDev.upstream-info.deps.clang)
+  ) ''
+    # Apply patch needed to build llvm versions from non-stable chromium
+    # TODO Also apply this for chromium stable once it's needed
+    patch -p2 -i ${./fix-out-of-tree-build.patch}
   '';
 
   # hacky fix: created binaries need to be run before installation