Skip to content

Commit

Permalink
chromium: Refactor updater entirely in Nix
Browse files Browse the repository at this point in the history 
The update.sh shell script now is only a call to nix-build, which does
all the hard work of updating the Chromium source channels and the
plugins. It results in a store path with the new sources.nix that
replaces the already existing sources.nix.

Along the way, this has led to a quite massive workaround, which abuses
MD5 collisions to detect whether an URL is existing, because something
like builtins.tryEval (builtins.fetchurl url) unfortunately doesn't
work. Further explanations and implementation details are documented in
the actual implementation.

The drawback of this is that we don't have nice status messages anymore,
but on the upside we have a more robust generation of the sources.nix
file, which now also should work properly on missing upstream
sources/binaries.

This also makes it much easier to implement fetching non-GNU/Linux
versions of Chromium and we have all values from omahaproxy available as
an attribute set (see the csv2nix and channels attributes in the update
attribute).

Signed-off-by: aszlig <aszlig@redmoonstudios.org>
  • Loading branch information
@aszlig
aszlig committed Feb 26, 2016
1 parent f6c3b13 commit 28b289e
Show file tree
Hide file tree
Showing 2 changed files with 190 additions and 189 deletions.
275 changes: 187 additions & 88 deletions pkgs/applications/networking/browsers/chromium/source/update.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,129 +3,228 @@
let
inherit (import ../../../../../../. {
inherit system;
}) lib writeText stdenv;
}) lib runCommand writeText stdenv curl cacert nix;

sources = if builtins.pathExists ./sources.nix
then import ./sources.nix
else null;
else {};

bucketURL = "https://commondatastorage.googleapis.com/"
+ "chromium-browser-official";

mkVerURL = version: "${bucketURL}/chromium-${version}.tar.xz";

debURL = "https://dl.google.com/linux/chrome/deb/pool/main/g";

getDebURL = channelName: version: arch: mirror: let
packageSuffix = if channelName == "dev" then "unstable" else channelName;
packageName = "google-chrome-${packageSuffix}";
in "${mirror}/${packageName}/${packageName}_${version}-1_${arch}.deb";

# Untrusted mirrors, don't try to update from them!
debMirrors = [
"http://95.31.35.30/chrome/pool/main/g"
"http://mirror.pcbeta.com/google/chrome/deb/pool/main/g"
"http://repo.fdzh.org/chrome/deb/pool/main/g"
];

tryChannel = channel: let
chan = builtins.getAttr channel sources;
in if sources != null then ''
oldver="${chan.version}";
echo -n "Checking if $oldver ($channel) is up to date..." >&2;
if [ "x$(get_newest_ver "$version" "$oldver")" != "x$oldver" ];
then
echo " no, getting sha256 for new version $version:" >&2;
sha256="$(prefetch_sha "$channel" "$version")" || return 1;
else
echo " yes, keeping old sha256." >&2;
sha256="${chan.sha256}";
${if (chan ? sha256bin32 && chan ? sha256bin64) then ''
sha256="$sha256.${chan.sha256bin32}.${chan.sha256bin64}";
'' else ''
sha256="$sha256.$(prefetch_deb_sha "$channel" "$version")";
''}
fi;
'' else ''
sha256="$(prefetch_sha "$channel" "$version")" || return 1;
'';

caseChannel = channel: ''
${channel}) ${tryChannel channel};;
'';

in rec {
getChannel = channel: let
chanAttrs = builtins.getAttr channel sources;
in {
inherit (chanAttrs) version;

main = {
url = "${bucketURL}/chromium-${chanAttrs.version}.tar.xz";
url = mkVerURL chanAttrs.version;
inherit (chanAttrs) sha256;
};

binary = let
pname = if channel == "dev"
then "google-chrome-unstable"
else "google-chrome-${channel}";

mkUrls = arch: let
relpath = "${pname}/${pname}_${chanAttrs.version}-1_${arch}.deb";
in map (url: "${url}/${relpath}") ([ debURL ] ++ debMirrors);

mkURLForMirror = getDebURL channel chanAttrs.version arch;
in map mkURLForMirror ([ debURL ] ++ debMirrors);
in if stdenv.is64bit && chanAttrs ? sha256bin64 then {
urls = mkUrls "amd64";
sha256 = chanAttrs.sha256bin64;
} else if stdenv.is32bit && chanAttrs ? sha256bin32 then {
urls = mkUrls "i386";
sha256 = chanAttrs.sha256bin64;
sha256 = chanAttrs.sha256bin32;
} else throw "No Chrome plugins are available for your architecture.";
};

updateHelpers = writeText "update-helpers.sh" ''
prefetch_main_sha()
{
nix-prefetch-url "${bucketURL}/chromium-$2.tar.xz";
}
prefetch_deb_sha()
{
channel="$1";
version="$2";
case "$1" in
dev) pname="google-chrome-unstable";;
*) pname="google-chrome-$channel";;
esac;
deb_pre="${debURL}/$pname/$pname";
deb32="$(nix-prefetch-url "''${deb_pre}_$version-1_i386.deb")" || :
deb64="$(nix-prefetch-url "''${deb_pre}_$version-1_amd64.deb")" || :
if [ -n "$deb32" -o -n "$deb64" ]; then
echo "$deb32.$deb64";
return 0
else
return 1
fi
}
prefetch_sha()
{
main_sha="$(prefetch_main_sha "$@")" || return 1;
deb_sha="$(prefetch_deb_sha "$@")" || return 1;
echo "$main_sha.$deb_sha";
return 0;
}
get_sha256()
{
channel="$1";
version="$2";
case "$channel" in
${lib.concatMapStrings caseChannel [ "stable" "dev" "beta" ]}
esac;
sha_insert "$version" "$sha256";
echo "$sha256";
return 0;
}
update = let
csv2nix = name: src: import (runCommand "${name}.nix" {
src = builtins.fetchurl src;
} ''
esc() { echo "\"$(echo "$1" | sed -e 's/"\\$/\\&/')\""; }
IFS=, read -r -a headings <<< "$(head -n1 "$src")"
echo "[" > "$out"
tail -n +2 "$src" | while IFS=, read -r -a line; do
echo " {"
for idx in "''${!headings[@]}"; do
echo " $(esc "''${headings[idx]}") = $(esc ''${line[$idx]});"
done
echo " }"
done >> "$out"
echo "]" >> "$out"
'');

channels = lib.fold lib.recursiveUpdate {} (map (attrs: {
${attrs.os}.${attrs.channel} = attrs // {
history = let
drvName = "omahaproxy-${attrs.os}.${attrs.channel}-info";
history = csv2nix drvName "http://omahaproxy.appspot.com/history";
cond = h: attrs.os == h.os && attrs.channel == h.channel
&& lib.versionOlder h.version attrs.current_version;
# Note that this is a *reverse* sort!
sorter = a: b: lib.versionOlder b.version a.version;
sorted = builtins.sort sorter (lib.filter cond history);
in map (lib.flip removeAttrs ["os" "channel"]) sorted;
version = attrs.current_version;
};
}) (csv2nix "omahaproxy-info" "http://omahaproxy.appspot.com/all?csv=1"));

/*
XXX: This is essentially the same as:
builtins.tryEval (builtins.fetchurl url)
... except that tryEval on fetchurl isn't working and doesn't catch errors
for fetchurl, so we go for a different approach.
We only have fixed-output derivations that can have networking access, so
we abuse MD5 and its weaknesses to forge a fixed-output derivation which
is not so fixed, because it emits different contents that have the same
MD5 hash.
Using this method, we can distinguish whether the URL is available or
whether it's not based on the actual content.
So let's use tryEval as soon as it's working with fetchurl in Nix.
*/
tryFetch = url: let

This comment has been minimized.

Sign in to view

Copy link
@aszlig

aszlig Feb 26, 2016

Author Member

@edolstra: Just to be sure: Should builtins.tryEval (builtins.fetchurl url) work or is this a bug in Nix?

This comment has been minimized.

Sign in to view

Copy link
@edolstra

edolstra Feb 26, 2016

Member

Well, the semantics of tryEval are poorly specified (in fact it's not even documented), so who can say. But I have a vague memory of making builtins.fetchurl errors uncatchable on purpose.

This comment has been minimized.

Sign in to view

Copy link
@aszlig

aszlig Feb 27, 2016

Author Member

So is there a better way to implement this rather than relying on MD5 collisions?
mkBin = b: runCommand "binary-blurb" { inherit b; } ''
h="$(echo "$b" | sed -e ':r;N;$!br;s/[^ \n][^ \n]/\\x&/g;s/[ \n]//g')"
echo -ne "$h" > "$out"
'';

# Both MD5 hash collision examples are from:
# https://en.wikipedia.org/wiki/MD5#Collision_vulnerabilities
hashCollTrue = mkBin ''
d131dd02c5e6eec4 693d9a0698aff95c 2fcab58712467eab 4004583eb8fb7f89
55ad340609f4b302 83e488832571415a 085125e8f7cdc99f d91dbdf280373c5b
d8823e3156348f5b ae6dacd436c919c6 dd53e2b487da03fd 02396306d248cda0
e99f33420f577ee8 ce54b67080a80d1e c69821bcb6a88393 96f9652b6ff72a70
'';

hashCollFalse = mkBin ''
d131dd02c5e6eec4 693d9a0698aff95c 2fcab50712467eab 4004583eb8fb7f89
55ad340609f4b302 83e4888325f1415a 085125e8f7cdc99f d91dbd7280373c5b
d8823e3156348f5b ae6dacd436c919c6 dd53e23487da03fd 02396306d248cda0
e99f33420f577ee8 ce54b67080280d1e c69821bcb6a88393 96f965ab6ff72a70
'';

cacheVal = let
urlHash = builtins.hashString "sha256" url;
timeSlice = builtins.currentTime / 600;
in "${urlHash}-${toString timeSlice}";

successBin = stdenv.mkDerivation {
name = "tryfetch-${cacheVal}";
inherit url;

outputHash = "79054025255fb1a26e4bc422aef54eb4";
outputHashMode = "flat";
outputHashAlgo = "md5";

buildInputs = [ curl ];
preferLocalBuild = true;

buildCommand = ''
if SSL_CERT_FILE="${cacert}/etc/ssl/certs/ca-bundle.crt" \
curl -s -L -f -I "$url" > /dev/null; then
cat "${hashCollTrue}" > "$out"
else
cat "${hashCollFalse}" > "$out"
fi
'';

impureEnvVars = [
"http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy"
];
};

in {
success = builtins.readFile successBin == builtins.readFile hashCollTrue;
value = builtins.fetchurl url;
};

fetchLatest = channel: let
result = tryFetch (mkVerURL channel.version);
in if result.success then result.value else fetchLatest (channel // {
version = if channel.history != []
then (lib.head channel.history).version
else throw "Unfortunately there's no older version than " +
"${channel.version} available for channel " +
"${channel.channel} on ${channel.os}.";
history = lib.tail channel.history;
});

getHash = path: import (runCommand "gethash.nix" {
inherit path;
buildInputs = [ nix ];
} ''
sha256="$(nix-hash --flat --base32 --type sha256 "$path")"
echo "\"$sha256\"" > "$out"
'');

isLatest = channel: version: let
ourVersion = sources.${channel}.version or null;
in if ourVersion == null then false
else lib.versionAtLeast version sources.${channel}.version;

# We only support GNU/Linux right now.
linuxChannels = let
genLatest = channelName: channel: let
newUpstream = {
inherit (channel) version;
sha256 = getHash (fetchLatest channel);
};
keepOld = let
oldChannel = sources.${channelName};
in {
inherit (oldChannel) version sha256;
} // lib.optionalAttrs (oldChannel ? sha256bin32) {
inherit (oldChannel) sha256bin32;
} // lib.optionalAttrs (oldChannel ? sha256bin64) {
inherit (oldChannel) sha256bin64;
};
in if isLatest channelName channel.version then keepOld else newUpstream;
in lib.mapAttrs genLatest channels.linux;

getLinuxFlash = channelName: channel: let
inherit (channel) version;
fetchArch = arch: tryFetch (getDebURL channelName version arch debURL);
packages = lib.genAttrs ["i386" "amd64"] fetchArch;
isNew = arch: attr: !(builtins.hasAttr attr channel)
&& packages.${arch}.success;
in channel // lib.optionalAttrs (isNew "i386" "sha256bin32") {
sha256bin32 = getHash (packages.i386.value);
} // lib.optionalAttrs (isNew "amd64" "sha256bin64") {
sha256bin64 = getHash (packages.amd64.value);
};

newChannels = lib.mapAttrs getLinuxFlash linuxChannels;

dumpAttrs = indent: attrs: let
mkVal = val: if lib.isAttrs val then dumpAttrs (indent + 1) val
else "\"${lib.escape ["$" "\\" "\""] (toString val)}\"";
mkIndent = level: lib.concatStrings (builtins.genList (_: " ") level);
mkAttr = key: val: "${mkIndent (indent + 1)}${key} = ${mkVal val};\n";
attrLines = lib.mapAttrsToList mkAttr attrs;
in "{\n" + (lib.concatStrings attrLines) + (mkIndent indent) + "}";
in writeText "chromium-new-sources.nix" ''
# This file is autogenerated from update.sh in the parent directory.
${dumpAttrs 0 newChannels}
'';
}
Loading

2 comments on commit 28b289e

@edolstra
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hm, this may be time for the periodic reminder that Nix is not intended as a general purpose language :-)

@shlevy
Copy link
Member

@shlevy shlevy commented on 28b289e Aug 15, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

NixOS/nix#1033 could alleviate the md5 hackery

Please sign in to comment.