Skip to content

Commit

Permalink
Add support for ghcjs 8.8 and making ghcjs bundles (#828)
Browse files Browse the repository at this point in the history 
Fixes ghcjs 8.6.5 and 8.8.4.

Uses emscripten for ghcjs 8.8 and adds scripts for building ghcjs 8.8.

Many fixes for ghcjs test failures.

Includes relocatableConfigFiles for making relocatable ghcjs bundles.
  • Loading branch information
@hamishmack
hamishmack committed Nov 17, 2020
1 parent 028eeb1 commit 7078b14
Show file tree
Hide file tree
Showing 92 changed files with 12,836 additions and 136 deletions.
5 changes: 3 additions & 2 deletions builder/comp-builder.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ let
]
);

setupGhcOptions = lib.optional (package.ghcOptions != null) '' --ghc-options="${package.ghcOptions}"'';
setupGhcOptions = lib.optional (package.ghcOptions != null) '' --ghc${lib.optionalString (stdenv.hostPlatform.isGhcjs) "js"}-options="${package.ghcOptions}"'';

executableToolDepends =
(lib.concatMap (c: if c.isHaskell or false
Expand All @@ -184,7 +184,7 @@ let
else abort "shellHook should be a string or a function";

exeExt = if stdenv.hostPlatform.isGhcjs then ".jsexe/all.js" else
if stdenv.hostPlatform.isWindows then ".exe" else "";
stdenv.hostPlatform.extensions.executable;
exeName = componentId.cname + exeExt;
testExecutable = "dist/build/${componentId.cname}/${exeName}";

Expand Down Expand Up @@ -246,6 +246,7 @@ let
inherit (package) identifier;
config = component;
inherit configFiles executableToolDepends cleanSrc exeName;
exePath = drv + "/bin/${exeName}";
env = shellWrappers;
profiled = self (drvArgs // { enableLibraryProfiling = true; });
} // lib.optionalAttrs (haskellLib.isLibrary componentId) ({
Expand Down
2 changes: 1 addition & 1 deletion builder/default.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ in {
inherit haskellLib ghc comp-builder setup-builder;
};

inherit shellFor;
inherit shellFor makeConfigFiles;

ghcWithPackages = withPackages { withHoogle = false; };
ghcWithHoogle = withPackages { withHoogle = true; };
Expand Down
19 changes: 14 additions & 5 deletions builder/make-config-files.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,13 +56,21 @@ let
(p: "${p.configFiles}")
libDeps;
in lib.concatStringsSep "\" \"" xs;
libs = lib.concatMapStringsSep "\" \"" (p: "${p}") libDeps;
in
runCommand "${ghc.targetPrefix}${fullName}-config" {
libs = lib.concatMapStringsSep "\" \"" (p: "${p}") libDeps;
drv = runCommand "${ghc.targetPrefix}${fullName}-config" {
nativeBuildInputs = [ghc];
passthru = {
inherit (ghc) targetPrefix;
inherit ghcCommand ghcCommandCaps libDir packageCfgDir;
inherit ghcCommand ghcCommandCaps libDir packageCfgDir component;
# Use ''${pkgroot} relative paths so that we can relocate the package database
# along with referenced packages and still have it work on systems with
# or without nix installed.
relocatableConfigFiles = runCommand "${ghc.targetPrefix}${fullName}-relocatable-config" ''
cp -r ${drv} $out
chmod -R +w $out
sed -i 's|/nix/store/|''${pkgroot}/../../../|' $out/${packageCfgDir}/*.conf
${target-pkg} -v0 --package-db $out/${packageCfgDir} recache
'';
};
} (''
mkdir -p $out
Expand Down Expand Up @@ -186,4 +194,5 @@ in
done
'' + ''
${target-pkg} -v0 --package-db $out/${packageCfgDir} recache
'')
'');
in drv
7 changes: 5 additions & 2 deletions ci.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,9 @@
# We need to use the actual nixpkgs version we're working with here, since the values
# of 'lib.systems.examples' are not understood between all versions
let lib = nixpkgs.lib;
in lib.optionalAttrs (system == "x86_64-linux" && (nixpkgsName == "R2009" || (compiler-nix-name != "ghc8101" && compiler-nix-name != "ghc8102" && compiler-nix-name != "ghc8102-experimental"))) {
in lib.optionalAttrs (nixpkgsName == "R2003" && (__elem compiler-nix-name ["ghc865" "ghc884"])) {
inherit (lib.systems.examples) ghcjs;
} // lib.optionalAttrs (system == "x86_64-linux" && (nixpkgsName == "R2009" || (!(__elem compiler-nix-name ["ghc8101" "ghc8102" "ghc8102-experimental"])))) {
# Windows cross compilation is currently broken on macOS
inherit (lib.systems.examples) mingwW64;
} // lib.optionalAttrs (system == "x86_64-linux") {
Expand Down Expand Up @@ -90,7 +92,8 @@ dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgs-pin:
} // pkgs.lib.optionalAttrs (runTests && crossSystemName != "aarch64-multiplatform") {
# Tests are broken on aarch64 cross https://github.com/input-output-hk/haskell.nix/issues/513
inherit (build) tests;
}) // pkgs.lib.optionalAttrs (ifdLevel >= 2) {
}) // pkgs.lib.optionalAttrs (ifdLevel >= 2 && crossSystemName != "ghcjs") {
# GHCJS builds its own template haskell runner.
remote-iserv = pkgs.ghc-extra-packages."${compiler-nix-name}".remote-iserv.components.exes.remote-iserv;
iserv-proxy = pkgs.ghc-extra-packages."${compiler-nix-name}".iserv-proxy.components.exes.iserv-proxy;
} // pkgs.lib.optionalAttrs (ifdLevel >= 3) {
Expand Down
4 changes: 2 additions & 2 deletions compiler/ghcjs/ghcjs-src.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"url": "https://github.com/ghcjs/ghcjs",
"rev": "e87195eaa2bc7e320e18cf10386802bc90b7c874",
"sha256": "02mwkf7aagxqi142gcmq048244apslrr72p568akcab9s0fn2gvy",
"rev": "31a54f7320b1f9700eb6389c2eb2d50c712e8371",
"sha256": "1cj1gfqcj0ygc23xj5gbm0ym9wmh8ini57zlgd6s25avgjbrwga6",
"fetchSubmodules": true
}
256 changes: 234 additions & 22 deletions compiler/ghcjs/ghcjs.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,18 @@
, ghcjsSrc ? pkgs.buildPackages.fetchgit (builtins.fromJSON (builtins.readFile ghcjsSrcJson))
, ghcjsVersion ? "8.6.0.1"
, ghcVersion ? "8.6.5"
, compiler-nix-name ? if builtins.compareVersions ghcjsVersion "8.8.0.0" > 0
then "ghc884"
else "ghc865"
, ghc ? pkgs.buildPackages.ghc
, happy ? pkgs.buildPackages.haskellPackages.happy
, alex ? pkgs.buildPackages.haskellPackages.alex
, cabal-install ? pkgs.buildPackages.cabal-install
}:
let
isGhcjs88 = builtins.compareVersions ghcjsVersion "8.8.0.0" > 0;

project = pkgs.buildPackages.haskell-nix.ghcjsProject {
src = ghcjsSrc;
inherit ghc ghcjsVersion ghcVersion happy alex cabal-install;
index-state = "2019-12-10T00:00:00Z";
inherit ghcjsVersion compiler-nix-name;
index-state = "2020-04-25T00:00:00Z";
# plan-sha256 = "1wy2lr08maxyi7r8jiwf2gj6pdayk5vxxwh42bj4s2gg4035z0yc";
# materialized = ../../materialized/ghcjs;
};
Expand All @@ -24,45 +26,255 @@ let
paths = [
ghcjs.components.exes.ghcjs
ghcjs.components.exes.ghcjs-pkg
ghcjs.components.exes.ghcjs-boot
ghcjs.components.exes.ghcjs-dumparchive
] ++ (if isGhcjs88
then [
ghcjs.components.exes.haddock
ghcjs.components.exes.private-ghcjs-run
ghcjs.components.exes.private-ghcjs-unlit
ghcjs.components.exes.private-ghcjs-hsc2hs
]
else [
ghcjs.components.exes.haddock-ghcjs
ghcjs.components.exes.hsc2hs-ghcjs
ghcjs.components.exes.ghcjs-boot
ghcjs.components.exes.ghcjs-run
ghcjs.components.exes.ghcjs-dumparchive
];
]);
};
libexec = "libexec/${builtins.replaceStrings ["darwin" "i686"] ["osx" "i386"] pkgs.stdenv.buildPlatform.system}-${ghc.name}/ghcjs-${ghcVersion}";
booted-ghcjs = pkgs.stdenv.mkDerivation {
name = "ghcjs-${ghcVersion}";
src = project.configured-src;

nativeBuildInputs = project.bootInputs;
nativeBuildInputs = project.bootInputs
++ pkgs.lib.optional isGhcjs88 pkgs.buildPackages.procps;
passthru = {
inherit all-ghcjs;
inherit all-ghcjs bundled-ghcjs project;
inherit (project) configured-src;
# Used to detect non haskell-nix compilers (accidental use of nixpkgs compilers can lead to unexpected errors)
isHaskellNixCompiler = true;
} // ghcjs.components.exes;
dontConfigure = true;
dontInstall = true;
buildPhase = ''
export HOME=$TMP
mkdir $HOME/.cabal
touch $HOME/.cabal/config
cd lib/boot
mkdir -p $out/bin
mkdir -p $out/lib/ghcjs-${ghcVersion}
lndir ${all-ghcjs}/${libexec} $out/bin
export HOME=$TMP
mkdir $HOME/.cabal
touch $HOME/.cabal/config
cd lib/boot
${if isGhcjs88
then ''
mkdir -p $out/bin
mkdir -p $out/lib
lndir ${all-ghcjs}/bin $out/bin
chmod -R +w $out/bin
rm $out/bin/ghcjs-boot
cp ${ghcjs.components.exes.ghcjs-boot}/bin/ghcjs-boot $out/bin
rm $out/bin/haddock
cp ${ghcjs.components.exes.haddock}/bin/haddock $out/bin
wrapProgram $out/bin/ghcjs --add-flags "-B$out/lib/ghcjs-${ghcVersion}"
wrapProgram $out/bin/haddock-ghcjs --add-flags "-B$out/lib/ghcjs-${ghcVersion}"
wrapProgram $out/bin/ghcjs-pkg --add-flags "--global-package-db=$out/lib/ghcjs-${ghcVersion}/package.conf.d"
wrapProgram $out/bin/ghcjs --add-flags "-B$out/lib"
wrapProgram $out/bin/haddock --add-flags "-B$out/lib"
wrapProgram $out/bin/ghcjs-pkg --add-flags "--global-package-db=$out/lib/package.conf.d"
''
else ''
mkdir -p $out/bin
mkdir -p $out/lib/ghcjs-${ghcVersion}
lndir ${all-ghcjs}/${libexec} $out/bin
env PATH=$out/bin:$PATH $out/bin/ghcjs-boot -j1 --with-ghcjs-bin $out/bin
wrapProgram $out/bin/ghcjs --add-flags "-B$out/lib/ghcjs-${ghcVersion}"
wrapProgram $out/bin/haddock-ghcjs --add-flags "-B$out/lib/ghcjs-${ghcVersion}"
wrapProgram $out/bin/ghcjs-pkg --add-flags "--global-package-db=$out/lib/ghcjs-${ghcVersion}/package.conf.d"
''
}
# Avoid timeouts while unix package runs hsc2hs (it does not print anything
# for more than 900 seconds).
{
for n in {1..50}; do
if [ ! -f $TMP/done ]; then
sleep 300
echo Keep alive $n
fi
done
} &
${ if isGhcjs88
then
# Unsets NIX_CFLAGS_COMPILE so the osx version of iconv.h is not used by mistake
''
env -u NIX_CFLAGS_COMPILE PATH=$out/bin:$PATH \
PYTHON=${pkgs.buildPackages.python3}/bin/python3 \
$out/bin/ghcjs-boot -j1 --with-emsdk=${project.emsdk} --no-prof --no-haddock \
|| (echo failed > $TMP/done; false)
''
else ''
env PATH=$out/bin:$PATH $out/bin/ghcjs-boot -j1 --with-ghcjs-bin $out/bin \
|| (echo failed > $TMP/done; false)
''
}
echo ok > $TMP/done
'';
# We hard code -j1 as a temporary workaround for
# https://github.com/ghcjs/ghcjs/issues/654
# enableParallelBuilding = true;
};
ghcjs-relocatable-bin = pkgs.stdenv.mkDerivation {
name = "ghcjs-relocatable-bin-${ghcVersion}";
src = all-ghcjs;
dontConfigure = true;
dontInstall = true;
dontPatchShebangs = true;
dontPatchELF = true;
buildPhase = ''
# Copy the ghcjs exectuables
mkdir -p $out/bin
cp $src/${libexec}/* $out/bin
# Add readlink (needed by bundleRootDir)
cp ${pkgs.coreutils}/bin/readlink $out/bin
'' + (pkgs.lib.optionalString pkgs.stdenv.isDarwin ''
# Make the executables location independent using install_name_tool and @executable_path
# Copy the libraries needed into place
cp ${pkgs.gmp}/lib/libgmp.10.dylib $out/bin
cp ${pkgs.ncurses}/lib/libncursesw.6.dylib $out/bin
cp ${pkgs.libffi}/lib/libffi.6.dylib $out/bin
# Set the ID of the libraries
chmod -R +w $out/bin
install_name_tool -id "@executable_path/libgmp.10.dylib" "$out/bin/libgmp.10.dylib"
install_name_tool -id "@executable_path/libncursesw.6.dylib" "$out/bin/libncursesw.6.dylib"
install_name_tool -id "@executable_path/libffi.6.dylib" "$out/bin/libffi.6.dylib"
# Modify all the references so we look for the libraries in the system location or
# @executable_path (the directory containin the exetubable itself).
for fn in $out/bin/*; do
install_name_tool -change "${pkgs.libiconv}/lib/libiconv.dylib" /usr/lib/libiconv.dylib "$fn"
install_name_tool -change "${pkgs.stdenv.libc}/lib/libSystem.B.dylib" /usr/lib/libSystem.B.dylib "$fn"
install_name_tool -change "${pkgs.gmp}/lib/libgmp.10.dylib" "@executable_path/libgmp.10.dylib" "$fn"
install_name_tool -change "${pkgs.ncurses}/lib/libncursesw.6.dylib" "@executable_path/libncursesw.6.dylib" "$fn"
install_name_tool -change "${pkgs.libffi}/lib/libffi.6.dylib" "@executable_path/libffi.6.dylib" "$fn"
done
'')

+ (pkgs.lib.optionalString pkgs.stdenv.isLinux ''
# Make the executables location independent using patchelf and $ORIGIN.
chmod -R +w $out/bin
# This interpreter setting will not work on nixOS, but this bundle is
# not really needed on nixOS systems.
patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 $out/bin/*
patchelf --set-rpath '$ORIGIN' $out/bin/*
# Link the libraries needed into place
ln -s ${pkgs.gmp}/lib/libgmp.so.* $out/bin
ln -s ${pkgs.ncurses}/lib/libncursesw.so.* $out/bin
ln -s ${pkgs.libffi}/lib/libffi.so.* $out/bin
'');
};
# This is a way to find the location of the root directory of this bundle
# when one of the wrapper scripts runs. By using readlink (to avoid
# issues with symlinks that might be made to the script) and dirname
# we can find the directory even when it may have been moved.
bundleRootDir = ''"$(dirname "$(dirname "$(readlink -f "$0")")")"'';
bundled-ghcjs = {
compilerName ? "ghcjs", # Name for the compiler wrapper
db ? null, # A ghcjs package database this argument should can
# be `project.(shellFor { ... }).configFiles` or
# the result of a `makeConfigFiles` call.
hostDb ? null # Like db, but this will be passed as the `-host-package-db`.
}:
let
libDeps = pkgs.lib.concatMapStrings (lib: "${lib} ") (pkgs.lib.unique (
[booted-ghcjs ghc db hostDb pkgs.ncurses pkgs.gmp pkgs.libffi]
++ (pkgs.haskell-nix.haskellLib.flatLibDepends db.component)
++ (pkgs.haskell-nix.haskellLib.flatLibDepends hostDb.component)
));
in pkgs.stdenv.mkDerivation {
name = "${compilerName}-${ghcVersion}-bundle";
src = booted-ghcjs;
nativeBuildInputs = [ pkgs.makeWrapper pkgs.xorg.lndir ];
dontConfigure = true;
dontInstall = true;
dontPatchShebangs = true;
dontPatchELF = true;
buildPhase = ''
# Copy the ghcjs exectuables
mkdir -p $out/bin
lndir ${ghcjs-relocatable-bin}/bin $out/bin
# Make the executables writeable for patchelf and install_name_tool
chmod -R +w $out/bin
# And links for the /lib directory of all the dependencies
# including the booted ghcjs
for lib in ${ libDeps }; do
if [ -d $lib/lib ]; then
mkdir -p $out/$(basename $lib)
lndir -silent $lib $out/$(basename $lib)
fi
done
'' + (pkgs.lib.optionalString pkgs.stdenv.isDarwin ''
rm -rf $out/$(basename ${hostDb})/lib/links
cp -rL ${hostDb}/lib/links $out/$(basename ${hostDb})/lib
chmod -R +w $out/$(basename ${hostDb})/lib/links
# Modify all the references so we look for the libraries in the system location or
# @executable_path (the directory containin the exetubable itself).
for fn in $out/$(basename ${hostDb})/lib/links/*; do
install_name_tool -change "${pkgs.libiconv}/lib/libiconv.dylib" /usr/lib/libiconv.dylib "$fn"
install_name_tool -change "${pkgs.stdenv.libc}/lib/libSystem.B.dylib" /usr/lib/libSystem.B.dylib "$fn"
install_name_tool -change "${pkgs.gmp}/lib/libgmp.10.dylib" "@executable_path/libgmp.10.dylib" "$fn"
install_name_tool -change "${pkgs.ncurses}/lib/libncursesw.6.dylib" "@executable_path/libncursesw.6.dylib" "$fn"
install_name_tool -change "${pkgs.libffi}/lib/libffi.6.dylib" "@executable_path/libffi.6.dylib" "$fn"
done
'') + ''
# Wrap the programs to add the ghcjs library dir and package DB directories
wrapProgram $out/bin/ghcjs \
--add-flags '"-B${bundleRootDir}/$(basename ${booted-ghcjs})/lib/ghcjs-${ghcVersion}"' \
--add-flags '"-package-db ${db}/${db.packageCfgDir}"' ${
pkgs.lib.optionalString (hostDb != null)
" --add-flags '-host-package-db=${hostDb}/${hostDb.packageCfgDir}'"
}
wrapProgram $out/bin/ghcjs-pkg --add-flags '"--global-package-db=${db}/${db.packageCfgDir}"'
wrapProgram $out/bin/haddock-ghcjs --add-flags '"-B${bundleRootDir}/$(basename ${booted-ghcjs})/lib/ghcjs-${ghcVersion}"'
# Fix the bang pattern to use the systems bash.
# Replace the absolute output path ($out) with bundleRootDir
# (this will fix the references to the unwarpped executables).
# Replace the `/nix/store` refs (in the package DB paths) with
# bundleRootDir.
sed -i \
-e 's|${pkgs.stdenv.shell}|/usr/bin/env -S bash|' \
-e "s|$out/|"'${bundleRootDir}/|g' \
-e 's|/nix/store/|${bundleRootDir}/|g' \
$out/bin/ghcjs $out/bin/haddock-ghcjs $out/bin/ghcjs-pkg
# Update the ghcjs and ghc settings files so that `cc` looked up in the PATH.
rm $out/$(basename ${booted-ghcjs})/lib/ghcjs-${ghcVersion}/settings
sed -e 's|/nix/store/.*/bin/cc|cc|' \
< ${booted-ghcjs}/lib/ghcjs-${ghcVersion}/settings \
> $out/$(basename ${booted-ghcjs})/lib/ghcjs-${ghcVersion}/settings
rm $out/$(basename ${ghc})/lib/ghc-${ghcVersion}/settings
sed -e 's|/nix/store/.*/bin/cc|cc|' \
< ${ghc}/lib/ghc-${ghcVersion}/settings \
> $out/$(basename ${ghc})/lib/ghc-${ghcVersion}/settings
# Update the ghcjs settings files so that `node` looked up in the PATH.
rm $out/$(basename ${booted-ghcjs})/lib/ghcjs-${ghcVersion}/nodeSettings.json
sed -e 's|/nix/store/.*/bin/node|node|' \
< ${booted-ghcjs}/lib/ghcjs-${ghcVersion}/nodeSettings.json \
> $out/$(basename ${booted-ghcjs})/lib/ghcjs-${ghcVersion}/nodeSettings.json
# Update the ghcjs settings files so that `node` looked up in the PATH.
rm $out/$(basename ${booted-ghcjs})/lib/ghcjs-${ghcVersion}/ghc_libdir
sed -e 's|/nix/store/|../../../|' \
< ${booted-ghcjs}/lib/ghcjs-${ghcVersion}/ghc_libdir \
> $out/$(basename ${booted-ghcjs})/lib/ghcjs-${ghcVersion}/ghc_libdir
'' + (pkgs.lib.optionalString (compilerName != "ghcjs") ''
# Rename the wrappers based on the `compilerName` arg
mv $out/bin/ghcjs $out/bin/${compilerName}
mv $out/bin/ghcjs-pkg $out/bin/${compilerName}-pkg
mv $out/bin/haddock-ghcjs $out/bin/haddock-${compilerName}
'');
};
in booted-ghcjs

0 comments on commit 7078b14

Please sign in to comment.