Merge remote-tracking branch 'dezgeg/shuffle-outputs' into staging

#14766

doc/multiple-output.xml

@@ -29,15 +29,15 @@
 
 <section><title>Using a split package</title>
   <para>In the Nix language the individual outputs can be reached explicitly as attributes, e.g. <varname>coreutils.info</varname>, but the typical case is just using packages as build inputs.</para>
-  <para>When a multiple-output derivation gets into a build input of another derivation, the first output is added (<varname>.dev</varname> by convention) and also <varname>propagatedBuildOutputs</varname> of that package which by default contain <varname>$outputBin</varname> and <varname>$outputLib</varname>. (See <xref linkend="multiple-output-file-type-groups" />.)</para>
+  <para>When a multiple-output derivation gets into a build input of another derivation, the <varname>dev</varname> output is added if it exists, otherwise the first output is added. In addition to that, <varname>propagatedBuildOutputs</varname> of that package which by default contain <varname>$outputBin</varname> and <varname>$outputLib</varname> are also added. (See <xref linkend="multiple-output-file-type-groups" />.)</para>
 </section>
 
 
 <section><title>Writing a split derivation</title>
   <para>Here you find how to write a derivation that produces multiple outputs.</para>
   <para>In nixpkgs there is a framework supporting multiple-output derivations. It tries to cover most cases by default behavior.  You can find the source separated in &lt;<filename>nixpkgs/pkgs/build-support/setup-hooks/multiple-outputs.sh</filename>&gt;; it's relatively well-readable.  The whole machinery is triggered by defining the <varname>outputs</varname> attribute to contain the list of desired output names (strings).</para>
-  <programlisting>outputs = [ "dev" "out" "bin" "doc" ];</programlisting>
-  <para>Often such a single line is enough.  For each output an equally named environment variable is passed to the builder and contains the path in nix store for that output.  By convention, the first output should usually be <varname>dev</varname>; typically you also want to have the main <varname>out</varname> output, as it catches any files that didn't get elsewhere.</para>
+  <programlisting>outputs = [ "bin" "dev" "out" "doc" ];</programlisting>
+  <para>Often such a single line is enough.  For each output an equally named environment variable is passed to the builder and contains the path in nix store for that output.  By convention, the first output should contain the executable programs provided by the package as that output is used by Nix in string conversions, allowing references to binaries like <literal>${pkgs.perl}/bin/perl</literal> to always work. Typically you also want to have the main <varname>out</varname> output, as it catches any files that didn't get elsewhere.</para>
 
   <note><para>There is a special handling of the <varname>debug</varname> output, described at <xref linkend="stdenv-separateDebugInfo" />.</para></note>
 

lib/attrsets.nix

@@ -454,6 +454,8 @@ rec {
   getLib = getOutput "lib";
   getDev = getOutput "dev";
 
+  /* Pick the outputs of packages to place in buildInputs */
+  chooseDevOutputs = drvs: builtins.map (drv: if drv.outputUnspecified or false then drv.dev or drv else drv) drvs;
 
   /*** deprecated stuff ***/
 

nixos/doc/manual/release-notes/rl-1609.xml

@@ -34,6 +34,17 @@ following incompatible changes:</para>
 
 <itemizedlist>
 
+  <listitem>
+    <para>A large number of packages have been converted to use the multiple outputs feature
+      of Nix to greatly reduce the amount of required disk space. This may require changes
+      to any custom packages to make them build again; see the relevant chapter in the
+      Nixpkgs manual for more information. (Additional caveat to packagers: some packaging conventions
+      related to multiple-output packages
+      <link xlink:href="https://github.com/NixOS/nixpkgs/pull/14766">were changed</link>
+      late (August 2016) in the release cycle and differ from the initial introduction of multiple outputs.)
+    </para>
+  </listitem>
+
   <listitem>
     <para>Shell aliases for systemd sub-commands
     <link xlink:href="https://github.com/NixOS/nixpkgs/pull/15598">were dropped</link>:

pkgs/applications/audio/ardour/ardour3.nix

@@ -48,7 +48,7 @@ stdenv.mkDerivation rec {
 
   patchPhase = ''
     printf '#include "libs/ardour/ardour/revision.h"\nnamespace ARDOUR { const char* revision = \"${revision}\"; }\n' > libs/ardour/revision.cc
-    sed 's|/usr/include/libintl.h|${glibc}/include/libintl.h|' -i wscript
+    sed 's|/usr/include/libintl.h|${glibc.dev}/include/libintl.h|' -i wscript
     patchShebangs ./tools/
   '';
 

pkgs/applications/audio/flac/default.nix

@@ -12,7 +12,7 @@ stdenv.mkDerivation rec {
 
   #doCheck = true; # takes lots of time
 
-  outputs = [ "dev" "out" "bin" "doc" ];
+  outputs = [ "bin" "dev" "out" "doc" ];
 
   meta = with stdenv.lib; {
     homepage = http://xiph.org/flac/;

pkgs/applications/graphics/ImageMagick/default.nix

@@ -43,7 +43,7 @@ stdenv.mkDerivation rec {
 
   patches = [ ./imagetragick.patch ] ++ cfg.patches;
 
-  outputs = [ "dev" "out" "doc" ]; # bin/ isn't really big
+  outputs = [ "out" "dev" "doc" ]; # bin/ isn't really big
   outputMan = "out"; # it's tiny
 
   enableParallelBuilding = true;

pkgs/applications/misc/djvulibre/default.nix

@@ -8,7 +8,7 @@ stdenv.mkDerivation rec {
     sha256 = "0psh3zl9dj4n4r3lx25390nx34xz0bg0ql48zdskhq354ljni5p6";
   };
 
-  outputs = [ "dev" "out" "bin" ];
+  outputs = [ "bin" "dev" "out" ];
 
   buildInputs = [ libjpeg libtiff librsvg ] ++ libintlOrEmpty;
 

pkgs/applications/misc/mupdf/default.nix

@@ -30,7 +30,7 @@ stdenv.mkDerivation rec {
   makeFlags = [ "prefix=$(out)" ];
   nativeBuildInputs = [ pkgconfig ];
   buildInputs = [ zlib libX11 libXcursor libXext harfbuzz mesa libXrandr libXinerama freetype libjpeg jbig2dec openjpeg ];
-  outputs = [ "out" "bin" "doc" ];
+  outputs = [ "bin" "out" "doc" ];
 
   preConfigure = ''
     # Don't remove mujs because upstream version is incompatible

pkgs/applications/misc/pgadmin/default.nix

@@ -18,7 +18,7 @@ stdenv.mkDerivation rec {
   '';
 
   configureFlags = [
-    "--with-libxml2=${libxml2}"
+    "--with-libxml2=${libxml2.dev}"
     "--with-libxslt=${libxslt.dev}"
   ];
 

pkgs/applications/misc/taskjuggler/default.nix

@@ -46,8 +46,8 @@ stdenv.mkDerivation rec {
 
   configureFlags = "
     --without-arts --disable-docs
-    --x-includes=${libX11}/include
-    --x-libraries=${libX11}/lib
+    --x-includes=${libX11.dev}/include
+    --x-libraries=${libX11.out}/lib
     --with-qt-dir=${qt3}
   ";
 

pkgs/applications/science/electronics/tkgate/1.x.nix

@@ -17,8 +17,8 @@ stdenv.mkDerivation rec {
   patchPhase = ''
     sed -i config.h \
       -e 's|.*#define.*TKGATE_TCLTK_VERSIONS.*|#define TKGATE_TCLTK_VERSIONS "${tcl.release}"|' \
-      -e 's|.*#define.*TKGATE_INCDIRS.*|#define TKGATE_INCDIRS "${tcl}/include ${tk}/include ${libiconvInc} ${libX11}/include"|' \
-      -e 's|.*#define.*TKGATE_LIBDIRS.*|#define TKGATE_LIBDIRS "${tcl}/lib ${tk}/lib ${libiconvLib} ${libX11}/lib"|' \
+      -e 's|.*#define.*TKGATE_INCDIRS.*|#define TKGATE_INCDIRS "${tcl}/include ${tk}/include ${libiconvInc} ${libX11.dev}/include"|' \
+      -e 's|.*#define.*TKGATE_LIBDIRS.*|#define TKGATE_LIBDIRS "${tcl}/lib ${tk}/lib ${libiconvLib} ${libX11.out}/lib"|' \
       \
       -e '20 i #define TCL_LIBRARY "${tcl}/lib"' \
       -e '20 i #define TK_LIBRARY "${tk}/lib/${tk.libPrefix}"' \

pkgs/applications/science/math/calc/default.nix

@@ -3,7 +3,7 @@
 with stdenv.lib;
 let
   makeFlags = ''
-    INCDIR=${glibc}/include \
+    INCDIR=${glibc.dev}/include \
     BINDIR=$out/bin LIBDIR=$out/lib CALC_INCDIR=$out/include/calc CALC_SHAREDIR=$out/share/calc MANDIR=$out/share/man/man1 \
     USE_READLINE=-DUSE_READLINE READLINE_LIB=-lreadline READLINE_EXTRAS='-lhistory -lncurses' \
     TERMCONTROL=-DUSE_TERMIOS \

pkgs/applications/version-management/redmine/default.nix

@@ -48,7 +48,7 @@ in stdenv.mkDerivation rec {
     mkdir -p vendor/cache
     ${stdenv.lib.concatStrings (map (gem: "ln -s ${gem} vendor/cache/${gem.name};") gemspec)}
 
-    bundle config build.nokogiri --use-system-libraries --with-iconv-dir="${libiconv}" --with-xslt-dir="${libxslt.dev}" --with-xml2-dir="${libxml2}"
+    bundle config build.nokogiri --use-system-libraries --with-iconv-dir="${libiconv}" --with-xslt-dir="${libxslt.dev}" --with-xml2-dir="${libxml2.dev}"
 
     bundle install --verbose --local --deployment
 

pkgs/applications/version-management/subversion/default.nix

@@ -27,7 +27,7 @@ let
     };
 
   # Can't do separate $lib and $bin, as libs reference bins
-  outputs = [ "dev" "out" "man" ];
+  outputs = [ "out" "dev" "man" ];
 
     buildInputs = [ zlib apr aprutil sqlite ]
       ++ stdenv.lib.optional httpSupport serf

pkgs/build-support/setup-hooks/multiple-outputs.sh

@@ -160,8 +160,7 @@ _multioutDevs() {
     done
 }
 
-# Make the first output (typically "dev") propagate other outputs needed for development.
-# Take the first, because that's what one gets when putting the package into buildInputs.
+# Make the "dev" propagate other outputs needed for development.
 # Note: with current cross-building setup, all packages are "native" if not cross-building;
 # however, if cross-building, the outputs are non-native. We have to choose the right file.
 _multioutPropagateDev() {
@@ -171,13 +170,17 @@ _multioutPropagateDev() {
     for outputFirst in $outputs; do
         break
     done
+    local propagaterOutput="$outputDev"
+    if [ -z "$propagaterOutput" ]; then
+        propagaterOutput="$outputFirst"
+    fi
 
     # Default value: propagate binaries, includes and libraries
     if [ -z "${propagatedBuildOutputs+1}" ]; then
         local po_dirty="$outputBin $outputInclude $outputLib"
         set +o pipefail
         propagatedBuildOutputs=`echo "$po_dirty" \
-            | tr -s ' ' '\n' | grep -v -F "$outputFirst" \
+            | tr -s ' ' '\n' | grep -v -F "$propagaterOutput" \
             | sort -u | tr '\n' ' ' `
         set -o pipefail
     fi
@@ -187,16 +190,16 @@ _multioutPropagateDev() {
         return
     fi
 
-    mkdir -p "${!outputFirst}"/nix-support
     local propagatedBuildInputsFile
     if [ -z "$crossConfig" ]; then
         propagatedBuildInputsFile=propagated-native-build-inputs
     else
         propagatedBuildInputsFile=propagated-build-inputs
     fi
 
+    mkdir -p "${!propagaterOutput}"/nix-support
     for output in $propagatedBuildOutputs; do
-        echo -n " ${!output}" >> "${!outputFirst}"/nix-support/$propagatedBuildInputsFile
+        echo -n " ${!output}" >> "${!propagaterOutput}"/nix-support/$propagatedBuildInputsFile
     done
 }
 

pkgs/desktops/gnome-2/platform/GConf/default.nix

@@ -11,7 +11,7 @@ stdenv.mkDerivation {
     sha256 = "09ch709cb9fniwc4221xgkq0jf0x0lxs814sqig8p2dcll0llvzk";
   };
 
-  outputs = [ "dev" "out" "doc" ];
+  outputs = [ "out" "dev" "doc" ];
 
   buildInputs = [ ORBit2 dbus_libs dbus_glib libxml2 ]
     # polkit requires pam, which requires shadow.h, which is not available on

pkgs/desktops/gnome-2/platform/ORBit2/default.nix

@@ -12,7 +12,7 @@ stdenv.mkDerivation rec {
   nativeBuildInputs = [ pkgconfig ];
   propagatedBuildInputs = [ glib libIDL ] ++ libintlOrEmpty;
 
-  outputs = [ "dev" "out" ];
+  outputs = [ "out" "dev" ];
 
   preBuild = ''
     sed 's/-DG_DISABLE_DEPRECATED//' -i linc2/src/Makefile

pkgs/desktops/gnome-2/platform/gnome-vfs/default.nix

@@ -10,7 +10,7 @@ stdenv.mkDerivation rec {
     sha256 = "1ajg8jb8k3snxc7rrgczlh8daxkjidmcv3zr9w809sq4p2sn9pk2";
   };
 
-  outputs = [ "dev" "out" ];
+  outputs = [ "out" "dev" ];
 
   buildInputs =
     [ pkgconfig libxml2 bzip2 openssl samba dbus_glib fam cdparanoia

pkgs/desktops/gnome-2/platform/libbonobo/default.nix

@@ -10,7 +10,7 @@ stdenv.mkDerivation rec {
     sha256 = "0swp4kk6x7hy1rvd1f9jba31lvfc6qvafkvbpg9h0r34fzrd8q4i";
   };
 
-  outputs = [ "dev" "out" ];
+  outputs = [ "out" "dev" ];
 
   preConfigure = # still using stuff deprecated in new glib versions
     "sed 's/-DG_DISABLE_DEPRECATED//g' -i configure activation-server/Makefile.in";

pkgs/desktops/gnome-2/platform/libglade/default.nix

@@ -8,7 +8,7 @@ stdenv.mkDerivation {
     sha256 = "1v2x2s04jry4gpabws92i0wq2ghd47yr5n9nhgnkd7c38xv1wdk4";
   };
 
-  outputs = [ "dev" "out" ];
+  outputs = [ "out" "dev" ];
 
   buildInputs = [ pkgconfig gtk python gettext ];
 

pkgs/desktops/gnome-2/platform/libgnome/default.nix

@@ -11,7 +11,7 @@ stdenv.mkDerivation rec {
     sha256 = "197pnq8y0knqjhm2fg4j6hbqqm3qfzfnd0irhwxpk1b4hqb3kimj";
   };
 
-  outputs = [ "dev" "out" ];
+  outputs = [ "out" "dev" ];
 
   patches = [ ./new-glib.patch ];
 

pkgs/desktops/gnome-2/platform/libgnomecanvas/default.nix

@@ -9,7 +9,7 @@ stdenv.mkDerivation rec {
     sha256 = "0h6xvswbqspdifnyh5pm2pqq55yp3kn6yrswq7ay9z49hkh7i6w5";
   };
 
-  outputs = [ "dev" "out" ];
+  outputs = [ "out" "dev" ];
 
   buildInputs = [ libglade ];
   nativeBuildInputs = [ pkgconfig intltool ];

pkgs/desktops/gnome-3/3.20/apps/evolution/default.nix

@@ -27,7 +27,7 @@ in stdenv.mkDerivation rec {
   configureFlags = [ "--disable-spamassassin" "--disable-pst-import" "--disable-autoar"
                      "--disable-libcryptui" ];
 
-  NIX_CFLAGS_COMPILE = "-I${nspr.dev}/include/nspr -I${nss}/include/nss -I${glib.dev}/include/gio-unix-2.0";
+  NIX_CFLAGS_COMPILE = "-I${nspr.dev}/include/nspr -I${nss.dev}/include/nss -I${glib.dev}/include/gio-unix-2.0";
 
   enableParallelBuilding = true;
 

pkgs/desktops/gnome-3/3.20/core/epiphany/default.nix

@@ -20,7 +20,7 @@ stdenv.mkDerivation rec {
                   gdk_pixbuf gnome3.defaultIconTheme librsvg which gnome_common
                   gcr avahi gnome3.gsettings_desktop_schemas gnome3.dconf ];
 
-  NIX_CFLAGS_COMPILE = "-I${nspr.dev}/include/nspr -I${nss}/include/nss -I${glib.dev}/include/gio-unix-2.0";
+  NIX_CFLAGS_COMPILE = "-I${nspr.dev}/include/nspr -I${nss.dev}/include/nss -I${glib.dev}/include/gio-unix-2.0";
 
   enableParallelBuilding = true;
 

pkgs/desktops/gnome-3/3.20/core/folks/default.nix

@@ -23,7 +23,7 @@ stdenv.mkDerivation rec {
 
   configureFlags = "--disable-fatal-warnings";
 
-  NIX_CFLAGS_COMPILE = ["-I${nspr.dev}/include/nspr" "-I${nss}/include/nss"
+  NIX_CFLAGS_COMPILE = ["-I${nspr.dev}/include/nspr" "-I${nss.dev}/include/nss"
                         "-I${dbus_glib.dev}/include/dbus-1.0" "-I${dbus_libs.dev}/include/dbus-1.0"];
 
   enableParallelBuilding = true;

pkgs/desktops/gnome-3/3.20/core/libcroco/default.nix

@@ -8,7 +8,7 @@ stdenv.mkDerivation rec {
     sha256 = "0mm0wldbi40am5qn0nv7psisbg01k42rwzjxl3gv11l5jj554aqk";
   };
 
-  outputs = [ "dev" "out" ];
+  outputs = [ "out" "dev" ];
   outputBin = "dev";
 
   configureFlags = stdenv.lib.optional stdenv.isDarwin "--disable-Bsymbolic";

pkgs/desktops/kde-5/plasma/plasma-desktop/default.nix

@@ -36,6 +36,6 @@ plasmaPackage rec {
   NIX_CFLAGS_COMPILE = [ "-I${xorgserver.dev}/include/xorg" ];
   cmakeFlags = [
     "-DEvdev_INCLUDE_DIRS=${xf86inputevdev.dev}/include/xorg"
-    "-DSynaptics_INCLUDE_DIRS=${xf86inputsynaptics}/include/xorg"
+    "-DSynaptics_INCLUDE_DIRS=${xf86inputsynaptics.dev}/include/xorg"
   ];
 }

pkgs/desktops/xfce/core/exo.nix

@@ -12,7 +12,7 @@ stdenv.mkDerivation rec {
   };
   name = "${p_name}-${ver_maj}.${ver_min}";
 
-  outputs = [ "dev" "out" "docdev" ];
+  outputs = [ "out" "dev" "docdev" ];
     # lib/xfce4/exo-1/exo-compose-mail-1 is a perl script :-/
 
   nativeBuildInputs = [ pkgconfig intltool ];

pkgs/desktops/xfce/core/garcon.nix

@@ -12,7 +12,7 @@ stdenv.mkDerivation rec {
     sha256 = "0wm9pjbwq53s3n3nwvsyf0q8lbmhiy2ln3bn5ncihr9vf5cwhzbq";
   };
 
-  outputs = [ "dev" "out" ];
+  outputs = [ "out" "dev" ];
 
   buildInputs = [ pkgconfig intltool glib libxfce4util gtk libxfce4ui ];
 

pkgs/desktops/xfce/core/libxfce4ui.nix

@@ -16,7 +16,7 @@ stdenv.mkDerivation rec {
     sha256 = "3d619811bfbe7478bb984c16543d980cadd08586365a7bc25e59e3ca6384ff43";
   };
 
-  outputs = [ "dev" "out" "docdev" ];
+  outputs = [ "out" "dev" "docdev" ];
 
   nativeBuildInputs = [ pkgconfig intltool ];
 

pkgs/desktops/xfce/core/libxfce4util.nix

@@ -12,7 +12,7 @@ stdenv.mkDerivation rec {
     sha256 = "07c8r3xwx5is298zk77m3r784gmr5y4mh8bbca5zdjqk5vxdwsw7";
   };
 
-  outputs = [ "dev" "out" "docdev" ];
+  outputs = [ "out" "dev" "docdev" ];
 
   buildInputs = [ pkgconfig glib intltool ];
 

pkgs/desktops/xfce/core/libxfcegui4.nix

@@ -13,7 +13,7 @@ stdenv.mkDerivation rec {
     sha256 = "0cs5im0ib0cmr1lhr5765yliqjfyxvk4kwy8h1l8bn3mj6bzk0ib";
   };
 
-  outputs = [ "dev" "out" "docdev" ];
+  outputs = [ "out" "dev" "docdev" ];
 
   #TODO: gladeui
   # By default, libxfcegui4 tries to install into libglade's prefix.

pkgs/desktops/xfce/core/tumbler.nix

@@ -13,7 +13,7 @@ stdenv.mkDerivation rec {
     sha256 = "0wvip28gm2w061hn84zp2q4dv947ihylrppahn4cjspzff935zfh";
   };
 
-  outputs = [ "dev" "out" "docdev" ];
+  outputs = [ "out" "dev" "docdev" ];
 
   buildInputs = [
     pkgconfig intltool dbus_glib gdk_pixbuf curl freetype

pkgs/desktops/xfce/core/xfce4-panel.nix

@@ -20,7 +20,7 @@ stdenv.mkDerivation rec {
   patches = [ ./xfce4-panel-datadir.patch ];
   patchFlags = "-p1";
 
-  outputs = [ "dev" "out" "docdev" ];
+  outputs = [ "out" "dev" "docdev" ];
 
   buildInputs =
     [ pkgconfig intltool gtk libxfce4util exo libwnck

pkgs/desktops/xfce/core/xfconf.nix

@@ -12,7 +12,7 @@ stdenv.mkDerivation rec {
     sha256 = "0mmi0g30aln3x98y5p507g17pipq0dj0bwypshan8cq5hkmfl44r";
   };
 
-  outputs = [ "dev" "out" "docdev" ];
+  outputs = [ "out" "dev" "docdev" ];
 
   #TODO: no perl bingings yet (ExtUtils::Depends, ExtUtils::PkgConfig, Glib)
   buildInputs = [ pkgconfig intltool glib libxfce4util ];

pkgs/development/compilers/orc/default.nix

@@ -8,7 +8,7 @@ stdenv.mkDerivation rec {
     sha256 = "1lak3hyvvb0w9avzmf0a8vayb7vqhj4m709q1czlhvgjb15dbcf1";
   };
 
-  outputs = [ "dev" "out" ];
+  outputs = [ "out" "dev" ];
   outputBin = "dev"; # compilation tools
 
   postInstall = ''

pkgs/development/compilers/terra/default.nix

@@ -19,7 +19,7 @@ stdenv.mkDerivation rec {
     sha256 = "1c2i9ih331304bh31c5gh94fx0qa49rsn70pvczvdfhi8pmcms6g";
   };
 
-  outputs = [ "dev" "out" "bin" "static" ];
+  outputs = [ "bin" "dev" "out" "static" ];
 
   postPatch = ''
     substituteInPlace Makefile --replace \