WIP: Feature: creation of relocatable tarballs from installed packages

lib/spack/docs/binary_caches.rst

@@ -0,0 +1,66 @@
+.. _binary_caches:
+
+Binary caches
+============================
+
+.. warning:: The feature of binary caches is still experimental
+             and chosen conventions still may evolve over time.
+
+Some sites may encourage users to set up their own test environments
+before carrying out central installations, or some users prefer to set
+up these environments on their own motivation. To reduce the load of
+recompiling otherwise identical package specs in different installations,
+created build artifacts can be put into binary tarballs, uploaded onto 
+your spack mirror and then downloaded and installed by others.
+
+
+Creating binary tarballs
+-----------------------------
+
+Tarballs of sofware built can be created via ``spack create-tarball``.
+It allows either to tar up a single package or a package including all
+its dependencies (``-r``, ``--recurse``). The location for the tarballs
+can be given via the ``--directory`` option:
+
+.. code-block:: sh
+
+   $ spack create-tarball -d ~/caches -r bison
+   ==> recursing dependencies
+   ==> adding dependency bison@3.0.4%gcc@5.3.1=linux-x86_64^libsigsegv@2.10%gcc@5.3.1=linux-x86_64^m4@1.4.17%gcc@5.3.1+sigsegv=linux-x86_64
+   ==> adding dependency m4@1.4.17%gcc@5.3.1+sigsegv=linux-x86_64^libsigsegv@2.10%gcc@5.3.1=linux-x86_64
+   ==> adding dependency libsigsegv@2.10%gcc@5.3.1=linux-x86_64
+   ==> creating tarball for package bison@3.0.4%gcc@5.3.1=linux-x86_64^libsigsegv@2.10%gcc@5.3.1=linux-x86_64^m4@1.4.17%gcc@5.3.1+sigsegv=linux-x86_64 
+   ==> /home/hegner/caches/ubuntu16_04-x86_64/gcc-5.3.1/bison/ubuntu16_04-x86_64-bison-3.0.4-n6naf2v2wt2p5tg3jdveuqufhjwlba7o.tar.gz
+   ==> creating tarball for package libsigsegv@2.10%gcc@5.3.1=linux-x86_64 
+   ==> /home/hegner/caches/ubuntu16_04-x86_64/gcc-5.3.1/libsigsegv/ubuntu16_04-x86_64-libsigsegv-2.10-klc6t4jq2w6ochuz6xosu6vaujbwszds.tar.gz
+   ==> creating tarball for package m4@1.4.17%gcc@5.3.1+sigsegv=linux-x86_64^libsigsegv@2.10%gcc@5.3.1=linux-x86_64 
+   ==> /home/hegner/caches/ubuntu16_04-x86_64/gcc-5.3.1/m4/ubuntu16_04-x86_64-m4-1.4.17-6hpdn55vhztd25vxwuamxqo7edmootwv.tar.gz
+
+
+The created tarballs are put into the directory structure expected for the
+spack mirror.
+
+
+Installing binary tarballs
+--------------------------------
+
+To install binary tarballs, one has to add the corresponding spack mirror
+with ``spack mirror add <name> <url>``. Afterwards binaries can be installed
+via:
+
+.. code-block:: sh
+
+   $ spack install-tarball bison
+
+The package bison and all its dependencies will be downloaded from the 
+specified mirror(s). It fails if a package cannot be downloaded.
+
+
+Relocation
+-------------------------------
+
+Initial build and later installation do not necessarily happen at the same
+location. Spack provides a very basic relocation capability and corrects for
+RPATHs and non-relocatable scripts. However, many packages compile paths into
+binary artificats directly. In such cases, the build instructions of this
+package would need to be adjusted for better re-locatability.

lib/spack/docs/index.rst

@@ -65,6 +65,7 @@ or refer to the full manual below.
    repositories
    command_index
    package_list
+   binary_caches
 
 .. toctree::
    :maxdepth: 2

lib/spack/spack/binary_distribution.py

@@ -0,0 +1,230 @@
+# "Benedikt Hegner (CERN)"
+
+import os
+import platform
+import yaml
+
+import llnl.util.tty as tty
+from llnl.util.filesystem import mkdirp
+
+from spack.util.executable import which
+import spack.cmd
+import spack
+from spack.stage import Stage
+import spack.fetch_strategy as fs
+import spack.relocate
+
+
+def get_full_system_from_platform():
+    import platform
+    import re
+    system = platform.system()
+    if system == "Linux":
+        pf = platform.linux_distribution(full_distribution_name=0)[0]
+        version = platform.linux_distribution(full_distribution_name=0)[1]
+        if pf != 'Ubuntu':
+            # For non-Ubuntu major version number is enough
+            # to understand compatibility
+            version = version.split('.')[0]
+    elif system == "Darwin":
+        pf = "macos10"
+        version = platform.mac_ver()[0].split(".")[1]
+    else:
+        raise "System %s not supported" % system
+    sys_type = pf + version + '-' + platform.machine()
+    sys_type = re.sub(r'[^\w-]', '_', sys_type)
+    return sys_type.lower()
+
+
+def prepare():
+    """
+    Install patchelf as pre-requisite to the
+    required relocation of binary packages
+    """
+    if platform.system() == 'Darwin':
+        return
+    dir = os.getcwd()
+    patchelf_spec = spack.cmd.parse_specs("patchelf", concretize=True)[0]
+    if not spack.store.layout.check_installed(patchelf_spec):
+        patchelf = spack.repo.get(patchelf_spec)
+        patchelf.do_install()
+    os.chdir(dir)
+
+
+def buildinfo_file_name(spec):
+    """
+    Filename of the binary package meta-data file
+    """
+    return os.path.join(spec.prefix, ".spack", "binary_distribution")
+
+
+def read_buildinfo_file(package):
+    """
+    Read buildinfo file
+    """
+    filename = buildinfo_file_name(package)
+    with open(filename, 'r') as inputfile:
+        content = inputfile.read()
+        buildinfo = yaml.load(content)
+    return buildinfo
+
+
+def write_buildinfo_file(spec):
+    """
+    Create a cache file containing information
+    required for the relocation
+    """
+    text_to_relocate = []
+    binary_to_relocate = []
+    blacklist = (".spack", "man")
+    for root, dirs, files in os.walk(spec.prefix, topdown=True):
+        dirs[:] = [d for d in dirs if d not in blacklist]
+        for filename in files:
+            path_name = os.path.join(root, filename)
+            filetype = spack.relocate.get_filetype(path_name)
+            if spack.relocate.needs_binary_relocation(filetype):
+                rel_path_name = os.path.relpath(path_name, spec.prefix)
+                binary_to_relocate.append(rel_path_name)
+            elif spack.relocate.needs_text_relocation(filetype):
+                rel_path_name = os.path.relpath(path_name, spec.prefix)
+                text_to_relocate.append(rel_path_name)
+
+    # Create buildinfo data and write it to disk
+    buildinfo = {}
+    buildinfo['buildpath'] = spack.store.layout.root
+    buildinfo['relocate_textfiles'] = text_to_relocate
+    buildinfo['relocate_binaries'] = binary_to_relocate
+    filename = buildinfo_file_name(spec)
+    with open(filename, 'w') as outfile:
+        outfile.write(yaml.dump(buildinfo, default_flow_style=True))
+
+
+def tarball_directory_name(spec):
+    """
+    Return name of the tarball directory according to the convention
+    <os>-<architecture>/<compiler>/<package>/
+    """
+    return "%s/%s/%s" % (get_full_system_from_platform(),
+                         str(spec.compiler).replace("@", "-"),
+                         spec.name)
+
+
+def tarball_name(spec):
+    """
+    Return the name of the tarfile according to the convention
+    <os>-<architecture>-<package>-<dag_hash>.tar.gz
+    """
+    return "%s-%s-%s-%s.tar.gz" % (get_full_system_from_platform(),
+                                   spec.name,
+                                   spec.version,
+                                   spec.dag_hash())
+
+
+def tarball_path_name(spec):
+    """
+    Return the full path+name for a given spec according to the convention
+    <tarball_directory_name>/<tarball_name>
+    """
+    return os.path.join(tarball_directory_name(spec),
+                        tarball_name(spec))
+
+
+def build_tarball(spec, outdir, force=False):
+    """
+    Build a tarball from given spec and put it into the directory structure
+    used at the mirror (following <tarball_directory_name>).
+    """
+    tarfile_dir = os.path.join(outdir, tarball_directory_name(spec))
+    tarfile = os.path.join(outdir, tarball_path_name(spec))
+    if os.path.exists(tarfile):
+        if force:
+            os.remove(tarfile)
+        else:
+            tty.warn("file exists, use -f to force overwrite: %s" % tarfile)
+            return
+    if not os.path.exists(tarfile_dir):
+        mkdirp(tarfile_dir)
+
+    tar = which('tar', required=True)
+    dirname = os.path.dirname(spec.prefix)
+    basename = os.path.basename(spec.prefix)
+
+    # handle meta-data
+    cp = which("cp", required=True)
+    spec_file = os.path.join(spec.prefix, ".spack", "spec.yaml")
+    target_spec_file = tarfile + ".yaml"
+    cp(spec_file, target_spec_file)
+
+    # create info for later relocation and create tar
+    write_buildinfo_file(spec)
+    tar("--directory=%s" % dirname, "-czf", tarfile, basename)
+    tty.msg(tarfile)
+
+
+def download_tarball(package):
+    """
+    Download binary tarball for given package into stage area
+    Return True if successful
+    """
+    mirrors = spack.config.get_config('mirrors')
+    if len(mirrors) == 0:
+        tty.die("Please add a spack mirror to allow " +
+                "download of pre-compiled packages.")
+    tarball = tarball_path_name(package.spec)
+    for key in mirrors:
+        url = mirrors[key] + "/" + tarball
+        # print url
+        # stage the tarball into standard place
+        stage = Stage(url, name=package.stage.path)
+        try:
+            stage.fetch()
+            return True
+        except fs.FetchError:
+            next
+    return False
+
+
+def extract_tarball(package):
+    """
+    extract binary tarball for given package into install area
+    """
+    tarball = tarball_name(package.spec)
+    tar = which("tar")
+    local_tarball = package.stage.path + "/" + tarball
+    mkdirp('%s' % package.prefix)
+    tar("--strip-components=1",
+        "-C%s" % package.prefix,
+        "-xf",
+        local_tarball)
+
+
+def relocate_package(package):
+    """
+    Relocate the given package
+    """
+    buildinfo = read_buildinfo_file(package)
+    new_path = spack.store.layout.root
+    old_path = buildinfo['buildpath']
+    if old_path == new_path:
+        return True  # No need to relocate
+
+    tty.warn("Using experimental feature for relocating package from",
+             "%s to %s." % (old_path, new_path))
+
+    # as we may need patchelf, find out where it is
+    patchelf_executable = ''
+    if platform.system() != 'Darwin':
+        patchelf_spec = spack.cmd.parse_specs("patchelf", concretize=True)[0]
+        patchelf = spack.repo.get(patchelf_spec)
+        patchelf_executable = os.path.join(patchelf.prefix, "bin", "patchelf")
+
+    # now do the actual relocation
+    for filename in buildinfo['relocate_binaries']:
+        path_name = os.path.join(package.prefix, filename)
+        spack.relocate.relocate_binary(path_name,
+                                       old_path,
+                                       new_path,
+                                       patchelf_executable)
+    for filename in buildinfo['relocate_textfiles']:
+        path_name = os.path.join(package.prefix, filename)
+        spack.relocate.relocate_text(path_name, old_path, new_path)

lib/spack/spack/cmd/create_tarball.py

@@ -0,0 +1,68 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+import argparse
+
+import llnl.util.tty as tty
+
+import spack
+import spack.cmd
+from spack.binary_distribution import build_tarball
+
+
+description = "Create binary cache tarballs for given packages"
+section = "packaging"
+level = "long"
+
+
+def setup_parser(subparser):
+    subparser.add_argument('-r', '--recurse', action='store_true',
+                           help="also make tarballs for dependencies.")
+    subparser.add_argument('-f', '--force', action='store_true',
+                           help="overwrite tarball if it exists.")
+    subparser.add_argument('-d', '--directory', default=".",
+                           help="directory in which to save the tarballs.")
+
+    subparser.add_argument(
+        'packages', nargs=argparse.REMAINDER,
+        help="specs of packages to package")
+
+
+def create_tarball(parser, args):
+    if not args.packages:
+        tty.die("binary cache tarball creation requires at least one package argument")
+
+    pkgs = set(args.packages)
+    specs = set()
+    for pkg in pkgs:
+        for spec in spack.cmd.parse_specs(pkg, concretize=True):
+            specs.add(spec)
+            if args.recurse:
+                tty.msg('recursing dependencies')
+                for d, node in spec.traverse(order='pre', depth=True):
+                    tty.msg('adding dependency %s' % node)
+                    specs.add(node)
+    for spec in specs:
+        tty.msg('creating binary cache tarball for package %s ' % spec)
+        build_tarball(spec, args.directory, args.force)