Refactor package specific code into cloudbio module

cloudbio/package/__init__.py

@@ -0,0 +1,2 @@
+"""Install software and configure package managers.
+"""

cloudbio/package/deb.py

@@ -0,0 +1,137 @@
+"""Automated installation on debian package systems with apt.
+"""
+from fabric.api import *
+from fabric.contrib.files import *
+
+from cloudbio.package.shared import _yaml_to_packages
+
+def _apt_packages(to_install):
+    """Install packages available via apt-get.
+    """
+    env.logger.info("Update and install all packages")
+    pkg_config_file = os.path.join(env.config_dir, "packages.yaml")
+    subs_pkg_config_file = os.path.join(env.config_dir, "packages-%s.yaml" %
+                                   env.distribution)
+    if not os.path.exists(subs_pkg_config_file): subs_pkg_config_file = None
+    sudo("apt-get update") # Always update
+    env.edition.apt_upgrade_system()
+    # Retrieve final package names
+    (packages, _) = _yaml_to_packages(pkg_config_file, to_install,
+                                      subs_pkg_config_file)
+    # At this point allow the Edition to rewrite the package list - 
+    # this is shared within and between editions.
+    # Ref:  https://github.com/chapmanb/cloudbiolinux/pull/10#issuecomment-1616423
+    packages = env.edition.rewrite_config_items("packages", packages)
+
+    # At this point allow the Flavor to rewrite the package list
+    packages = env.flavor.rewrite_config_items("packages", packages)
+
+    # A single line install is much faster - note that there is a max
+    # for the command line size, so we do 30 at a time
+    group_size = 30
+    i = 0
+    env.logger.info("Updating %i packages" % len(packages))
+    while i < len(packages):
+        sudo("apt-get -y --force-yes install %s" % " ".join(packages[i:i+group_size]))
+        i += group_size
+    sudo("apt-get clean")
+
+def _add_apt_gpg_keys():
+    """Adds GPG keys from all repositories
+    """
+    env.logger.info("Update GPG keys for repositories")
+    standalone = [
+        "http://archive.cloudera.com/debian/archive.key",
+        'http://download.virtualbox.org/virtualbox/debian/oracle_vbox.asc'
+    ]
+    keyserver = [
+            ("keyserver.ubuntu.com", "7F0CEB10"),
+            ("keyserver.ubuntu.com", "E084DAB9"),
+            ("keyserver.ubuntu.com", "D67FC6EAE2A11821"),
+        ]
+    standalone, keyserver = env.edition.rewrite_apt_keys(standalone, keyserver)
+    for key in standalone:
+        sudo("wget -q -O- %s | apt-key add -" % key)
+    for url, key in keyserver:
+        sudo("apt-key adv --keyserver %s --recv %s" % (url, key))
+
+def _setup_apt_automation():
+    """Setup the environment to be fully automated for tricky installs.
+
+    Sun Java license acceptance:
+    http://www.davidpashley.com/blog/debian/java-license
+
+    MySQL root password questions; install with empty root password:
+    http://snowulf.com/archives/540-Truly-non-interactive-unattended-apt-get-install.html
+
+    Postfix, setup for no configuration. See more on issues here:
+    http://www.uluga.ubuntuforums.org/showthread.php?p=9120196
+    """
+    interactive_cmd = "export DEBIAN_FRONTEND=noninteractive"
+    if not contains(env.shell_config, interactive_cmd):
+        append(env.shell_config, interactive_cmd)
+    package_info = [
+            "postfix postfix/main_mailer_type select No configuration",
+            "postfix postfix/mailname string notusedexample.org",
+            "mysql-server-5.1 mysql-server/root_password string '(password omitted)'",
+            "mysql-server-5.1 mysql-server/root_password_again string '(password omitted)'",
+            "sun-java6-jdk shared/accepted-sun-dlj-v1-1 select true",
+            "sun-java6-jre shared/accepted-sun-dlj-v1-1 select true",
+            "sun-java6-bin shared/accepted-sun-dlj-v1-1 select true",
+            "grub-pc grub2/linux_cmdline string ''",
+            "grub-pc grub-pc/install_devices_empty boolean true",
+            "acroread acroread/default-viewer boolean false",
+            ]
+    package_info = env.edition.rewrite_apt_automation(package_info)
+    cmd = ""
+    for l in package_info:
+        cmd += "echo %s | /usr/bin/debconf-set-selections ; " % l
+    sudo(cmd)
+
+def _setup_apt_sources():
+    """Add sources for retrieving library packages.
+       Using add-apt-repository allows processing PPAs (on Ubuntu)
+
+       This method modifies the apt sources file.
+
+       Uses python-software-properties, which provides an abstraction of apt repositories
+    """
+
+    # It may be sudo is not installed - which has fab fail - therefor
+    # we'll try to install it by default, assuming we have root access
+    # already (e.g. on EC2). Fab will fail anyway, otherwise.
+    if not exists('/usr/bin/sudo'):
+        run('apt-get update')
+        run('apt-get -y --force-yes install sudo')
+
+    env.logger.debug("_setup_apt_sources " + env.sources_file + " " + env.edition.name)
+    env.edition.check_packages_source()
+    comment = "# This file was modified for "+ env.edition.name
+    # Setup apt download policy (default is None)
+    # (see also https://help.ubuntu.com/community/PinningHowto)
+    preferences = env.edition.rewrite_apt_preferences([])
+    if len(preferences):
+        # make sure it exists, and is empty
+        sudo("rm -f %s" % env.apt_preferences_file)
+        sudo("touch %s" % env.apt_preferences_file)
+        append(env.apt_preferences_file, comment, use_sudo=True)
+        lines = "\n".join(preferences)
+        env.logger.debug("Policy %s" % lines)
+        # append won't duplicate, so we use echo
+        sudo("/bin/echo -e \"%s\" >> %s" % (lines, env.apt_preferences_file))
+        # check there is no error parsing the file
+        env.logger.debug(sudo("apt-cache policy"))
+
+    # Make sure a source file exists
+    if not exists(env.sources_file):
+        sudo("touch %s" % env.sources_file)
+    # Add a comment
+    if not contains(env.sources_file, comment):
+        append(env.sources_file, comment, use_sudo=True)
+    for source in env.edition.rewrite_apt_sources_list(env.std_sources):
+        env.logger.debug("Source %s" % source)
+        if source.startswith("ppa:"):
+            sudo("apt-get install -y --force-yes python-software-properties")
+            sudo("add-apt-repository '%s'" % source)
+        elif not contains(env.sources_file, source): # FIXME: append never adds dups!
+            append(env.sources_file, source, use_sudo=True)

cloudbio/package/nix.py

@@ -0,0 +1,41 @@
+"""Install software with the Nix package manager.
+"""
+from fabric.api import *
+from fabric.contrib.files import *
+
+from cloudbio.package.shared import _yaml_to_packages
+
+def _setup_nix_sources():
+    if env.nixpkgs:
+        # first override the path
+        append("/root/.bashrc", "export PATH=$HOME/.nix-profile/bin:$PATH", use_sudo=True)
+        env.logger.info("Checking NixPkgs")
+        if not exists("/nix/store"):
+            if not exists("/usr/bin/nix-env"):
+               # install Nix (standard Debian release)
+               nix_deb = "nix_0.16-1_i386.deb"
+               if not exists(nix_deb):
+                   run("wget http://hydra.nixos.org/build/565031/download/1/nix_0.16-1_i386.deb")
+                   sudo("dpkg -i "+nix_deb)
+            # Set sources
+            sudo("nix-channel --add http://nixos.org/releases/nixpkgs/channels/nixpkgs-unstable")
+            sudo("nix-channel --update")
+            # upgrade Nix to latest (and remove the older version, as it is much slower)
+            sudo("nix-env -b -i nix")
+            if exists("/usr/bin/nix-env"):
+                env.logger.info("uninstall older Nix (Debian release)")
+                sudo("dpkg -r nix")
+
+def _nix_packages(to_install):
+    """Install packages available via nixpkgs (optional)
+    """
+    if env.nixpkgs:
+        env.logger.info("Update and install NixPkgs packages")
+        pkg_config_file = os.path.join(env.config_dir, "packages-nix.yaml")
+        sudo("nix-channel --update")
+        # Retrieve final package names
+        (packages, _) = _yaml_to_packages(pkg_config_file, to_install)
+        packages = env.edition.rewrite_config_items("packages", packages)
+        packages = env.flavor.rewrite_config_items("packages", packages)
+        for p in packages:
+            sudo("nix-env -b -i %s" % p)

cloudbio/package/rpm.py

@@ -0,0 +1,38 @@
+"""Automated installation on RPM systems with the yum package manager.
+"""
+from fabric.api import *
+from fabric.contrib.files import *
+
+from cloudbio.package.shared import _yaml_to_packages
+
+def _yum_packages(to_install):
+    """Install rpm packages available via yum.
+    """
+    pkg_config = os.path.join(env.config_dir, "packages-yum.yaml")
+    with settings(warn_only=True):
+        sudo("yum check-update")
+    sudo("yum -y upgrade")
+    # Retrieve packages to get and install each of them
+    (packages, _) = _yaml_to_packages(pkg_config, to_install)
+    # At this point allow the Flavor to rewrite the package list
+    packages = env.flavor.rewrite_config_items("packages", packages)
+    for package in packages:
+        sudo("yum -y install %s" % package)
+
+def _setup_yum_bashrc():
+    """Fix the user bashrc to update compilers.
+    """
+    to_include = ["export CC=gcc44", "export CXX=g++44", "export FC=gfortran44",
+                  "export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:/usr/lib/pkgconfig"]
+    fname = run("ls %s" % env.shell_config)
+    for line in to_include:
+        if not contains(fname, line.split("=")[0]):
+            append(fname, line)
+
+def _setup_yum_sources():
+    """Add additional useful yum repositories.
+    """
+    repos = ["http://download.fedora.redhat.com/pub/epel/5/x86_64/epel-release-5-4.noarch.rpm"]
+    for repo in repos:
+        with settings(warn_only=True):
+            sudo("rpm -Uvh %s" % repo)

cloudbio/package/shared.py

@@ -0,0 +1,56 @@
+"""Shared functionality useful for multiple package managers.
+"""
+import yaml
+from fabric.api import *
+from fabric.contrib.files import *
+
+def _yaml_to_packages(yaml_file, to_install, subs_yaml_file = None):
+    """Read a list of packages from a nested YAML configuration file.
+    """
+    # allow us to check for packages only available on 64bit machines
+    machine = run("uname -m")
+    is_64bit = machine.find("_64") > 0
+    env.logger.info("Reading %s" % yaml_file)
+    with open(yaml_file) as in_handle:
+        full_data = yaml.load(in_handle)
+    if subs_yaml_file is not None:
+        with open(subs_yaml_file) as in_handle:
+            subs = yaml.load(in_handle)
+    else:
+        subs = {}
+    # filter the data based on what we have configured to install
+    data = [(k, v) for (k, v) in full_data.iteritems()
+            if to_install is None or k in to_install]
+    data.sort()
+    packages = []
+    pkg_to_group = dict()
+    while len(data) > 0:
+        cur_key, cur_info = data.pop(0)
+        if cur_info:
+            if isinstance(cur_info, (list, tuple)):
+                packages.extend(_filter_subs_packages(cur_info, subs))
+                for p in cur_info:
+                    pkg_to_group[p] = cur_key
+            elif isinstance(cur_info, dict):
+                for key, val in cur_info.iteritems():
+                    # if we are okay, propagate with the top level key
+                    if key != 'needs_64bit' or is_64bit:
+                        data.append((cur_key, val))
+            else:
+                raise ValueError(cur_info)
+    env.logger.debug("Packages:")
+    env.logger.debug(",".join(packages))
+    return packages, pkg_to_group
+
+def _filter_subs_packages(initial, subs):
+    """Rename and filter package list with subsitutions; for similar systems.
+    """
+    final = []
+    for p in initial:
+        try:
+            new_p = subs[p]
+        except KeyError:
+            new_p = p
+        if new_p:
+            final.append(new_p)
+    return sorted(final)