From 887cbe40412c963be5a14d68534cd4cb2076a8bb Mon Sep 17 00:00:00 2001 From: chapmanb Date: Wed, 12 Oct 2011 07:56:19 -0400 Subject: [PATCH] Refactor package specific code into cloudbio module --- cloudbio/package/__init__.py | 2 + cloudbio/package/deb.py | 137 ++++++++++++++++++ cloudbio/package/nix.py | 41 ++++++ cloudbio/package/rpm.py | 38 +++++ cloudbio/package/shared.py | 56 ++++++++ fabfile.py | 263 +---------------------------------- 6 files changed, 281 insertions(+), 256 deletions(-) create mode 100644 cloudbio/package/__init__.py create mode 100644 cloudbio/package/deb.py create mode 100644 cloudbio/package/nix.py create mode 100644 cloudbio/package/rpm.py create mode 100644 cloudbio/package/shared.py diff --git a/cloudbio/package/__init__.py b/cloudbio/package/__init__.py new file mode 100644 index 000000000..ca3fdd794 --- /dev/null +++ b/cloudbio/package/__init__.py @@ -0,0 +1,2 @@ +"""Install software and configure package managers. +""" diff --git a/cloudbio/package/deb.py b/cloudbio/package/deb.py new file mode 100644 index 000000000..da0ef43bf --- /dev/null +++ b/cloudbio/package/deb.py @@ -0,0 +1,137 @@ +"""Automated installation on debian package systems with apt. +""" +from fabric.api import * +from fabric.contrib.files import * + +from cloudbio.package.shared import _yaml_to_packages + +def _apt_packages(to_install): + """Install packages available via apt-get. + """ + env.logger.info("Update and install all packages") + pkg_config_file = os.path.join(env.config_dir, "packages.yaml") + subs_pkg_config_file = os.path.join(env.config_dir, "packages-%s.yaml" % + env.distribution) + if not os.path.exists(subs_pkg_config_file): subs_pkg_config_file = None + sudo("apt-get update") # Always update + env.edition.apt_upgrade_system() + # Retrieve final package names + (packages, _) = _yaml_to_packages(pkg_config_file, to_install, + subs_pkg_config_file) + # At this point allow the Edition to rewrite the package list - + # this is shared within and between editions. + # Ref: https://github.com/chapmanb/cloudbiolinux/pull/10#issuecomment-1616423 + packages = env.edition.rewrite_config_items("packages", packages) + + # At this point allow the Flavor to rewrite the package list + packages = env.flavor.rewrite_config_items("packages", packages) + + # A single line install is much faster - note that there is a max + # for the command line size, so we do 30 at a time + group_size = 30 + i = 0 + env.logger.info("Updating %i packages" % len(packages)) + while i < len(packages): + sudo("apt-get -y --force-yes install %s" % " ".join(packages[i:i+group_size])) + i += group_size + sudo("apt-get clean") + +def _add_apt_gpg_keys(): + """Adds GPG keys from all repositories + """ + env.logger.info("Update GPG keys for repositories") + standalone = [ + "http://archive.cloudera.com/debian/archive.key", + 'http://download.virtualbox.org/virtualbox/debian/oracle_vbox.asc' + ] + keyserver = [ + ("keyserver.ubuntu.com", "7F0CEB10"), + ("keyserver.ubuntu.com", "E084DAB9"), + ("keyserver.ubuntu.com", "D67FC6EAE2A11821"), + ] + standalone, keyserver = env.edition.rewrite_apt_keys(standalone, keyserver) + for key in standalone: + sudo("wget -q -O- %s | apt-key add -" % key) + for url, key in keyserver: + sudo("apt-key adv --keyserver %s --recv %s" % (url, key)) + +def _setup_apt_automation(): + """Setup the environment to be fully automated for tricky installs. + + Sun Java license acceptance: + http://www.davidpashley.com/blog/debian/java-license + + MySQL root password questions; install with empty root password: + http://snowulf.com/archives/540-Truly-non-interactive-unattended-apt-get-install.html + + Postfix, setup for no configuration. See more on issues here: + http://www.uluga.ubuntuforums.org/showthread.php?p=9120196 + """ + interactive_cmd = "export DEBIAN_FRONTEND=noninteractive" + if not contains(env.shell_config, interactive_cmd): + append(env.shell_config, interactive_cmd) + package_info = [ + "postfix postfix/main_mailer_type select No configuration", + "postfix postfix/mailname string notusedexample.org", + "mysql-server-5.1 mysql-server/root_password string '(password omitted)'", + "mysql-server-5.1 mysql-server/root_password_again string '(password omitted)'", + "sun-java6-jdk shared/accepted-sun-dlj-v1-1 select true", + "sun-java6-jre shared/accepted-sun-dlj-v1-1 select true", + "sun-java6-bin shared/accepted-sun-dlj-v1-1 select true", + "grub-pc grub2/linux_cmdline string ''", + "grub-pc grub-pc/install_devices_empty boolean true", + "acroread acroread/default-viewer boolean false", + ] + package_info = env.edition.rewrite_apt_automation(package_info) + cmd = "" + for l in package_info: + cmd += "echo %s | /usr/bin/debconf-set-selections ; " % l + sudo(cmd) + +def _setup_apt_sources(): + """Add sources for retrieving library packages. + Using add-apt-repository allows processing PPAs (on Ubuntu) + + This method modifies the apt sources file. + + Uses python-software-properties, which provides an abstraction of apt repositories + """ + + # It may be sudo is not installed - which has fab fail - therefor + # we'll try to install it by default, assuming we have root access + # already (e.g. on EC2). Fab will fail anyway, otherwise. + if not exists('/usr/bin/sudo'): + run('apt-get update') + run('apt-get -y --force-yes install sudo') + + env.logger.debug("_setup_apt_sources " + env.sources_file + " " + env.edition.name) + env.edition.check_packages_source() + comment = "# This file was modified for "+ env.edition.name + # Setup apt download policy (default is None) + # (see also https://help.ubuntu.com/community/PinningHowto) + preferences = env.edition.rewrite_apt_preferences([]) + if len(preferences): + # make sure it exists, and is empty + sudo("rm -f %s" % env.apt_preferences_file) + sudo("touch %s" % env.apt_preferences_file) + append(env.apt_preferences_file, comment, use_sudo=True) + lines = "\n".join(preferences) + env.logger.debug("Policy %s" % lines) + # append won't duplicate, so we use echo + sudo("/bin/echo -e \"%s\" >> %s" % (lines, env.apt_preferences_file)) + # check there is no error parsing the file + env.logger.debug(sudo("apt-cache policy")) + + # Make sure a source file exists + if not exists(env.sources_file): + sudo("touch %s" % env.sources_file) + # Add a comment + if not contains(env.sources_file, comment): + append(env.sources_file, comment, use_sudo=True) + for source in env.edition.rewrite_apt_sources_list(env.std_sources): + env.logger.debug("Source %s" % source) + if source.startswith("ppa:"): + sudo("apt-get install -y --force-yes python-software-properties") + sudo("add-apt-repository '%s'" % source) + elif not contains(env.sources_file, source): # FIXME: append never adds dups! + append(env.sources_file, source, use_sudo=True) diff --git a/cloudbio/package/nix.py b/cloudbio/package/nix.py new file mode 100644 index 000000000..696e0b707 --- /dev/null +++ b/cloudbio/package/nix.py @@ -0,0 +1,41 @@ +"""Install software with the Nix package manager. +""" +from fabric.api import * +from fabric.contrib.files import * + +from cloudbio.package.shared import _yaml_to_packages + +def _setup_nix_sources(): + if env.nixpkgs: + # first override the path + append("/root/.bashrc", "export PATH=$HOME/.nix-profile/bin:$PATH", use_sudo=True) + env.logger.info("Checking NixPkgs") + if not exists("/nix/store"): + if not exists("/usr/bin/nix-env"): + # install Nix (standard Debian release) + nix_deb = "nix_0.16-1_i386.deb" + if not exists(nix_deb): + run("wget http://hydra.nixos.org/build/565031/download/1/nix_0.16-1_i386.deb") + sudo("dpkg -i "+nix_deb) + # Set sources + sudo("nix-channel --add http://nixos.org/releases/nixpkgs/channels/nixpkgs-unstable") + sudo("nix-channel --update") + # upgrade Nix to latest (and remove the older version, as it is much slower) + sudo("nix-env -b -i nix") + if exists("/usr/bin/nix-env"): + env.logger.info("uninstall older Nix (Debian release)") + sudo("dpkg -r nix") + +def _nix_packages(to_install): + """Install packages available via nixpkgs (optional) + """ + if env.nixpkgs: + env.logger.info("Update and install NixPkgs packages") + pkg_config_file = os.path.join(env.config_dir, "packages-nix.yaml") + sudo("nix-channel --update") + # Retrieve final package names + (packages, _) = _yaml_to_packages(pkg_config_file, to_install) + packages = env.edition.rewrite_config_items("packages", packages) + packages = env.flavor.rewrite_config_items("packages", packages) + for p in packages: + sudo("nix-env -b -i %s" % p) diff --git a/cloudbio/package/rpm.py b/cloudbio/package/rpm.py new file mode 100644 index 000000000..d36867dcb --- /dev/null +++ b/cloudbio/package/rpm.py @@ -0,0 +1,38 @@ +"""Automated installation on RPM systems with the yum package manager. +""" +from fabric.api import * +from fabric.contrib.files import * + +from cloudbio.package.shared import _yaml_to_packages + +def _yum_packages(to_install): + """Install rpm packages available via yum. + """ + pkg_config = os.path.join(env.config_dir, "packages-yum.yaml") + with settings(warn_only=True): + sudo("yum check-update") + sudo("yum -y upgrade") + # Retrieve packages to get and install each of them + (packages, _) = _yaml_to_packages(pkg_config, to_install) + # At this point allow the Flavor to rewrite the package list + packages = env.flavor.rewrite_config_items("packages", packages) + for package in packages: + sudo("yum -y install %s" % package) + +def _setup_yum_bashrc(): + """Fix the user bashrc to update compilers. + """ + to_include = ["export CC=gcc44", "export CXX=g++44", "export FC=gfortran44", + "export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:/usr/lib/pkgconfig"] + fname = run("ls %s" % env.shell_config) + for line in to_include: + if not contains(fname, line.split("=")[0]): + append(fname, line) + +def _setup_yum_sources(): + """Add additional useful yum repositories. + """ + repos = ["http://download.fedora.redhat.com/pub/epel/5/x86_64/epel-release-5-4.noarch.rpm"] + for repo in repos: + with settings(warn_only=True): + sudo("rpm -Uvh %s" % repo) diff --git a/cloudbio/package/shared.py b/cloudbio/package/shared.py new file mode 100644 index 000000000..16221e7b4 --- /dev/null +++ b/cloudbio/package/shared.py @@ -0,0 +1,56 @@ +"""Shared functionality useful for multiple package managers. +""" +import yaml +from fabric.api import * +from fabric.contrib.files import * + +def _yaml_to_packages(yaml_file, to_install, subs_yaml_file = None): + """Read a list of packages from a nested YAML configuration file. + """ + # allow us to check for packages only available on 64bit machines + machine = run("uname -m") + is_64bit = machine.find("_64") > 0 + env.logger.info("Reading %s" % yaml_file) + with open(yaml_file) as in_handle: + full_data = yaml.load(in_handle) + if subs_yaml_file is not None: + with open(subs_yaml_file) as in_handle: + subs = yaml.load(in_handle) + else: + subs = {} + # filter the data based on what we have configured to install + data = [(k, v) for (k, v) in full_data.iteritems() + if to_install is None or k in to_install] + data.sort() + packages = [] + pkg_to_group = dict() + while len(data) > 0: + cur_key, cur_info = data.pop(0) + if cur_info: + if isinstance(cur_info, (list, tuple)): + packages.extend(_filter_subs_packages(cur_info, subs)) + for p in cur_info: + pkg_to_group[p] = cur_key + elif isinstance(cur_info, dict): + for key, val in cur_info.iteritems(): + # if we are okay, propagate with the top level key + if key != 'needs_64bit' or is_64bit: + data.append((cur_key, val)) + else: + raise ValueError(cur_info) + env.logger.debug("Packages:") + env.logger.debug(",".join(packages)) + return packages, pkg_to_group + +def _filter_subs_packages(initial, subs): + """Rename and filter package list with subsitutions; for similar systems. + """ + final = [] + for p in initial: + try: + new_p = subs[p] + except KeyError: + new_p = p + if new_p: + final.append(new_p) + return sorted(final) diff --git a/fabfile.py b/fabfile.py index 09811bb29..26b7989fa 100755 --- a/fabfile.py +++ b/fabfile.py @@ -33,6 +33,12 @@ from cloudbio.distribution import _setup_distribution_environment from cloudbio.utils import _setup_logging, _update_biolinux_log from cloudbio.cloudman import (_configure_cloudman, _cleanup_ec2) +from cloudbio.package.shared import _yaml_to_packages +from cloudbio.package.deb import (_apt_packages, _add_apt_gpg_keys, + _setup_apt_automation, _setup_apt_sources) +from cloudbio.package.rpm import (_yum_packages, _setup_yum_bashrc, + _setup_yum_sources) +from cloudbio.package.nix import _setup_nix_sources, _nix_packages # ## Utility functions for establishing our build environment @@ -218,63 +224,12 @@ def install_custom(p, automated=False, pkg_to_group=None): % (p, pkg_to_group[p])) fn(env) -def _yaml_to_packages(yaml_file, to_install, subs_yaml_file = None): - """Read a list of packages from a nested YAML configuration file. - """ - # allow us to check for packages only available on 64bit machines - machine = run("uname -m") - is_64bit = machine.find("_64") > 0 - env.logger.info("Reading %s" % yaml_file) - with open(yaml_file) as in_handle: - full_data = yaml.load(in_handle) - if subs_yaml_file is not None: - with open(subs_yaml_file) as in_handle: - subs = yaml.load(in_handle) - else: - subs = {} - # filter the data based on what we have configured to install - data = [(k, v) for (k,v) in full_data.iteritems() - if to_install is None or k in to_install] - data.sort() - packages = [] - pkg_to_group = dict() - while len(data) > 0: - cur_key, cur_info = data.pop(0) - if cur_info: - if isinstance(cur_info, (list, tuple)): - packages.extend(_filter_subs_packages(cur_info, subs)) - for p in cur_info: - pkg_to_group[p] = cur_key - elif isinstance(cur_info, dict): - for key, val in cur_info.iteritems(): - # if we are okay, propagate with the top level key - if key != 'needs_64bit' or is_64bit: - data.append((cur_key, val)) - else: - raise ValueError(cur_info) - env.logger.debug("Packages:") - env.logger.debug(",".join(packages)) - return packages, pkg_to_group - -def _filter_subs_packages(initial, subs): - """Rename and filter package list with subsitutions; for similar systems. - """ - final = [] - for p in initial: - try: - new_p = subs[p] - except KeyError: - new_p = p - if new_p: - final.append(new_p) - return sorted(final) - def _read_main_config(yaml_file=None): """Pull a list of groups to install based on our main configuration YAML. Reads 'main.yaml' and returns packages and libraries """ - if yaml_file==None: + if yaml_file is None: yaml_file = os.path.join(env.config_dir, "main.yaml") with open(yaml_file) as in_handle: full_data = yaml.load(in_handle) @@ -421,210 +376,6 @@ def _do_library_installs(to_install): config = yaml.load(in_handle) lib_installers[iname](config) -# ### Automated installation on apt systems - -def _apt_packages(to_install): - """Install packages available via apt-get. - """ - env.logger.info("Update and install all packages") - pkg_config_file = os.path.join(env.config_dir, "packages.yaml") - subs_pkg_config_file = os.path.join(env.config_dir, "packages-%s.yaml" % - env.distribution) - if not os.path.exists(subs_pkg_config_file): subs_pkg_config_file = None - sudo("apt-get update") # Always update - env.edition.apt_upgrade_system() - # Retrieve final package names - (packages, _) = _yaml_to_packages(pkg_config_file, to_install, - subs_pkg_config_file) - # At this point allow the Edition to rewrite the package list - - # this is shared within and between editions. - # Ref: https://github.com/chapmanb/cloudbiolinux/pull/10#issuecomment-1616423 - packages = env.edition.rewrite_config_items("packages", packages) - - # At this point allow the Flavor to rewrite the package list - packages = env.flavor.rewrite_config_items("packages", packages) - - # A single line install is much faster - note that there is a max - # for the command line size, so we do 30 at a time - group_size = 30 - i = 0 - env.logger.info("Updating %i packages" % len(packages)) - while i < len(packages): - sudo("apt-get -y --force-yes install %s" % " ".join(packages[i:i+group_size])) - i += group_size - sudo("apt-get clean") - -def _add_apt_gpg_keys(): - """Adds GPG keys from all repositories - """ - env.logger.info("Update GPG keys for repositories") - standalone = [ - "http://archive.cloudera.com/debian/archive.key", - 'http://download.virtualbox.org/virtualbox/debian/oracle_vbox.asc' - ] - keyserver = [ - ("keyserver.ubuntu.com", "7F0CEB10"), - ("keyserver.ubuntu.com", "E084DAB9"), - ("keyserver.ubuntu.com", "D67FC6EAE2A11821"), - ] - standalone, keyserver = env.edition.rewrite_apt_keys(standalone, keyserver) - for key in standalone: - sudo("wget -q -O- %s | apt-key add -" % key) - for url, key in keyserver: - sudo("apt-key adv --keyserver %s --recv %s" % (url, key)) - -def _setup_apt_automation(): - """Setup the environment to be fully automated for tricky installs. - - Sun Java license acceptance: - http://www.davidpashley.com/blog/debian/java-license - - MySQL root password questions; install with empty root password: - http://snowulf.com/archives/540-Truly-non-interactive-unattended-apt-get-install.html - - Postfix, setup for no configuration. See more on issues here: - http://www.uluga.ubuntuforums.org/showthread.php?p=9120196 - """ - interactive_cmd = "export DEBIAN_FRONTEND=noninteractive" - if not contains(env.shell_config, interactive_cmd): - append(env.shell_config, interactive_cmd) - package_info = [ - "postfix postfix/main_mailer_type select No configuration", - "postfix postfix/mailname string notusedexample.org", - "mysql-server-5.1 mysql-server/root_password string '(password omitted)'", - "mysql-server-5.1 mysql-server/root_password_again string '(password omitted)'", - "sun-java6-jdk shared/accepted-sun-dlj-v1-1 select true", - "sun-java6-jre shared/accepted-sun-dlj-v1-1 select true", - "sun-java6-bin shared/accepted-sun-dlj-v1-1 select true", - "grub-pc grub2/linux_cmdline string ''", - "grub-pc grub-pc/install_devices_empty boolean true", - "acroread acroread/default-viewer boolean false", - ] - package_info = env.edition.rewrite_apt_automation(package_info) - cmd = "" - for l in package_info: - cmd += "echo %s | /usr/bin/debconf-set-selections ; " % l - sudo(cmd) - -def _setup_apt_sources(): - """Add sources for retrieving library packages. - Using add-apt-repository allows processing PPAs (on Ubuntu) - - This method modifies the apt sources file. - - Uses python-software-properties, which provides an abstraction of apt repositories - """ - - # It may be sudo is not installed - which has fab fail - therefor - # we'll try to install it by default, assuming we have root access - # already (e.g. on EC2). Fab will fail anyway, otherwise. - if not exists('/usr/bin/sudo'): - run('apt-get update') - run('apt-get -y --force-yes install sudo') - - env.logger.debug("_setup_apt_sources " + env.sources_file + " " + env.edition.name) - env.edition.check_packages_source() - comment = "# This file was modified for "+ env.edition.name - # Setup apt download policy (default is None) - # (see also https://help.ubuntu.com/community/PinningHowto) - preferences = env.edition.rewrite_apt_preferences([]) - if len(preferences): - # make sure it exists, and is empty - sudo("rm -f %s" % env.apt_preferences_file) - sudo("touch %s" % env.apt_preferences_file) - append(env.apt_preferences_file, comment, use_sudo=True) - lines = "\n".join(preferences) - env.logger.debug("Policy %s" % lines) - # append won't duplicate, so we use echo - sudo("/bin/echo -e \"%s\" >> %s" % (lines, env.apt_preferences_file)) - # check there is no error parsing the file - env.logger.debug(sudo("apt-cache policy")) - - # Make sure a source file exists - if not exists(env.sources_file): - sudo("touch %s" % env.sources_file) - # Add a comment - if not contains(env.sources_file, comment): - append(env.sources_file, comment, use_sudo=True) - for source in env.edition.rewrite_apt_sources_list(env.std_sources): - env.logger.debug("Source %s" % source) - if source.startswith("ppa:"): - sudo("apt-get install -y --force-yes python-software-properties") - sudo("add-apt-repository '%s'" % source) - elif not contains(env.sources_file, source): # FIXME: append never adds dups! - append(env.sources_file, source, use_sudo=True) - -# ### Automated installation on systems with yum package manager - -def _yum_packages(to_install): - """Install rpm packages available via yum. - """ - pkg_config = os.path.join(env.config_dir, "packages-yum.yaml") - with settings(warn_only=True): - sudo("yum check-update") - sudo("yum -y upgrade") - # Retrieve packages to get and install each of them - (packages, _) = _yaml_to_packages(pkg_config, to_install) - # At this point allow the Flavor to rewrite the package list - packages = env.flavor.rewrite_config_items("packages", packages) - for package in packages: - sudo("yum -y install %s" % package) - -def _setup_yum_bashrc(): - """Fix the user bashrc to update compilers. - """ - to_include = ["export CC=gcc44", "export CXX=g++44", "export FC=gfortran44", - "export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:/usr/lib/pkgconfig"] - fname = run("ls %s" % env.shell_config) - for line in to_include: - if not contains(fname, line.split("=")[0]): - append(fname, line) - -def _setup_yum_sources(): - """Add additional useful yum repositories. - """ - repos = ["http://download.fedora.redhat.com/pub/epel/5/x86_64/epel-release-5-4.noarch.rpm"] - for repo in repos: - with settings(warn_only=True): - sudo("rpm -Uvh %s" % repo) - -# ### Nix Packages specific - -def _setup_nix_sources(): - if env.nixpkgs: - # first override the path - append("/root/.bashrc", "export PATH=$HOME/.nix-profile/bin:$PATH", use_sudo=True) - env.logger.info("Checking NixPkgs") - if not exists("/nix/store"): - if not exists("/usr/bin/nix-env"): - # install Nix (standard Debian release) - nix_deb = "nix_0.16-1_i386.deb" - if not exists(nix_deb): - run("wget http://hydra.nixos.org/build/565031/download/1/nix_0.16-1_i386.deb") - sudo("dpkg -i "+nix_deb) - # Set sources - sudo("nix-channel --add http://nixos.org/releases/nixpkgs/channels/nixpkgs-unstable") - sudo("nix-channel --update") - # upgrade Nix to latest (and remove the older version, as it is much slower) - sudo("nix-env -b -i nix") - if exists("/usr/bin/nix-env"): - env.logger.info("uninstall older Nix (Debian release)") - sudo("dpkg -r nix") - -def _nix_packages(to_install): - """Install packages available via nixpkgs (optional) - """ - if env.nixpkgs: - env.logger.info("Update and install NixPkgs packages") - pkg_config_file = os.path.join(env.config_dir, "packages-nix.yaml") - sudo("nix-channel --update") - # Retrieve final package names - (packages, _) = _yaml_to_packages(pkg_config_file, to_install) - packages = env.edition.rewrite_config_items("packages", packages) - packages = env.flavor.rewrite_config_items("packages", packages) - for p in packages: - sudo("nix-env -b -i %s" % p) - # ### CloudBioLinux specific scripts def _freenx_scripts():