Permalink
Browse files

Generalize to have BII only deployment target; reuse cloudbiolinux fu…

…nctionality instead of replication and pull in library; improve README
  • Loading branch information...
chapmanb committed Sep 6, 2011
1 parent 7b4c986 commit 0033b0ecb2d2812326583fcb2256e2fefc8e7e10
View
@@ -3,15 +3,38 @@ Automated deployment for [Galaxy][1] and the
## Galaxy and BII Installation
-- Install [VirtualBox][5], [Vagrant][4] and Python
+### Preparation
-- Build libraries
+- Install Python (version 2.6 or better, not Python 3)
+
+- Build libraries and install Python library pre-requisites
cd ~/hsph/projects/scde_deploy
python setup.py build && sudo python setup.py install
- Edit `config/scde.yaml` to specify system directories and passwords
+### Deployment
+
+- Run Fabric script for deployment:
+
+ fab -f ~/hsph/projects/scde_deploy/scde_fabfile.py -H your_remote_host install_scde
+
+ You can also install only BII:
+
+ fab -f ~/hsph/projects/scde_deploy/scde_fabfile.py -H your_remote_host install_bii
+
+ Or use the script to start servers:
+
+ fab -f ~/hsph/projects/scde_deploy/scde_fabfile.py -H your_remote_host install_bii
+
+### Vagrant testing
+
+Pre-deployment testing uses a local virtual machine managed with
+[Vagrant][4]:
+
+- Install [VirtualBox][5] and [Vagrant][4]
+
- Install CloudBioLinux box with Vagrant. Creates a `Vagrantfile` for
configuring and running the virtual machine
@@ -1,65 +0,0 @@
-"""Functions to help with setting up Fabric build environments.
-"""
-import subprocess
-
-from fabric.api import env
-
-def setup_vagrant_environment():
- """Use vagrant commands to get connection information.
- https://gist.github.com/1d4f7c3e98efdf860b7e
- """
- raw_ssh_config = subprocess.Popen(["vagrant", "ssh-config"],
- stdout=subprocess.PIPE).communicate()[0]
- ssh_config = dict([l.strip().split() for l in raw_ssh_config.split("\n") if l])
- env.user = ssh_config["User"]
- env.hosts = [ssh_config["HostName"]]
- env.port = ssh_config["Port"]
- env.host_string = "%s@%s:%s" % (env.user, env.hosts[0], env.port)
- env.key_filename = ssh_config["IdentityFile"]
-
-def setup_centos():
- env.python_version_ext = "2.6"
- if not env.has_key("java_home"):
- env.java_home = "/etc/alternatives/java_sdk"
-
-def setup_ubuntu():
- default_dist = "maverick"
- default_version = "10.10"
- env.sources_file = "/etc/apt/sources.list"
- version = (env.get("dist_name", default_dist),
- env.get("dist_version", default_version))
- sources = [
- "deb http://us.archive.ubuntu.com/ubuntu/ %s universe",
- "deb-src http://us.archive.ubuntu.com/ubuntu/ %s universe",
- "deb http://us.archive.ubuntu.com/ubuntu/ %s-updates universe",
- "deb-src http://us.archive.ubuntu.com/ubuntu/ %s-updates universe",
- "deb http://us.archive.ubuntu.com/ubuntu/ %s multiverse",
- "deb-src http://us.archive.ubuntu.com/ubuntu/ %s multiverse",
- "deb http://us.archive.ubuntu.com/ubuntu/ %s-updates multiverse",
- "deb-src http://us.archive.ubuntu.com/ubuntu/ %s-updates multiverse",
- "ppa:sun-java-community-team/sun-java6", # sun-java
- "deb http://downloads.mongodb.org/distros/ubuntu % s 10gen", # mongodb
- "deb http://cran.stat.ucla.edu/bin/linux/ubuntu %s/", # lastest R versions
- "deb http://nebc.nox.ac.uk/bio-linux/ unstable bio-linux", # Bio-Linux
- "deb http://archive.cloudera.com/debian %s-cdh3 contrib", # Hadoop
- "deb http://ppa.launchpad.net/freenx-team/ppa/ubuntu lucid main", # FreeNX PPA
- "deb http://download.virtualbox.org/virtualbox/debian %s contrib", # virtualbox
- ]
- env.std_sources = _add_source_versions(version, sources)
- env.python_version_ext = ""
- if not env.has_key("java_home"):
- # XXX look for a way to find JAVA_HOME automatically
- env.java_home = "/usr/lib/jvm/java-6-openjdk"
-
-def _add_source_versions(version, sources):
- name, num = version
- final = []
- for s in sources:
- if s.find("%s") > 0:
- s = s % name
- elif s.find("% s") > 0:
- s = s % num
- final.append(s)
- return final
-
-
@@ -5,7 +5,7 @@
from fabric.api import *
from fabric.contrib.files import *
-from bcbio.deploy.shared import (_if_not_installed, _fetch_and_unpack)
+from cloudbio.custom.shared import (_if_not_installed, _fetch_and_unpack)
def _java_dir(base_dir):
java_dir = os.path.join(base_dir, "java")
@@ -2,149 +2,10 @@
"""
import os
import time
-from contextlib import contextmanager
from fabric.api import *
from fabric.contrib.files import *
-# -- decorators and context managers
-
-def _if_not_installed(pname):
- """Decorator that checks if a callable program is installed.
- """
- def argcatcher(func):
- def decorator(*args, **kwargs):
- with settings(
- hide('warnings', 'running', 'stdout', 'stderr'),
- warn_only=True):
- result = run(pname)
- if result.return_code == 127:
- return func(*args, **kwargs)
- return decorator
- return argcatcher
-
-def _if_not_python_lib(library):
- """Decorator that checks if a python library is installed.
- """
- def argcatcher(func):
- def decorator(*args, **kwargs):
- with settings(warn_only=True):
- pyver = env.python_version_ext if env.has_key("python_version_ext") else ""
- result = run("python%s -c 'import %s'" % (pyver, library))
- if result.failed:
- return func(*args, **kwargs)
- return decorator
- return argcatcher
-
-@contextmanager
-def _make_tmp_dir():
- home_dir = run("echo $HOME")
- work_dir = os.path.join(home_dir, "tmp", "cloudbiolinux")
- if not exists(work_dir):
- run("mkdir -p %s" % work_dir)
- yield work_dir
- if exists(work_dir):
- run("rm -rf %s" % work_dir)
-
-# -- Standard build utility simplifiers
-
-def _get_expected_file(url):
- tar_file = os.path.split(url)[-1]
- tar_file = tar_file.split("?")[0] # remove any extra arguments
- safe_tar = "--pax-option='delete=SCHILY.*,delete=LIBARCHIVE.*'"
- exts = {(".tar.gz", ".tgz") : "tar %s -xzpf" % safe_tar,
- (".tar.bz2",): "tar %s -xjpf" % safe_tar,
- (".zip",) : "unzip"}
- for ext_choices, tar_cmd in exts.iteritems():
- for ext in ext_choices:
- if tar_file.endswith(ext):
- return tar_file, tar_file[:-len(ext)], tar_cmd
- raise ValueError("Did not find extract command for %s" % url)
-
-def _safe_dir_name(dir_name, need_dir=True):
- replace_try = ["", "-src"]
- for replace in replace_try:
- check = dir_name.replace(replace, "")
- if exists(check):
- return check
- # still couldn't find it, it's a nasty one
- first_part = dir_name.split("-")[0].split("_")[0]
- with settings(warn_only=True):
- dirs = run("ls -d1 *%s*/" % first_part).split("\n")
- if len(dirs) == 1:
- return dirs[0]
- if need_dir:
- raise ValueError("Could not find directory %s" % dir_name)
-
-def _fetch_and_unpack(url, need_dir=True):
- if url.startswith(("git", "svn", "hg", "cvs")):
- base = os.path.basename(url.split()[-1])
- dirname = os.path.splitext(base)[0]
- if not exists(dirname):
- run(url)
- return dirname
- else:
- tar_file, dir_name, tar_cmd = _get_expected_file(url)
- if not exists(tar_file):
- run("wget --no-check-certificate -O %s '%s'" % (tar_file, url))
- if not exists(dir_name):
- run("%s %s" % (tar_cmd, tar_file))
- return _safe_dir_name(dir_name, need_dir)
-
-def _configure_make(env):
- run("./configure --prefix=%s " % env.system_install)
- run("make")
- sudo("make install")
-
-def _make_copy(find_cmd=None, premake_cmd=None, do_make=True):
- def _do_work(env):
- if premake_cmd:
- premake_cmd()
- if do_make:
- run("make")
- if find_cmd:
- install_dir = os.path.join(env.system_install, "bin")
- for fname in run(find_cmd).split("\n"):
- sudo("mv -f %s %s" % (fname.rstrip("\r"), install_dir))
- return _do_work
-
-def _get_install(url, env, make_command):
- """Retrieve source from a URL and install in our system directory.
- """
- with _make_tmp_dir() as work_dir:
- with cd(work_dir):
- dir_name = _fetch_and_unpack(url)
- with cd(dir_name):
- make_command(env)
-
-def _get_install_local(url, env, make_command):
- """Build and install in a local directory.
- """
- (_, test_name, _) = _get_expected_file(url)
- test1 = os.path.join(env.local_install, test_name)
- test2, _ = test1.rsplit("-", 1)
- if not exists(test1) and not exists(test2):
- with _make_tmp_dir() as work_dir:
- with cd(work_dir):
- dir_name = _fetch_and_unpack(url)
- if not exists(os.path.join(env.local_install, dir_name)):
- with cd(dir_name):
- make_command(env)
- run("mv %s %s" % (dir_name, env.local_install))
-
-# --- Language specific utilities
-
-def _symlinked_java_version_dir(pname, version):
- base_dir = os.path.join(env.system_install, "share", "java", pname)
- install_dir = "%s-%s" % (base_dir, version)
- if not exists(install_dir):
- sudo("mkdir -p %s" % install_dir)
- if exists(base_dir):
- sudo("rm -f %s" % base_dir)
- sudo("ln -s %s %s" % (install_dir, base_dir))
- return install_dir
- return None
-
# --- Running servers and daemons
def _is_running(cmd):
@@ -1,5 +1,7 @@
---
distribution: ubuntu
+dist_name: natty
+# distribution: centos
base_install: /usr/local
bii_dirname: bioinvindex
bii_lucene_index: index
@@ -4,8 +4,12 @@
from fabric.contrib.files import *
import yaml
-from bcbio.deploy import environ, java
-from bcbio.deploy.shared import (_fetch_and_unpack, _run_in_screen)
+from cloudbio import distribution
+from cloudbio.utils import _setup_logging
+from cloudbio.custom.shared import _fetch_and_unpack
+
+from bcbio.deploy import java
+from bcbio.deploy.shared import _run_in_screen
# ## Install targets
@@ -15,7 +19,7 @@ def install_scde(configfile=None):
config = _install_prereqs(config)
_configure_system(config)
galaxy_servers = _install_galaxy(config)
- bii_servers = _install_bii(config)
+ bii_servers = _compile_install_bii(config)
# ## Deployed servers
@@ -25,8 +29,17 @@ def start_servers(config, server_info):
# ## BII installation
-def _install_bii(config):
- """Main target for installing the bioinvestigator index.
+def install_bii(configfile=None):
+ """Install BII along with prerequisites, copying EAR file to jboss.
+ """
+ config = _read_config(configfile)
+ _setup_environment(config)
+ config = _install_prereqs(config)
+ _configure_system(config)
+ bii_servers = _compile_install_bii(config)
+
+def _compile_install_bii(config):
+ """Internal target for installing the bioinvestigator index.
"""
dirs = _install_bii_tools(config)
_configure_bii(dirs, config)
@@ -295,14 +308,11 @@ def _install_prereqs(config):
return config
def _setup_environment(config):
- if config["distribution"] == "centos":
- environ.setup_centos()
- elif config["distribution"] == "ubuntu":
- environ.setup_ubuntu()
- else:
- raise NotImplementedError
- if env.hosts == ["vagrant"]:
- environ.setup_vagrant_environment()
+ env.distribution = config["distribution"]
+ env.dist_name = config.get("dist_name", None)
+ env.system_install = config["base_install"]
+ _setup_logging(env)
+ distribution._setup_distribution_environment()
result = run("echo $HOSTNAME")
config["host"] = result.strip()
run("chmod a+rx $HOME")
View
@@ -18,7 +18,8 @@
},
install_requires = [
"biopy-isatab >= 0.1",
+ "cloudbiolinux >= 0.1",
"PyYAML >= 3.09",
- "fabric >= 1.0.1",
+ "fabric >= 1.1.1",
"boto >= 2.0b4",
])

0 comments on commit 0033b0e

Please sign in to comment.