Permalink
Find file
Fetching contributors…
Cannot retrieve contributors at this time
224 lines (196 sloc) 7.01 KB
"""
Build ceph packages
Unit tests:
py.test -v -s tests/test_buildpackages.py
Integration tests:
teuthology-openstack --verbose --key-name myself --key-filename ~/Downloads/myself --ceph infernalis --suite teuthology/buildpackages
"""
import copy
import logging
import os
import types
from teuthology import packaging
from teuthology import misc
from teuthology.config import config as teuth_config
from teuthology.openstack import OpenStack
log = logging.getLogger(__name__)
class LocalGitbuilderProject(packaging.GitbuilderProject):
def __init__(self):
pass
def get_pkg_type(os_type):
if os_type in ('centos', 'fedora', 'opensuse', 'rhel', 'sles'):
return 'rpm'
else:
return 'deb'
def apply_overrides(ctx, config):
if config is None:
config = {}
else:
config = copy.deepcopy(config)
assert isinstance(config, dict), \
"task install only supports a dictionary for configuration"
project, = config.get('project', 'ceph'),
log.debug('project %s' % project)
overrides = ctx.config.get('overrides')
if overrides:
install_overrides = overrides.get('install', {})
misc.deep_merge(config, install_overrides.get(project, {}))
return config
def get_config_install(ctx, config):
config = apply_overrides(ctx, config)
log.debug('install config %s' % config)
return [(config.get('flavor', 'basic'),
config.get('tag', ''),
config.get('branch', ''),
config.get('sha1'))]
def get_config_install_upgrade(ctx, config):
log.debug('install.upgrade config before override %s' % config)
configs = []
for (role, role_config) in config.iteritems():
if role_config is None:
role_config = {}
o = apply_overrides(ctx, role_config)
log.debug('install.upgrade config ' + str(role_config) +
' and with overrides ' + str(o))
# for install.upgrade overrides are actually defaults
configs.append((o.get('flavor', 'basic'),
role_config.get('tag', o.get('tag', '')),
role_config.get('branch', o.get('branch', '')),
role_config.get('sha1', o.get('sha1'))))
return configs
GET_CONFIG_FUNCTIONS = {
'install': get_config_install,
'install.upgrade': get_config_install_upgrade,
}
def lookup_configs(ctx, node):
configs = []
if type(node) is types.ListType:
for leaf in node:
configs.extend(lookup_configs(ctx, leaf))
elif type(node) is types.DictType:
for (key, value) in node.iteritems():
if key in ('install', 'install.upgrade'):
configs.extend(GET_CONFIG_FUNCTIONS[key](ctx, value))
elif key in ('overrides',):
pass
else:
configs.extend(lookup_configs(ctx, value))
return configs
def get_sha1(ref):
url = teuth_config.get_ceph_git_url()
ls_remote = misc.sh("git ls-remote " + url + " " + ref)
return ls_remote.split()[0]
def task(ctx, config):
"""
Build Ceph packages. This task will automagically be run
before the task that need to install packages (this is taken
care of by the internal teuthology task).
The config should be as follows:
buildpackages:
good_machine:
disk: 40 # GB
ram: 48000 # MB
cpus: 16
min_machine:
disk: 40 # GB
ram: 8000 # MB
cpus: 1
example:
tasks:
- buildpackages:
good_machine:
disk: 40 # GB
ram: 15000 # MB
cpus: 16
min_machine:
disk: 40 # GB
ram: 8000 # MB
cpus: 1
- install:
When a buildpackages task is already included, the values it contains can be
overriden with:
overrides:
buildpackages:
good_machine:
disk: 20 # GB
ram: 2000 # MB
cpus: 2
min_machine:
disk: 10 # GB
ram: 1000 # MB
cpus: 1
"""
log.info('Beginning buildpackages...')
if config is None:
config = {}
assert isinstance(config, dict), \
'task only accepts a dict for config not ' + str(config)
overrides = ctx.config.get('overrides', {})
misc.deep_merge(config, overrides.get('buildpackages', {}))
d = os.path.join(os.path.dirname(__file__), 'buildpackages')
os_type = misc.get_distro(ctx)
os_version = misc.get_distro_version(ctx)
arch = ctx.config.get('arch', OpenStack().get_default_arch())
dist = LocalGitbuilderProject()._get_distro(distro=os_type,
version=os_version)
pkg_type = get_pkg_type(os_type)
misc.sh(
"flock --close /tmp/buildpackages " +
"make -C " + d + " " + os.environ['HOME'] + "/.ssh_agent")
for (flavor, tag, branch, sha1) in lookup_configs(ctx, ctx.config):
if tag:
sha1 = get_sha1(tag)
elif branch:
sha1 = get_sha1(branch)
log.info("building flavor = " + flavor + "," +
" tag = " + tag + "," +
" branch = " + branch + "," +
" sha1 = " + sha1)
target = ('ceph-' +
pkg_type + '-' +
dist + '-' +
arch + '-' +
flavor + '-' +
sha1)
openstack = OpenStack()
openstack.set_provider()
if openstack.provider == 'ovh':
select = '^(vps|hg)-.*ssd'
else:
select = ''
network = openstack.net()
if network != "":
network = " OPENSTACK_NETWORK='" + network + "' "
openstack.image(os_type, os_version, arch) # create if it does not exist
build_flavor = openstack.flavor_range(
config['min_machine'], config['good_machine'], arch, select)
default_arch = openstack.get_default_arch()
http_flavor = openstack.flavor({
'disk': 30, # GB
'ram': 1024, # MB
'cpus': 1,
}, default_arch, select)
lock = "/tmp/buildpackages-" + sha1 + "-" + os_type + "-" + os_version
cmd = (". " + os.environ['HOME'] + "/.ssh_agent ; " +
" flock --close " + lock +
" make -C " + d +
network +
" CEPH_GIT_URL=" + teuth_config.get_ceph_git_url() +
" CEPH_PKG_TYPE=" + pkg_type +
" CEPH_OS_TYPE=" + os_type +
" CEPH_OS_VERSION=" + os_version +
" CEPH_DIST=" + dist +
" CEPH_ARCH=" + arch +
" CEPH_SHA1=" + sha1 +
" CEPH_TAG=" + tag +
" CEPH_BRANCH=" + branch +
" CEPH_FLAVOR=" + flavor +
" BUILD_FLAVOR=" + build_flavor +
" HTTP_FLAVOR=" + http_flavor +
" HTTP_ARCH=" + default_arch +
" " + target +
" ")
log.info("buildpackages: " + cmd)
misc.sh(cmd)
teuth_config.gitbuilder_host = openstack.get_ip('packages-repository', '')
log.info('Finished buildpackages')