devstack-vm-update-image.py

#!/usr/bin/env python

# Update the base image that is used for devstack VMs.

# Copyright (C) 2011-2012 OpenStack LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.


import sys
import os
import commands
import time
import subprocess
import traceback
import socket
import pprint

import vmdatabase
import utils
from sshclient import SSHClient

WORKSPACE = os.environ['WORKSPACE']
DEVSTACK_GATE_PREFIX = os.environ.get('DEVSTACK_GATE_PREFIX', '')
DEVSTACK = os.path.join(WORKSPACE, 'devstack')
PROVIDER_NAME = sys.argv[1]
JENKINS_SSH_KEY = os.environ.get('JENKINS_SSH_KEY', False)

if JENKINS_SSH_KEY:
    PUPPET_CLASS = ("class {'openstack_project::slave_template': "
                    "install_users => false, ssh_key => '%s', }" %
                    JENKINS_SSH_KEY)
else:
    PUPPET_CLASS = "class {'openstack_project::slave_template': }"

PROJECTS = ['openstack/nova',
            'openstack/glance',
            'openstack/keystone',
            'openstack/horizon',
            'openstack/cinder',
            'openstack/python-cinderclient',
            'openstack/swift',
            'openstack/tempest',
            'openstack/quantum',
            'openstack/python-glanceclient',
            'openstack/python-keystoneclient',
            'openstack/python-novaclient',
            'openstack/python-openstackclient',
            'openstack/python-quantumclient',
            'openstack-dev/devstack',
            'openstack-infra/devstack-gate']


def run_local(cmd, status=False, cwd='.', env={}):
    print "Running:", cmd
    newenv = os.environ
    newenv.update(env)
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd,
                         stderr=subprocess.STDOUT, env=newenv)
    (out, nothing) = p.communicate()
    if status:
        return (p.returncode, out.strip())
    return out.strip()


def git_branches():
    branches = []
    for branch in run_local(['git', 'branch', '-a'], cwd=DEVSTACK).split("\n"):
        branch = branch.strip()
        if not branch.startswith('remotes/origin'):
            continue
        branches.append(branch)
    return branches


def tokenize(fn, tokens, distribution, comment=None):
    for line in open(fn):
        if 'dist:' in line and ('dist:%s' % distribution not in line):
            continue
        if 'qpid' in line:
            continue  # TODO: explain why this is here
        if comment and comment in line:
            line = line[:line.rfind(comment)]
        line = line.strip()
        if line and line not in tokens:
            tokens.append(line)


def local_prep(distribution):
    branches = []
    for branch in git_branches():
        branch_data = {'name': branch}
        print 'Branch: ', branch
        run_local(['git', 'checkout', branch], cwd=DEVSTACK)
        run_local(['git', 'pull', '--ff-only', 'origin'], cwd=DEVSTACK)

        pips = []
        pipdir = os.path.join(DEVSTACK, 'files', 'pips')
        if os.path.exists(pipdir):
            for fn in os.listdir(pipdir):
                fn = os.path.join(pipdir, fn)
                tokenize(fn, pips, distribution)
        branch_data['pips'] = pips

        debs = []
        debdir = os.path.join(DEVSTACK, 'files', 'apts')
        for fn in os.listdir(debdir):
            fn = os.path.join(debdir, fn)
            tokenize(fn, debs, distribution, comment='#')
        branch_data['debs'] = debs

        images = []
        for line in open(os.path.join(DEVSTACK, 'stackrc')):
            line = line.strip()
            if line.startswith('IMAGE_URLS'):
                if '#' in line:
                    line = line[:line.rfind('#')]
                if line.endswith(';;'):
                    line = line[:-2]
                line = line.split('=', 1)[1].strip()
                if line.startswith('${IMAGE_URLS:-'):
                    line = line[len('${IMAGE_URLS:-'):]
                if line.endswith('}'):
                    line = line[:-1]
                if line[0] == line[-1] == '"':
                    line = line[1:-1]
                images += [x.strip() for x in line.split(',')]
        branch_data['images'] = images
        branches.append(branch_data)
    return branches


def bootstrap_server(provider, server, admin_pass, key):
    client = server.manager.api
    ip = utils.get_public_ip(server)
    if not ip and 'os-floating-ips' in utils.get_extensions(client):
        utils.add_public_ip(server)
        ip = utils.get_public_ip(server)
    if not ip:
        raise Exception("Unable to find public ip of server")

    ssh_kwargs = {}
    if key:
        ssh_kwargs['pkey'] = key
    else:
        ssh_kwargs['password'] = admin_pass

    for username in ['root', 'ubuntu']:
        client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600)
        if client:
            break

    if not client:
        raise Exception("Unable to log in via SSH")

    # hpcloud can't reliably set the hostname
    gerrit_url = 'https://review.openstack.org/p/openstack-infra/config.git'
    client.ssh("set hostname", "sudo hostname %s" % server.name)
    client.ssh("get puppet repo deb",
               "sudo /usr/bin/wget "
               "http://apt.puppetlabs.com/puppetlabs-release-"
               "`lsb_release -c -s`.deb -O /root/puppet-repo.deb")
    client.ssh("install puppet repo deb", "sudo dpkg -i /root/puppet-repo.deb")
    client.ssh("update apt cache", "sudo apt-get update")
    client.ssh("upgrading system packages",
               'sudo DEBIAN_FRONTEND=noninteractive apt-get '
               '--option "Dpkg::Options::=--force-confold"'
               ' --assume-yes upgrade')
    client.ssh("install git and puppet",
               'sudo DEBIAN_FRONTEND=noninteractive apt-get '
               '--option "Dpkg::Options::=--force-confold"'
               ' --assume-yes install git puppet')
    client.ssh("clone puppret repo",
               "sudo git clone %s /root/config" % gerrit_url)
    client.ssh("install puppet modules",
               "sudo /bin/bash /root/config/install_modules.sh")
    client.ssh("run puppet",
               "sudo puppet apply --modulepath=/root/config/modules:"
               "/etc/puppet/modules "
               '-e "%s"' % PUPPET_CLASS)


def configure_server(server, branches):
    client = SSHClient(utils.get_public_ip(server), 'jenkins')
    client.ssh('make file cache directory', 'mkdir -p ~/cache/files')
    client.ssh('make pip cache directory', 'mkdir -p ~/cache/pip')
    client.ssh('install build-essential',
               'sudo DEBIAN_FRONTEND=noninteractive '
               'apt-get --option "Dpkg::Options::=--force-confold"'
               ' --assume-yes install build-essential python-dev '
               'linux-headers-virtual linux-headers-`uname -r`')

    for branch_data in branches:
        if branch_data['debs']:
            client.ssh('cache debs for branch %s' % branch_data['name'],
                       'sudo apt-get -y -d install %s' %
                       ' '.join(branch_data['debs']))

        if branch_data['pips']:
            venv = client.ssh('get temp dir for venv', 'mktemp -d').strip()
            client.ssh('create venv',
                       'virtualenv --no-site-packages %s' % venv)
            client.ssh('cache pips for branch %s' % branch_data['name'],
                       'source %s/bin/activate && '
                       'PIP_DOWNLOAD_CACHE=~/cache/pip pip install %s' %
                       (venv, ' '.join(branch_data['pips'])))
            client.ssh('remove venv', 'rm -fr %s' % venv)

        for url in branch_data['images']:
            fname = url.split('/')[-1]
            try:
                client.ssh('check for %s' % fname,
                           'ls ~/cache/files/%s' % fname)
            except:
                client.ssh('download image %s' % fname,
                    'wget -nv -c %s -O ~/cache/files/%s' % (url, fname))

    client.ssh('clear workspace', 'rm -rf ~/workspace-cache')
    client.ssh('make workspace', 'mkdir -p ~/workspace-cache')
    for project in PROJECTS:
        sp = project.split('/')[0]
        client.ssh('clone %s' % project,
            'cd ~/workspace-cache && '
            'git clone https://review.openstack.org/p/%s' % project)

    script = os.environ.get('DEVSTACK_GATE_CUSTOM_SCRIPT', '')
    if script and os.path.isfile(script):
        bn = os.path.basename(script)
        client.scp(script, '/tmp/%s' % bn)
        client.ssh('run custom script %s' % bn,
            'chmod +x /tmp/%s && sudo /tmp/%s' % (bn, bn))

    client.ssh('sync', 'sync && sleep 5')


def snapshot_server(client, server, name):
    print 'Saving image'
    if hasattr(client.images, 'create'):  # v1.0
        image = client.images.create(server, name)
    else:
        # TODO: fix novaclient so it returns an image here
        # image = server.create_image(name)
        uuid = server.manager.create_image(server, name)
        image = client.images.get(uuid)
    print "Waiting for image ID %s" % image.id
    image = utils.wait_for_resource(image)
    return image


def build_image(provider, client, base_image, image,
                flavor, name, branches, timestamp):
    print "Building image %s" % name

    create_kwargs = dict(image=image, flavor=flavor, name=name)

    key = None
    key_name = '%sdevstack-%i' % (DEVSTACK_GATE_PREFIX, time.time())
    if 'os-keypairs' in utils.get_extensions(client):
        print "Adding keypair"
        key, kp = utils.add_keypair(client, key_name)
        create_kwargs['key_name'] = key_name

    server = client.servers.create(**create_kwargs)
    snap_image = base_image.newSnapshotImage(name=name,
                                             version=timestamp,
                                             external_id=None,
                                             server_external_id=server.id)
    admin_pass = server.adminPass
    try:
        print "Waiting for server ID %s" % server.id
        server = utils.wait_for_resource(server)
        bootstrap_server(provider, server, admin_pass, key)
        configure_server(server, branches)
        remote_snap_image = snapshot_server(client, server, name)
        snap_image.external_id = remote_snap_image.id
        snap_image.state = vmdatabase.READY
        # We made the snapshot, try deleting the server, but it's okay
        # if we fail.  The reap script will find it and try again.
        try:
            utils.delete_server(server)
        except:
            print "Exception encountered deleting server:"
            traceback.print_exc()
    except Exception, real_error:
        # Something went wrong, try our best to mark the server in error
        # then delete the server, then delete the db record for it.
        # If any of this fails, the reap script should catch it.  But
        # having correct info in the DB will help it do its job faster.
        try:
            snap_image.state = vmdatabase.ERROR
            try:
                utils.delete_server(server)
                snap_image.delete()
            except Exception, delete_error:
                print "Exception encountered deleting server:"
                traceback.print_exc()
        except Execption, database_error:
            print "Exception encountered marking server in error:"
            traceback.print_exc()
        # Raise the important exception that started this
        raise


def main():
    if '-n' in sys.argv:
        dry = True
    else:
        dry = False

    db = vmdatabase.VMDatabase()
    provider = db.getProvider(PROVIDER_NAME)
    print "Working with provider %s" % provider.name
    client = utils.get_client(provider)

    for base_image in provider.base_images:
        if base_image.min_ready <= 0:
            continue
        print "Working on base image %s" % base_image.name

        flavor = utils.get_flavor(client, base_image.min_ram)
        print "Found flavor", flavor

        branches = local_prep(base_image.name)
        pprint.pprint(branches)

        remote_base_image = client.images.find(name=base_image.external_id)
        if not dry:
            timestamp = int(time.time())
            remote_snap_image_name = ('%sdevstack-%s-%s.template.openstack.org' %
                                      (DEVSTACK_GATE_PREFIX,
                                       base_image.name, str(timestamp)))
            remote_snap_image = build_image(provider, client, base_image,
                                            remote_base_image, flavor,
                                            remote_snap_image_name,
                                            branches, timestamp)


if __name__ == '__main__':
    main()