reproducer-quickstart.sh.j2

#!/bin/bash

# See documentation for using the reproducer script:
# README-reproducer-quickstart.html
# (in the same top-level logs directory as this reproducer script).

: ${WORKSPACE:=$(mktemp -d -t reproduce-tmp.XXXXX)}
{% if 'ovb' in toci_jobtype %}
: ${CREATE_VIRTUALENV:=true}
{% else %}
: ${CREATE_VIRTUALENV:=false}
{% endif %}
: ${REMOVE_STACKS_KEYPAIRS:=false}
: ${NODESTACK_PREFIX:=""}
: ${AUTORUN:=0}
: ${TIMEOUT:=240}
: ${LIBVIRT:=0}
: ${TOCI_JOBTYPE:="{{ toci_jobtype }}"}
: ${NODES_FILE:="{{ nodes_config }}"}
: ${ZUUL_CHANGES:="{{ zuul_changes }}"}
: ${EXTRA_PARAMS:=""}
: ${IN_PLACE:=false}

SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

usage () {
    echo "Usage: $0 [options]"
    echo ""
    echo "Options:"
    echo "  -w, --workspace <dir>"
    echo "                      directory where the virtualenv, inventory files, etc."
    echo "                      are created. Defaults to creating a directory in /tmp"
    echo "  -i, --in-place"
    echo "                      Avoid wiping tq[e] repositories if they already exist"
    echo "                      and just pull --rebase them. [IN_PLACE=true]"
    echo "  -v, --create-virtualenv"
    echo "                      create a virtualenv to install Ansible and dependencies."
    echo "                      Options to pass true/false. Defaults to true for OVB. "
    echo "                      Defaults to false for other deployment types."
    echo "  -r, --remove-stacks-keypairs"
    echo "                      delete all Heat stacks (both Multinode and OVB created) "
    echo "                      in the tenant before deployment."
    echo "                      Will also delete associated keypairs if they exist."
    echo "                      Options to pass true/false. Defaults to false."
    echo "  -p, --nodestack-prefix"
    echo "                      add a unique prefix for multinode and singlenode stacks"
    echo "                      Defaults to empty."
    echo "  -a, --autorun"
    echo "                      Run job on prepared environment automatically"
    echo "                      Default is to stop after environment is ready"
    echo "  -t, --timeout"
    echo "                      Timeout in minutes for the deployment"
    echo "                      Default is 240 minutes"
    echo "  -l, --libvirt"
    echo "                      Runs a 2-node multinode job or singlenode job on a"
    echo "                      single virthost using libvirt to create the nodes."
    echo "                      The script sets the virthost to be 127.0.0.2"
    echo "                      so the reproducer must be run from the virthost."
    echo "                      If a singlenode reproducer is run, two VMs will still be created."
    echo "  -n, --nodes-file"
    echo "                      Node config file of the topology needed"
    echo "                      default: $NODES_FILE"
    echo "  -z, --zuul-changes"
    echo "                      Refspecs with changes to test"
    echo "                      default: $ZUUL_CHANGES"
    echo "  -j, --toci-jobtype"
    echo "                      TOCI description of the job to run"
    echo "                      default: $TOCI_JOBTYPE"
    echo "  -e, --extra-params"
    echo "                      File or/and parameters used to override default"
    echo "                      parameters for playbooks. Multiple files"
    echo "                      can be passed [-e @file1.yml -e @file2.yml ...]"
    echo "                      and arguments [-e var=value -e var2=value2 ...]"
    echo "  -h, --help          print this help and exit"
}

set -e

# Input argument assignments
while [ "x$1" != "x" ]; do

    case "$1" in
        --workspace|-w)
            # realpath fails if /some/path doesn't exist. It is created later
            WORKSPACE=$(realpath $2 || echo -n $2)
            shift
            ;;

        --in-place|-i)
            IN_PLACE=true
            shift
            ;;

        --create-virtualenv|-v)
            CREATE_VIRTUALENV=$2
            shift
            ;;

        --remove-stacks-keypairs|-r)
            REMOVE_STACKS_KEYPAIRS=$2
            shift
            ;;

        --nodestack-prefix|-p)
            NODESTACK_PREFIX=$2
            shift
            ;;

        --autorun|-a)
            AUTORUN=1
            ;;

        --timeout|-t)
            TIMEOUT=$2
            shift
            ;;

        --libvirt|-l)
            LIBVIRT=1
            ;;


        --extra-params|-e)
            [[ ${2::1} == "@" ]] && EXTRA_PARAMS+=("-e @$(realpath ${2#@}) ") || EXTRA_PARAMS+=("-e ${2} ")
            shift
            ;;

        --nodes-file|-n)
            NODES_FILE=$2
            shift
            ;;

        --zuul-changes|-z)
            ZUUL_CHANGES=$2
            shift
            ;;

        --toci-jobtype|-j)
            TOCI_JOBTYPE=$2
            shift
            ;;

        --help|-h)
            usage
            exit
            ;;

        --) shift
            break
            ;;

        -*) echo "ERROR: unknown option: $1" >&2
            usage >&2
            exit 2
            ;;

        *)    break
            ;;
    esac

    shift
done


function check_installed {
    local binary=$1
    if ! which $binary >/dev/null ; then
        echo "Error: Could not find required binary $binary. Ensure you have
            it installed before trying again."
        return 1
    fi
}
BOOTSTRAP_DEPENDENCIES=("git" "virtualenv" "ansible" "pip" "openstack" "heat")
function check_bootstrap_dependencies {
    status=0
    for dependency in ${BOOTSTRAP_DEPENDENCIES[@]} ; do
        check_installed $dependency || status=1
    done
    return $status
}

# Check that tenant credentials have been sourced
# when deploying on a host cloud
if [[ (! -v OS_TENANT_NAME) && ("$LIBVIRT" != "1") ]]; then
    # TODO(aschultz): work around for LP#1750455
    if [[ -v OS_PROJECT_NAME ]]; then
        export OS_TENANT_NAME=$OS_PROJECT_NAME
    else
        echo "Tenant credentials are not sourced."
        exit 1;
    fi
fi
set -x


# Exit if running ovb-fakeha-caserver
# This test is not converted to run with tripleo-quickstart
if [[ "$TOCI_JOBTYPE" == *"ovb-fakeha-caserver"* ]]; then
    echo "
        ovb-fakeha-caserver is not run with tripleo-quickstart.
        It can not be reproduced using this script.
    "
    exit 1;
fi

# Exit of the user request a Libvirt reproducer for an OVB job
if [[ "$TOCI_JOBTYPE" == *"ovb"* && "$LIBVIRT" == "1" ]]; then
    echo "
        The Libvirt reproducer option is only valid for
        Multinode and Singlenode jobs."
    exit 1;
fi

set -x

# Export set variables, they make visible to ansible roles
export WORKSPACE
export ZUUL_CHANGES

# Start from a clean workspace
mkdir -p $WORKSPACE
cd $WORKSPACE
[ "$IN_PLACE" == "true" ] || \
    rm -rf tripleo-quickstart tripleo-quickstart-extras

# Clone quickstart and quickstart-extras
for REPO in tripleo-quickstart tripleo-quickstart-extras
do
    if [ -d "$REPO" ]; then
        pushd $REPO
        git pull --rebase
        popd
    else
        git clone https://github.com/openstack/$REPO
    fi
done

# Set up a virtual env if requested
if [ "$CREATE_VIRTUALENV" = "true" ]; then
    check_installed virtualenv
    virtualenv $WORKSPACE/venv_ansible
    source $WORKSPACE/venv_ansible/bin/activate
    pip install --upgrade setuptools pip
    pip install -Ur $WORKSPACE/tripleo-quickstart/requirements.txt
fi

if [ "$REMOVE_STACKS_KEYPAIRS" = "true" ]; then
    # The cleanup templates expects there to be in a /bin dir in the workspace # from quickstart setup.
    # To use the clients sourced from venv
    sed -i "s#{.*/bin/##g" $WORKSPACE/tripleo-quickstart-extras/roles/ovb-manage-stack/templates/cleanup-stacks-keypairs.sh.j2
fi

# sanity check the env has ansible/pip/virtualenv/openstack/etc bootstrap dependencies
check_bootstrap_dependencies

# Export our roles path so that we can use the roles from our workspace
export ANSIBLE_ROLES_PATH=$ANSIBLE_ROLES_PATH:$WORKSPACE/tripleo-quickstart/roles:$WORKSPACE/tripleo-quickstart-extras/roles
export ANSIBLE_HOST_KEY_CHECKING=False
export ANSIBLE_LIBRARY=$WORKSPACE/tripleo-quickstart/library
export ANSIBLE_STDOUT_CALLBACK=${ANSIBLE_STDOUT_CALLBACK:-debug}

{% if 'ovb' in toci_jobtype %}
ansible-playbook $WORKSPACE/tripleo-quickstart-extras/playbooks/ovb-create-stack.yml \
    -e local_working_dir=$WORKSPACE \
    -e virthost=localhost \
    -e @$WORKSPACE/tripleo-quickstart-extras/config/environments/rdocloud.yml \
    -e ssh_extra_args="" \
    -e ovb_dump_hosts=true \
    -e ovb_setup_user=true \
    -e ansible_python_interpreter="/usr/bin/python" \
    -e cleanup_stacks_keypairs=$REMOVE_STACKS_KEYPAIRS \
    -e @$WORKSPACE/tripleo-quickstart/$NODES_FILE \
    ${EXTRA_PARAMS[@]}

# Run the playbook to setup the undercloud/subnodes to look like nodepool nodes
ansible-playbook $WORKSPACE/tripleo-quickstart-extras/playbooks/nodepool-setup.yml \
    -i $WORKSPACE/ovb_hosts \
    ${EXTRA_PARAMS[@]}

# Copy the nodes.json file to the undercloud
export $(awk '/subnode-0/ {print $2}' ovb_hosts)
scp ${SSH_OPTS}  $WORKSPACE/nodes.json zuul@$ansible_host:/home/zuul/
{% endif %}

{% set jobtype = toci_jobtype.replace('periodic-','').split('-')[0] %}
{% set types = [ "multinode", "singlenode", "standalone" ] %}
{% if jobtype in types %}

if [[ "$LIBVIRT" == "1" ]]; then
    echo "Running a libvirt reproducer ..."

    # Run from the Virthost
    export VIRTHOST=127.0.0.2
    echo -e "[virthost]\n$VIRTHOST ansible_connection=local" > $WORKSPACE/virthost

    ansible-playbook $WORKSPACE/tripleo-quickstart/playbooks/libvirt-nodepool.yml \
        -i $WORKSPACE/virthost \
        -e ansible_python_interpreter="/usr/bin/python" \
        -e local_working_dir=$WORKSPACE \
        -e working_dir=$WORKSPACE \
        -e non_root_user=$USER \
        -e non_root_group=$USER \
        ${EXTRA_PARAMS[@]}

    HOSTS_FILE="hosts"
else
    # Calculate subnode_count
    if  [[ -z "$NODES_FILE" ]]; then
        SUBNODE_COUNT=1
    else
        SUBNODE_COUNT=$(( $( awk '/node_count: / {print $2}' $WORKSPACE/tripleo-quickstart/$NODES_FILE ) +1 ))
    fi

    ansible-playbook $WORKSPACE/tripleo-quickstart-extras/playbooks/provision_multinodes.yml \
        -e local_working_dir=$WORKSPACE \
        -e subnode_count=$SUBNODE_COUNT \
        -e ansible_python_interpreter="/usr/bin/python" \
        -e prefix=$NODESTACK_PREFIX \
        ${EXTRA_PARAMS[@]}

    HOSTS_FILE="multinode_hosts"
fi

# Run the playbook to setup the undercloud/subnodes to look like nodepool nodes
ansible-playbook $WORKSPACE/tripleo-quickstart-extras/playbooks/nodepool-setup.yml \
    -i $WORKSPACE/$HOSTS_FILE \
    ${EXTRA_PARAMS[@]}

# Get ansible_host
export $(awk '/subnode-0/ {print $2}' $WORKSPACE/$HOSTS_FILE)
{% endif %}

{% if dlrn_hash_newest is defined or ('undercloud' in hostvars and 'dlrn_hash_newest' in hostvars['undercloud']) %}
EXTRA_VARS="$EXTRA_VARS --extra-vars dlrn_hash_tag_newest={{ hostvars['undercloud'].dlrn_hash_newest }} "
{% endif %}

{% if dlrn_hash is defined or ('undercloud' in hostvars and 'dlrn_hash' in hostvars['undercloud']) %}
EXTRA_VARS="$EXTRA_VARS --extra-vars dlrn_hash_tag={{ hostvars['undercloud'].dlrn_hash }}"
{% endif %}

# Create the env_vars_to_source file and copy it to the undercloud
cat >"{{ env_vars_to_source_file }}" <<EOF
export ZUUL_CHANGES="${ZUUL_CHANGES}"
export NODES_FILE="${NODES_FILE}"
export TOCI_JOBTYPE="${TOCI_JOBTYPE}"
# for periodic jobs enable the following
# export EXTRA_VARS="$EXTRA_VARS -e to_build=True "
# export EXTRA_VARS="$EXTRA_VARS -e docker_registry_host=trunk.registry.rdoproject.org "
export EXTRA_VARS="${EXTRA_VARS}"
export REMAINING_TIME="$TIMEOUT"
export PLAYBOOKS="$PLAYBOOKS"
EOF

{% if stable_release != '' %}
cat >>"{{ env_vars_to_source_file }}" <<EOF
export STABLE_RELEASE="{{ stable_release }}"
EOF
{% endif %}

{% if 'ovb' in toci_jobtype %}
cat >>"{{ env_vars_to_source_file }}" <<EOF
export TE_DATAFILE=/home/zuul/nodes.json
EOF
{% endif %}

if [[ "$LIBVIRT" == "1" ]]; then
cat >>"{{ env_vars_to_source_file }}" <<EOF
# Uncomment this if you have established docker proxy on virthost
# export NODEPOOL_DOCKER_REGISTRY_PROXY=$NODEPOOL_DOCKER_REGISTRY_PROXY
export NODEPOOL_CENTOS_MIRROR=$NODEPOOL_CENTOS_MIRROR
export NODEPOOL_MIRROR_HOST=mirror.mtl01.inap.openstack.org
EOF
fi

scp ${SSH_OPTS} "$WORKSPACE/{{ env_vars_to_source_file }}" zuul@$ansible_host:/home/zuul/
if [[ "$AUTORUN" == "1" ]]; then
    ssh ${SSH_OPTS} zuul@$ansible_host -tt \
    "screen -LS ci -h 25000 -dm bash -c 'set -m; source ~/{{ env_vars_to_source_file }}; ANSIBLE_FORCE_COLOR=true /opt/stack/tripleo-ci/toci_gate_test.sh 2>&1 | tee ~/console.log; exec bash' && screen -RD"
else
    # Remove -x so that the instructions don't print twice
    set +x

    # Instruct the user to execute toci_gate_test.sh on the undercloud
    echo "
        # Now complete the test execution on the undercloud:
        # ssh to the undercloud:
        ssh ${SSH_OPTS} zuul@$ansible_host
        # Source the environment settings file and run the toci gate script
        source /home/zuul/env_vars_to_src.sh
        /opt/stack/tripleo-ci/toci_gate_test.sh 2>&1 | tee console.log

        # To avoid timeouts, you can start a screen session before executing the commands:
        screen -S ci
    "
fi