common.sh

#!/usr/bin/env bash

# Copyright 2021 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# hack script for preparing devstack to run cluster-api-provider-openstack e2e
# This script is invoked by devstack-on-*-project-install.sh

set -x
set -o errexit -o nounset -o pipefail

CLUSTER_NAME=${CLUSTER_NAME:-"capo-e2e"}

OPENSTACK_RELEASE=${OPENSTACK_RELEASE:-"wallaby"}
OPENSTACK_ENABLE_HORIZON=${OPENSTACK_ENABLE_HORIZON:-"false"}
# Flavors are default or preinstalled:
# * default: installs devstack via cloud-init
#   * OPENSTACK_RELEASE only works on default
# * preinstalled: uses a already installed devstack
FLAVOR=${FLAVOR:="default"}

# Devstack will create a provider network using this range
# We create a route to it with sshuttle
FLOATING_RANGE=${FLOATING_RANGE:-"172.24.4.0/24"}

# Servers will be directly attached to the private network
# We create a route to it with sshuttle
PRIVATE_NETWORK_CIDR=${PRIVATE_NETWORK_CIDR:-"10.0.2.0/24"}
CONTROLLER_IP=${CONTROLLER_IP:-"10.0.2.15"}
WORKER_IP=${WORKER_IP:-"10.0.2.16"}

# For apt-get
export DEBIAN_FRONTEND=noninteractive

REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/../../
cd "${REPO_ROOT}" || exit 1
REPO_ROOT_ABSOLUTE=$(pwd)
ARTIFACTS=${ARTIFACTS:-/tmp/${CLUSTER_NAME}-artifacts}
devstackdir="${ARTIFACTS}/devstack"
mkdir -p "${devstackdir}"

# retry $1 times with $2 sleep in between
function retry {
    attempt=0
    max_attempts=${1}
    interval=${2}
    shift; shift
    until [[ ${attempt} -ge "${max_attempts}" ]] ; do
        attempt=$((attempt+1))
        set +e
        eval "$*" && return || echo "failed ${attempt} times: $*"
        set -e
        sleep "${interval}"
    done
    echo "error: reached max attempts at retry($*)"
    return 1
}

function ensure_openstack_client {
    if ! command -v openstack;
    then
        apt-get install -y python3-dev
        # install PyYAML first because otherwise we get an error because pip3 doesn't upgrade PyYAML to the correct version
        # ERROR: Cannot uninstall 'PyYAML'. It is a distutils installed project and thus we cannot accurately determine which
        # files belong to it which would lead to only a partial uninstall.
        pip3 install --ignore-installed PyYAML
        pip3 install python-cinderclient python-glanceclient python-keystoneclient python-neutronclient python-novaclient python-openstackclient python-octaviaclient
    fi
}

function wait_for_ssh {
    local ip=$1 && shift

    retry 10 30 "$(get_ssh_cmd) ${ip} -- true"
}

function start_sshuttle {
    if ! command -v sshuttle;
    then
        # Install sshuttle from source because we need: https://github.com/sshuttle/sshuttle/pull/661
        # TODO(sbueringer) install via pip after the next release after 1.0.5 via:
        # pip3 install sshuttle
        cd /tmp
        git clone https://github.com/sshuttle/sshuttle.git
        cd sshuttle
        pip3 install .
        cd "${REPO_ROOT_ABSOLUTE}" || exit 1
    fi

    kill_sshuttle

    # Open tunnel
    public_ip=$(get_public_ip)
    wait_for_ssh "${public_ip}"
    echo "Opening tunnel to ${PRIVATE_NETWORK_CIDR} and ${FLOATING_RANGE} via ${public_ip}"
    sshuttle -r "${public_ip}" "${PRIVATE_NETWORK_CIDR}" "${FLOATING_RANGE}" --ssh-cmd="$(get_ssh_cmd)" -l 0.0.0.0 -D || exit 1
}

function kill_sshuttle {
    sshuttle_pidfile="${REPO_ROOT_ABSOLUTE}/sshuttle.pid"
    if [ -f "$sshuttle_pidfile" ]; then
        sshuttle_pid=$(cat $sshuttle_pidfile)
        kill $sshuttle_pid
        while [ -d "/proc/$sshuttle_pid" ]; do
            echo "Waiting for sshuttle pid $sshuttle_pid to die"
            sleep 1
        done
    fi
}

function wait_for_devstack {
    local name=$1 && shift
    local ip=$1 && shift

    ssh_cmd=$(get_ssh_cmd)

    # Wait until cloud-init is done
    wait_for_ssh "${ip}"

    retry 120 30 "$ssh_cmd ${ip} -- cat /var/lib/cloud/instance/boot-finished"

    # Continuously capture devstack logs until killed
    $ssh_cmd "${ip}" -- sudo journalctl -b -u 'devstack@*' -f > "${devstackdir}/${name}-devstack.log" &

    # Capture cloud-init logs
    # Devstack logs are in cloud-final
    for service in cloud-config cloud-final cloud-init-local cloud-init; do
        $ssh_cmd ${ip} -- sudo journalctl -u ${service} > "${devstackdir}/${name}-${service}.log"

        # Fail early if any cloud-init service failed
        $ssh_cmd ${ip} -- sudo systemctl status ${service} || exit 1
    done
}

function get_ssh_cmd {
    echo "ssh -i $(get_ssh_private_key_file) -l ubuntu " \
         "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=yes -o PasswordAuthentication=no " \
         "-o ControlMaster=auto -o ControlPath=~/.ssh/ssh-%r@%h:%p -o ControlPersist=1m"
}

function create_devstack {
    local name=$1 && shift
    local ip=$1 && shift
    local flavor=$1 && shift
    local public=${1:-} && shift

    cloud_init="${devstackdir}/cloud-init-${flavor}.yaml"

    if [[ ${OPENSTACK_ENABLE_HORIZON} = "true" ]]
    then
        OPENSTACK_ADDITIONAL_SERVICES="${OPENSTACK_ADDITIONAL_SERVICES},horizon"
    fi

    if ! command -v envsubst;
    then
        apt-get update && apt-get install -y gettext
    fi

    # Ensure cloud-init is empty
    truncate --size 0 "${cloud_init}"

    for tpl in common ${flavor}; do
        SSH_PUBLIC_KEY=$(get_ssh_public_key) \
        OPENSTACK_ADDITIONAL_SERVICES=${OPENSTACK_ADDITIONAL_SERVICES:-""} \
        OPENSTACK_RELEASE=${OPENSTACK_RELEASE} \
        HOST_IP=${ip} \
        CONTROLLER_IP=${CONTROLLER_IP} \
        FLOATING_RANGE=${FLOATING_RANGE} \
            envsubst '${SSH_PUBLIC_KEY} ${OPENSTACK_ADDITIONAL_SERVICES} ${OPENSTACK_RELEASE} ${HOST_IP} ${FLOATING_RANGE} ${CONTROLLER_IP}' \
                < ./hack/ci/cloud-init/${tpl}.yaml.tpl >> "${cloud_init}"
    done

    create_vm "${CLUSTER_NAME}-${name}" "${ip}" "${cloud_init}" "${flavor}" "${public}"
}

function cleanup {
    kill_sshuttle
    cloud_cleanup
    exit 0
}

function create_worker {
    # Create the worker machine synchronously
    create_devstack worker "${WORKER_IP}" worker

    # Wait and run post-install tasks asynchronously
    wait_for_devstack worker "${WORKER_IP}" > "${devstackdir}/worker-build.log" 2>&1 &
}

function main() {
    if [ "${1:-}" == "cleanup" ]; then
        cleanup
    fi

    # Initialize the necessary infrastructure requirements
    if [[ -n "${SKIP_INIT_INFRA:-}" ]]; then
        echo "Skipping infrastructure initialization..."
    else
        init_infrastructure
    fi

    # Create devstack VM.
    # devstack initialisation proceeds asynchronously in the VM.
    create_devstack controller "${CONTROLLER_IP}" "${FLAVOR}" public

    # Install some local dependencies we later need in the meantime (we have to wait for cloud init anyway)
    ensure_openstack_client
    start_sshuttle

    wait_for_devstack controller "${CONTROLLER_IP}"

    # Let worker creation proceed asynchronously
    create_worker

    cat << EOF > "${REPO_ROOT_ABSOLUTE}/clouds.yaml"
clouds:
  ${CLUSTER_NAME}:
    auth:
      username: demo
      password: secretadmin
      user_domain_id: default
      auth_url: http://${CONTROLLER_IP}/identity
      domain_id: default
      project_name: demo
    verify: false
    region_name: RegionOne
  ${CLUSTER_NAME}-admin:
    auth:
      username: admin
      password: secretadmin
      user_domain_id: default
      auth_url: http://${CONTROLLER_IP}/identity
      domain_id: default
      project_name: admin
    verify: false
    region_name: RegionOne
EOF

    export OS_CLOUD="${CLUSTER_NAME}-admin"

    # Wait until the OpenStack API is reachable
    retry 5 30 "openstack versions show"

    openstack hypervisor stats show
    openstack host list
    openstack usage list
    openstack project list
    openstack network list
    openstack subnet list
    openstack image list
    openstack flavor list
    openstack server list
    openstack availability zone list
    openstack domain list

    # the flavors are created in a way that we can execute at least 2 e2e tests in parallel (overall we have 32 vCPUs)
    openstack flavor delete m1.tiny
    openstack flavor create --ram 512 --disk 1 --vcpus 1 --public --id 1 m1.tiny --property hw_rng:allowed='True'
    openstack flavor delete m1.small
    openstack flavor create --ram 4192 --disk 10 --vcpus 2 --public --id 2 m1.small --property hw_rng:allowed='True'
    openstack flavor delete m1.medium
    openstack flavor create --ram 6144 --disk 10 --vcpus 4 --public --id 3 m1.medium --property hw_rng:allowed='True'

    # Adjust the CPU quota
    openstack quota set --cores 32 demo
    openstack quota set --secgroups 50 demo

    echo "${REPO_ROOT_ABSOLUTE}/clouds.yaml:"
    cat "${REPO_ROOT_ABSOLUTE}/clouds.yaml"
}