build.yaml

# vim: ai ts=4 sts=4 et sw=4 ft=yaml fdm=indent et foldlevel=0
#
# build.yaml
#
# This file contains the Jenkins job definitions for the Flocker project.
#
# We add new or reconfigure existing jobs in Jenkins using the Job DSL plugin.
# https://github.com/jenkinsci/job-dsl-plugin
#
# That plugin consumes 'groovy Job DSL' code through FreeStyle Jenkins job
# types which contain a 'Process Job DSLs' step.
#
# As part of the provisioning process for the Jenkins Master, we configure a
# job called 'setup_ClusterHQ_Flocker' which is responsible for querying the
# github clusterhq/flocker respository and retrieving this build.yaml file, as
# well as the jobs.groovy.j2 jinja file.
#
# As part of the provisioning process for the Jenkins Master, we also deploy
# a small python script '/usr/local/bin/render.py', this script simply contains
# code to read a YAML file into a dictionary and expand a Jinja2 template using
# the k,v in that dict.
#
# Our job 'setup_ClusterHQ_Flocker' when running, will run the 'render.py' with
# this build.yaml file and produce a jobs.groovy file.
# We do this, because we'd rather configure our jobs with YAML than with Groovy
#
# The next step in the job is a 'Process JOB DSL's' step which consumes the
# jobs.groovy file, and generates all the jenkins folders and jobs.
#
#
# We pass the branch name as a parameter to the setup_clusterhq_flocker job,
# the parameter is shown as 'RECONFIGURE_BRANCH'.
#
# The setup job only produces jobs for a single branch . We don't produce jobs
# for every branch due to the large number of branches in the repository, which
# would generate over 16000 jobs and take over an hour to run.
#
# The workflow is that, when a developer is working on a feature branch and is
# happy to start testing some of his code they will execute the job
# setup_clusterhq_flocker passing his branch as the parameter.
#
# Their jobs will then be available under the path:
# /ClusterHQ-Flocker/<branch>/
#
# Inside that folder there will be a large number of jobs, where at the top
# she/he can see a job called '_main_multijob'. This job is responsible for
# executing all other jobs in parallel and collecting the produced artifacts
# from each job after its execution.
#
# The artifacts in this case are trial logs, coverage xml reports, subunit
# reports.
#
# Those artifacts are consumed by the _main_multijob to produce an overall
# coverage report, and an aggregated summary of all the executed tests and
# their failures/skips/successes.
#

# The project contains the github owner and repository to build
project: 'ClusterHQ/flocker'

# git_url, contains the full HTTPS url for the repository to build
git_url: 'https://github.com/ClusterHQ/flocker.git'


# We use a set of YAML aliases and anchors to define the different steps
# in our jobs.
# This helps us to keep some of the code DRY, we are not forced to use YAML
# operators, a different approach could be used:
#  - bash functions
#  - python functions
#  - Rust or D code
#
common_cli:
  hashbang: &hashbang |
    #!/bin/bash -l
    # don't leak secrets
    set +x
    set -e


  # add_shell_functions, contains our the bash functions consumed by the
  # the build script.
  # We set the shebang to /bin/bash. Jenkins 'should' respect this.
  # Note:
  # We noticed that our Ubuntu /bin/bash call were being executed as /bin/sh.
  # So we as part of the slave image build process symlinked
  # /bin/sh -> /bin/bash.
  # TODO: https://clusterhq.atlassian.net/browse/FLOC-2986
  add_shell_functions: &add_shell_functions |

    # set default AWS region for S3 operations
    export S3_REGION=us-west-2
    # Docs buckets (clusterhq-staging-docs, doc-dev.clusterhq.com) are hosted in us-east-1
    export S3_DOCS_REGION=us-east-1

    # The long directory names where we build our code cause pip to fail.
    # https://gitlab.com/gitlab-org/gitlab-ci-multi-runner/issues/20
    # http://stackoverflow.com/questions/10813538/shebang-line-limit-in-bash-and-linux-kernel
    # https://github.com/spotify/dh-virtualenv/issues/10
    # so we set our virtualenv dir to live in /tmp/<random number>
    #
    export venv=/tmp/${RANDOM}
    # make sure the virtualenv doesn't already exist
    while [ -e ${venv} ]
    do
      export venv=/tmp/${RANDOM}
    done

    function os {
      # Let's figure out if this is OSX or Linux
      # and return the OS type (or the distribution for a Linux OS)
      _nix_flavour=$(uname)
      if [ 'Darwin' == "${_nix_flavour}" ]; then
        echo "OSX"
      else
        # Determine OS platform
        source /etc/os-release
        echo ${ID}
      fi
    }
    function is_ubuntu {
      if [ 'ubuntu' == "$(os)" ]; then
        return 0
      else
        return 1
      fi
    }
    function is_centos {
      if [ 'centos' == "$(os)" ]; then
        return 0
      else
        return 1
      fi
    }
    function is_osx {
      if [ 'OSX' == "$(os)" ]; then
        return 0
      else
        return 1
      fi
    }

    # fix the docker permission issue, the base image doesn't have the correct
    # permissions/owners.
    # This is to be tackled as part of:
    # https://clusterhq.atlassian.net/browse/FLOC-2689
    test -e /var/run/docker.sock && sudo chmod 777 /var/run/docker.sock

    # Returns the ip address for eth0
    # We consume this as part or the vagrant build tests.
    # Those tests run on our Mesos Cluster, the job connects to a local nginx
    # instance running on the Mesos Slave where the job is running.
    export eth0_ip=$( ip -o -4 addr show eth0 |awk '{ print $4 '} | cut -f 1 -d "/")

    # pass the exit code from the previous and return the the aggregated
    # exit status for the whole job.
    function updateExitStatus {
      # if we had previous failures, we just fail here.
      if [ "${JOB_EXIT_STATUS}" != "0" ] && [ "${JOB_EXIT_STATUS}" != "" ]; then
        echo 1
      else
         if [ "$1" !=  "0" ]; then
         # first failure on the job, lets return 1, which will set
         # JOB_EXIT_STATUS to 1 and mark the job as failed.
             echo 1
         fi
      fi
    }

    # Retries a command if it fails, using the exponential backoff algorithm.
    # Usage:
    # retry_n_times_with_timeout <max retries> <timeout in seconds> command
    #
    function retry_n_times_with_timeout {
      set +e
      max_tries=${1}
      seconds=${2}
      shift 2

      let delay=15
      let tries=0
      while [ ${tries} -lt ${max_tries} ]; do
        echo "executing...  ${*}"
        if (timeout --foreground --signal=SIGKILL ${seconds} bash -c "${*}"); then
            break
        fi
        echo Command failed, waiting ${delay}...
        sleep ${delay}
        let delay=delay*2
        let tries=tries+1

        # abort on too many failures
        if [ ${tries} -eq ${max_tries} ]; then
          echo "too many failed retries... aborting"
          exit 1
        fi

      done
    }

    # uploads a file to S3, retrying on failure.
    # usage:
    # s3_upload <bucket> <filename>
    function s3_upload() {
      echo "uploading ${2} to S3 bucket ${1} ..."
      retry_n_times_with_timeout 3 900 \
        aws --region ${S3_REGION} s3 cp "${2}" s3://${1}/
    }

    # creates a metadata file to be used with vagrant up; vagrant box update
    # usage: create_vagrant_metadata_file
    #   <filename> <box_name> <description> <version> <url> <sha1>
    function create_vagrant_metadata_file() {
      cat <<EOF_METADATA > ${1}
      {
        "name": "${2}",
        "description": "${3}",
        "versions": [{
          "version": "${4}",
          "providers": [{
            "name": "virtualbox",
            "url": "${5}",
            "checksum_type": "sha1",
            "checksum": "${6}"
          }]
        }]
      }
    EOF_METADATA
    }

    # destroy the vagrant vm and aborts the build
    # this is used when one the of the intermediate steps has failed
    # usage:
    # <some code> || abort_build
    function abort_build() {
      echo "aborting ..."
      vagrant_destroy
      exit 1
    }

    # attempts a vagrant up, and aborts the build on failure
    # usage:
    # vagrant_up
    function vagrant_up() {
      (
        retry_n_times_with_timeout 3 1200 \
          vagrant up
      ) || abort_build
    }

    # attempts to refresh the local vagrant box, and aborts the build on error
    # make sure we are using the latest available version of this vagrant box
    # usage:
    # vagrant_box_update
    function vagrant_box_update() {
      (
        retry_n_times_with_timeout 2 1800 \
          vagrant box update
      ) || abort_build
    }

    # destroys the vagrant vm
    # usage:
    # vagrant_destroy
    function vagrant_destroy() {
      vagrant destroy -f
    }

  # TODO: do we need to clean up old files on ubuntu and centos or
  # does the pre-scm plugin does this correctly for us ?
  # https://clusterhq.atlassian.net/browse/FLOC-3139
  cleanup: &cleanup |
    export PATH=/usr/local/bin:${PATH}
    # clean up the stuff from previous runs
    # due to the length of the jobname workspace, we are hitting limits in
    # our sheebang path name in pip.
    # https://github.com/spotify/dh-virtualenv/issues/10
    # http://stackoverflow.com/questions/10813538/shebang-line-limit-in-bash-and-linux-kernel
    # https://gitlab.com/gitlab-org/gitlab-ci-multi-runner/issues/20
    # So we will place the virtualenv in /tmp/v instead
    #
    rm -rf ${venv}

  setup_venv: &setup_venv |
    # Set up the new venv.
    virtualenv -p python2.7 --clear ${venv}
    . ${venv}/bin/activate
    # Report the version of Python we're using, to aid debugging.
    ${venv}/bin/python --version

  setup_pip_cache: &setup_pip_cache |
    # exports the PIP_INDEX_URL, TRUSTED_HOST variables to the environment.
    # The /tmp/pip.sh file is copied to the Jenkins Slave by the Jenkins Master
    # during the bootstrapping of the slave.
    # This file is located in /etc/jenkins_slave on the Master box.
    # These variables contain the PIP URL and IP address of our caching server.
    if [ -e /tmp/pip.sh ]; then
      . /tmp/pip.sh
    fi

  setup_flocker_modules: &setup_flocker_modules |
    # installs all the required python modules as well as the flocker code.
    # But first, we need to upgrade pip to 7.1.
    # We have a devpi cache in AWS which we will consume instead of going
    # upstream to the PyPi servers.
    # We specify that devpi caching server using -i $PIP_INDEX_URL
    # which requires as to include --trusted-host as we are not (yet) using
    # SSL on our caching box.
    # The --trusted-host option is only available with pip 7.
    #
    if [ ${PIP_INDEX_URL} ]; then
        PIP_ADDITIONAL_OPTIONS="${PIP_ADDITIONAL_OPTIONS} -i ${PIP_INDEX_URL} "
    fi
    if [ ${TRUSTED_HOST} ]; then
        PIP_ADDITIONAL_OPTIONS="${PIP_ADDITIONAL_OPTIONS} \
                                --trusted-host ${TRUSTED_HOST} "
    fi

    # using the caching-layer, install all the dependencies
    pip install --upgrade pip
    pip install . ${PIP_ADDITIONAL_OPTIONS}
    # Install Flocker. Use --process-dependency-links so we can temporarily use
    # testtools fork. See FLOC-3498.
    pip install --process-dependency-links  ".[dev]" ${PIP_ADDITIONAL_OPTIONS}
    # install junix for our coverage report
    pip install python-subunit junitxml ${PIP_ADDITIONAL_OPTIONS}

  setup_aws_env_vars: &setup_aws_env_vars |
    # set vars and run tests
    export FLOCKER_FUNCTIONAL_TEST_CLOUD_CONFIG_FILE=/tmp/acceptance.yaml
    FLOCKER_FUNCTIONAL_TEST_AWS_AVAILABILITY_ZONE="`wget -q -O - \
        http://169.254.169.254/latest/meta-data/placement/availability-zone`"
    export FLOCKER_FUNCTIONAL_TEST_AWS_AVAILABILITY_ZONE
    export FLOCKER_FUNCTIONAL_TEST_CLOUD_PROVIDER=aws

  setup_rackspace_env_vars: &setup_rackspace_env_vars |
    # set vars and run tests
    # The /tmp/acceptance.yaml file is deployed to the jenkins slave during
    # bootstrapping. These are copied from the Jenkins Master /etc/slave_config
    # directory.
    export FLOCKER_FUNCTIONAL_TEST_CLOUD_CONFIG_FILE=/tmp/acceptance.yaml
    export FLOCKER_FUNCTIONAL_TEST_CLOUD_PROVIDER=openstack
    export FLOCKER_FUNCTIONAL_TEST_OPENSTACK_REGION=dfw

  setup_coverage: &setup_coverage |
    # we install the python coverage module so generate coverage.xml files
    # which Jenkins will process through the Jenkins Cobertura plugin.
    # https://wiki.jenkins-ci.org/display/JENKINS/Cobertura+Plugin
    # This plugin allows for producing coverage reports over time for a
    # particular job, or aggregated reports across a set of related jobs.
    # This is achieved by downloading the coverage.xml artifacts from the
    # downstream child jobs to the parent job and processing those files
    # one last time through the cobertura plugin.
    # The resulting report wil contain stats from every single job.
    pip install coverage  ${PIP_ADDITIONAL_OPTIONS}

  run_coverage: &run_coverage |
    # run coverage and produce a report
    coverage xml --include=flocker*

  convert_results_to_junit: &convert_results_to_junit |
    # pip the trial.log results through subunit and export them as junit in xml
    cat trial.log | subunit-1to2 | subunit2junitxml \
      --no-passthrough --output-to=results.xml

  run_sphinx: &run_sphinx |
    ${venv}/bin/python setup.py --version
    cd docs
    # check spelling
    ${venv}/bin/sphinx-build -d _build/doctree -b spelling . _build/spelling
    # check links
    ${venv}/bin/sphinx-build -d _build/doctree -b linkcheck . _build/linkcheck
    # build html pages
    ${venv}/bin/sphinx-build -d _build/doctree -b html . _build/html

  # flocker artifacts contains the list of files we want to collect from our
  # _main_multijob. These are used to produce the coverage, test reports.
  flocker_artifacts: &flocker_artifacts
    - results.xml
    - _trial_temp/test.log
    - coverage.xml

  # acceptance-test-artifacts contains the list of files we want to collect
  # from our _main_multijob.
  # These are the remote logs from the acceptance tests.
  acceptance_tests_artifacts: &acceptance_tests_artifacts
    - run-acceptance-tests.log
    - _trial_temp/test.log
    - remote_logs.log

  # Ubuntu acceptance tests do not collect the logs from the remote nodes
  # https://clusterhq.atlassian.net/browse/FLOC-2560
  acceptance_tests_artifacts_ubuntu_special_case: &acceptance_tests_artifacts_ubuntu_special_case
    - run-acceptance-tests.log
    - remote_logs.log
    - _trial_temp/test.log

  run_trial_with_coverage: &run_trial_with_coverage |
    # The jobs.groovy.j2 file produces jobs that contain a parameterized job
    # type. These type of jobs always require a parameter to be passed on in
    # order for they to be executed.
    # We grab the value from the 'with_modules:' dictionary in the yaml job
    # defintion, and feed it to the job configuration as the default value for
    # the 'MODULE' parameter.
    # This is how we tell trial which flocker module to call.
    coverage run ${venv}/bin/trial \
      --debug-stacktraces --reporter=subunit \
      ${MODULE} 2>&1 | tee trial.log

  run_trial_with_coverage_as_root: &run_trial_with_coverage_as_root |
    # The jobs.groovy.j2 file produces jobs that contain a parameterized job
    # type. These type of jobs always require a parameter to be passed on in
    # order for they to be executed.
    # We grab the value from the 'with_modules:' dictionary in the yaml job
    # defintion, and feed it to the job configuration as the default value for
    # the 'MODULE' parameter.
    # This is how we tell trial which flocker module to call.
    sudo ${venv}/bin/coverage run ${venv}/bin/trial \
      --debug-stacktraces --reporter=subunit \
      ${MODULE} 2>&1 | tee trial.log

  run_trial_for_storage_drivers_with_coverage: &run_trial_for_storage_drivers_with_coverage |
    # The jobs.groovy.j2 file produces jobs that contain a parameterized job
    # type. These type of jobs always require a parameter to be passed on in
    # order for they to be executed.
    # We grab the value from the 'with_modules:' dictionary in the yaml job
    # defintion, and feed it to the job configuration as the default value for
    # Consume the MODULE parameter set in the job configuration
    sudo -E ${venv}/bin/coverage run ${venv}/bin/trial \
      --debug-stacktraces --reporter=subunit \
      --testmodule ${MODULE} 2>&1 | tee trial.log

  setup_authentication: &setup_authentication |
    # acceptance tests rely on this file existing
    touch ${HOME}/.ssh/known_hosts
    # remove existing keys
    rm -f ${HOME}/.ssh/id_rsa*
    cp /tmp/id_rsa ${HOME}/.ssh/id_rsa
    chmod -R 0700 ${HOME}/.ssh
    ssh-keygen -N '' -f ${HOME}/.ssh/id_rsa_flocker
    eval `ssh-agent -s`
    ssh-add ${HOME}/.ssh/id_rsa

  run_acceptance_aws_tests: &run_acceptance_aws_tests |
    # We gather the return code but make sure we come out of these tests with 0
    # we store that code and pass it to the end of the job execution,
    # as part of the JOB_EXIT_STATUS variable.
    #
    # The admin/run-acceptance-tests will provision a flocker cluster of
    # several nodes. These nodes will install the flocker packages (RPM/DEB)
    # during the provisioning process by that tool. These packages are fetched
    # from a repository on the network through a common apt-get/yum install.
    # The jenkins slave will be the repository host containing those packages
    # which are made available through a webserver running on port 80.
    # We pass the URL of our Jenkins Slave to the acceptance test nodes
    # through the --build-server parameter below.
    ${venv}/bin/python admin/run-acceptance-tests \
    --distribution ${DISTRIBUTION_NAME} \
    --provider aws --dataset-backend aws --branch ${TRIGGERED_BRANCH} \
    --build-server  \
    http://$(wget -qO- http://instance-data/latest/meta-data/public-ipv4)  \
    --config-file /tmp/acceptance.yaml \
    ${ACCEPTANCE_TEST_MODULE}
    JOB_EXIT_STATUS="$( updateExitStatus $? )"

  run_acceptance_loopback_tests: &run_acceptance_loopback_tests |
    # We gather the return code but make sure we come out of these tests with 0
    # we store that code and pass it to the end of the job execution,
    # as part of the JOB_EXIT_STATUS variable.
    #
    # The admin/run-acceptance-tests will provision a flocker cluster with a
    # single node since the loopback backend doesn't support moving data across
    # hosts.
    ${venv}/bin/python admin/run-acceptance-tests \
    --distribution ${DISTRIBUTION_NAME} --number-of-nodes 1 \
    --provider aws --dataset-backend loopback --branch ${TRIGGERED_BRANCH} \
    --build-server  \
    http://$(wget -qO- http://instance-data/latest/meta-data/public-ipv4)  \
    --config-file /tmp/acceptance.yaml \
    ${ACCEPTANCE_TEST_MODULE}
    JOB_EXIT_STATUS="$( updateExitStatus $? )"

  run_acceptance_rackspace_tests: &run_acceptance_rackspace_tests |
    # We gather the return code but make sure we come out of these tests with 0
    # we store that code and pass it to the end of the job execution,
    # as part of the JOB_EXIT_STATUS variable.
    #
    # The admin/run-acceptance-tests will provision a flocker cluster of
    # several nodes. These nodes will install the flocker packages (RPM/DEB)
    # during the provisioning process by that tool. These packages are fetched
    # from a repository on the network through a common apt-get/yum install.
    # The jenkins slave will be the repository host containing those packages
    # which are made available through a webserver running on port 80.
    # We pass the URL of our Jenkins Slave to the acceptance test nodes
    # through the --build-server parameter below.
    ${venv}/bin/python admin/run-acceptance-tests \
    --distribution ${DISTRIBUTION_NAME} \
    --provider rackspace --dataset-backend rackspace \
    --branch ${TRIGGERED_BRANCH} \
    --build-server  \
    http://$(wget -qO- http://instance-data/latest/meta-data/public-ipv4)  \
    --config-file /tmp/acceptance.yaml \
    ${ACCEPTANCE_TEST_MODULE}
    JOB_EXIT_STATUS="$( updateExitStatus $? )"

  run_client_tests: &run_client_tests |
    # We gather the return code but make sure we come out of these tests with 0
    # we store that code and pass it to the end of the job execution,
    # as part of the JOB_EXIT_STATUS variable.
    ${venv}/bin/python admin/run-client-tests \
    --distribution ${DISTRIBUTION_NAME} \
    --branch ${TRIGGERED_BRANCH} \
    --build-server  \
    http://$(wget -qO- http://instance-data/latest/meta-data/public-ipv4)
    JOB_EXIT_STATUS="$( updateExitStatus $? )"

  disable_selinux: &disable_selinux |
    sudo /usr/sbin/setenforce 0

  check_version: &check_version |
    export FLOCKER_VERSION=$(${venv}/bin/python setup.py --version)

  build_sdist: &build_sdist  |
    # package the goodies
    ${venv}/bin/python setup.py sdist

  build_package: &build_package  |
    # and build a rpm/deb package using docker
    ${venv}/bin/python admin/build-package \
    --destination-path repo \
    --distribution ${DISTRIBUTION_NAME} \
    /flocker/dist/Flocker-${FLOCKER_VERSION}.tar.gz

  build_homebrew_package: &build_homebrew_package  |
    # and build a homebrew recipe and a package
    SDIST_URL=file://${PWD}/dist/Flocker-${FLOCKER_VERSION}.tar.gz
    RECIPE_FILE=${PWD}/Flocker${GIT_COMMIT}.rb
    ${venv}/bin/python -m admin.homebrew --flocker-version ${GIT_COMMIT} \
      --sdist ${SDIST_URL} --output-file ${RECIPE_FILE}

  install_homebrew_package: &install_homebrew_package |
    # make sure we're no longer in our virtualenv
    deactivate
    # install the new freshly baked package
    brew update
    brew tap ClusterHQ/tap
    brew install ${RECIPE_FILE}
    brew test ${RECIPE_FILE}

  build_repo_metadata: &build_repo_metadata |
    # the acceptance tests look for a package in a yum repository,
    # we provide one by starting a webserver and pointing the tests
    # to look over there
    REPO_PATH=/results/omnibus/${TRIGGERED_BRANCH}/${DISTRIBUTION_NAME}
    DOC_ROOT=/usr/share/nginx/html
    sudo rm -rf ${DOC_ROOT}/${REPO_PATH}
    sudo mkdir -p ${DOC_ROOT}/${REPO_PATH}
    sudo cp repo/* ${DOC_ROOT}/${REPO_PATH}
    cd ${DOC_ROOT}/${REPO_PATH}
    # create a repo on either centos or ubuntu
    if [ "${DISTRIBUTION_NAME}" == "ubuntu-14.04" ]; then
      sudo sh -c 'dpkg-scanpackages --multiversion . | gzip > Packages.gz'
    fi
    if [ "${DISTRIBUTION_NAME}" == "ubuntu-15.10" ]; then
      sudo sh -c 'dpkg-scanpackages --multiversion . | gzip > Packages.gz'
    fi
    if [ "${DISTRIBUTION_NAME}" == "centos-7" ]; then
      sudo createrepo .
    fi
    cd -

  clean_packages: &clean_packages |
    # jenkins is unable to clean the git repository as some files are owned
    # by root, so we make sure we delete the repo files we created
    sudo rm -rf repo/

  exit_with_return_code_from_test: &exit_with_return_code_from_test |
    # this is where we make sure we exit with the correct return code
    # from the tests we executed above.
    exit ${JOB_EXIT_STATUS}

  push_image_to_dockerhub: &push_image_to_dockerhub |
    # the /tmp/dockerhub_creds is copied from the Jenkins Master on
    # /etc/slave_config to the slave during the bootstrap process of the slave.
    # This contains the login details for our dockerhub instance, which is
    # deployed as part of our caching platform.
    #
    export D_USER=$( cat /tmp/dockerhub_creds | cut -f 1 -d ":" )
    export D_PASSWORD=$( cat /tmp/dockerhub_creds | cut -f 2 -d ":" )
    export D_EMAIL=$( cat /tmp/dockerhub_creds | cut -f 3 -d ":" )
    docker login -u ${D_USER} -p ${D_PASSWORD} -e ${D_EMAIL}
    echo y | docker push ${DOCKER_IMAGE}

  build_docker_image: &build_docker_image |
    # we want to make sure we are fetching the latest OS updates every time
    # so we build the docker image using --no-cache, this way apt-get updates
    # are actually executed.
    # See: https://github.com/docker/docker/issues/3313
    docker build --no-cache -t $DOCKER_IMAGE .

  # These are the docker images we will be using during our tests.
  # We build them every 24 hours, making sure we have the latest OS updates
  # installed on those images.
  # By doing this we speed up the bootstrapping of our client/acceptance tests.
  #
  build_dockerfile_centos7: &build_dockerfile_centos7 |
    # Download the latest pip requirements file from flocker
    wget -c \
    https://raw.githubusercontent.com/ClusterHQ/flocker/${TRIGGERED_BRANCH}/requirements.txt
    # don't waste time installing ruby or fpm, use an image containing fpm
    # https://github.com/alanfranz/fpm-within-docker
    echo "FROM alanfranz/fwd-centos-7:latest" > Dockerfile
    echo "MAINTAINER ClusterHQ <contact@clusterhq.com>" >> Dockerfile
    echo "# URLGRABBER_DEBUG=1 to log low-level network info \
          - see FLOC-2640" >> Dockerfile
    echo "RUN env URLGRABBER_DEBUG=1 yum groupinstall \
          --assumeyes 'Development Tools'" >> Dockerfile
    echo "RUN env URLGRABBER_DEBUG=1 yum install \
          --assumeyes epel-release" >> Dockerfile
    echo "RUN env URLGRABBER_DEBUG=1 yum install --assumeyes \
          git ruby-devel python-devel libffi-devel openssl-devel \
          python-pip rpmlint" >> Dockerfile
    echo "RUN env URLGRABBER_DEBUG=1 yum update --assumeyes" >> Dockerfile
    echo "COPY requirements.txt /tmp/" >> Dockerfile
    echo "RUN pip install -r /tmp/requirements.txt" >> Dockerfile

  build_dockerfile_ubuntu_trusty: &build_dockerfile_ubuntu_trusty |
    # Download the latest pip requirements file from flocker
    wget -c \
    https://raw.githubusercontent.com/ClusterHQ/flocker/${TRIGGERED_BRANCH}/requirements.txt
    # don't waste time installing ruby or fpm, use an image containing fpm
    # https://github.com/alanfranz/fpm-within-docker
    echo "FROM alanfranz/fwd-ubuntu-trusty:latest" > Dockerfile
    echo "MAINTAINER ClusterHQ <contact@clusterhq.com>" >> Dockerfile
    echo "RUN apt-get update" >> Dockerfile
    echo "RUN apt-get install --no-install-recommends -y git ruby-dev \
          libffi-dev libssl-dev build-essential python-pip \
          python2.7-dev lintian" >> Dockerfile
    echo "COPY requirements.txt /tmp/" >> Dockerfile
    echo "RUN pip install -r /tmp/requirements.txt" >> Dockerfile

  build_dockerfile_ubuntu_wily: &build_dockerfile_ubuntu_wily |
    # Download the latest pip requirements file from flocker
    wget -c \
    https://raw.githubusercontent.com/ClusterHQ/flocker/${TRIGGERED_BRANCH}/requirements.txt
    # don't waste time installing ruby or fpm, use an image containing fpm
    # https://github.com/alanfranz/fpm-within-docker
    echo "FROM alanfranz/fwd-ubuntu-wily:latest" > Dockerfile
    echo "MAINTAINER ClusterHQ <contact@clusterhq.com>" >> Dockerfile
    echo "RUN apt-get update" >> Dockerfile
    echo "RUN apt-get install --no-install-recommends -y \
          git ruby-dev libffi-dev libssl-dev build-essential python-pip \
          python2.7-dev lintian" >> Dockerfile
    echo "COPY requirements.txt /tmp/" >> Dockerfile
    echo "RUN pip install -r /tmp/requirements.txt" >> Dockerfile

  build_vagrant_tutorial_basebox: &build_vagrant_tutorial_basebox |
    vagrant destroy -f
    (
      retry_n_times_with_timeout 3 2400 \
        $venv/bin/python admin/build-vagrant-box \
          --box tutorial \
          --branch ${TRIGGERED_BRANCH} \
          --build-server http://${eth0_ip}
    )|| abort_build

    (
      retry_n_times_with_timeout 2 1200 \
        vagrant box add --force "clusterhq/flocker-tutorial" \
          vagrant/tutorial/flocker-tutorial-${FLOCKER_VERSION}.box
    ) || abort_build

  do_not_abort_on_errors: &do_not_abort_on_errors |
    # make sure we don't abort the job on the first error we find.
    #
    # jenkins will execute our shellscript with -e, which will cause the
    # script to terminate at the first error and mark the job as failed.
    # This is fine for the majority of the build cases, but in some situations
    # we don't want to terminate the job straight away. For example, a job
    # which generates a new Virtual Machine where the tests are executed.
    # With '-e' defined, jenkins would abort the job and leave an orphan VM
    # behind.
    # To avoid it, we set '+e' on this shell.
    set +e

  cleanup_old_jenkins_osx_yosemite_boxes: &cleanup_old_jenkins_osx_yosemite_boxes |
    admin/ci-tools/cleanup-vagrant-boxes clusterhq/jenkins-osx-yosemite

  new_vagrantfile_for_osx_yosemite: &new_vagrantfile_for_osx_yosemite |
    # this builds a Vagrantfile which will use the latest jenkins-osx-yosemite
    # vagrant box built overnight.
    cat <<EOF_VAGRANTFILE >Vagrantfile
    Vagrant.configure(2) do |config|
       config.vm.box = 'clusterhq/jenkins-osx-yosemite'
       # "vagrant plugin install vagrant-s3auth" to enable support for s3 box URLs
       config.vm.box_url = "s3://${S3_BUCKET}/metadata.json"
       config.vm.synced_folder '.', '/vagrant', disabled: true
       config.vm.provider 'virtualbox' do |vb|
          vb.memory = '1024'
          # VirtualBox 5 now supports para-virtualization, let's use it.
          vb.customize("pre-boot", ["modifyvm", :id, "--paravirtprovider", "KVM"])
       end
    end
    EOF_VAGRANTFILE


  source_git_commit: &source_git_commit |
    # we use a script 'git-commit.sh' to pass any the git commit sha1
    # to our vagrant machine. This file is copied and sourced as part of our
    # build.sh run inside the virtual machine.
    chmod 755 git-commit.sh
    . git-commit.sh

  set_home_variable_on_mesos: &set_home_variable_on_mesos |
    # mesos is not setting HOME, so we set it here.
    # it needs to be set to /root, so that we can re-use the existing
    # vagrant base images which live under /root/.vagrant.d
    export HOME=/root

  # this is a wrapper that we can use for building a script that will be
  # populated with different cli actions.
  # we copy the resulting build.sh file to the vagrant box for execution.
  begin_build_sh_EOF: &begin_build_sh_EOF cat <<'EOF_BUILD_SH' > build.sh

  end_build_sh_EOF: &end_build_sh_EOF |
     EOF_BUILD_SH
     chmod 755 build.sh

  run_build_sh_script_on_vagrant_box: &run_build_sh_script_on_vagrant_box |
    # copy project files to the vagrant box
    vagrant ssh-config > ssh-config
    rsync -ave 'ssh -F ssh-config' . default:.

    # this sets some variables used by the tests, since the tests run inside
    # virtualbox we store them to a file and copy them to the vagrant box.
    echo "export GIT_COMMIT=${GIT_COMMIT}" > git-commit.sh
    for item in /tmp/pip.sh build.sh git-commit.sh
    do
      vagrant scp ${item} default:${item}
      vagrant ssh -c "chmod 755 ${item}"
    done

    # make sure we don't abort on the first error
    set +e
    # run the build.sh script inside our vagrant box
    vagrant ssh -c 'bash build.sh' || abort_build

    # this removes the secrets files we used during provisioning
    for item in "/tmp/pip.sh build.sh git-commit.sh"
    do
      vagrant ssh -c "rm -f ${item}"
    done


  install_pkgs_on_vanilla_osx_yosemite_box: &install_pkgs_on_vanilla_osx_yosemite_box |
    # permissions are not right, lets fix them here
    sudo chown -R vagrant /users/vagrant
    # do a bunch of steps before attempting to install Xcode command line tools
    xcode-select --install
    sudo xcodebuild -license accept
    sudo /usr/sbin/DevToolsSecurity --enable
    # installs XCode CommandLineTools from the CLI
    # https://github.com/chcokr/osx-init/blob/master/install.sh#L25
    touch /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress
    PROD=$(softwareupdate -l |grep '\\*.*Command Line' |head -n 1 \
     | awk -F"*" '{print $2}' |sed -e 's/^ *//' |tr -d "\n")
    softwareupdate -i "${PROD}" -v

    # the basebox we use for OSX is missing quite a number of software pieces.
    brew update
    brew install rsync
    brew install python
    pip install virtualenv
    brew install libffi
    export PKG_CONFIG_PATH=/usr/local/Cellar/libffi/3.0.13/lib/pkgconfig/; pip install bcrypt
    # brew test is failling unless we install these packages
    pip install docutils
    pip install botocore
    pip install boto

  package_jenkins_osx_yosemite_box: &package_jenkins_osx_yosemite_box |
    # we now proceed to package our vagrant box
    # let's stop it and wait a bit, as sometimes the box is not fully down
    # when the vagrant halt command finishes.
    vagrant halt ; sleep 60

    # Write the status of the VM into the console output.  It's not clear
    # whether it makes a difference or not what the status is at this point --
    # ``vagrant package`` supposedly shuts down the VM automatically if
    # necessary -- but some packaging problems suggest this might not be
    # happening.  Having the extra information may be handy for debugging.
    vagrant status

    BOX_NAME="jenkins-osx-yosemite-${FLOCKER_VERSION}.box"
    (
      retry_n_times_with_timeout 2 1800 \
        vagrant package --output "${BOX_NAME}"
    ) || abort_build

  start_vanilla_osx_yosemite_vagrant_box: &start_vanilla_osx_yosemite_vagrant_box |
    cat <<EOF_VAGRANTFILE >Vagrantfile
    Vagrant.configure(2) do |config|
      config.vm.box = "AndrewDryga/vagrant-box-osx"
      config.vm.provider "virtualbox" do |vb|
        # increase the virtualbox instance RAM size
        # our mesos executor will have 2GB, we'll stick to 1GB for the vagrant instance
        vb.memory = "1024"
        # VirtualBox 5 now supports para-virtualization.
        vb.customize("pre-boot", ["modifyvm", :id, "--paravirtprovider", "KVM"])
      end
    end
    EOF_VAGRANTFILE

    (
      retry_n_times_with_timeout 3 1200 \
        vagrant up
    ) || abort_build

  install_aws_cli: &install_aws_cli |
    # installs the AWS python based cli
    # we use the aws cli to upload the docs files to S3
    pip install awscli ${PIP_ADDITIONAL_OPTIONS}

  get_s3_creds_for_bucket_staging_docs: &get_s3_creds_for_bucket_staging_docs |
    # collects the S3 credentials that allows us to upload files to the S3
    # staging bucket for our docs.
    # the file with the credentials are deployed by the jenkins master to the
    # /tmp of the slave when the slave is instantiated.
    . /tmp/s3_staging_docs_clusterhq_com.sh

  get_s3_creds_for_bucket_vagrant_jenkins_boxes: &get_s3_creds_for_bucket_vagrant_jenkins_boxes |
    # collects the S3 credentials that allows us to upload files to the S3
    # vagrant-jenkins_boxes for our vagrant boxes
    # the file with the credentials is deployed by the jenkins master to the
    # /tmp of the slave when the slave is instantiated.
    . /tmp/s3_vagrant_jenkins_boxes_clusterhq.com

  get_s3_creds_for_clusterhq_dev_archive: &get_s3_creds_for_clusterhq_dev_archive |
    # collects the S3 credentials that allows us to upload files to the S3
    # clusterhq-dev-archive for our vagrant boxes
    # the file with the credentials is deployed by the jenkins master to the
    # /tmp of the slave when the slave is instantiated.
    . /tmp/s3_dev_archive_clusterhq.com

  upload_new_docs_html_to_s3_staging_docs: &upload_new_docs_html_to_s3_staging_docs |
    # uploads the new generated html to a folder on the S3 staging-docs bucket
    cd _build/html
    aws --region ${S3_DOCS_REGION} s3 sync . s3://clusterhq-staging-docs/$TRIGGERED_BRANCH
    echo "This branch is available at :"
    echo "http://clusterhq-staging-docs.s3.amazonaws.com/$TRIGGERED_BRANCH/index.html"

  sync_master_docs_to_doc_dev: &sync_master_docs_to_doc_dev |
    # ensure master docs at doc-dev are up-to-date
    [ "$TRIGGERED_BRANCH" != master ] || aws --region ${S3_DOCS_REGION} s3 sync --acl public-read \
      s3://clusterhq-staging-docs/master s3://doc-dev.clusterhq.com

  upload_vagrant_tutorial_basebox_to_s3: &upload_vagrant_tutorial_basebox_to_s3 |
    # uploads the generated vagrant box to a folder on the S3
    # clusterhq-dev-archive bucket.
    FLOCKER_TUTORIAL=flocker-tutorial-${FLOCKER_VERSION}
    BOX_NAME=${FLOCKER_TUTORIAL}.box
    BOX_PATH=vagrant/tutorial
    SHA1=$( sha1sum ${BOX_NAME} | cut -f 1 -d " " )
    HTTP_BOX_URL=http://${S3_BUCKET}.s3.amazonaws.com/${BOX_PATH}/${BOX_NAME}

    # the JSON metadata for our box is flocker-tutorial.version.box.json
    HTTP_BOX_JSON=http://${S3_BUCKET}.s3.amazonaws.com/${BOX_PATH}/${BOX_NAME}.json

    # the JSON url below is what the vagrant file will consume
    HTTP_BOX_JSON_LATEST=http://${S3_BUCKET}.s3.amazonaws.com/${BOX_PATH}/metadata.json

    # vagrant doesn't support fully semantic versioning, numbering must be x.y.z
    # https://github.com/mitchellh/vagrant/issues/6052
    VAGRANT_BOX_VERSION="$( echo ${FLOCKER_VERSION} | cut -f 1,2,3 -d "." )"

    cd $BOX_PATH

    # generates the metadata.json, it used by vagrant box update operations.
    create_vagrant_metadata_file \
      metadata.json \
      clusterhq/flocker-tutorial \
      'This box contains the flocker-tutorial VM.' \
      ${VAGRANT_BOX_VERSION} \
      ${HTTP_BOX_URL} \
      ${SHA1}

    cp metadata.json ${BOX_NAME}.json

    # uploads the vagrant box and metadata files to S3
    # We won't overwrite the json metadata files if the upload of the box
    # failed
    box_files="${BOX_NAME} ${BOX_NAME}.json metadata.json"
    (
      s3_upload \
        ${S3_BUCKET}/${BOX_PATH}/ "${BOX_NAME}" && \
      s3_upload \
        ${S3_BUCKET}/${BOX_PATH}/ "${BOX_NAME}.json" && \
      s3_upload \
        ${S3_BUCKET}/${BOX_PATH}/ "metadata.json"
    ) ||  abort_build

    cd -
    echo "This vagrant box is available at : ${HTTP_BOX_JSON_LATEST}"

  upload_vagrant_basebox_osx_yosemite_to_s3: &upload_vagrant_basebox_osx_yosemite_to_s3 |
    # uploads the generated vagrant box to a folder on the S3
    # clusterhq-vagrant-jenkins-boxes bucket.
    BOX_NAME="jenkins-osx-yosemite-${FLOCKER_VERSION}.box"
    SHA1=$( sha1sum ${BOX_NAME} | cut -f 1 -d " " )

    S3_BOX_URL=s3://${S3_BUCKET}/${BOX_NAME}

    # the JSON metadata for our box is jenkins-osx-yosemite-version.box.json
    S3_BOX_JSON=s3://${S3_BUCKET}/${BOX_PATH}/${BOX_NAME}.json

    # the JSON url below is what the vagrant file will consume
    S3_BOX_JSON_LATEST=s3://${S3_BUCKET}/${BOX_PATH}/metadata.json

    # vagrant doesn't support fully semantic versioning, numbering must be x.y.z
    # https://github.com/mitchellh/vagrant/issues/6052
    VAGRANT_BOX_VERSION="$( date +%s).0.0"

    # generates the metadata.json, it used by vagrant box update operations.
    create_vagrant_metadata_file \
      metadata.json \
      clusterhq/jenkins-osx-yosemite \
      'This box contains the OSX box used for the client tests.' \
      ${VAGRANT_BOX_VERSION} \
      ${S3_BOX_URL} \
      ${SHA1}

    cp metadata.json ${BOX_NAME}.json

    # uploads the vagrant box and metadata.json files
    # We won't overwrite the json metadata files if the upload of the box
    # failed
    (
      s3_upload ${S3_BUCKET} "${BOX_NAME}" &&  \
      s3_upload ${S3_BUCKET} "${BOX_NAME}.json" &&  \
      s3_upload ${S3_BUCKET} "metadata.json"
    ) || abort_build

    echo "This vagrant box is available at : ${S3_BOX_JSON_LATEST}"

    # The box is supposed to be a gzipped tar file.  Validate that here.  Only
    # do this *after* the upload so that the corrupt box is available online
    # for inspection to debug the build failure.
    gzip --verbose --test "${BOX_NAME}" || abort_build
    tar --verbose --list --file "${BOX_NAME}" || abort_build

  run_lint: &run_lint |
    # we need to unset PIP_INDEX_URL since it causes tox to fail.
    # in order to use our pip caching we need to append --trusted-hosts
    # which we currently can't do through tox.
    unset PIP_INDEX_URL
    # run flake8 lint tests on Flocker source code
    tox -e lint

  install_flake8: &install_flake8 |
    # flake8 is used by lint tests on Flocker source code
    pip install flake8 ${PIP_ADDITIONAL_OPTIONS}

#-----------------------------------------------------------------------------#
# Job Definitions below this point
#-----------------------------------------------------------------------------#
# Job Types:
#
# * run_trial
# * run_trial_for_storage_driver (ebs/cinder)
# * run_sphinx (old docs job)
# * run_acceptance (tests)
# * cronly_jobs (builds docker images every 24 hours)
#
# Toggles:
#   * archive_artifacts: ( define if there are files to be archived)
#   * coverage_report: ( enable if this job produces a coverage report file)
#   * clean_repo: (enable if we need to clean old files owned by root)
#

# run_trial_modules contains a list of all the modules we want to execute
# through trial.
run_trial_modules: &run_trial_modules
  - admin
  - benchmark
  - flocker.acceptance
  - flocker.apiclient
  - flocker.ca.functional
  - flocker.ca.test
  - flocker.cli
  - flocker.common
  - flocker.control
  - flocker.restapi
  - flocker.docs
  - flocker.dockerplugin
  - flocker.node.test
  - flocker.node.functional.test_docker
  - flocker.node.functional.test_script
  - flocker.node.functional.test_deploy
  - flocker.provision
  - flocker.restapi
  - flocker.route
  - flocker.test
  - flocker.testtools
run_trial_as_root_modules: &run_trial_as_root_modules
  # flocker.volume needs to run as root due to ZFS calls
  # so we run it as part of the run_trial_on_<Cloud>_<OS>_as_root jobs
  - flocker.volume
  # flocker.node.agent needs to run as root due to mounting filesystems
  - flocker.node.agents


# run_trial_cli contains a list of all the CLI yaml anchors we want to
# execute as part of our run_trial_tasks
run_trial_cli: &run_trial_cli [
  *hashbang,
  *add_shell_functions,
  *setup_pip_cache,
  *cleanup,
  *setup_venv,
  *setup_flocker_modules,
  *setup_coverage,
  *setup_aws_env_vars,
  *run_trial_with_coverage,
  *run_coverage,
  *convert_results_to_junit ]

run_trial_cli_as_root: &run_trial_cli_as_root [
  *hashbang,
  *add_shell_functions,
  *setup_pip_cache,
  *cleanup,
  *setup_venv,
  *setup_flocker_modules,
  *setup_coverage,
  *setup_aws_env_vars,
  *run_trial_with_coverage_as_root,
  *run_coverage,
  *convert_results_to_junit ]

# run_acceptance_modules contains the list of Flocker modules to be executed
# during the acceptance tests
run_acceptance_modules: &run_acceptance_modules
  - flocker.acceptance.endtoend
  - flocker.acceptance.integration
  - flocker.acceptance.obsolete

# When we have acceptance tests that run quickly enough (as they do on loopback)
# we just run them all in one builder:
run_full_acceptance_modules: &run_full_acceptance_modules
  - flocker.acceptance

# flocker.node.functional is hanging, so we don't run it
job_type:
  run_trial:
    # http://build.clusterhq.com/builders/flocker-centos-7
    run_trial_on_AWS_CentOS_7:
      on_nodes_with_labels: 'aws-centos-7-SELinux-T2Small'
      with_modules: *run_trial_modules
      with_steps:
        - { type: 'shell', cli: *run_trial_cli }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      timeout: 30

    run_trial_on_AWS_CentOS_7_as_root:
      on_nodes_with_labels: 'aws-centos-7-SELinux-T2Small'
      with_modules: *run_trial_as_root_modules
      with_steps:
        - { type: 'shell', cli: *run_trial_cli_as_root }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      timeout: 30

    # http://build.clusterhq.com/builders/flocker-admin
    # http://build.clusterhq.com/builders/flocker-ubuntu-14.04
    run_trial_on_AWS_Ubuntu_Trusty:
      on_nodes_with_labels: 'aws-ubuntu-trusty-T2Small'
      with_modules: *run_trial_modules
      with_steps:
        - { type: 'shell', cli: *run_trial_cli }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      timeout: 30

    run_trial_on_AWS_Ubuntu_Trusty_as_root:
      on_nodes_with_labels: 'aws-ubuntu-trusty-T2Small'
      with_modules: *run_trial_as_root_modules
      with_steps:
        - { type: 'shell', cli: *run_trial_cli_as_root }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      timeout: 30


  run_trial_for_storage_driver:
    run_trial_for_ebs_storage_driver_on_CentOS_7:
      on_nodes_with_labels: 'aws-centos-7-SELinux-T2Small'
      with_modules:
        - flocker/node/agents/ebs.py
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,  *setup_pip_cache,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *setup_coverage, *setup_aws_env_vars,
                   'export FLOCKER_FUNCTIONAL_TEST=TRUE',
                   *run_trial_for_storage_drivers_with_coverage, *run_coverage,
                   *convert_results_to_junit ]
          }
      archive_artifacts: *flocker_artifacts
      coverage_report: true
      clean_repo: true
      timeout: 45

    run_trial_for_ebs_storage_driver_on_Ubuntu_trusty:
      on_nodes_with_labels: 'aws-ubuntu-trusty-T2Small'
      with_modules:
        - flocker/node/agents/ebs.py
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,  *setup_pip_cache,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *setup_coverage, *setup_aws_env_vars,
                   'export FLOCKER_FUNCTIONAL_TEST=TRUE',
                   *run_trial_for_storage_drivers_with_coverage, *run_coverage,
                   *convert_results_to_junit ]
          }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      timeout: 45


    run_trial_for_cinder_storage_driver_on_CentOS_7:
      on_nodes_with_labels: 'rackspace-jenkins-slave-centos7-selinux-standard-4-dfw'
      with_modules:
        - flocker/node/agents/cinder.py
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,  *setup_pip_cache,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *setup_coverage, *setup_rackspace_env_vars,
                   'export FLOCKER_FUNCTIONAL_TEST=TRUE',
                   *run_trial_for_storage_drivers_with_coverage, *run_coverage,
                   *convert_results_to_junit ]
          }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      # Give this job a slightly longer timeout because it has to interact with
      # the cloud-supplied Cinder which can be a bit sluggish even in the best
      # of times.
      timeout: 90

    run_trial_for_cinder_storage_driver_on_Ubuntu_trusty:
      on_nodes_with_labels: 'rackspace-jenkins-slave-ubuntu14-standard-4-dfw'
      with_modules:
        - flocker/node/agents/cinder.py
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,  *setup_pip_cache,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *setup_coverage, *setup_rackspace_env_vars,
                   'export FLOCKER_FUNCTIONAL_TEST=TRUE',
                   *run_trial_for_storage_drivers_with_coverage, *run_coverage,
                   *convert_results_to_junit ]
          }
      archive_artifacts: *flocker_artifacts
      publish_test_results: true
      coverage_report: true
      clean_repo: true
      # Reasoning for run_trial_for_cinder_storage_driver_on_CentOS_7's timeout
      # applies here as well.
      timeout: 90


  # http://build.clusterhq.com/builders/flocker-docs
  run_sphinx:
    run_sphinx:
      on_nodes_with_labels: 'aws-ubuntu-trusty-T2Small'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,  *setup_pip_cache,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *install_aws_cli, *run_sphinx,
                   *get_s3_creds_for_bucket_staging_docs,
                   *upload_new_docs_html_to_s3_staging_docs,
                   *sync_master_docs_to_doc_dev ]
          }
      timeout: 10


  # http://build.clusterhq.com/builders/flocker%2Facceptance%2Faws%2Fcentos-7%2Faws
  run_acceptance:
    run_acceptance_loopback_on_AWS_CentOS_7_for:
      on_nodes_with_labels: 'aws-centos-7-SELinux-T2Medium'
      with_modules: *run_full_acceptance_modules
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,  *setup_pip_cache,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   'export DISTRIBUTION_NAME=centos-7',
                   *setup_aws_env_vars, *check_version,
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *setup_authentication,
                   'export ACCEPTANCE_TEST_MODULE=${MODULE}',
                   *run_acceptance_loopback_tests,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      archive_artifacts: *acceptance_tests_artifacts
      timeout: 45

    run_acceptance_loopback_on_AWS_Ubuntu_Trusty_for:
      on_nodes_with_labels: 'aws-ubuntu-trusty-T2Medium'
      with_modules: *run_full_acceptance_modules
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions, *setup_pip_cache,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *setup_aws_env_vars, *check_version,
                   'export DISTRIBUTION_NAME=ubuntu-14.04',
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *setup_authentication,
                   'export ACCEPTANCE_TEST_MODULE=${MODULE}',
                   *run_acceptance_loopback_tests,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      archive_artifacts: *acceptance_tests_artifacts_ubuntu_special_case
      timeout: 45


  run_client:
    run_client_installation_on_Ubuntu_Trusty:
      on_nodes_with_labels: 'aws-ubuntu-trusty-T2Medium'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,  *setup_pip_cache,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   'export DISTRIBUTION_NAME=ubuntu-14.04',
                   *setup_aws_env_vars, *check_version,
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *run_client_tests,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      timeout: 30
    run_client_installation_on_Ubuntu_Wily:
      on_nodes_with_labels: 'aws-ubuntu-trusty-T2Medium'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,  *setup_pip_cache,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   'export DISTRIBUTION_NAME=ubuntu-15.10',
                   *setup_aws_env_vars, *check_version,
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *run_client_tests,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      timeout: 30
    run_client_installation_on_OSX:
      on_nodes_with_labels: 'mesos-4GB'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang,
                   *add_shell_functions,

                   *begin_build_sh_EOF,
                   *hashbang,
                   *add_shell_functions,
                   *setup_pip_cache,
                   *cleanup,
                   *setup_venv,
                   *setup_flocker_modules,
                   'export DISTRIBUTION_NAME=OSX',
                   *source_git_commit,
                   *check_version,
                   *build_sdist,
                   *build_homebrew_package,
                   *install_homebrew_package,
                   *end_build_sh_EOF,

                   '# Execution starts here:',
                   *set_home_variable_on_mesos,
                   *do_not_abort_on_errors,
                   *get_s3_creds_for_bucket_vagrant_jenkins_boxes,
                   *cleanup_old_jenkins_osx_yosemite_boxes,
                   *new_vagrantfile_for_osx_yosemite,
                   'vagrant_box_update',
                   'vagrant_up',
                   *run_build_sh_script_on_vagrant_box,
                   'vagrant_destroy']
          }
      clean_repo: false
      # FLOC-3597: Importing base box 'jenkins-osx-yosemite' step can
      # sometimes take ~20 minutes, leaving us not quite enough time to
      # actually run the tests. Also has to be less than the timeout for the
      # main multijob, see jobs.groovy.j2:545.
      timeout: 50


  run_lint:
    run_lint:
      on_nodes_with_labels: 'aws-centos-7-T2Small'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,  *setup_pip_cache,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *install_flake8, *run_lint ]
          }
      timeout: 10


  cronly_jobs:
    run_docker_build_centos7_fpm:
      at: '0 0 * * *'
      on_nodes_with_labels: 'aws-centos-7-SELinux-T2Medium'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   'export DOCKER_IMAGE=clusterhqci/fpm-centos-7',
                   *build_dockerfile_centos7,
                   *build_docker_image,
                   *push_image_to_dockerhub ]
          }
      timeout: 30
    run_docker_build_ubuntu_trusty_fpm:
      at: '0 1 * * *'
      on_nodes_with_labels: 'aws-centos-7-SELinux-T2Medium'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   'export DOCKER_IMAGE=clusterhqci/fpm-ubuntu-trusty',
                   *build_dockerfile_ubuntu_trusty,
                   *build_docker_image,
                   *push_image_to_dockerhub ]
          }
      timeout: 30
    run_docker_build_ubuntu_wily_fpm:
      at: '0 3 * * *'
      on_nodes_with_labels: 'aws-centos-7-SELinux-T2Medium'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,
                   'export DOCKER_IMAGE=clusterhqci/fpm-ubuntu-wily',
                   *build_dockerfile_ubuntu_wily,
                   *build_docker_image,
                   *push_image_to_dockerhub ]
          }
      timeout: 30
    build_vagrant_basebox_for_flocker_tutorial:
      at: '0 4 * * *'
      on_nodes_with_labels: 'mesos-2GB'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions, *setup_pip_cache,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *check_version,
                   'export DISTRIBUTION_NAME=centos-7',
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   'export HOME=/root',
                   *build_vagrant_tutorial_basebox,

                   # upload this new vagrant box to S3
                   *install_aws_cli,
                   *get_s3_creds_for_clusterhq_dev_archive,
                   *upload_vagrant_tutorial_basebox_to_s3,
                   *clean_packages,
                   'vagrant_destroy'
                ]
          }
      timeout: 60
    build_vagrant_basebox_for_osx_yosemite:
      at: '0 5 * * *'
      on_nodes_with_labels: 'mesos-2GB'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang,
                   *add_shell_functions,
                   *set_home_variable_on_mesos,
                   *do_not_abort_on_errors,

                   # we create a build.sh script which will be used to
                   # provision the OSX machine.
                   # it installs XCode command line tools and other required
                   # packages.
                   *begin_build_sh_EOF,
                   *add_shell_functions,
                   *install_pkgs_on_vanilla_osx_yosemite_box,
                   *end_build_sh_EOF,

                   # starts and provisions an OSX vagrant machine
                   # using the build.sh script created above
                   *start_vanilla_osx_yosemite_vagrant_box,
                   *run_build_sh_script_on_vagrant_box,

                   # packages the vagrant box so that we can upload it to S3
                   # we need to install pip and flocker deps, as we need
                   # to find out the Flocker version
                   *setup_pip_cache,
                   *cleanup,
                   *setup_venv,
                   *setup_flocker_modules,
                   *check_version,
                   *package_jenkins_osx_yosemite_box,

                   # upload this new vagrant box to S3
                   *install_aws_cli,
                   *get_s3_creds_for_bucket_vagrant_jenkins_boxes,
                   *upload_vagrant_basebox_osx_yosemite_to_s3,

                   # cleanup
                   *clean_packages,
                   'vagrant_destroy'
            ]
          }
      timeout: 120

    run_acceptance_on_AWS_CentOS_7_with_EBS:
      at: '0 5 * * *'
      on_nodes_with_labels: 'aws-centos-7-SELinux-T2Medium'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,  *setup_pip_cache,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   'export DISTRIBUTION_NAME=centos-7',
                   *setup_aws_env_vars, *check_version,
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *setup_authentication,
                   'export ACCEPTANCE_TEST_MODULE=flocker.acceptance',
                   *run_acceptance_aws_tests,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      archive_artifacts: *acceptance_tests_artifacts
      # Give the acceptance test suite a nice long time to run.  Give it even
      # longer on CentOS than Ubuntu because Docker configuration on CentOS
      # causes some things to be particularly slow.  This value is just a guess
      # at what a reasonable upper-bound for the runtime of the suite might be
      # as of Dec 2015.
      timeout: 120

    run_acceptance_on_AWS_Ubuntu_Trusty_with_EBS:
      at: '0 6 * * *'
      on_nodes_with_labels: 'aws-ubuntu-trusty-T2Medium'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions, *setup_pip_cache,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *setup_aws_env_vars, *check_version,
                   'export DISTRIBUTION_NAME=ubuntu-14.04',
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *setup_authentication,
                   'export ACCEPTANCE_TEST_MODULE=flocker.acceptance',
                   *run_acceptance_aws_tests,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      archive_artifacts: *acceptance_tests_artifacts_ubuntu_special_case
      # Similar to the reasoning for run_acceptance_on_AWS_CentOS_7_with_EBS
      # but slightly shorter since Ubuntu runs the tests faster.
      timeout: 90

    run_acceptance_on_Rackspace_CentOS_7_with_Cinder:
      at: '0 5 * * *'
      # flocker.provision is responsible for creating the test nodes on
      # Rackspace, so we can actually run run-acceptance-tests from AWS
      on_nodes_with_labels: 'aws-centos-7-SELinux-T2Medium'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions,  *setup_pip_cache,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   'export DISTRIBUTION_NAME=centos-7',
                   *setup_aws_env_vars, *check_version,
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *setup_authentication,
                   'export ACCEPTANCE_TEST_MODULE=flocker.acceptance',
                   *run_acceptance_rackspace_tests,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      archive_artifacts: *acceptance_tests_artifacts
      # Reasoning as for run_acceptance_on_AWS_CentOS_7_with_EBS
      timeout: 120

    run_acceptance_on_Rackspace_Ubuntu_Trusty_with_Cinder:
      at: '0 6 * * *'
      # flocker.provision is responsible for creating the test nodes on
      # Rackspace, so we can actually run run-acceptance-tests from AWS
      on_nodes_with_labels: 'aws-ubuntu-trusty-T2Medium'
      with_steps:
        - { type: 'shell',
            cli: [ *hashbang, *add_shell_functions, *setup_pip_cache,
                   *cleanup, *setup_venv, *setup_flocker_modules,
                   *setup_aws_env_vars, *check_version,
                   'export DISTRIBUTION_NAME=ubuntu-14.04',
                   *build_sdist, *build_package,
                   *build_repo_metadata,
                   *setup_authentication,
                   'export ACCEPTANCE_TEST_MODULE=flocker.acceptance',
                   *run_acceptance_rackspace_tests,
                   *clean_packages,
                   *exit_with_return_code_from_test ]
          }
      clean_repo: true
      archive_artifacts: *acceptance_tests_artifacts_ubuntu_special_case
      # Reasoning as for run_acceptance_on_AWS_Ubuntu_Trusty_with_EBS
      timeout: 90

#-----------------------------------------------------------------------------#
# View definitions below this point
#-----------------------------------------------------------------------------#

# Use the section below to define multiple 'views/tabs' that appear at the top
# of the jenkins build page for every branch.
# These views allow for filtering of particular jobs according to its type,
# cloud, operating system, ...
#
#
# The views are built from :
# * the dictionary 'key', which is the view name.
# * description, containing a description of the jobs in this view
# * regex, containing a regular expression that is applied to build the view.
#
# the site below can help with regular expressions:
# http://pythex.org/

views:
  trial:
    description: 'All jobs that are executed directly using trial'
    regex: 'run_(?i)trial_.*.*'
  acceptance:
    description: 'All Acceptance jobs'
    regex: 'run_(?i)acceptance_.*.*'
  client:
    description: 'All client installation jobs'
    regex: 'run_(?i)client_.*.*'
  code_quality:
    description: 'All code quality jobs'
    regex: 'run_(?i)lint.*.*'
  docs:
    description: 'Documentation jobs'
    regex: 'run_(?i)sphinx.*.*'
  cinder:
    description: 'cinder jobs'
    regex: '.*_(?i)cinder_.*.*'
  ebs:
    description: 'ebs jobs'
    regex: '.*_(?i)ebs_.*.*'
  storage_drivers:
    description: 'All Storage Driver Jobs'
    regex: '.*_(?i)storage_(?i)driver_.*.*'
  on_AWS:
    description: 'All AWS Jobs'
    regex: '(.*_(?i)ebs_.*.*|.*_(?i)aws_.*.*)'
  on_Rackspace:
    description: 'All Rackspace Jobs'
    regex: '(.*_(?i)cinder_.*.*|.*_(?i)rackspace_.*.*)'
  CentOS_7:
    description: 'All CentOS jobs'
    regex: '.*_(?i)centos_.*.*'
  Ubuntu_Trusty_14_04_LTS:
    description: 'All Ubuntu Trusty LTS (14.04) jobs'
    regex: '.*_(?i)trusty_.*.*'
  Ubuntu_Wily_15_10:
    description: 'All Ubuntu Wily (15.10) jobs'
    regex: '.*_(?i)wily_.*.*'
  cron:
    description: 'All Nightly Cron jobs'
    regex: '^_[^_](?i)[a-z]*.*'