diff --git a/.gitignore b/.gitignore index 04336a6a..5c7dbe5a 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,7 @@ jenkins/credentials.xml # Include the deployment files !deployment !deployment/** + +# Include the Jenkins Job Builder files +!jobs-builder +!jobs-builder/** diff --git a/README.md b/README.md index 0029df7a..63112c3d 100644 --- a/README.md +++ b/README.md @@ -57,3 +57,7 @@ You can check for any known CI system issues [via this link](http://jenkins.kata The Jenkins CI jobs can be controlled (such as re-triggered or skipped) in a number of ways. Details of the control trigger phrases are listed on [a community repo wiki page](https://github.com/kata-containers/community/wiki/Controlling-the-CI). + +# Jobs builder + +See [README](jobs-builder/README.md) for information about the use of Jenkins Job Builder (JJB) to make the CI jobs. diff --git a/jobs-builder/README.md b/jobs-builder/README.md new file mode 100644 index 00000000..da6f7818 --- /dev/null +++ b/jobs-builder/README.md @@ -0,0 +1,86 @@ +# Overview + +Manage the Jenkins jobs with help of the [Jenkins Job Builder](https://docs.openstack.org/infra/jenkins-job-builder) (JJB). + +The JJB converts jobs and views from YAML representations into the Jenkins XML +configuration files. Also the tool is able to manage jobs, for example, publish +the generated jobs and views in a running Jenkins instance. + +# Getting started + +First of all, you need to install Jenkins Job Builder in your environment. The +instructions can be found [here](https://docs.openstack.org/infra/jenkins-job-builder/installation.html). + +To use the Jenkins Job Builder a configuration file is needed which contains +the Jenkins URL, user and token API to manage Jenkins, among other information +that sets the tool's behavior. That file can copied from the `jjb.conf.template` +template then filling out the fields marked with *XXX*. + +Bootstrapping your environment: +```bash +$ pip install --user jenkins-job-builder +$ cp jjb.conf.template jjb.conf +$ sed -i 's/user=XXX/user=my_user/' jjb.conf +$ sed -i 's/password=XXX/password=my_user_token/' jjb.conf +``` + +# Managing the jobs + +Use the `publish_jobs.sh` to update all the Jenkins Job Builder managed jobs. + +Example of use: +```bash +$ ./publish_jobs.sh -c jjb.conf +``` + +If you only want to check the jobs can be generated but not actually publish +them all then do: +```bash +$ ./publish_jobs.sh -c jjb.conf -t +``` + +Run `./publish_jobs.sh -h` to see all the available options of the script. + +# Checking your changes on a local Jenkins + +Often you will need to see how the jobs look like on the Jenkins UI in order +to check that the generated configurations are correct. The easiest way to +accomplish that is to publish the jobs on a local Jenkins instance. Use the +following instructions to setup a sandbox locally, but beware that it won't be +exactly alike the instance on production, although for the purpose of checking +the configurations it works out. + + 1. Start the Jenkins container + +The following command creates the Jenkins container, and the instance service +will be accessible through the port 8080 on the local host. + +```bash +$ docker run --rm -p 8080:8080 --name=jenkins-container -d jenkins/jenkins +``` + + 2. Give an initial configuration + +Using your web browser, access Jenkins from [http://localhost:8080](http://localhost:8080). + +The first displayed page asks for the initial administrator password, which can +be obtained with the following command: + +```bash +$ docker exec jenkins-container cat /var/jenkins_home/secrets/initialAdminPassword +``` + +Paste that token on the "administrator password" field then continue with the +setup. You will be asked to install plugins (select to install all) and finally +to create an account. + + 3. Create the API token + +Access your new user account (on the top-right menus) then go to "Configure". +Click "Add new Token" then on the "Generate" button. Save the generated token. + + 4. Create the JJB configuration file + +Now you need to create the `jjb.conf` as explained on [Getting started](#getting-started). Use +the user name and API token created on the previous steps, and don't forget to set +the `url` property to `http://localhost:8080`. diff --git a/jobs-builder/jjb.conf.template b/jobs-builder/jjb.conf.template new file mode 100644 index 00000000..f445b307 --- /dev/null +++ b/jobs-builder/jjb.conf.template @@ -0,0 +1,20 @@ +# Copyright (c) 2020 Red Hat, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Jenkins Job Builder configuration file (template). +# +[job_builder] +ignore_cache=True +keep_descriptions=False +include_path=.:scripts:~/git/ +recursive=False +exclude=.*:manual:./development +allow_duplicates=False +update=all + +[jenkins] +user=XXX +password=XXX +url=https://jenkins.katacontainers.io +query_plugins_info=False diff --git a/jobs-builder/jobs/dependencies.yaml b/jobs-builder/jobs/dependencies.yaml new file mode 100644 index 00000000..c6045234 --- /dev/null +++ b/jobs-builder/jobs/dependencies.yaml @@ -0,0 +1,157 @@ +# Copyright (c) 2021 Red Hat, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# This file contains the configurations to generate the Kernel and QEMU jobs. +# +--- +### +# Define shareable YAML snippets. +### +- common_job_properties: &common_job_properties + name: 'common_job_properties' + project-type: freestyle + disabled: false + concurrent: true + logrotate: + daysToKeep: 30 + numToKeep: 5 + # Convert the os variable to label name. + node: !include-jinja2: include/os2node.yaml.inc + scm: + - git: + url: https://github.com/kata-containers/kata-containers + branches: + - '*/{branch}' + wrappers: + - ansicolor: + colormap: "xterm" + - openstack: + single-use: True + - timestamps + - timeout: + timeout: 20 + type: no-activity +- default_publishers: &default_publishers + name: 'default_publishers' + publishers: + - archive: + artifacts: "artifacts/*" +### +# Define jobs templates. +### +- job-template: + name: kata-containers-{branch}-kernel-{flavor}-{arch} + <<: *common_job_properties + maintainers: + - Kata Containers CI team (#kata-ci at Slack) + description: + !j2: | +

Build Kernel for usage in Kata Containers CI jobs.

+

You should not use these artifacts on production.

+ + Maintainers: + + triggers: + - timed: 'H 0-23/6 * * 1-5' + builders: + - shell: + !j2: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + [ -n "$BASH_VERSION" ] && set -o errtrace + {% raw %} + [ -n "${DEBUG:-}" ] && set -o xtrace + {% endraw %} + + export GOPATH="${WORKSPACE}/go" + export GOROOT="/usr/local/go" + export PATH="$PATH:$GOPATH/bin:$GOROOT/bin" + + source ci/lib.sh + export branch="{{branch}}" + export target_branch="$branch" + {% if flavor == "experimental" -%} + export build_type="experimental" + {% endif %} + clone_tests_repo + ci/install_go.sh + cd "$tests_repo_dir" + ./.ci/install_kata_kernel.sh + ./.ci/ci_cache_components.sh -k + <<: *default_publishers +- job-template: + name: kata-containers-{branch}-qemu-{arch} + <<: *common_job_properties + maintainers: + - Kata Containers CI team (#kata-ci at Slack) + description: + # Passing a Jinja2 template. + !j2: | +

Build QEMU for usage in Kata Containers CI jobs.

+

You should not use these artifacts on production.

+ + Maintainers: + + triggers: + - github + builders: + - shell: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + [ -n "$BASH_VERSION" ] && set -o errtrace + [ -n "${{DEBUG:-}}" ] && set -o xtrace + + export GOPATH="$WORKSPACE/go" + export GOROOT="/usr/local/go" + export PATH="$PATH:$GOPATH/bin:$GOROOT/bin" + + source ci/lib.sh + export branch="{branch}" + export target_branch="$branch" + clone_tests_repo + ci/install_go.sh + + cd "$tests_repo_dir" + ./.ci/setup_env_ubuntu.sh "default" + ./cmd/container-manager/manage_ctr_mgr.sh docker install -f + ./.ci/install_qemu.sh + ./.ci/ci_cache_components.sh -q + <<: *default_publishers +### +# Define the projects +### +- project: + name: "Create Kernel cache jobs" + flavor: + - vanilla + - experimental + branch: + - main + arch: + - x86_64 + os: ubuntu1804 + jobs: + - "kata-containers-{branch}-kernel-{flavor}-{arch}" +- project: + name: "Create QEMU cache jobs" + branch: + - main + - stable-2.3 + arch: + - x86_64 + os: ubuntu1804 + jobs: + - "kata-containers-{branch}-qemu-{arch}" diff --git a/jobs-builder/jobs/include/os2node.yaml.inc b/jobs-builder/jobs/include/os2node.yaml.inc new file mode 100644 index 00000000..5f267fa7 --- /dev/null +++ b/jobs-builder/jobs/include/os2node.yaml.inc @@ -0,0 +1,17 @@ +{# +# Copyright (c) 2020 Red Hat, Inc. +# SPDX-License-Identifier: Apache-2.0 +# +# Convert OS name to node label string. +#} +{%- if os == "centos8" -%} +centos8_azure +{%- elif os == "fedora35" -%} +fedora35_azure +{%- elif os == "ubuntu1804" -%} +ubuntu1804_azure || ubuntu1804-azure +{%- elif os == "ubuntu-20.04" -%} +ubuntu_20.04 +{%- elif os == "ubuntu-20.04-ARM" -%} +arm_node || arm-ubuntu-2004 +{%- endif %} diff --git a/jobs-builder/publish_jobs.sh b/jobs-builder/publish_jobs.sh new file mode 100755 index 00000000..43ec76fc --- /dev/null +++ b/jobs-builder/publish_jobs.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# +# Copyright (c) 2020 Red Hat, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Use this script to publish the jobs on Jenkins. +# +set -o errexit +set -o nounset +set -o pipefail + +[ -n "$BASH_VERSION" ] && set -o errtrace + +[ -n "${DEBUG:-}" ] && set -o xtrace + +script_dir="$(realpath $(dirname $0))" +jobs_dir="$script_dir/jobs" +readonly cmd="jenkins-jobs" +test_only=0 +config_file="" + +function die +{ + local msg="$*" + echo "ERROR: $msg" >&2 + exit 1 +} + +function usage +{ + cat <<-EOF + This script uses the Jenkins Job Builder to manage the Kata Containers + CI jobs on Jenkins. + + It needs the jenkins-jobs command installed as well as a configuration + file that contains information about how to access the Jenkins + instance. See the README.md for further details. + + Usage $0: -c CONFIG [-t] [-h], where: + -c Path to the configuration file. + -h Print this message. + -t Do not publish jobs, only test they can be generated. + EOF +} + +function parse_args +{ + while getopts "c:ht" opt; do + case ${opt} in + c) config_file="${OPTARG}" ;; + h) usage; exit 0 ;; + t) test_only=1 ;; + *) usage; exit 1 ;; + esac + done + + if [ -z "$config_file" ]; then + usage + die "missing the config file" + fi +} + +function main +{ + parse_args "$@" + command -v "$cmd" || die "$cmd command is needed" + + # First test it can generate the jobs. + $cmd test "$jobs_dir" || die "some jobs cannot be generated" + + if [ $test_only -eq 0 ]; then + # Going to update the managed jobs. + $cmd --conf "$config_file" update "$jobs_dir" + fi +} + +main "$@"