main | Introduce Jenkins Job Builder for CI

.gitignore

@@ -42,3 +42,7 @@ jenkins/credentials.xml
 # Include the deployment files
 !deployment
 !deployment/**
+
+# Include the Jenkins Job Builder files
+!jobs-builder
+!jobs-builder/**

jobs-builder/README.md

@@ -0,0 +1,86 @@
+# Overview
+
+Manage the Jenkins jobs with help of the [Jenkins Job Builder](https://docs.openstack.org/infra/jenkins-job-builder) (JJB).
+
+The JJB converts jobs and views from YAML representations into the Jenkins XML
+configuration files. Also the tool is able to manage jobs, for example, publish
+the generated jobs and views in a running Jenkins instance.
+
+# Getting started
+
+First of all, you need to install Jenkins Job Builder in your environment. The
+instructions can be found [here](https://docs.openstack.org/infra/jenkins-job-builder/installation.html).
+
+To use the Jenkins Job Builder a configuration file is needed which contains
+the Jenkins URL, user and token API to manage Jenkins, among other information
+that sets the tool's behavior. That file can copied from the `jjb.conf.template`
+template then filling out the fields marked with *XXX*.
+
+Bootstrapping your environment:
+```bash
+$ pip install --user jenkins-job-builder
+$ cp jjb.conf.template jjb.conf
+$ sed -i 's/user=XXX/user=my_user/' jjb.conf
+$ sed -i 's/password=XXX/password=my_user_token/' jjb.conf
+```
+
+# Managing the jobs
+
+Use the `publish_jobs.sh` to update all the Jenkins Job Builder managed jobs.
+
+Example of use:
+```bash
+$ ./publish_jobs.sh -c jjb.conf
+```
+
+If you only want to check the jobs can be generated but not actually publish
+them all then do:
+```bash
+$ ./publish_jobs.sh -c jjb.conf -t
+```
+
+Run `./publish_jobs.sh -h` to see all the available options of the script.
+
+# Checking your changes on a local Jenkins
+
+Often you will need to see how the jobs look like on the Jenkins UI in order
+to check that the generated configurations are correct. The easiest way to
+accomplish that is to publish the jobs on a local Jenkins instance. Use the
+following instructions to setup a sandbox locally, but beware that it won't be
+exactly alike the instance on production, although for the purpose of checking
+the configurations it works out.
+
+ 1. Start the Jenkins container
+
+The following command creates the jenkins container, and the instance service
+will be accessible through the port 8080 on localhost.
+
+```bash
+$ docker run --rm -p 8080:8080 --name=jenkins-container -d jenkins/jenkins
+```
+
+ 2. Give an initial configuration
+
+Using your web browser, access Jenkins from http://localhost:8080.
+
+The first displayed page asks for the initial administrator password, which can
+be obtained with the following command:
+
+```bash
+$ docker exec jenkins-container cat /var/jenkins_home/secrets/initialAdminPassword
+```
+
+Paste that token on the "administrator password" field then continue with the
+setup. You will be asked to install plugins (select to install all) and finally
+to create an account.
+
+ 3. Create the API token
+
+Access your new user account (on the top-right menus) then go to "Configure".
+Click "Add new Token" then on the "Generate" button. Save the generated token.
+
+ 4. Create the JJB configuration file
+
+Now you need to create the jjb.conf as explained on [Getting started](#getting-started). Use
+the username and API token created on the previous steps, and don't forget to set
+the *url* property to `http://localhost:8080`.

jobs-builder/jjb.conf.template

@@ -0,0 +1,20 @@
+# Copyright (c) 2020 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Jenkins Job Builder configuration file (template).
+#
+[job_builder]
+ignore_cache=True
+keep_descriptions=False
+include_path=.:scripts:~/git/
+recursive=False
+exclude=.*:manual:./development
+allow_duplicates=False
+update=all
+
+[jenkins]
+user=XXX
+password=XXX
+url=https://jenkins.katacontainers.io
+query_plugins_info=False

jobs-builder/jobs/dependencies.yaml

@@ -0,0 +1,153 @@
+# Copyright (c) 2021 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# This file contains the configurations to generate the Kernel and QEMU jobs.
+#
+---
+###
+# Define shareable YAML snippets.
+###
+- common_job_properties: &common_job_properties
+    name: 'common_job_properties'
+    project-type: freestyle
+    disabled: false
+    concurrent: true
+    logrotate:
+      daysToKeep: 30
+      numToKeep: 5
+    # Convert the os variable to label name.
+    node: !include-jinja2: include/os2node.yaml.inc
+    scm:
+      - git:
+          url: https://github.com/kata-containers/kata-containers
+          branches:
+            - '*/{branch}'
+    wrappers:
+      - ansicolor:
+          colormap: "xterm"
+      - openstack:
+          single-use: True
+      - timestamps
+      - timeout:
+          timeout: 20
+          type: no-activity
+- default_publishers: &default_publishers
+    name: 'default_publishers'
+    publishers:
+      - archive:
+          artifacts: "artifacts/*"
+###
+# Define jobs templates.
+###
+- job-template:
+    name: kata-containers-{branch}-kernel-{flavor}-{arch}
+    <<: *common_job_properties
+    maintainers:
+      - Wainer dos Santos Moschetta <wainersm@redhat.com>
+    description:
+      !j2: |
+        Build Kernel for usage in Kata Containers CI jobs.
+        You should not use these artifacts on production.
+
+        Maintainers:
+        {% for i in maintainers -%}
+        - {{ i }}
+        {% endfor %}
+    triggers:
+      - timed: 'H 0-23/6 * * 1-5'
+    builders:
+      - shell:
+          !j2: |
+            #!/bin/bash
+            set -o errexit
+            set -o nounset
+            set -o pipefail
+            [ -n "$BASH_VERSION" ] && set -o errtrace
+            {% raw %}
+            [ -n "${DEBUG:-}" ] && set -o xtrace
+            {% endraw %}
+
+            export GOPATH="${WORKSPACE}/go"
+            export GOROOT="/usr/local/go"
+            export PATH="$PATH:$GOPATH/bin:$GOROOT/bin"
+
+            source ci/lib.sh
+            export branch="{{branch}}"
+            export target_branch="$branch"
+            {% if flavor == "experimental" -%}
+            export build_type="experimental"
+            {% endif %}
+            clone_tests_repo
+            ci/install_go.sh
+            cd "$tests_repo_dir"
+            ./.ci/install_kata_kernel.sh
+            ./.ci/ci_cache_components.sh -k
+    <<: *default_publishers
+- job-template:
+    name: kata-containers-{branch}-qemu-{arch}
+    <<: *common_job_properties
+    maintainers:
+      - Wainer dos Santos Moschetta <wainersm@redhat.com>
+    description:
+      # Passing a Jinja2 template.
+      !j2: |
+        Build QEMU for usage in Kata Containers CI jobs.
+        You should not use these artifacts on production.
+
+        Maintainers:
+        {% for i in maintainers -%}
+        - {{ i }}
+        {% endfor %}
+    triggers:
+      - github
+    builders:
+      - shell: |
+          #!/bin/bash
+          set -o errexit
+          set -o nounset
+          set -o pipefail
+          [ -n "$BASH_VERSION" ] && set -o errtrace
+          [ -n "${{DEBUG:-}}" ] && set -o xtrace
+
+          export GOPATH="$WORKSPACE/go"
+          export GOROOT="/usr/local/go"
+          export PATH="$PATH:$GOPATH/bin:$GOROOT/bin"
+
+          source ci/lib.sh
+          export branch="{branch}"
+          export target_branch="$branch"
+          clone_tests_repo
+          ci/install_go.sh
+
+          cd "$tests_repo_dir"
+          ./.ci/setup_env_ubuntu.sh "default"
+          ./cmd/container-manager/manage_ctr_mgr.sh docker install -f
+          ./.ci/install_qemu.sh
+          ./.ci/ci_cache_components.sh -q
+    <<: *default_publishers
+###
+# Define the projects
+###
+- project:
+    name: "Create Kernel cache jobs"
+    flavor:
+      - vanilla
+      - experimental
+    branch:
+      - main
+    arch:
+      - x86_64
+    os: ubuntu1804
+    jobs:
+      - "kata-containers-{branch}-kernel-{flavor}-{arch}"
+- project:
+    name: "Create QEMU cache jobs"
+    branch:
+      - main
+      - stable-2.3
+    arch:
+      - x86_64
+    os: ubuntu1804
+    jobs:
+      - "kata-containers-{branch}-qemu-{arch}"

jobs-builder/jobs/include/os2node.yaml.inc

@@ -0,0 +1,17 @@
+{#
+# Copyright (c) 2020 Red Hat, Inc.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Convert OS name to node label string.
+#}
+{% if os == "centos8" -%}
+centos8_azure
+{% elif os == "fedora32" -%}
+fedora32_azure
+{% elif os == "ubuntu1804" -%}
+ubuntu1804_azure || ubuntu1804-azure
+{% elif os == "ubuntu-20.04" -%}
+ubuntu_20.04
+{% elif os == "ubuntu-20.04-ARM" -%}
+arm_node || arm-ubuntu-2004
+{% endif %}

jobs-builder/publish_jobs.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+#
+# Copyright (c) 2020 Red Hat, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Use this script to publish the jobs on Jenkins.
+#
+set -o errexit
+set -o nounset
+set -o pipefail
+
+[ -n "$BASH_VERSION" ] && set -o errtrace
+
+[ -n "${DEBUG:-}" ] && set -o xtrace
+
+script_dir="$(realpath $(dirname $0))"
+jobs_dir="$script_dir/jobs"
+readonly cmd="jenkins-jobs"
+test_only=0
+config_file=""
+
+function die
+{
+	local msg="$*"
+	echo "ERROR: $msg" >&2
+	exit 1
+}
+
+function usage
+{
+	cat <<-EOF
+	This script uses the Jenkins Job Builder to manage the Kata Containers
+	CI jobs on Jenkins.
+
+	It needs the jenkins-jobs command installed as well as a configuration
+	file that contains information about how to access the Jenkins
+	instance. See the README.md for further details.
+
+	Usage $0: -c CONFIG [-t] [-h], where:
+	  -c           Path to the configuration file.
+	  -h           Print this message.
+	  -t           Do not publish jobs, only test they can be generated.
+	EOF
+}
+
+function parse_args
+{
+	while getopts "c:ht" opt; do
+		case ${opt} in
+			c) config_file="${OPTARG}" ;;
+			h) usage; exit 0 ;;
+			t) test_only=1 ;;
+			*) usage; exit 1 ;;
+		esac
+	done
+
+	if [ -z "$config_file" ]; then
+		usage
+		die "missing the config file"
+	fi
+}
+
+function main
+{
+	parse_args "$@"
+	command -v "$cmd" || die "$cmd command is needed"
+
+	# First test it can generate the jobs.
+	$cmd test "$jobs_dir" || die "some jobs cannot be generated"
+
+	if [ $test_only -eq 0 ]; then
+		# Going to update the managed jobs.
+		$cmd --conf "$config_file" update "$jobs_dir"
+	fi
+}
+
+main "$@"