Redesigned CI infrastructure

The Travis CI infrastructure now directly sets Gerrit labels instead of the TravisPy library being used. - This avoids dependency on 3rd party libraries. - Logs are posted directly as comments in case of failure - Removed python3 tests as it needs to be tested from F28+ Change-Id: Ie399f612655f3172c065ccd626a9dd41b588ee59
dogtagpki · Apr 11, 2018 · 910ee1d · 910ee1d
1 parent 3e0f7df
commit 910ee1d
Show file tree

Hide file tree

Showing 12 changed files with 245 additions and 142 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -4,68 +4,56 @@ language: python
 services:
   - docker
 cache: pip
-env:
-  global:
-    - CONTAINER=pkitest
-    - SCRIPTDIR=/tmp/workdir/pki/.travis
-    - PKI_LOG=pki_log.log
-    - CI_RESULTS_LOG="ci_results_${TRAVIS_BRANCH}.log"
-    - CI_RUNNER_LOG_ARCHIVE="dogtag-ci-job-${TRAVIS_JOB_NUMBER}.tar.gz"
-    - IMAGE=dogtagpki/pki-ci:f26_105
-    # F25 is outdated
-    # - IMAGE=dogtagpki/pki-ci:f25_104
 
-    # rawhide repo is unstable
-    # - IMAGE=dogtagpki/pki-ci:rawhide
-  matrix:
-    - TASK_TO_RUN="run-dogtag-build"
-
-    - TASK_TO_RUN="run-ipa-test"
-      IPA_TEST_RUNNER_IMAGE="dogtagpki/pki-ci:f26_ipa_4-5"
-      RPMS_LOCATION=/tmp/workdir/packages/RPMS
-      DOGTAG_PKI_RPMS=${TRAVIS_BUILD_DIR}/dogtag_rpms
-
-before_install:
-  - pyenv global system 3.6
-  - docker pull ${IMAGE}
-  - >
-    docker run
-    --detach
-    --name=${CONTAINER}
-    --hostname='pki.test'
-    --privileged
-    --tmpfs /tmp
-    --tmpfs /run
-    -v /sys/fs/cgroup:/sys/fs/cgroup:ro
-    -v $(pwd):/tmp/workdir/pki
-    -e BUILDUSER_UID=$(id -u)
-    -e BUILDUSER_GID=$(id -g)
-    -e TRAVIS=${TRAVIS}
-    -e TRAVIS_JOB_NUMBER=${TRAVIS_JOB_NUMBER}
-    -ti
-    ${IMAGE}
-
-install:
-  - docker exec -ti ${CONTAINER} /bin/ls -la /tmp/workdir
-
-  - docker exec -ti ${CONTAINER} ${SCRIPTDIR}/00-init
-  - docker exec -ti ${CONTAINER} ${SCRIPTDIR}/10-compose-rpms
-
-  # IPA related installs
-  - pip install --upgrade pip
-  - pip3 install --upgrade pip
-  - pip install pep8
-
-script:
-  # It is time to run FreeIPA tests with new Dogtag RPMS built
-  - travis_wait 20 ./.travis_run_task.sh
-after_script:
-  - docker kill ${CONTAINER}
-  - docker rm ${CONTAINER}
-
-# The errors can be in either dogtag container or freeipa container
-after_failure:
-  - docker exec ${CONTAINER} journalctl  -l > ${PKI_LOG}
-  - echo "Uploading CI Logs to transfer.sh ..."
-  - curl --upload-file ./${PKI_LOG} https://transfer.sh/dogtag_build_logs.txt
-  - curl --upload-file ./${CI_RESULTS_LOG} https://transfer.sh/freeipa-integration.txt
+jobs:
+  include:
+    - env: TASK_TO_RUN="pki-test"
+      before_install:
+        - set -a && source .travis/global_variables
+        - echo -e $gerrit_ssh_key >> ~/.ssh/id_rsa && chmod 600 ~/.ssh/id_rsa
+        # Post the Travis Build URL
+        - TRAVIS_BUILD_URL="https://travis-ci.org/$TRAVIS_REPO_SLUG/builds/$TRAVIS_BUILD_ID"
+        - MESSAGE="$(printf "Build Started.\nTravis Build:"${TRAVIS_BUILD_URL})"
+        - .travis/set_gerrit_message.sh -m "${MESSAGE}"
+      install: .travis/init_task.sh | tee /dev/tty | grep -Eo '(http|https)://transfer.sh/[a-zA-Z0-9./?=_-]*.txt' | uniq >> ${TRANSFER_SH_URLS}
+      script:
+        - set -o pipefail
+        - travis_wait 20 .travis/run_task.sh | tee /dev/tty | grep -Eo '(http|https)://transfer.sh/[a-zA-Z0-9./?=_-]*.txt' | uniq >> ${TRANSFER_SH_URLS}
+      after_failure:
+        # Post the URL of Travis Job that failed
+        - TRAVIS_JOB_URL="https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$TRAVIS_JOB_ID"
+        - MESSAGE="$(printf "Job 1 Failed\nTravis Job:"${TRAVIS_JOB_URL}"\n\nLogs:\n" )"$'\n'"$(cat ${TRANSFER_SH_URLS})"
+        - .travis/set_gerrit_message.sh -v -1 -m "${MESSAGE}"
+        - cat ${TRANSFER_SH_URLS}
+      after_script:
+        - docker kill ${CONTAINER}
+        - docker rm ${CONTAINER}
+
+    - env: TASK_TO_RUN="ipa-test"
+      before_install:
+        - set -a && source .travis/global_variables
+        - echo -e $gerrit_ssh_key >> ~/.ssh/id_rsa && chmod 600 ~/.ssh/id_rsa
+      install: .travis/init_task.sh
+      script: travis_wait 20 .travis/run_task.sh
+      after_script:
+        - docker kill ${CONTAINER}
+        - docker rm ${CONTAINER}
+      after_failure:
+        - docker exec ${CONTAINER} journalctl  -l > ${PKI_LOG}
+        - echo "Uploading CI Logs to transfer.sh ..."
+        - curl -w "\n" --upload-file ./${PKI_LOG} https://transfer.sh/dogtag_build_logs.txt > ${TRANSFER_SH_URLS}
+        - curl -w "\n" --upload-file ./${CI_RESULTS_LOG} https://transfer.sh/freeipa-integration.txt >> ${TRANSFER_SH_URLS}
+        # Post the URL of Travis Job that failed
+        - TRAVIS_JOB_URL="https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$TRAVIS_JOB_ID"
+        - MESSAGE="$(printf "Job 2 Failed\nTravis Job:"${TRAVIS_JOB_URL}"\n\nLogs:\n" )"$'\n'"$(cat ${TRANSFER_SH_URLS})"
+        - .travis/set_gerrit_message.sh -v -1 -m "${MESSAGE}"
+        - cat ${TRANSFER_SH_URLS}
+
+    - stage: Verification Label
+      before_install: echo -e $gerrit_ssh_key >> ~/.ssh/id_rsa && chmod 600 ~/.ssh/id_rsa
+      script:
+        - MESSAGE="$(printf "Travis Build Successful.")"
+        - .travis/set_gerrit_message.sh -v +1 -m "${MESSAGE}"
+      after_script:
+        # Add remote repo in SSH form. Using existing 'origin' will result in user/pass authentication
+        - .travis/delete_branch.sh
diff --git a/.travis/00-init b/.travis/00-init
@@ -31,6 +31,3 @@ dnf makecache
 
 # update, container might be outdated
 dnf update -y
-
-## prepare additional build dependencies
-dnf builddep -y ${BUILDDIR}/pki/specs/pki-core.spec
diff --git a/.travis/01-install-dependencies b/.travis/01-install-dependencies
@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e
+
+## prepare additional build dependencies
+dnf builddep -y --spec ${BUILDDIR}/pki/specs/$1.spec
diff --git a/.travis/10-compose-rpms b/.travis/10-compose-rpms
@@ -1,31 +1,32 @@
 #!/bin/bash
 set -e
 
-BUILDLOG=/tmp/compose.log
+BUILDLOG=/tmp/compose_$1.log
 
 function compose {
     pushd ${BUILDDIR}/pki
     # run make with --quiet to reduce log verbosity.
     sudo -u ${BUILDUSER} MAKEFLAGS="-j2 --quiet" -- \
-        ./scripts/compose_pki_core_packages rpms
+        ./scripts/$1 rpms
     popd
 }
 
 function upload {
     if test -f $BUILDLOG; then
         echo "Uploading build log to transfer"
-        curl --upload-file $BUILDLOG https://transfer.sh/pkitravis.txt
+        curl --upload-file $BUILDLOG https://transfer.sh/pkitravis_$1.txt
+        # Add new line for readability of logs
+        printf "\n\n=====================================\n\n"
     fi
 }
 
 if test "${TRAVIS}" != "true"; then
     compose
 else
-    trap upload EXIT
-    echo "Runing compose_pki_core_packages rpms."
+    trap "upload $1" EXIT
+    echo "Runing $1 rpms."
     echo "Build log will be posted to transfer.sh"
     echo $(date) > $BUILDLOG
     echo "Travis job ${TRAVIS_JOB_NUMBER}" >> $BUILDLOG
-    compose >>$BUILDLOG 2>&1
-fi
-
+    compose $1>>$BUILDLOG 2>&1
+fi
diff --git a/.travis/20-install-rpms b/.travis/20-install-rpms
@@ -1,6 +1,12 @@
 #!/bin/bash
 set -e
+PACKAGES=`find ${BUILDDIR}/packages/RPMS/ -name '*.rpm' -and -not -name '*debuginfo*'`
 
-find ${BUILDDIR}/packages/RPMS/ -name '*.rpm' -and -not -name '*debuginfo*' \
-    | xargs dnf install -y --best --allowerasing
+# To list all packages that are available. Useful for debug purposes
+echo ${PACKAGES}
 
+dnf install -y --best --allowerasing ${PACKAGES}
+
+# Remove the RPMs once installed. They are not required anymore. This will ensure
+# that there is no issue generating next set of RPMs
+rm ${PACKAGES}
diff --git a/.travis/delete_branch.sh b/.travis/delete_branch.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+# Authors:
+#     Dinesh Prasanth M K <dmoluguw@redhat.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright (C) 2018 Red Hat, Inc.
+# All rights reserved.
+#
+set -e
+if [[ -n "${GERRIT_URL}" ]]
+then
+    git remote add bot git@github.com:${TRAVIS_REPO_SLUG}.git
+    git push -d bot ${TRAVIS_BRANCH}
+else
+    echo "Skip deleting branch"
+fi
diff --git a/.travis/global_variables b/.travis/global_variables
@@ -0,0 +1,10 @@
+CONTAINER=pkitest
+SCRIPTDIR=/tmp/workdir/pki/.travis
+PKI_LOG=pki_log.log
+CI_RESULTS_LOG="ci_results_${TRAVIS_BRANCH}.log"
+CI_RUNNER_LOG_ARCHIVE="dogtag-ci-job-${TRAVIS_JOB_NUMBER}.tar.gz"
+PKI_IMAGE=${IMAGE_REPO:-dogtagpki/pki-ci}:f26_105
+IPA_IMAGE=${IMAGE_REPO:-dogtagpki/pki-ci}:f26_ipa_4-5
+RPMS_LOCATION=/tmp/workdir/packages/RPMS
+DOGTAG_PKI_RPMS=${TRAVIS_BUILD_DIR}/dogtag_rpms
+TRANSFER_SH_URLS=message.txt
diff --git a/.travis/init_task.sh b/.travis/init_task.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Authors:
+#     Dinesh Prasanth M K <dmoluguw@redhat.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright (C) 2018 Red Hat, Inc.
+# All rights reserved.
+#
+set -e
+
+pyenv global system 3.6
+docker pull ${PKI_IMAGE}
+docker run \
+    --detach \
+    --name=${CONTAINER} \
+    --hostname='pki.test' \
+    --privileged \
+    --tmpfs /tmp \
+    --tmpfs /run \
+    -v /sys/fs/cgroup:/sys/fs/cgroup:ro \
+    -v $(pwd):/tmp/workdir/pki \
+    -e BUILDUSER_UID=$(id -u) \
+    -e BUILDUSER_GID=$(id -g) \
+    -e TRAVIS=${TRAVIS} \
+    -e TRAVIS_JOB_NUMBER=${TRAVIS_JOB_NUMBER} \
+    -i \
+    ${PKI_IMAGE}
+
+
+docker exec -i ${CONTAINER} /bin/ls -la /tmp/workdir
+docker exec -i ${CONTAINER} ${SCRIPTDIR}/00-init
+docker exec -i ${CONTAINER} ${SCRIPTDIR}/01-install-dependencies pki-core
+docker exec -i ${CONTAINER} ${SCRIPTDIR}/10-compose-rpms compose_pki_core_packages
+
+# IPA related installs
+pip install --upgrade pip
+pip3 install --upgrade pip
+pip install pep8
diff --git a/.test_runner_config.yaml → .travis/ipa-test.yaml b/.test_runner_config.yaml → .travis/ipa-test.yaml
@@ -41,7 +41,7 @@ steps:
   - echo "Skipping Build. Nothing to do..."
   builddep:
   - rm -rf /var/cache/dnf/*
-  - "dnf makecache fast || :"
+  - "dnf makecache || :"
   cleanup:
   - chown -R ${uid}:${gid} ${container_working_dir}
   - journalctl -b --no-pager > systemd_journal.log
@@ -54,16 +54,16 @@ steps:
       /var/log/pki
       systemd_journal.log
   - chown ${uid}:${gid} ${container_working_dir}/var_log.tar
+  - curl --upload ${container_working_dir}/var_log.tar https://transfer.sh/var_log.tar
   configure:
   - dnf repolist
   install_packages:
   - echo "Installing recently built dogtag RPMs..."
-  - sysctl net.ipv6.conf.lo.disable_ipv6=0
-  - find /freeipa/dogtag_rpms/ -name '*.rpm' -and -not -name '*debuginfo*'   | xargs dnf install -y --best --allowerasing
-  - rpm -qa freeipa-*
-  - rpm -qa pki-*
+  - find ${container_working_dir}/dogtag_rpms/ -name '*.rpm' -and -not -name '*debuginfo*'   | xargs dnf install -y
+  - rpm -qa tomcat* pki-* freeipa-* | sort
   install_server:
   - echo "Installing ipa-server..."
+  - sysctl net.ipv6.conf.lo.disable_ipv6=0
   - ipa-server-install -U --domain ${server_domain} --realm ${server_realm} -p ${server_password}
     -a ${server_password} --setup-dns --setup-kra --auto-forwarders
   - systemctl restart httpd.service

diff --git a/.travis/py3rewrite b/.travis/py3rewrite