From c7a591f758639875fb746ddd7cf78a200b35aaac Mon Sep 17 00:00:00 2001 From: "Mark A. Grondona" Date: Thu, 20 Sep 2018 06:41:33 -0700 Subject: [PATCH] travis-ci: rewrite travis.yml to use docker builds Rewrite the .travis.yml to utilize the new docker infrastructure (Dockerfiles, docker-run-checks.sh, and travis_run.sh). Other changes to the Travis builds include: - builds run on Ubuntu 18.04 (bionic) GCC 8 and Clang 6.0 as well as CentOS 7 (default gcc) - cppcheck and pylint are disabled for now becuase newer versions of these commands on Ubuntu 18.04 cause unadressed errors - re-titled builders for a clearer description on the travis website - If DOCKER_DEPLOY=t, then deploy the resulting docker image for docker hub Co-authored-by: Tom Scogland --- .travis.yml | 186 ++++++++++++++++++++++------------------------------ 1 file changed, 79 insertions(+), 107 deletions(-) diff --git a/.travis.yml b/.travis.yml index 152a9661df19..0e1c438040a4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,132 +1,104 @@ +sudo: required +services: docker +dist: trusty language: c -sudo: true - matrix: include: - - compiler: gcc - env: LUA_VERSION=5.1 - - compiler: clang - env: LUA_VERSION=5.1 - - compiler: gcc - env: LUA_VERSION=5.1 COVERAGE=t ARGS="--with-flux-security --enable-caliper --enable-pylint" - - compiler: gcc - env: LUA_VERSION=5.1 T_INSTALL=t - - compiler: clang - env: LUA_VERSION=5.1 CPPCHECK=t ARGS="--with-flux-security --enable-sanitizer" CC=clang-3.8 CXX=clang++-3.8 - - compiler: gcc - env: LUA_VERSION=5.2 CC=gcc-8 CXX=g++-8 ARGS="--with-flux-security" chain_lint=t - - compiler: clang - env: LUA_VERSION=5.2 ARGS="--with-flux-security --enable-caliper" CC=clang-3.8 CXX=clang++-3.8 - -cache: - directories: - - $HOME/local - - $HOME/.ccache - - $HOME/.local - -addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-precise-3.8 - - george-edison55-precise-backports # CMake 3.0 - packages: - - cmake - - cmake-data - - clang-3.8 - - gcc-8 - - g++-8 - - libmunge-dev - - uuid-dev - - aspell - - aspell-en - - ccache - - apport # for coredumps - - gdb - - valgrind - - libyaml-cpp-dev - - libboost-dev # for yaml-cpp 0.5.1. >=0.5.2 no longer need boost - - coverity_scan: - project: - name: "grondo/flux-core" - description: "Build submitted via Travis CI" - notification_email: mark.grondona@gmail.com - build_command_prepend: "./autogen.sh && ./configure" - build_command: "make -j 4" - branch_pattern: coverity_scan + - name: "Unbuntu" + compiler: gcc + - name: "Ubuntu: gcc-8 --with-flux-security/caliper, distcheck" + compiler: gcc-8 + env: + - CC=gcc-8 + - CXX=g++-8 + - ARGS="--with-flux-security --enable-caliper" + - DISTCHECK=t + - name: "Ubuntu: clang-6.0 chain-lint --with-flux-security" + compiler: clang-6.0 + env: + - CC=clang-6.0 + - CXX=clang++-6.0 + - chain_lint=t + - ARGS="--with-flux-security" + - name: "Ubuntu: COVERAGE=t, --with-flux-security --enable-caliper" + compiler: gcc + env: + - COVERAGE=t + - ARGS="--with-flux-security --enable-caliper" + - name: "Ubuntu: TEST_INSTALL" + compiler: gcc + env: + - ARGS="--with-flux-security --enable-caliper" + - TEST_INSTALL=t + - DOCKER_TAG=t + - name: "Centos 7: --with-flux-security --enable-caliper" + compiler: gcc + env: + - ARGS="--with-flux-security --enable-caliper" + - IMG=centos7-base env: global: - # The next declaration is the encrypted COVERITY_SCAN_TOKEN, created - # via the "travis encrypt" command using the project repo's public key - - secure: "T06LmG6hAwmRculQGfmV06A0f2i9rPoc8itDwyWkmg2CbtCqDyYV93V53jsVknqKA1LA9+Yo7Q3bJnnEAWI7kWAptTcL5ipRycYkl5FqoYawkwRdQW3giZwbs9zRchrwFmZ03N0hRfl31IntXhDmj5EkHOZoduMpkQFFGo0XDC4=" - -before_install: - # Do not run coverity-scan on every build in matrix - - test "${TRAVIS_BRANCH}" != 'coverity_scan' -o "${TRAVIS_JOB_NUMBER##*.}" = '1' || exit 0 - # - - test "$TRAVIS_PULL_REQUEST" == "false" || export CCACHE_READONLY=1 - - if test "$CC" = "clang"; then export CCACHE_CPP2=1; fi + - DOCKERREPO=fluxrm/flux-core + - DOCKER_USERNAME=travisflux + - secure: "Uga2i1Yu0PvWMFzOYvM9yxnAMDTgY17ZqeFlIN8MV3uoTCy6y61GULrMkKuhuI1sUfyugpFWVKIJo5jwTpsfG84f3o9lUTRgLPpTA2Xls8A/rmurF/QacVv6hZ2Zs2LQVlrM8BkT36TpT2NfWW2D2238kovqz3l5gIZKMClMvyk=" - - rm -rf $HOME/.luarocks - - $(./src/test/travis-dep-builder.sh --printenv) - - ./src/test/travis-dep-builder.sh --cachedir=$HOME/local/.cache +cache: + directories: + - $HOME/.ccache +before_install: # coveralls-lcov required only for coveralls upload: - if test "$COVERAGE" = "t" ; then gem install coveralls-lcov; fi - # Ensure we always use internal by placing a dummy file - # in the same path as ZMQ_FLAGS: - - mkdir -p $HOME/local/include/flux - - echo '#error Non-build-tree flux/core.h!' > $HOME/local/include/flux/core.h - script: - # Skip build if we already ran coverity-scan - - test "${TRAVIS_BRANCH}" != 'coverity_scan' || exit 0 - # Force git to update the shallow clone and include tags so git-describe works - - git fetch --unshallow --tags - - ulimit -c unlimited - - export CC="ccache $CC" - - export MAKECMDS="make distcheck" - # Ensure travis builds libev such that libfaketime will work: - # (force libev to *not* use syscall interface for clock_gettime()) - - export CPPFLAGS="$CPPFLAGS -DEV_USE_CLOCK_SYSCALL=0 -DEV_USE_MONOTONIC=1" - - # Travis has limited resources, even though number of processors might - # might appear to be large. Limit session size for testing to 5 to avoid - # spurious timeouts. - - export FLUX_TEST_SIZE_MAX=5 - - # Invoke MPI tests - - export TEST_MPI=t - - # Enable coverage for $CC-coverage build - # We can't use distcheck here, it doesn't play well with coverage testing: - - if test "$COVERAGE" = "t" ; then ARGS="$ARGS --enable-code-coverage"; MAKECMDS="make -j 2 && make -j 2 check-code-coverage && lcov -l flux*-coverage.info"; fi - # Use make install for T_INSTALL: - - if test "$T_INSTALL" = "t" ; then ARGS="$ARGS --prefix=/tmp/flux"; MAKECMDS="make && make install && /tmp/flux/bin/flux keygen && FLUX_TEST_INSTALLED_PATH=/tmp/flux/bin make -j 2 check"; fi - # Use src/test/cppcheck.sh instead of make?: - - if test "$CPPCHECK" = "t" ; then MAKECMDS="sh -x src/test/cppcheck.sh && make"; fi - - - export FLUX_TESTS_LOGFILE=t - - export DISTCHECK_CONFIGURE_FLAGS=${ARGS} - - if test "$COVERITY_SCAN_BRANCH" != "1" ; then ./autogen.sh && ./configure ${ARGS} && eval ${MAKECMDS}; fi + - if test -z "${IMG}"; then IMG="bionic-base"; fi + # + # Tag image if this build is on master or result of a tag: + - | + if test "$DOCKER_TAG" = "t" \ + -a "$TRAVIS_REPO_SLUG" = "grondo/flux-core" \ + -a "$TRAVIS_PULL_REQUEST" = "false" \ + -a \( "$TRAVIS_BRANCH" = "master" -o -n "$TRAVIS_TAG" \); then + export TAGNAME="${DOCKERREPO}:${IMG}-${TRAVIS_TAG:-latest}" + echo "Tagging new image $TAGNAME" + fi + - | + src/test/docker/docker-run-checks.sh -j2 \ + --quiet \ + --image=${IMG} \ + ${TAGNAME:+--tag=${TAGNAME}} \ + -- ${ARGS} after_success: - ccache -s - # Upload results to coveralls.io for first job of N - - if test "$COVERAGE" = "t" ; then coveralls-lcov flux*-coverage.info; bash <(curl -s https://codecov.io/bash) ; fi + # Upload coverage results for COVERAGE run + - | + if test "$COVERAGE" = "t"; \ + then coveralls-lcov flux*-coverage.info; \ + bash <(curl -s https://codecov.io/bash); \ + fi + # Deploy resulting docker image to Docker Hub with appropriate tag + - | + if test -n "$TAGNAME"; then + echo "$DOCKER_PASSWORD" | \ + docker login -u "$DOCKER_USERNAME" --password-stdin && \ + docker push ${TAGNAME} + # If this is the bionic-base build, then also tag without image name: + if echo "$TAGNAME" | grep -q "bionic-base"; then + t="${DOCKERREPO}:${TRAVIS_TAG:-latest}" + docker tag "$TAGNAME" ${t} && \ + docker push ${t} + fi + fi after_failure: - find . -name test-suite.log | xargs -i sh -c 'printf "===XXX {} XXX===";cat {}' - find . -name t[0-9]*.output | xargs -i sh -c 'printf "\033[31mFound {}\033[39m\n";cat {}' - find . -name *.broker.log | xargs -i sh -c 'printf "\033[31mFound {}\033[39m\n";cat {}' - src/test/backtrace-all.sh - - grep -q 'configure: exit 1' config.log && cat config.log - # Issue #997 debugging - - cat src/common/libflux/test_reactor.log + - grep -q 'configure. exit 1' config.log && cat config.log before_deploy: # Get anchor (formatted properly) and base URI for latest tag in NEWS file