MDEV-16213: Further improvements to the Travis config

* Exclude some storage engines from Travis to conserve build time and disk usage per job. Exluded: TOKUDB MROONGA SPIDER OQGRAPH PERFSCHEMA SPHINX * Increase travis_wait from default 20m to 30 for MTR * Use travis_wait for long running MTR command (wait 30m instead of default 20m) * Increase testcase-timeout to 20m for OSX, 2m for Linux * Set ccache size only on Linux, adjust timeout again * Increase cache push timeout to 5 mins * Remove AWS defines, not needed * Remove commented out ASAN rules, has been disabled previously since it has a significant impact on job runtime, should be used more in buildbot instead * Misc cleanup and fixes
MariaDB · Jun 29, 2018 · 7b6e867 · 7b6e867
1 parent 1d5220a
commit 7b6e867
Show file tree

Hide file tree

Showing 2 changed files with 56 additions and 47 deletions.
diff --git a/.travis.compiler.sh b/.travis.compiler.sh
@@ -1,45 +1,51 @@
 #!/bin/sh
 set -v -x
+
+# Exclude modules from build not directly affecting the current
+# test suites found in $MYSQL_TEST_SUITES, to conserve job time
+# as well as disk usage
+
+function exclude_modules() {
+# excludes for all
+CMAKE_OPT="${CMAKE_OPT} -DPLUGIN_TOKUDB=NO -DPLUGIN_MROONGA=NO -DPLUGIN_SPIDER=NO -DPLUGIN_OQGRAPH=NO -DPLUGIN_PERFSCHEMA=NO -DPLUGIN_SPHINX=NO"
+# exclude storage engines not being tested in current job
+if [[ ! "${MYSQL_TEST_SUITES}" =~ "archive" ]]; then
+CMAKE_OPT="${CMAKE_OPT} -DPLUGIN_ARCHIVE=NO"
+fi
+if [[ ! "${MYSQL_TEST_SUITES}" =~ "rocksdb" ]]; then
+CMAKE_OPT="${CMAKE_OPT} -DPLUGIN_ROCKSDB=NO"
+fi
+}
+
 if [[ "${TRAVIS_OS_NAME}" == 'linux' ]]; then
+  TEST_CASE_TIMEOUT=2
+  exclude_modules;
+  if which ccache ; then
+    CMAKE_OPT="${CMAKE_OPT} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache"
+    ccache --max-size=1200M
+  fi
   if [[ "${CXX}" == 'clang++' ]]; then
-    CMAKE_OPT="-DWITHOUT_TOKUDB_STORAGE_ENGINE=ON -DWITHOUT_MROONGA_STORAGE_ENGINE=ON"
-    #CMAKE_OPT="${CMAKE_OPT} -DWITH_ASAN=ON"
-    if which ccache ; then
-      CMAKE_OPT="${CMAKE_OPT} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache"
-    fi
     export CXX CC=${CXX/++/}
   elif [[ "${CXX}" == 'g++' ]]; then
-    CMAKE_OPT=""
-    if [[ "${MYSQL_TEST_SUITES}" == 'rpl' ]]; then
-      CMAKE_OPT="${CMAKE_OPT} -DWITHOUT_TOKUDB_STORAGE_ENGINE=TRUE"
-      CMAKE_OPT="${CMAKE_OPT} -DWITHOUT_MROONGA_STORAGE_ENGINE=TRUE"
-    fi
     export CXX=g++-${CC_VERSION}
     export CC=gcc-${CC_VERSION}
   fi
   if [[ ${CC_VERSION} == 6 ]]; then
-         wget http://mirrors.kernel.org/ubuntu/pool/universe/p/percona-xtradb-cluster-galera-2.x/percona-xtradb-cluster-galera-2.x_165-0ubuntu1_amd64.deb ;
-         ar vx percona-xtradb-cluster-galera-2.x_165-0ubuntu1_amd64.deb
-         tar -xJvf data.tar.xz
-         export WSREP_PROVIDER=$PWD/usr/lib/libgalera_smm.so
-         MYSQL_TEST_SUITES="${MYSQL_TEST_SUITES},wsrep"
-  #elif [[ ${CC_VERSION} != 5 ]]; then
-    #CMAKE_OPT="${CMAKE_OPT} -DWITH_ASAN=ON"
+    wget http://mirrors.kernel.org/ubuntu/pool/universe/p/percona-xtradb-cluster-galera-2.x/percona-xtradb-cluster-galera-2.x_165-0ubuntu1_amd64.deb ;
+    ar vx percona-xtradb-cluster-galera-2.x_165-0ubuntu1_amd64.deb
+    tar -xJvf data.tar.xz
+    export WSREP_PROVIDER=$PWD/usr/lib/libgalera_smm.so
+    MYSQL_TEST_SUITES="${MYSQL_TEST_SUITES},wsrep"
   fi
-else
-  # osx_image based tests
-  CMAKE_OPT="-DOPENSSL_ROOT_DIR=/usr/local/opt/openssl"
-  #CMAKE_OPT="${CMAKE_OPT} -DWITH_ASAN=ON"
+fi
+
+if [[ "${TRAVIS_OS_NAME}" == 'osx' ]]; then
+  TEST_CASE_TIMEOUT=20
+  exclude_modules;
+  CMAKE_OPT="${CMAKE_OPT} -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl"
   if which ccache ; then
     CMAKE_OPT="${CMAKE_OPT} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache"
   fi
-  if [[ "${MYSQL_TEST_SUITES}" == 'rpl' ]]; then
-    CMAKE_OPT="${CMAKE_OPT} -DWITHOUT_TOKUDB_STORAGE_ENGINE=ON"
-  fi
-  CMAKE_OPT="${CMAKE_OPT} -DWITHOUT_MROONGA_STORAGE_ENGINE=ON"
-  if [[ "${TYPE}" == "Debug" ]]; then
-    CMAKE_OPT="${CMAKE_OPT} -DWITHOUT_TOKUDB_STORAGE_ENGINE=ON"
-  fi
 fi
 
 set +v +x
diff --git a/.travis.yml b/.travis.yml
@@ -17,19 +17,20 @@ compiler:
   - clang
 
 cache:
+  timeout: 300
   apt: true
   ccache: true
   directories:
     - /usr/local/Cellar # Fails do to permission error: https://github.com/travis-ci/travis-ci/issues/8092
 
 env:
   matrix:
-    - CC_VERSION=5   TYPE=RelWithDebInfo MYSQL_TEST_SUITES=main
-    - CC_VERSION=5   TYPE=RelWithDebInfo MYSQL_TEST_SUITES=archive,optimizer_unfixed_bugs,parts,sys_vars,unit,vcol,innodb,innodb_gis,innodb_zip,innodb_fts
-    - CC_VERSION=5   TYPE=RelWithDebInfo MYSQL_TEST_SUITES=rpl
-    - CC_VERSION=6   TYPE=RelWithDebInfo MYSQL_TEST_SUITES=binlog,binlog_encryption,encryption
-    - CC_VERSION=6   TYPE=RelWithDebInfo MYSQL_TEST_SUITES=rocksdb
-    - CC_VERSION=6   TYPE=RelWithDebInfo MYSQL_TEST_SUITES=csv,federated,funcs_1,funcs_2,gcol,handler,heap,json,maria,perfschema,plugins,multi_source,roles
+    - CC_VERSION=5 TYPE=RelWithDebInfo MYSQL_TEST_SUITES=main
+    - CC_VERSION=5 TYPE=RelWithDebInfo MYSQL_TEST_SUITES=archive,optimizer_unfixed_bugs,parts,sys_vars,unit,vcol,innodb,innodb_gis,innodb_zip,innodb_fts
+    - CC_VERSION=5 TYPE=RelWithDebInfo MYSQL_TEST_SUITES=rpl
+    - CC_VERSION=6 TYPE=RelWithDebInfo MYSQL_TEST_SUITES=binlog,binlog_encryption,encryption
+    - CC_VERSION=6 TYPE=RelWithDebInfo MYSQL_TEST_SUITES=rocksdb
+    - CC_VERSION=6 TYPE=RelWithDebInfo MYSQL_TEST_SUITES=csv,federated,funcs_1,funcs_2,gcol,handler,heap,json,maria,perfschema,plugins,multi_source,roles
 
 matrix:
   exclude:
@@ -85,26 +86,26 @@ matrix:
         - export MTR_MEM=/tmp
         - env DEB_BUILD_OPTIONS="parallel=4" debian/autobake-deb.sh;
         - ccache --show-stats
-  # Until OSX becomes a bit more stable: MDEV-12435
+  # Until OSX becomes a bit more stable: MDEV-12435 MDEV-16213
   allow_failures:
     - os: osx
       compiler: clang
-      env: CC_VERSION=5   TYPE=RelWithDebInfo MYSQL_TEST_SUITES=rpl
+      env: CC_VERSION=5 TYPE=RelWithDebInfo MYSQL_TEST_SUITES=rpl
     - os: osx
       compiler: clang
-      env: CC_VERSION=5   TYPE=RelWithDebInfo MYSQL_TEST_SUITES=main
+      env: CC_VERSION=5 TYPE=RelWithDebInfo MYSQL_TEST_SUITES=main
     - os: osx
       compiler: clang
-      env: CC_VERSION=5   TYPE=RelWithDebInfo MYSQL_TEST_SUITES=archive,optimizer_unfixed_bugs,parts,sys_vars,unit,vcol,innodb,innodb_gis,innodb_zip,innodb_fts
+      env: CC_VERSION=5 TYPE=RelWithDebInfo MYSQL_TEST_SUITES=archive,optimizer_unfixed_bugs,parts,sys_vars,unit,vcol,innodb,innodb_gis,innodb_zip,innodb_fts
     - os: osx
       compiler: clang
-      env: CC_VERSION=6   TYPE=RelWithDebInfo MYSQL_TEST_SUITES=binlog,binlog_encryption,encryption
+      env: CC_VERSION=6 TYPE=RelWithDebInfo MYSQL_TEST_SUITES=binlog,binlog_encryption,encryption
     - os: osx
       compiler: clang
-      env: CC_VERSION=6   TYPE=RelWithDebInfo MYSQL_TEST_SUITES=rocksdb
+      env: CC_VERSION=6 TYPE=RelWithDebInfo MYSQL_TEST_SUITES=rocksdb
     - os: osx
       compiler: clang
-      env: CC_VERSION=6   TYPE=RelWithDebInfo MYSQL_TEST_SUITES=csv,federated,funcs_1,funcs_2,gcol,handler,heap,json,maria,perfschema,plugins,multi_source,roles
+      env: CC_VERSION=6 TYPE=RelWithDebInfo MYSQL_TEST_SUITES=csv,federated,funcs_1,funcs_2,gcol,handler,heap,json,maria,perfschema,plugins,multi_source,roles
 
 addons:
   apt:
@@ -121,7 +122,6 @@ addons:
       - llvm-5.0-dev
       - clang-6.0
       - llvm-6.0-dev
-#      - libasan0
       - bison
       - chrpath
       - cmake
@@ -160,10 +160,12 @@ before_install:
       brew link ccache;
     fi
 
-script:
+before_script:
+  - df -h
   - ccache --version
   - ccache --show-stats
-  - ccache --max-size=1000M
+
+script:
 # following modules are disabled after sourcing .travis.compiler.sh:
 # clang disabled: mroonga just generates too many warnings with clang and travis stops the job
 # cland disabled: tokudb has fatal warnings
@@ -172,16 +174,17 @@ script:
   - cmake .
            -DCMAKE_BUILD_TYPE=${TYPE}
            ${CMAKE_OPT}
-           -DWITH_SSL=system -DWITH_ZLIB=system -DPLUGIN_AWS_KEY_MANAGEMENT=DYNAMIC -DAWS_SDK_EXTERNAL_PROJECT=ON
+           -DWITH_SSL=system -DWITH_ZLIB=system
   - make -j 4
   - cd mysql-test
-# With ASAN --thread-stack=400K to account for overhead
+# With ASAN use --thread-stack=400K to account for overhead
 # Test timeout needs to be 10 (minutes) or less due to travis job timeout
-  - ./mtr --force --max-test-fail=20 --parallel=4 --testcase-timeout=4
+  - travis_wait 30 ./mtr --force --max-test-fail=20 --parallel=4 --testcase-timeout=${TEST_CASE_TIMEOUT}
          --suite=${MYSQL_TEST_SUITES}
          --skip-test-list=unstable-tests
          --skip-test=binlog.binlog_unsafe
   - ccache --show-stats
+  - df -h
 
 notifications:
   irc: