From f29abbee3e32ea2e274c9d277c6d85a2cd578306 Mon Sep 17 00:00:00 2001 From: John Whitlock Date: Wed, 17 May 2017 09:38:15 -0500 Subject: [PATCH 1/4] bug 1352232: Remove Ansible from py27 build Switch to TravisCI-provided MySQL without custom collation. This requires running tests with --no-migrations. The Docker build continues to use MySQL with the custom collation utf8_distinct_ci and migraations. Switch to the TravicCI-recommended method for installing and running ElasticSearch. Update caching so that node installed and downloaded (ElasticSearch and docker-compose) are fast when the version doesn't change. Move configuration environment variables to global, and use additional variables to control what gets installed (rather than TOXENV): * CREATE_DB=dbname - Runs MySQL database creation for Django tests * INSTALL_DOCKER_COMPOSE - Replaces docker_compose with the specific version. * INSTALL_ELASTICSEARCH=1 - Downloads and runs ElasticSearch, and ensures it is ready before starting tests * INSTALL_PIPELINE=1 - Installs node packages used by django-pipeline, so that "make build-static" will run successfully --- .travis.yml | 51 ++++++++++++++++++++++--------- requirements/travis.txt | 1 - scripts/travis-install | 67 +++++++++++++++++++++++++++-------------- tox.ini | 10 ++++-- 4 files changed, 88 insertions(+), 41 deletions(-) diff --git a/.travis.yml b/.travis.yml index 68f3890d7ac..9b40d5e18b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,38 +2,59 @@ sudo: required branches: only: - master -cache: pip +cache: + pip: True + directories: + - node_modules + - downloads language: python python: - "2.7" services: - docker - memcached + - mysql env: + global: + - CFLAGS=-O0 + - DATABASE_URL=mysql://root:@127.0.0.1:3306/kuma + - DJANGO_SETTINGS_MODULE=kuma.settings.travis + - DEBIAN_FRONTEND=noninteractive + - DOCKER_COMPOSE_VERSION=1.9.0 + # ES_VERSION must come before ES_DOWNLOAD_URL + - ES_VERSION=2.4.5 + - ES_DOWNLOAD_URL=https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/$ES_VERSION/elasticsearch-$ES_VERSION.tar.gz + - PIPELINE_CSSMIN_BINARY=$TRAVIS_BUILD_DIR/node_modules/.bin/cssmin + - PIPELINE_CSS_COMPRESSOR=pipeline.compressors.cssmin.CSSMinCompressor + - PIPELINE_JS_COMPRESSOR=pipeline.compressors.uglifyjs.UglifyJSCompressor + - PIPELINE_SASS_BINARY=$TRAVIS_BUILD_DIR/node_modules/.bin/node-sass + - PIPELINE_UGLIFYJS_BINARY=$TRAVIS_BUILD_DIR/node_modules/.bin/uglifyjs matrix: - TOXENV=py27 + CREATE_DB=kuma + INSTALL_PIPELINE=1 + INSTALL_ELASTICSEARCH=1 - TOXENV=flake8 - TOXENV=docs - - TOXENV=locales DOCKER_COMPOSE_VERSION=1.9.0 - - TOXENV=docker DOCKER_COMPOSE_VERSION=1.9.0 UID=0 + - TOXENV=locales + INSTALL_DOCKER_COMPOSE=1 + - TOXENV=docker + INSTALL_DOCKER_COMPOSE=1 + UID=0 - TOXENV=stylelint - global: - - DJANGO_SETTINGS_MODULE=kuma.settings.travis - - DEBIAN_FRONTEND=noninteractive - - DATABASE_URL=mysql://root:kuma@localhost:3306/kuma - - CFLAGS=-O0 matrix: allow_failures: - - env: TOXENV=locales DOCKER_COMPOSE_VERSION=1.9.0 -before_install: - - scripts/travis-install - - pip install -U pip + - env: TOXENV=locales INSTALL_DOCKER_COMPOSE=1 install: - - pip install -r requirements/travis.txt - - if [[ $TOXENV == 'py27' ]]; then ansible-playbook -vvv --tags=mysql,pipeline --connection=local --inventory-file=provisioning/inventory provisioning/travis.yml; fi - nvm install 6 - nvm use 6 -script: tox -v + - scripts/travis-install + - pip install -U pip + - pip install -r requirements/travis.txt + # Wait for ElasticSearch to be ready + - if [ ${INSTALL_ELASTICSEARCH:-0} -ne 0 ]; then wget -q --waitretry=1 --retry-connrefused -T 10 -O - http://127.0.0.1:9200; fi; +script: + tox -v; after_failure: - dmesg | tail after_success: diff --git a/requirements/travis.txt b/requirements/travis.txt index 1ddf2450f4f..6bedbb32907 100644 --- a/requirements/travis.txt +++ b/requirements/travis.txt @@ -1,6 +1,5 @@ # this is a standalone requirements file used in .travis.yml to install # Travis CI specific requirements -ansible==1.9.2 codecov==1.6.3 tox==2.3.1 coverage==4.0.3 diff --git a/scripts/travis-install b/scripts/travis-install index cbd1b7707da..c0763e12402 100755 --- a/scripts/travis-install +++ b/scripts/travis-install @@ -1,34 +1,55 @@ #!/bin/bash -set -e -set -x +set -e # Exit on non-zero status +set -x # Print commands run export LC_CTYPE=en_US.UTF-8 -if [ "$TOXENV" == "py27" ] +# Download ElasticSearch +if [ ${INSTALL_ELASTICSEARCH:-0} -ne 0 ] then - sudo apt-get update -qq - sudo apt-get -y install build-essential libxml2-dev libxslt-dev libjpeg8 libjpeg8-dev libfreetype6 libfreetype6-dev zlib1g-dev sqlite3 tidy libtidy-dev libtidy-0.99-0 python-dev libffi-dev libssl-dev - sudo ln -s /usr/lib/`uname -i`-linux-gnu/libfreetype.so /usr/lib - sudo ln -s /usr/lib/`uname -i`-linux-gnu/libz.so /usr/lib + mkdir -p downloads + wget -q -O downloads/elasticsearch-$ES_VERSION.tar.gz $ES_DOWNLOAD_URL + tar -zxf downloads/elasticsearch-$ES_VERSION.tar.gz +fi + +# Run ElasticSearch in background +(if [ ${INSTALL_ELASTICSEARCH:-0} -ne 0 ] +then + ./elasticsearch-$ES_VERSION/bin/elasticsearch 1> elasticsearch.log 2> elasticsearch.err +fi) & - # completely and utterly remove Travis' MySQL and let the Ansible role install it - sudo apt-get remove --purge 'mysql*' - sudo apt-get autoremove - sudo apt-get autoclean - sudo rm -rf /var/lib/mysql - # Install Elasticsearch - wget https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.2.deb - sudo dpkg -i elasticsearch-1.7.2.deb +# Get database ready +if [ -n "${CREATE_DB:-}" ] +then + mysql -e "CREATE DATABASE IF NOT EXISTS ${CREATE_DB};"; +fi - # limit elasticsearch / java memory usage to avoid OOM Killer - sudo service elasticsearch stop; - echo "ES_HEAP_SIZE=256m" | sudo tee --append /etc/default/elasticsearch - sudo service elasticsearch start; +# Install pipeline toolchain +if [ ${INSTALL_PIPELINE:-0} -ne 0 ] +then + npm install + npm install cssmin@0.4.3 + npm install uglify-js@2.4.13 fi -if [ "$TOXENV" == "docker" -o "$TOXENV" == "locales" ] +# Install docker-compose +if [ ${INSTALL_DOCKER_COMPOSE:--} -ne 0 ] then - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin + if [ -x $(command -v docker-compose) ] + then + echo "Overwriting existing docker-compose." + docker-compose -v + else + echo "Installing docker-compose ${DOCKER_COMPOSE_VERSION}." + fi + mkdir -p downloads + DOCKER_COMPOSE_FILE=downloads/docker-compose-${DOCKER_COMPOSE_VERSION} + if [ ! -f $DOCKER_COMPOSE_FILE ] + then + wget -q --waitretry=1 --retry-connrefused -T 10 \ + -O downloads/docker-compose-${DOCKER_COMPOSE_VERSION} \ + https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` + fi + chmod +x $DOCKER_COMPOSE_FILE + sudo cp $DOCKER_COMPOSE_FILE /usr/local/bin/docker-compose fi diff --git a/tox.ini b/tox.ini index 13d18a6ffdd..d2916fbfaf7 100644 --- a/tox.ini +++ b/tox.ini @@ -7,13 +7,19 @@ whitelist_externals = make deps = -rrequirements/dev.txt commands = - make compilejsi18n collectstatic coveragetest target={posargs:kuma} + make compilejsi18n collectstatic clean + py.test --no-migrations --cov=kuma kuma setenv = PYTHONPATH = . CFLAGS = -O0 # TODO: remove once http://bugzil.la/1127798 is fixed PYTHONHASHSEED = 0 -passenv=DJANGO_SETTINGS_MODULE DEBIAN_FRONTEND CFLAGS +passenv = + DJANGO_SETTINGS_MODULE + DEBIAN_FRONTEND + CFLAGS + DATABASE_URL + PIPELINE_* [testenv:flake8] basepython = python2.7 From e6d8b66ee06745c3ac60c7460057f0f01797a566 Mon Sep 17 00:00:00 2001 From: John Whitlock Date: Wed, 17 May 2017 11:13:08 -0500 Subject: [PATCH 2/4] bug 1352232: Less Travis-specific environment Drop PYTHONPATH, CFLAGS, and DEBIAN_FRONTED from the environment overrides. --- .travis.yml | 2 -- scripts/docker-compose.travis.yml | 2 -- tox.ini | 6 +----- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9b40d5e18b6..5bf413f630a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,10 +16,8 @@ services: - mysql env: global: - - CFLAGS=-O0 - DATABASE_URL=mysql://root:@127.0.0.1:3306/kuma - DJANGO_SETTINGS_MODULE=kuma.settings.travis - - DEBIAN_FRONTEND=noninteractive - DOCKER_COMPOSE_VERSION=1.9.0 # ES_VERSION must come before ES_DOWNLOAD_URL - ES_VERSION=2.4.5 diff --git a/scripts/docker-compose.travis.yml b/scripts/docker-compose.travis.yml index b5b198115fe..cd0642337d9 100644 --- a/scripts/docker-compose.travis.yml +++ b/scripts/docker-compose.travis.yml @@ -6,8 +6,6 @@ services: dockerfile: Dockerfile-base environment: - DJANGO_SETTINGS_MODULE - - DEBIAN_FRONTEND - - CFLAGS web: <<: *worker diff --git a/tox.ini b/tox.ini index d2916fbfaf7..e550e1a2ad1 100644 --- a/tox.ini +++ b/tox.ini @@ -10,15 +10,11 @@ commands = make compilejsi18n collectstatic clean py.test --no-migrations --cov=kuma kuma setenv = - PYTHONPATH = . - CFLAGS = -O0 # TODO: remove once http://bugzil.la/1127798 is fixed PYTHONHASHSEED = 0 passenv = - DJANGO_SETTINGS_MODULE - DEBIAN_FRONTEND - CFLAGS DATABASE_URL + DJANGO_SETTINGS_MODULE PIPELINE_* [testenv:flake8] From dbb0d94343c44b6c29176f6eedbd167fb63c3d24 Mon Sep 17 00:00:00 2001 From: John Whitlock Date: Wed, 17 May 2017 11:23:49 -0500 Subject: [PATCH 3/4] bug 1127798: Add build with random PYTHONHASHSEED Add a build (allowed to fail) that tests Kuma with a random PYTHONHASHSEED. --- .travis.yml | 14 +++++++++++++- tox.ini | 3 --- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5bf413f630a..a0712ea22f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,6 +32,7 @@ env: CREATE_DB=kuma INSTALL_PIPELINE=1 INSTALL_ELASTICSEARCH=1 + PYTHONHASHSEED=0 - TOXENV=flake8 - TOXENV=docs - TOXENV=locales @@ -40,9 +41,16 @@ env: INSTALL_DOCKER_COMPOSE=1 UID=0 - TOXENV=stylelint + # TODO: make this default when http://bugzil.la/1127798 is fixed + - TOXENV=py27 + CREATE_DB=kuma + INSTALL_PIPELINE=1 + INSTALL_ELASTICSEARCH=1 + matrix: allow_failures: - env: TOXENV=locales INSTALL_DOCKER_COMPOSE=1 + - env: TOXENV=py27 CREATE_DB=kuma INSTALL_PIPELINE=1 INSTALL_ELASTICSEARCH=1 install: - nvm install 6 - nvm use 6 @@ -52,7 +60,11 @@ install: # Wait for ElasticSearch to be ready - if [ ${INSTALL_ELASTICSEARCH:-0} -ne 0 ]; then wget -q --waitretry=1 --retry-connrefused -T 10 -O - http://127.0.0.1:9200; fi; script: - tox -v; + if [[ "${PYTHONHASHSEED:-}" == "0" ]]; then + tox -v --hashseed=noset; + else + tox -v; + fi after_failure: - dmesg | tail after_success: diff --git a/tox.ini b/tox.ini index e550e1a2ad1..dee475cdea7 100644 --- a/tox.ini +++ b/tox.ini @@ -9,9 +9,6 @@ deps = commands = make compilejsi18n collectstatic clean py.test --no-migrations --cov=kuma kuma -setenv = - # TODO: remove once http://bugzil.la/1127798 is fixed - PYTHONHASHSEED = 0 passenv = DATABASE_URL DJANGO_SETTINGS_MODULE From c0e6ec2275951fede758f3635451c1abef32a400 Mon Sep 17 00:00:00 2001 From: John Whitlock Date: Wed, 17 May 2017 16:25:53 -0500 Subject: [PATCH 4/4] bug 1352232: Use appropriate bash Use simple bash tests in .travis.yml Use set -u in scripts/travis-install, instead of checking that the right environment variables are set. --- .travis.yml | 4 ++-- scripts/travis-install | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a0712ea22f5..8905f19b812 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,9 +58,9 @@ install: - pip install -U pip - pip install -r requirements/travis.txt # Wait for ElasticSearch to be ready - - if [ ${INSTALL_ELASTICSEARCH:-0} -ne 0 ]; then wget -q --waitretry=1 --retry-connrefused -T 10 -O - http://127.0.0.1:9200; fi; + - if [[ "$INSTALL_ELASTICSEARCH" == "1" ]]; then wget -q --waitretry=1 --retry-connrefused -T 10 -O - http://127.0.0.1:9200; fi; script: - if [[ "${PYTHONHASHSEED:-}" == "0" ]]; then + if [[ "$PYTHONHASHSEED" == "0" ]]; then tox -v --hashseed=noset; else tox -v; diff --git a/scripts/travis-install b/scripts/travis-install index c0763e12402..7a4cf9e2927 100755 --- a/scripts/travis-install +++ b/scripts/travis-install @@ -1,6 +1,8 @@ #!/bin/bash set -e # Exit on non-zero status set -x # Print commands run +set -u # Treat unset variables as an error + export LC_CTYPE=en_US.UTF-8 # Download ElasticSearch