bug 1352232: Remove Ansible from py27 build

Switch to TravisCI-provided MySQL without custom collation. This
requires running tests with --no-migrations. The Docker build
continues to use MySQL with the custom collation utf8_distinct_ci and
migraations.

Switch to the TravicCI-recommended method for installing and running
ElasticSearch.

Update caching so that node installed and downloaded (ElasticSearch
and docker-compose) are fast when the version doesn't change.

Move configuration environment variables to global, and use additional
variables to control what gets installed (rather than TOXENV):

* CREATE_DB=dbname - Runs MySQL database creation for Django tests
* INSTALL_DOCKER_COMPOSE - Replaces docker_compose with the specific
  version.
* INSTALL_ELASTICSEARCH=1 - Downloads and runs ElasticSearch, and
  ensures it is ready before starting tests
* INSTALL_PIPELINE=1 - Installs node packages used by django-pipeline,
  so that "make build-static" will run successfully

.travis.yml

@@ -2,38 +2,59 @@ sudo: required
 branches:
     only:
         - master
-cache: pip
+cache:
+    pip: True
+    directories:
+      - node_modules
+      - downloads
 language: python
 python:
     - "2.7"
 services:
     - docker
     - memcached
+    - mysql
 env:
+    global:
+        - CFLAGS=-O0
+        - DATABASE_URL=mysql://root:@127.0.0.1:3306/kuma
+        - DJANGO_SETTINGS_MODULE=kuma.settings.travis
+        - DEBIAN_FRONTEND=noninteractive
+        - DOCKER_COMPOSE_VERSION=1.9.0
+          # ES_VERSION must come before ES_DOWNLOAD_URL
+        - ES_VERSION=2.4.5
+        - ES_DOWNLOAD_URL=https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/$ES_VERSION/elasticsearch-$ES_VERSION.tar.gz
+        - PIPELINE_CSSMIN_BINARY=$TRAVIS_BUILD_DIR/node_modules/.bin/cssmin
+        - PIPELINE_CSS_COMPRESSOR=pipeline.compressors.cssmin.CSSMinCompressor
+        - PIPELINE_JS_COMPRESSOR=pipeline.compressors.uglifyjs.UglifyJSCompressor
+        - PIPELINE_SASS_BINARY=$TRAVIS_BUILD_DIR/node_modules/.bin/node-sass
+        - PIPELINE_UGLIFYJS_BINARY=$TRAVIS_BUILD_DIR/node_modules/.bin/uglifyjs
     matrix:
         - TOXENV=py27
+          CREATE_DB=kuma
+          INSTALL_PIPELINE=1
+          INSTALL_ELASTICSEARCH=1
         - TOXENV=flake8
         - TOXENV=docs
-        - TOXENV=locales DOCKER_COMPOSE_VERSION=1.9.0
-        - TOXENV=docker DOCKER_COMPOSE_VERSION=1.9.0 UID=0
+        - TOXENV=locales
+          INSTALL_DOCKER_COMPOSE=1
+        - TOXENV=docker
+          INSTALL_DOCKER_COMPOSE=1
+          UID=0
         - TOXENV=stylelint
-    global:
-        - DJANGO_SETTINGS_MODULE=kuma.settings.travis
-        - DEBIAN_FRONTEND=noninteractive
-        - DATABASE_URL=mysql://root:kuma@localhost:3306/kuma
-        - CFLAGS=-O0
 matrix:
     allow_failures:
-        - env: TOXENV=locales DOCKER_COMPOSE_VERSION=1.9.0
-before_install:
-    - scripts/travis-install
-    - pip install -U pip
+        - env: TOXENV=locales INSTALL_DOCKER_COMPOSE=1
 install:
-    - pip install -r requirements/travis.txt
-    - if [[ $TOXENV == 'py27' ]]; then ansible-playbook -vvv --tags=mysql,pipeline --connection=local --inventory-file=provisioning/inventory provisioning/travis.yml; fi
     - nvm install 6
     - nvm use 6
-script: tox -v
+    - scripts/travis-install
+    - pip install -U pip
+    - pip install -r requirements/travis.txt
+    # Wait for ElasticSearch to be ready
+    - if [ ${INSTALL_ELASTICSEARCH:-0} -ne 0 ]; then wget -q --waitretry=1 --retry-connrefused -T 10 -O - http://127.0.0.1:9200; fi;
+script:
+    tox -v;
 after_failure:
     - dmesg | tail
 after_success:

requirements/travis.txt

@@ -1,6 +1,5 @@
 # this is a standalone requirements file used in .travis.yml to install
 # Travis CI specific requirements
-ansible==1.9.2
 codecov==1.6.3
 tox==2.3.1
 coverage==4.0.3

scripts/travis-install

@@ -1,34 +1,55 @@
 #!/bin/bash
-set -e
-set -x
+set -e  # Exit on non-zero status
+set -x  # Print commands run
 export LC_CTYPE=en_US.UTF-8
 
-if [ "$TOXENV" == "py27" ]
+# Download ElasticSearch
+if [ ${INSTALL_ELASTICSEARCH:-0} -ne 0 ]
 then
-    sudo apt-get update -qq
-    sudo apt-get -y install build-essential libxml2-dev libxslt-dev libjpeg8 libjpeg8-dev libfreetype6 libfreetype6-dev zlib1g-dev sqlite3 tidy libtidy-dev libtidy-0.99-0 python-dev libffi-dev libssl-dev
-    sudo ln -s /usr/lib/`uname -i`-linux-gnu/libfreetype.so /usr/lib
-    sudo ln -s /usr/lib/`uname -i`-linux-gnu/libz.so /usr/lib
+    mkdir -p downloads
+    wget -q -O downloads/elasticsearch-$ES_VERSION.tar.gz $ES_DOWNLOAD_URL
+    tar -zxf downloads/elasticsearch-$ES_VERSION.tar.gz
+fi
+
+# Run ElasticSearch in background
+(if [ ${INSTALL_ELASTICSEARCH:-0} -ne 0 ]
+then
+    ./elasticsearch-$ES_VERSION/bin/elasticsearch 1> elasticsearch.log 2> elasticsearch.err
+fi) &
 
-    # completely and utterly remove Travis' MySQL and let the Ansible role install it
-    sudo apt-get remove --purge 'mysql*'
-    sudo apt-get autoremove
-    sudo apt-get autoclean
-    sudo rm -rf /var/lib/mysql
 
-    # Install Elasticsearch
-    wget https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.2.deb
-    sudo dpkg -i elasticsearch-1.7.2.deb
+# Get database ready
+if [ -n "${CREATE_DB:-}" ]
+then
+    mysql -e "CREATE DATABASE IF NOT EXISTS ${CREATE_DB};";
+fi
 
-    # limit elasticsearch / java memory usage to avoid OOM Killer
-    sudo service elasticsearch stop;
-    echo "ES_HEAP_SIZE=256m" | sudo tee --append /etc/default/elasticsearch
-    sudo service elasticsearch start;
+# Install pipeline toolchain
+if [ ${INSTALL_PIPELINE:-0} -ne 0 ]
+then
+    npm install
+    npm install cssmin@0.4.3
+    npm install uglify-js@2.4.13
 fi
 
-if [ "$TOXENV" == "docker" -o "$TOXENV" == "locales" ]
+# Install docker-compose
+if [ ${INSTALL_DOCKER_COMPOSE:--} -ne 0 ]
 then
-    curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
-    chmod +x docker-compose
-    sudo mv docker-compose /usr/local/bin
+    if [ -x $(command -v docker-compose) ]
+    then
+        echo "Overwriting existing docker-compose."
+        docker-compose -v
+    else
+        echo "Installing docker-compose ${DOCKER_COMPOSE_VERSION}."
+    fi
+    mkdir -p downloads
+    DOCKER_COMPOSE_FILE=downloads/docker-compose-${DOCKER_COMPOSE_VERSION}
+    if [ ! -f $DOCKER_COMPOSE_FILE ]
+    then
+        wget -q --waitretry=1 --retry-connrefused -T 10 \
+            -O downloads/docker-compose-${DOCKER_COMPOSE_VERSION} \
+            https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m`
+    fi
+    chmod +x $DOCKER_COMPOSE_FILE
+    sudo cp $DOCKER_COMPOSE_FILE /usr/local/bin/docker-compose
 fi

tox.ini

@@ -7,13 +7,19 @@ whitelist_externals = make
 deps =
     -rrequirements/dev.txt
 commands =
-    make compilejsi18n collectstatic coveragetest target={posargs:kuma}
+    make compilejsi18n collectstatic clean
+    py.test --no-migrations --cov=kuma kuma
 setenv =
     PYTHONPATH = .
     CFLAGS = -O0
     # TODO: remove once http://bugzil.la/1127798 is fixed
     PYTHONHASHSEED = 0
-passenv=DJANGO_SETTINGS_MODULE DEBIAN_FRONTEND CFLAGS
+passenv =
+    DJANGO_SETTINGS_MODULE
+    DEBIAN_FRONTEND
+    CFLAGS
+    DATABASE_URL
+    PIPELINE_*
 
 [testenv:flake8]
 basepython = python2.7