fixes 1548591 - use docker-compose in travis

.flake8

@@ -0,0 +1,11 @@
+[flake8]
+exclude = **/migrations/**,.tox,*.egg,vendor
+# E501 - line too long (82 > 79 characters)
+# E731 - do not assign a lambda expression, use a def
+# F405 - name may be undefined, or defined from star imports: module
+# W504 - line break after binary operator
+#        conflicts with W503 (line break before binary operator)
+ignore = E501,E731,F405,W504
+# flake8-import-order settings
+import-order-style=edited
+application-import-names=kuma,pages,utils

.travis.yml

@@ -1,89 +1,36 @@
-sudo: required
 branches:
     only:
         - master
-        - potato-1490727
-cache:
-    pip: True
-    directories:
-      - node_modules
-      - downloads
 language: python
+python:
+    - "2.7"
 services:
     - docker
-    - mysql
-    - redis
-env:
-    global:
-        - DATABASE_URL=mysql://root:@127.0.0.1:3306/kuma
-        - DJANGO_SETTINGS_MODULE=kuma.settings.travis
-        - DOCKER_COMPOSE_VERSION=1.23.2
-        - ES_VERSION=6.7.1
-        - ES_DOWNLOAD_URL=https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.7.1.tar.gz
-        - PIPELINE_CLEANCSS_BINARY=$TRAVIS_BUILD_DIR/node_modules/.bin/cleancss
-        - PIPELINE_CLEANCSS_ARGUMENTS="-O1 --skip-rebase"
-        - PIPELINE_CSS_COMPRESSOR=kuma.core.pipeline.cleancss.CleanCSSCompressor
-        - PIPELINE_JS_COMPRESSOR=pipeline.compressors.uglifyjs.UglifyJSCompressor
-        - PIPELINE_SASS_BINARY=$TRAVIS_BUILD_DIR/node_modules/.bin/node-sass
-        - PIPELINE_UGLIFYJS_BINARY=$TRAVIS_BUILD_DIR/node_modules/.bin/uglifyjs
-
 matrix:
     include:
-        - python: "2.7"
+        - language: python
+          python: 3.6
+          cache: pip
+          script:
+            - scripts/travis-docs
+        - language: python
+          python: 2.7
           env:
-            TOXENV=pytest
-            CREATE_DB=kuma
-            INSTALL_ELASTICSEARCH=1
-            INSTALL_NPM_TOOLS=1
-        - python: "2.7"
-          env:
-            TOXENV=flake8
-        - python: "2.7"
-          env:
-            TOXENV=docs
-        - python: "2.7"
-          env:
-            TOXENV=locales
-            CREATE_DB=kuma
-            INSTALL_NPM_TOOLS=1
-        - python: "2.7"
-          env:
-            TOXENV=docker
-            INSTALL_DOCKER_COMPOSE=1
             UID=0
-        - python: "2.7"
-          env:
-            TOXENV=assetlint
-            INSTALL_NPM_TOOLS=1
-        - python: "3.6"
-          env:
-            TOXENV=pytest3
-            CREATE_DB=kuma
-            INSTALL_ELASTICSEARCH=1
-            INSTALL_NPM_TOOLS=1
-    allow_failures:
-        - python: "3.6"
-          env:
-            TOXENV=pytest3
-            CREATE_DB=kuma
-            INSTALL_ELASTICSEARCH=1
-            INSTALL_NPM_TOOLS=1
-install:
-    - nvm install 10
-    - nvm use 10
-    - scripts/travis-install
-    - pip install -U pip
-    - pip install -r requirements/travis.txt
-    # Wait for ElasticSearch to be ready
-    - if [[ "$INSTALL_ELASTICSEARCH" == "1" ]]; then wget -q --waitretry=1 --retry-connrefused -T 10 -O - http://127.0.0.1:9200; fi;
-script:
-    - tox -v
-after_failure:
-    - dmesg | tail
-notifications:
-    irc:
-        channels:
-            - "irc.mozilla.org#mdndev"
-        on_success: always
-        on_failure: always
-        use_notice: true
+            DOCKER_COMPOSE_VERSION=1.23.2
+          cache:
+            directories:
+              - downloads
+              - build
+              - static
+          script:
+            - set -e
+            - scripts/travis-python
+            - scripts/travis-localerefresh
+          after_success:
+            - bash <(curl -s https://codecov.io/bash)
+        - language: node_js
+          node_js: 10
+          cache: npm
+          script:
+            - scripts/travis-js

Makefile

@@ -34,7 +34,9 @@ test:
 	py.test $(target)
 
 coveragetest: clean
-	py.test --cov=$(target) $(target)
+	# py.test --cov=$(target) --no-cov-on-fail $(target)
+	@ echo "TEMPORARITLY RUNNING WITH -x"
+	@ py.test -x --cov=$(target) --no-cov-on-fail $(target)
 
 coveragetesthtml: coveragetest
 	coverage html
@@ -180,9 +182,15 @@ bash: up
 shell_plus: up
 	docker-compose exec web ./manage.py shell_plus
 
-lint:
+pylint:
 	flake8 kuma docs tests
 
+jslint:
+	npm run eslint
+	npm run stylelint
+
+lint: pylint jslint
+
 npmrefresh:
 	cd /tools
 	echo '{"lockfileVersion": 1}' > package-lock.json

docker/images/kuma_base/Dockerfile

@@ -123,4 +123,3 @@ ENV NODE_PATH=/tools/node_modules
 USER kuma
 WORKDIR /app
 EXPOSE 8000
-CMD ["gunicorn", "--bind", "0.0.0.0:8000", "--timeout=120", "--worker-class=meinheld.gmeinheld.MeinheldWorker", "kuma.wsgi:application"]

package.json

@@ -16,7 +16,7 @@
         "build-watch": "gulp",
         "webpack": "webpack",
         "webpack-watch": "webpack -w",
-        "stylelint": "./node_modules/.bin/stylelint \"kuma/static/styles/**/*.scss\"",
+        "stylelint": "stylelint \"kuma/static/styles/**/*.scss\"",
         "eslint": "eslint --ext .js --ext .jsx kuma/",
         "optim:svg": "svgo jinja2/includes/icons/* --disable={removeViewBox,removeTitle} --multipass --pretty",
         "precommit": "npm run eslint && npm run stylelint",

requirements/README.rst

@@ -10,7 +10,6 @@ Kuma.  The files are:
 * ``docs.txt`` - Requirements for building docs in `Read the Docs`_.
 * ``dev.txt`` - Requirements for Docker or native development.
 * ``test.txt`` - Requirements to run functional tests.
-* ``travis.txt`` - Requirements for testing in TravisCI.
 
 These files have two audiences:
 

requirements/constraints.txt

@@ -541,7 +541,6 @@ lxml==4.2.5 \
     --hash=sha256:fa39ea60d527fbdd94215b5e5552f1c6a912624521093f1384a491a8ad89ad8b
 
 # pytest-cov
-# Keep in sync with travis.txt
 coverage==4.3.4 \
     --hash=sha256:e1fb21a807aa0b5cc79806d8ef09078acaa83f994e15f0f7277489ca8eda51b7 \
     --hash=sha256:8a82664931a071399d703d65af2521e2202b34f2d8db20fa22a922fec0339022 \

requirements/dev.txt

@@ -1,6 +1,5 @@
 # These requirements are for development and include the default requirements
 -r default.txt
--r travis.txt
 
 # Extract messages from Django templates
 # Code: https://github.com/python-babel/django-babel

scripts/travis-docs

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e  # Exit on non-zero status
+set -u  # Treat unset variables as an error
+
+pip install -r requirements/docs.txt
+sphinx-build -b html -d doctrees docs html

scripts/travis-js

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e  # Exit on non-zero status
+set -u  # Treat unset variables as an error
+
+# XXX I think Travis automatically discovers that this is npm
+# based on the existence of a `package.json` & `package-lock.json`
+# so it automatically executes `npm ci`
+#npm install
+make jslint

scripts/travis-localerefresh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e  # Exit on non-zero status
+set -u  # Treat unset variables as an error
+
+# XXX Instead of using the "make-bundle" `localerefresh` we spell out
+# each and every one of these. Otherwise it causes to build-static again.
+## docker-compose exec -T web make localerefresh
+docker-compose exec -T web make localeextract
+docker-compose exec -T web make locale-populate-react
+docker-compose exec -T web make localetest
+docker-compose exec -T web make localecompile
+
+cd locale
+export GIT_PAGER=cat
+git diff -G "^msgid " templates/LC_MESSAGES

scripts/travis-python

@@ -0,0 +1,50 @@
+#!/bin/bash
+set -e  # Exit on non-zero status
+set -u  # Treat unset variables as an error
+
+# First install docker-compose (right version of it)
+if [ -x $(command -v docker-compose) ]
+then
+    echo "Overwriting existing docker-compose."
+    docker-compose -v
+else
+    echo "Installing docker-compose ${DOCKER_COMPOSE_VERSION}."
+fi
+mkdir -p downloads
+DOCKER_COMPOSE_FILE=downloads/docker-compose-${DOCKER_COMPOSE_VERSION}
+if [ ! -f $DOCKER_COMPOSE_FILE ]
+then
+    time wget -q --waitretry=1 --retry-connrefused -T 10 \
+        -O downloads/docker-compose-${DOCKER_COMPOSE_VERSION} \
+        https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m`
+fi
+chmod +x $DOCKER_COMPOSE_FILE
+sudo cp $DOCKER_COMPOSE_FILE /usr/local/bin/docker-compose
+
+
+# Now build the Docker container
+
+time VERSION=latest make build-base
+time VERSION=latest make build-kuma
+
+time docker-compose --version
+time docker-compose up -d
+# Hang tight until MySQL appears to be working
+time docker-compose exec -T web urlwait mysql://root:kuma@mysql:3306/developer_mozilla_org 30
+# Hang tight until Elasticsearch appears to be working
+time docker-compose exec -T web urlwait http://elasticsearch:9200 30
+
+# echo "docker-compose exec -T web make webpack"
+# docker-compose exec -T web make webpack
+time docker-compose exec -T web make clean
+time docker-compose exec -T web make localecompile
+time docker-compose exec -T web make build-static
+time docker-compose exec -T web make pylint
+time docker-compose exec -T web ./manage.py migrate
+# time docker-compose exec -T web make test
+time docker-compose exec -T web make coveragetest
+# time docker-compose exec -T web make coveragetesthtml
+
+# This one is deliberately commented out because of this bug:
+# https://bugzilla.mozilla.org/show_bug.cgi?id=1548669
+#docker-compose exec -T web ./manage.py makemigrations --check --dry-run