bug 1304101: Split Dockerfile into dev+prod version (#131). r=rail,jl…

…orenzo

Dockerfile

@@ -3,12 +3,10 @@ FROM python:2.7-slim
 MAINTAINER bhearsum@mozilla.com
 
 # Some versions of the python:2.7 Docker image remove libpcre3, which uwsgi needs for routing support to be enabled.
-# Node and npm are to build the frontend. nodejs-legacy is needed by this version of npm.
+# Node and npm are to build the frontend. nodejs-legacy is needed by this version of npm. These will get removed after building.
 # libmysqlclient-dev is required to use SQLAlchemy with MySQL, which we do in production.
-# libfontconfig1 is required by phantomjs
 RUN apt-get -q update \
     && apt-get -q --yes install libpcre3 libpcre3-dev nodejs nodejs-legacy npm libmysqlclient-dev \
-                                libfontconfig1 \
     && apt-get clean
 
 WORKDIR /app
@@ -19,27 +17,28 @@ WORKDIR /app
 COPY requirements.txt /app/
 RUN pip install -r requirements.txt
 
-# copy in sources after
 # Copying Balrog to /app instead of installing it means that production can run
 # it, and we can bind mount to override it for local development.
 COPY auslib/ /app/auslib/
 COPY ui/ /app/ui/
 COPY uwsgi/ /app/uwsgi/
-COPY scripts/ /app/scripts/
-COPY MANIFEST.in setup.py version.json /app/
-# These files are only needed for CI, but they're not very big so we may as
-# well just include them here to avoid forking the Dockerfile.
-COPY .coveragerc requirements-test.txt run-tests.sh tox.ini version.txt /app/
-COPY aus-data-snapshots/ /app/aus-data-snapshots/
+COPY scripts/manage-db.py scripts/run-batch-deletes.sh /app/scripts/
+COPY version.json /app/
+RUN rm -rf /app/auslib/test
 
 WORKDIR /app/ui
 RUN npm install
 RUN npm run build
 
+RUN find . -maxdepth 1 -not -name dist -exec rm -rf {} \;
+
+RUN apt-get -q --yes remove nodejs nodejs-legacy npm \
+    && apt-get clean
+
 WORKDIR /app
 
 # Using /bin/bash as the entrypoint works around some volume mount issues on Windows
 # where volume-mounted files do not have execute bits set.
 # https://github.com/docker/compose/issues/2301#issuecomment-154450785 has additional background.
-ENTRYPOINT ["/bin/bash", "/app/uwsgi/run.sh"]
+ENTRYPOINT ["/bin/bash", "/app/scripts/run.sh"]
 CMD ["public"]

Dockerfile.dev

@@ -0,0 +1,36 @@
+# NOTE: The only supported way to use the Balrog dev image is with a volume mount of /app.
+# Because of this, the only things we do in the Dockerfile are install things to
+# system locations (some system packages and some python packages).
+# Things that are fully local to /app (such as npm requirements and the built UI)
+# are done as needed in the entrypoint.
+FROM python:2.7-slim
+
+MAINTAINER bhearsum@mozilla.com
+
+# Some versions of the python:2.7 Docker image remove libpcre3, which uwsgi needs for routing support to be enabled.
+# Node and npm are to build the frontend. nodejs-legacy is needed by this version of npm.
+# mysql-client is needed to import sample data
+# libmysqlclient-dev is required to use SQLAlchemy with MySQL, which we do in production.
+# libfontconfig1 is required by phantomjs
+RUN apt-get -q update \
+    && apt-get -q --yes install libpcre3 libpcre3-dev nodejs nodejs-legacy npm libmysqlclient-dev mysql-client \
+                                libfontconfig1 \
+    && apt-get clean
+
+WORKDIR /app
+
+# The general app requirements and packages required to run Tox are installed into the system,
+# so we need to include them as part of the Docker build.
+# TODO: edit this comment
+COPY ./ /app/
+# install the requirements into the container first
+# these rarely change and is more cache friendly
+# ... really speeds up building new containers
+RUN pip install -r requirements.txt
+RUN pip install -r requirements-tox.txt
+
+# Using /bin/bash as the entrypoint works around some volume mount issues on Windows
+# where volume-mounted files do not have execute bits set.
+# https://github.com/docker/compose/issues/2301#issuecomment-154450785 has additional background.
+ENTRYPOINT ["/bin/bash", "/app/scripts/run.sh"]
+CMD ["public"]

docker-compose-test.yml

@@ -1,4 +1,4 @@
 balrogtest:
   build: .
-  entrypoint:
-    - /app/scripts/test-entrypoint.sh
+  dockerfile: Dockerfile.dev
+  command: test

docker-compose.yml

@@ -1,21 +1,23 @@
+# Some people have reported problems with the balrogui API proxy not working correctly.
+# This appears to be the same problem reported in https://github.com/docker/compose/issues/2172,
+# and seems to only occur with certain (older) versions of docker-compose.
 balrogadmin:
   build: .
+  dockerfile: Dockerfile.dev
   entrypoint:
     - bash
     - scripts/initdb_and_run.sh
   command: admin-dev
-  ports:
-    - "8080:8080"
   expose:
-    - "8080"
+    - "7070"
   volumes:
     - .:/app
   links:
     - balrogdb
   environment:
     - DBURI=mysql://balrogadmin:balrogadmin@balrogdb/balrog
     - SECRET_KEY=blahblah
-    - PORT=8080
+    - PORT=7070
     - LOG_FORMAT=plain
     - LOG_LEVEL=DEBUG
     # Grab mail information from the local environment
@@ -28,6 +30,7 @@ balrogadmin:
 
 balrogpub:
   build: .
+  dockerfile: Dockerfile.dev
   command: public
   ports:
     - "9090:9090"
@@ -49,13 +52,29 @@ balrogagent:
   links:
     - balrogadmin
   environment:
-    - BALROG_API_ROOT=http://balrogadmin:8080/api
+    - BALROG_API_ROOT=http://balrogadmin:7070/api
     - BALROG_USERNAME=balrogadmin
     - BALROG_PASSWORD=balrogadmin
     - TELEMETRY_API_ROOT=abc
     - LOG_FORMAT=plain
     - LOG_LEVEL=DEBUG
 
+balrogui:
+  image: node:0.10
+  ports:
+    - "8080:8080"
+  links:
+    - balrogadmin
+  volumes:
+    - ./ui:/app
+    - .cache:/cache
+  environment:
+    - WEB_PORT=8080
+  entrypoint:
+    - /bin/bash
+    - -c
+    - cd /app && /usr/local/bin/npm install && /usr/local/bin/npm build && /usr/local/bin/npm start
+
 balrogdb:
   image: mysql:5.7
   expose:

requirements-test.txt

@@ -1,8 +1,22 @@
-tox==2.3.1 \
-# --hash=sha256:bf7fcc140863820700d3ccd65b33820ba747b61c5fe4e2b91bb8c64cb21a47ee
-pluggy==0.3.1 \
-# --hash=sha256:159cc783e056c07da6552aa5aef6b1e6c0064b4f18bd49c531fd2d40aafb0ea3
-py==1.4.31 \
-# --hash=sha256:a6501963c725fc2554dabfece8ae9a8fb5e149c0ac0a42fd2b02c5c1c57fc114
-virtualenv==15.0.1 \
-# --hash=sha256:1a74278b8adb383ce4c7619e33c753b1eb7b58dc1e449601c096ca4b76125f84
+certifi==2016.8.31 \
+#    --hash=sha256:f7708a42d86f29ccc7c8c4ff9d34a8d854d8d78eb2973d1f28406bb43d6b8919
+coverage==4.2 \
+#    --hash=sha256:e312776d3ef04632ec742ce2d2b7048b635073e0245e4f44dfe8b08cc50ac656
+flake8==3.0.4 \
+#    --hash=sha256:b4c210c998f07d6ff24325dd91fbc011f2c37bcd6bf576b188de01d8656e970d
+mock==2.0.0 \
+#    --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba
+ordereddict==1.1 \
+#    --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f
+paste==2.0.3 \
+#    --hash=sha256:2346a347824c32641bf020c17967b49ae74d3310ec1bc9b958d4b84e2d985218
+pyflakes==1.3.0 \
+#    --hash=sha256:a4f93317c97a9d9ed71d6ecfe08b68e3de9fea3f4d94dcd1d9d83ccbf929bc31
+pytest==3.0.2 \
+#    --hash=sha256:64d8937626dd2a4bc15ef0edd307d26636a72a3f3f9664c424d78e40efb1e339
+pytest-catchlog==1.2.2 \
+#    --hash=sha256:4be15dc5ac1750f83960897f591453040dff044b5966fe24a91c2f7d04ecfcf0
+pytest-cov==2.3.1 \
+#    --hash=sha256:fa0a212283cdf52e2eecc24dd6459bb7687cc29adb60cb84258fab73be8dda0f
+pytest-xdist==1.15.0 \
+#    --hash=sha256:6238395f8bd050f9288a3b10f34330edece80f4424cf2b4204d6e7d622f0f00b

requirements-tox.txt

@@ -0,0 +1,8 @@
+pluggy==0.3.1 \
+# --hash=sha256:159cc783e056c07da6552aa5aef6b1e6c0064b4f18bd49c531fd2d40aafb0ea3
+py==1.4.31 \
+# --hash=sha256:a6501963c725fc2554dabfece8ae9a8fb5e149c0ac0a42fd2b02c5c1c57fc114
+tox==2.3.1 \
+# --hash=sha256:bf7fcc140863820700d3ccd65b33820ba747b61c5fe4e2b91bb8c64cb21a47ee
+virtualenv==15.0.1 \
+# --hash=sha256:1a74278b8adb383ce4c7619e33c753b1eb7b58dc1e449601c096ca4b76125f84

run-tests.sh

@@ -2,5 +2,7 @@
 
 # TODO: When we can run docker-compose in Taskcluster, we should use
 # docker-compose-test.yml instead of running docker directly.
-docker build --pull -t balrogtest .
-docker run --rm --entrypoint /app/scripts/test-entrypoint.sh balrogtest $@
+docker build  -t balrogtest -f Dockerfile.dev .
+# Using a volume mount here ensures that the tox cache dir ends up on the host,
+# which avoids re-installing test dependencies every single time.
+docker run --rm balrogtest test $@

scripts/initdb_and_run.sh

@@ -1,20 +1,20 @@
 set -xe
 
-apt-get -q update
-apt-get -q --yes install mysql-client
-apt-get clean
-
 if [ ! -e /app/.cache/mysql/db.done ]; then
-    sleep 30
+    # We need to sleep awhile for fresh databases because mysql will take longer to initialize.
+    # Ideally, this would find some better way to probe for mysql-readyness.
+    sleep 45
     echo "Initializing DB..."
     python scripts/manage-db.py -d mysql://balrogadmin:balrogadmin@balrogdb/balrog create
     bunzip2 -c /app/scripts/sample-data.sql.bz2 | mysql -h balrogdb -u balrogadmin --password=balrogadmin balrog
     mysql -h balrogdb -u balrogadmin --password=balrogadmin -e "insert into permissions (username, permission, data_version) values (\"balrogadmin\", \"admin\", 1)" balrog
     touch /app/.cache/mysql/db.done
     echo "Done"
 else
+    # We also should sleep for existing databases, but we don't need for nearly as long.
+    sleep 10
     python scripts/manage-db.py -d mysql://balrogadmin:balrogadmin@balrogdb/balrog upgrade
 fi
 
 # run the command passed from docker
-/app/uwsgi/run.sh $@
+/app/scripts/run.sh $@

uwsgi/run.sh → scripts/run.sh

@@ -7,17 +7,22 @@ build_front_end() {
     cd -
 }
 
+run_back_end_tests() {
+  cd /app
+  tox $@
+}
+
+run_front_end_tests() {
+  build_front_end
+  cd /app/ui/
+  npm test
+}
 
 if [ $1 == "public" ]; then
    exec uwsgi --ini /app/uwsgi/public.ini --python-autoreload 1
 elif [ $1 == "admin" ]; then
    exec uwsgi --ini /app/uwsgi/admin.ini --python-autoreload 1
 elif [ $1 == "admin-dev" ]; then
-    # When running the dev instance, we override /app by the developer's folder on his host
-    # machine. Building the frontend here makes sure we always serve the most up to date
-    # revision at each `docker-compose up`. You can still build it manually with, though
-    build_front_end
-
     exec uwsgi --ini /app/uwsgi/admin.dev.ini --ini /app/uwsgi/admin.ini --python-autoreload 1
 elif [ $1 == "upgrade-db" ]; then
     if [ -z "${DBURI}" ]; then
@@ -40,6 +45,28 @@ elif [ $1 == "cleanup-db" ]; then
     fi
 
     exec scripts/run-batch-deletes.sh $DBURI $MAX_AGE $DELETE_RUN_TIME
+elif [ $1 == "test" ]; then
+    shift
+    if [[ $1 == "backend" ]]; then
+        shift
+        run_back_end_tests $@
+    elif [[ $1 == "frontend" ]]; then
+        run_front_end_tests
+    else
+        run_back_end_tests $@
+        backend_rc=$?
+        run_front_end_tests
+        frontend_rc=$?
+        echo
+
+        if [[ $backend_rc == 0 && $frontend_rc == 0 ]]; then
+            echo "All tests pass!!!"
+            exit 0
+        else
+            echo "FAIL FAIL FAIL FAIL FAIL FAIL FAIL FAIL. Some tests failed, see above for details."
+            exit 1
+        fi
+    fi
 else
    echo "unknown mode: $1"
    exit 1

tox.ini

@@ -7,20 +7,7 @@ envlist = py27
 ignore_errors=True
 setenv =
     PYTHONDONTWRITEBYTECODE=1
-    PYTHONPATH={toxinidir}/vendor/lib/python
-deps =
-# test deps
-    certifi
-    coverage
-    flake8
-    mock
-    ordereddict
-    Paste
-    pyflakes
-    pytest
-    pytest-catchlog
-    pytest-cov
-    pytest-xdist
+deps = -rrequirements-test.txt
 
 commands=
     flake8 auslib scripts uwsgi

ui/config/application.js

@@ -35,8 +35,8 @@ module.exports = function(lineman) {
       apiProxy: {
         prefix: 'api',
         enabled: true,
-        host: 'localhost',
-        port: 8080
+        host: 'balrogadmin',
+        port: 7070
       }
     },