Permalink
Browse files

bug 1304101: Split Dockerfile into dev+prod version (#131). r=rail,jl…

…orenzo
  • Loading branch information...
1 parent d6f5f47 commit d75baafc94ad2f19aeb09a1912de76da0ad28981 @mozbhearsum mozbhearsum committed on GitHub Oct 5, 2016
View
@@ -3,12 +3,10 @@ FROM python:2.7-slim
MAINTAINER bhearsum@mozilla.com
# Some versions of the python:2.7 Docker image remove libpcre3, which uwsgi needs for routing support to be enabled.
-# Node and npm are to build the frontend. nodejs-legacy is needed by this version of npm.
+# Node and npm are to build the frontend. nodejs-legacy is needed by this version of npm. These will get removed after building.
# libmysqlclient-dev is required to use SQLAlchemy with MySQL, which we do in production.
-# libfontconfig1 is required by phantomjs
RUN apt-get -q update \
&& apt-get -q --yes install libpcre3 libpcre3-dev nodejs nodejs-legacy npm libmysqlclient-dev \
- libfontconfig1 \
&& apt-get clean
WORKDIR /app
@@ -19,27 +17,28 @@ WORKDIR /app
COPY requirements.txt /app/
RUN pip install -r requirements.txt
-# copy in sources after
# Copying Balrog to /app instead of installing it means that production can run
# it, and we can bind mount to override it for local development.
COPY auslib/ /app/auslib/
COPY ui/ /app/ui/
COPY uwsgi/ /app/uwsgi/
-COPY scripts/ /app/scripts/
-COPY MANIFEST.in setup.py version.json /app/
-# These files are only needed for CI, but they're not very big so we may as
-# well just include them here to avoid forking the Dockerfile.
-COPY .coveragerc requirements-test.txt run-tests.sh tox.ini version.txt /app/
-COPY aus-data-snapshots/ /app/aus-data-snapshots/
+COPY scripts/manage-db.py scripts/run-batch-deletes.sh /app/scripts/
+COPY version.json /app/
+RUN rm -rf /app/auslib/test
WORKDIR /app/ui
RUN npm install
RUN npm run build
+RUN find . -maxdepth 1 -not -name dist -exec rm -rf {} \;
+
+RUN apt-get -q --yes remove nodejs nodejs-legacy npm \
+ && apt-get clean
+
WORKDIR /app
# Using /bin/bash as the entrypoint works around some volume mount issues on Windows
# where volume-mounted files do not have execute bits set.
# https://github.com/docker/compose/issues/2301#issuecomment-154450785 has additional background.
-ENTRYPOINT ["/bin/bash", "/app/uwsgi/run.sh"]
+ENTRYPOINT ["/bin/bash", "/app/scripts/run.sh"]
CMD ["public"]
View
@@ -0,0 +1,36 @@
+# NOTE: The only supported way to use the Balrog dev image is with a volume mount of /app.
+# Because of this, the only things we do in the Dockerfile are install things to
+# system locations (some system packages and some python packages).
+# Things that are fully local to /app (such as npm requirements and the built UI)
+# are done as needed in the entrypoint.
+FROM python:2.7-slim
+
+MAINTAINER bhearsum@mozilla.com
+
+# Some versions of the python:2.7 Docker image remove libpcre3, which uwsgi needs for routing support to be enabled.
+# Node and npm are to build the frontend. nodejs-legacy is needed by this version of npm.
+# mysql-client is needed to import sample data
+# libmysqlclient-dev is required to use SQLAlchemy with MySQL, which we do in production.
+# libfontconfig1 is required by phantomjs
+RUN apt-get -q update \
+ && apt-get -q --yes install libpcre3 libpcre3-dev nodejs nodejs-legacy npm libmysqlclient-dev mysql-client \
+ libfontconfig1 \
+ && apt-get clean
+
+WORKDIR /app
+
+# The general app requirements and packages required to run Tox are installed into the system,
+# so we need to include them as part of the Docker build.
+# TODO: edit this comment
+COPY ./ /app/
+# install the requirements into the container first
+# these rarely change and is more cache friendly
+# ... really speeds up building new containers
+RUN pip install -r requirements.txt
+RUN pip install -r requirements-tox.txt
+
+# Using /bin/bash as the entrypoint works around some volume mount issues on Windows
+# where volume-mounted files do not have execute bits set.
+# https://github.com/docker/compose/issues/2301#issuecomment-154450785 has additional background.
+ENTRYPOINT ["/bin/bash", "/app/scripts/run.sh"]
+CMD ["public"]
@@ -1,4 +1,4 @@
balrogtest:
build: .
- entrypoint:
- - /app/scripts/test-entrypoint.sh
+ dockerfile: Dockerfile.dev
+ command: test
View
@@ -1,21 +1,23 @@
+# Some people have reported problems with the balrogui API proxy not working correctly.
+# This appears to be the same problem reported in https://github.com/docker/compose/issues/2172,
+# and seems to only occur with certain (older) versions of docker-compose.
balrogadmin:
build: .
+ dockerfile: Dockerfile.dev
entrypoint:
- bash
- scripts/initdb_and_run.sh
command: admin-dev
- ports:
- - "8080:8080"
expose:
- - "8080"
+ - "7070"
volumes:
- .:/app
links:
- balrogdb
environment:
- DBURI=mysql://balrogadmin:balrogadmin@balrogdb/balrog
- SECRET_KEY=blahblah
- - PORT=8080
+ - PORT=7070
- LOG_FORMAT=plain
- LOG_LEVEL=DEBUG
# Grab mail information from the local environment
@@ -28,6 +30,7 @@ balrogadmin:
balrogpub:
build: .
+ dockerfile: Dockerfile.dev
command: public
ports:
- "9090:9090"
@@ -49,13 +52,29 @@ balrogagent:
links:
- balrogadmin
environment:
- - BALROG_API_ROOT=http://balrogadmin:8080/api
+ - BALROG_API_ROOT=http://balrogadmin:7070/api
- BALROG_USERNAME=balrogadmin
- BALROG_PASSWORD=balrogadmin
- TELEMETRY_API_ROOT=abc
- LOG_FORMAT=plain
- LOG_LEVEL=DEBUG
+balrogui:
+ image: node:0.10
+ ports:
+ - "8080:8080"
+ links:
+ - balrogadmin
+ volumes:
+ - ./ui:/app
+ - .cache:/cache
+ environment:
+ - WEB_PORT=8080
+ entrypoint:
+ - /bin/bash
+ - -c
+ - cd /app && /usr/local/bin/npm install && /usr/local/bin/npm build && /usr/local/bin/npm start
+
balrogdb:
image: mysql:5.7
expose:
View
@@ -1,8 +1,22 @@
-tox==2.3.1 \
-# --hash=sha256:bf7fcc140863820700d3ccd65b33820ba747b61c5fe4e2b91bb8c64cb21a47ee
-pluggy==0.3.1 \
-# --hash=sha256:159cc783e056c07da6552aa5aef6b1e6c0064b4f18bd49c531fd2d40aafb0ea3
-py==1.4.31 \
-# --hash=sha256:a6501963c725fc2554dabfece8ae9a8fb5e149c0ac0a42fd2b02c5c1c57fc114
-virtualenv==15.0.1 \
-# --hash=sha256:1a74278b8adb383ce4c7619e33c753b1eb7b58dc1e449601c096ca4b76125f84
+certifi==2016.8.31 \
+# --hash=sha256:f7708a42d86f29ccc7c8c4ff9d34a8d854d8d78eb2973d1f28406bb43d6b8919
+coverage==4.2 \
+# --hash=sha256:e312776d3ef04632ec742ce2d2b7048b635073e0245e4f44dfe8b08cc50ac656
+flake8==3.0.4 \
+# --hash=sha256:b4c210c998f07d6ff24325dd91fbc011f2c37bcd6bf576b188de01d8656e970d
+mock==2.0.0 \
+# --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba
+ordereddict==1.1 \
+# --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f
+paste==2.0.3 \
+# --hash=sha256:2346a347824c32641bf020c17967b49ae74d3310ec1bc9b958d4b84e2d985218
+pyflakes==1.3.0 \
+# --hash=sha256:a4f93317c97a9d9ed71d6ecfe08b68e3de9fea3f4d94dcd1d9d83ccbf929bc31
+pytest==3.0.2 \
+# --hash=sha256:64d8937626dd2a4bc15ef0edd307d26636a72a3f3f9664c424d78e40efb1e339
+pytest-catchlog==1.2.2 \
+# --hash=sha256:4be15dc5ac1750f83960897f591453040dff044b5966fe24a91c2f7d04ecfcf0
+pytest-cov==2.3.1 \
+# --hash=sha256:fa0a212283cdf52e2eecc24dd6459bb7687cc29adb60cb84258fab73be8dda0f
+pytest-xdist==1.15.0 \
+# --hash=sha256:6238395f8bd050f9288a3b10f34330edece80f4424cf2b4204d6e7d622f0f00b
@@ -0,0 +1,8 @@
+pluggy==0.3.1 \
+# --hash=sha256:159cc783e056c07da6552aa5aef6b1e6c0064b4f18bd49c531fd2d40aafb0ea3
+py==1.4.31 \
+# --hash=sha256:a6501963c725fc2554dabfece8ae9a8fb5e149c0ac0a42fd2b02c5c1c57fc114
+tox==2.3.1 \
+# --hash=sha256:bf7fcc140863820700d3ccd65b33820ba747b61c5fe4e2b91bb8c64cb21a47ee
+virtualenv==15.0.1 \
+# --hash=sha256:1a74278b8adb383ce4c7619e33c753b1eb7b58dc1e449601c096ca4b76125f84
View
@@ -2,5 +2,7 @@
# TODO: When we can run docker-compose in Taskcluster, we should use
# docker-compose-test.yml instead of running docker directly.
-docker build --pull -t balrogtest .
-docker run --rm --entrypoint /app/scripts/test-entrypoint.sh balrogtest $@
+docker build -t balrogtest -f Dockerfile.dev .
+# Using a volume mount here ensures that the tox cache dir ends up on the host,
+# which avoids re-installing test dependencies every single time.
+docker run --rm balrogtest test $@
@@ -1,20 +1,20 @@
set -xe
-apt-get -q update
-apt-get -q --yes install mysql-client
-apt-get clean
-
if [ ! -e /app/.cache/mysql/db.done ]; then
- sleep 30
+ # We need to sleep awhile for fresh databases because mysql will take longer to initialize.
+ # Ideally, this would find some better way to probe for mysql-readyness.
+ sleep 45
echo "Initializing DB..."
python scripts/manage-db.py -d mysql://balrogadmin:balrogadmin@balrogdb/balrog create
bunzip2 -c /app/scripts/sample-data.sql.bz2 | mysql -h balrogdb -u balrogadmin --password=balrogadmin balrog
mysql -h balrogdb -u balrogadmin --password=balrogadmin -e "insert into permissions (username, permission, data_version) values (\"balrogadmin\", \"admin\", 1)" balrog
touch /app/.cache/mysql/db.done
echo "Done"
else
+ # We also should sleep for existing databases, but we don't need for nearly as long.
+ sleep 10
python scripts/manage-db.py -d mysql://balrogadmin:balrogadmin@balrogdb/balrog upgrade
fi
# run the command passed from docker
-/app/uwsgi/run.sh $@
+/app/scripts/run.sh $@
@@ -7,17 +7,22 @@ build_front_end() {
cd -
}
+run_back_end_tests() {
+ cd /app
+ tox $@
+}
+
+run_front_end_tests() {
+ build_front_end
+ cd /app/ui/
+ npm test
+}
if [ $1 == "public" ]; then
exec uwsgi --ini /app/uwsgi/public.ini --python-autoreload 1
elif [ $1 == "admin" ]; then
exec uwsgi --ini /app/uwsgi/admin.ini --python-autoreload 1
elif [ $1 == "admin-dev" ]; then
- # When running the dev instance, we override /app by the developer's folder on his host
- # machine. Building the frontend here makes sure we always serve the most up to date
- # revision at each `docker-compose up`. You can still build it manually with, though
- build_front_end
-
exec uwsgi --ini /app/uwsgi/admin.dev.ini --ini /app/uwsgi/admin.ini --python-autoreload 1
elif [ $1 == "upgrade-db" ]; then
if [ -z "${DBURI}" ]; then
@@ -40,6 +45,28 @@ elif [ $1 == "cleanup-db" ]; then
fi
exec scripts/run-batch-deletes.sh $DBURI $MAX_AGE $DELETE_RUN_TIME
+elif [ $1 == "test" ]; then
+ shift
+ if [[ $1 == "backend" ]]; then
+ shift
+ run_back_end_tests $@
+ elif [[ $1 == "frontend" ]]; then
+ run_front_end_tests
+ else
+ run_back_end_tests $@
+ backend_rc=$?
+ run_front_end_tests
+ frontend_rc=$?
+ echo
+
+ if [[ $backend_rc == 0 && $frontend_rc == 0 ]]; then
+ echo "All tests pass!!!"
+ exit 0
+ else
+ echo "FAIL FAIL FAIL FAIL FAIL FAIL FAIL FAIL. Some tests failed, see above for details."
+ exit 1
+ fi
+ fi
else
echo "unknown mode: $1"
exit 1
@@ -1,23 +0,0 @@
-#!/bin/bash
-
-run_back_end_tests() {
- pip install -r /app/requirements-test.txt
- tox $@
-}
-
-run_front_end_tests() {
- cd ui/
- npm test
-}
-
-type_of_tests="$1"
-
-if [[ $type_of_tests == "backend" ]]; then
- shift
- run_back_end_tests $@
-elif [[ $type_of_tests == "frontend" ]]; then
- run_front_end_tests
-else
- run_back_end_tests $@
- run_front_end_tests
-fi
View
@@ -7,20 +7,7 @@ envlist = py27
ignore_errors=True
setenv =
PYTHONDONTWRITEBYTECODE=1
- PYTHONPATH={toxinidir}/vendor/lib/python
-deps =
-# test deps
- certifi
- coverage
- flake8
- mock
- ordereddict
- Paste
- pyflakes
- pytest
- pytest-catchlog
- pytest-cov
- pytest-xdist
+deps = -rrequirements-test.txt
commands=
flake8 auslib scripts uwsgi
@@ -35,8 +35,8 @@ module.exports = function(lineman) {
apiProxy: {
prefix: 'api',
enabled: true,
- host: 'localhost',
- port: 8080
+ host: 'balrogadmin',
+ port: 7070
}
},

1 comment on commit d75baaf

Please sign in to comment.