Permalink
Browse files

bug 1304101: Split Dockerfile into dev+prod version (#131). r=rail,jl…

…orenzo
  • Loading branch information...
mozbhearsum committed Oct 5, 2016
1 parent d6f5f47 commit d75baafc94ad2f19aeb09a1912de76da0ad28981
View
@@ -3,12 +3,10 @@ FROM python:2.7-slim
MAINTAINER bhearsum@mozilla.com
# Some versions of the python:2.7 Docker image remove libpcre3, which uwsgi needs for routing support to be enabled.
# Node and npm are to build the frontend. nodejs-legacy is needed by this version of npm.
# Node and npm are to build the frontend. nodejs-legacy is needed by this version of npm. These will get removed after building.
# libmysqlclient-dev is required to use SQLAlchemy with MySQL, which we do in production.
# libfontconfig1 is required by phantomjs
RUN apt-get -q update \
&& apt-get -q --yes install libpcre3 libpcre3-dev nodejs nodejs-legacy npm libmysqlclient-dev \
libfontconfig1 \
&& apt-get clean
WORKDIR /app
@@ -19,27 +17,28 @@ WORKDIR /app
COPY requirements.txt /app/
RUN pip install -r requirements.txt
# copy in sources after
# Copying Balrog to /app instead of installing it means that production can run
# it, and we can bind mount to override it for local development.
COPY auslib/ /app/auslib/
COPY ui/ /app/ui/
COPY uwsgi/ /app/uwsgi/
COPY scripts/ /app/scripts/
COPY MANIFEST.in setup.py version.json /app/
# These files are only needed for CI, but they're not very big so we may as
# well just include them here to avoid forking the Dockerfile.
COPY .coveragerc requirements-test.txt run-tests.sh tox.ini version.txt /app/
COPY aus-data-snapshots/ /app/aus-data-snapshots/
COPY scripts/manage-db.py scripts/run-batch-deletes.sh /app/scripts/
COPY version.json /app/
RUN rm -rf /app/auslib/test
WORKDIR /app/ui
RUN npm install
RUN npm run build
RUN find . -maxdepth 1 -not -name dist -exec rm -rf {} \;
RUN apt-get -q --yes remove nodejs nodejs-legacy npm \
&& apt-get clean
WORKDIR /app
# Using /bin/bash as the entrypoint works around some volume mount issues on Windows
# where volume-mounted files do not have execute bits set.
# https://github.com/docker/compose/issues/2301#issuecomment-154450785 has additional background.
ENTRYPOINT ["/bin/bash", "/app/uwsgi/run.sh"]
ENTRYPOINT ["/bin/bash", "/app/scripts/run.sh"]
CMD ["public"]
View
@@ -0,0 +1,36 @@
# NOTE: The only supported way to use the Balrog dev image is with a volume mount of /app.
# Because of this, the only things we do in the Dockerfile are install things to
# system locations (some system packages and some python packages).
# Things that are fully local to /app (such as npm requirements and the built UI)
# are done as needed in the entrypoint.
FROM python:2.7-slim
MAINTAINER bhearsum@mozilla.com
# Some versions of the python:2.7 Docker image remove libpcre3, which uwsgi needs for routing support to be enabled.
# Node and npm are to build the frontend. nodejs-legacy is needed by this version of npm.
# mysql-client is needed to import sample data
# libmysqlclient-dev is required to use SQLAlchemy with MySQL, which we do in production.
# libfontconfig1 is required by phantomjs
RUN apt-get -q update \
&& apt-get -q --yes install libpcre3 libpcre3-dev nodejs nodejs-legacy npm libmysqlclient-dev mysql-client \
libfontconfig1 \
&& apt-get clean
WORKDIR /app
# The general app requirements and packages required to run Tox are installed into the system,
# so we need to include them as part of the Docker build.
# TODO: edit this comment
COPY ./ /app/
# install the requirements into the container first
# these rarely change and is more cache friendly
# ... really speeds up building new containers
RUN pip install -r requirements.txt
RUN pip install -r requirements-tox.txt
# Using /bin/bash as the entrypoint works around some volume mount issues on Windows
# where volume-mounted files do not have execute bits set.
# https://github.com/docker/compose/issues/2301#issuecomment-154450785 has additional background.
ENTRYPOINT ["/bin/bash", "/app/scripts/run.sh"]
CMD ["public"]
View
@@ -1,4 +1,4 @@
balrogtest:
build: .
entrypoint:
- /app/scripts/test-entrypoint.sh
dockerfile: Dockerfile.dev
command: test
View
@@ -1,21 +1,23 @@
# Some people have reported problems with the balrogui API proxy not working correctly.
# This appears to be the same problem reported in https://github.com/docker/compose/issues/2172,
# and seems to only occur with certain (older) versions of docker-compose.
balrogadmin:
build: .
dockerfile: Dockerfile.dev
entrypoint:
- bash
- scripts/initdb_and_run.sh
command: admin-dev
ports:
- "8080:8080"
expose:
- "8080"
- "7070"
volumes:
- .:/app
links:
- balrogdb
environment:
- DBURI=mysql://balrogadmin:balrogadmin@balrogdb/balrog
- SECRET_KEY=blahblah
- PORT=8080
- PORT=7070
- LOG_FORMAT=plain
- LOG_LEVEL=DEBUG
# Grab mail information from the local environment
@@ -28,6 +30,7 @@ balrogadmin:
balrogpub:
build: .
dockerfile: Dockerfile.dev
command: public
ports:
- "9090:9090"
@@ -49,13 +52,29 @@ balrogagent:
links:
- balrogadmin
environment:
- BALROG_API_ROOT=http://balrogadmin:8080/api
- BALROG_API_ROOT=http://balrogadmin:7070/api
- BALROG_USERNAME=balrogadmin
- BALROG_PASSWORD=balrogadmin
- TELEMETRY_API_ROOT=abc
- LOG_FORMAT=plain
- LOG_LEVEL=DEBUG
balrogui:
image: node:0.10
ports:
- "8080:8080"
links:
- balrogadmin
volumes:
- ./ui:/app
- .cache:/cache
environment:
- WEB_PORT=8080
entrypoint:
- /bin/bash
- -c
- cd /app && /usr/local/bin/npm install && /usr/local/bin/npm build && /usr/local/bin/npm start
balrogdb:
image: mysql:5.7
expose:
View
@@ -1,8 +1,22 @@
tox==2.3.1 \
# --hash=sha256:bf7fcc140863820700d3ccd65b33820ba747b61c5fe4e2b91bb8c64cb21a47ee
pluggy==0.3.1 \
# --hash=sha256:159cc783e056c07da6552aa5aef6b1e6c0064b4f18bd49c531fd2d40aafb0ea3
py==1.4.31 \
# --hash=sha256:a6501963c725fc2554dabfece8ae9a8fb5e149c0ac0a42fd2b02c5c1c57fc114
virtualenv==15.0.1 \
# --hash=sha256:1a74278b8adb383ce4c7619e33c753b1eb7b58dc1e449601c096ca4b76125f84
certifi==2016.8.31 \
# --hash=sha256:f7708a42d86f29ccc7c8c4ff9d34a8d854d8d78eb2973d1f28406bb43d6b8919
coverage==4.2 \
# --hash=sha256:e312776d3ef04632ec742ce2d2b7048b635073e0245e4f44dfe8b08cc50ac656
flake8==3.0.4 \
# --hash=sha256:b4c210c998f07d6ff24325dd91fbc011f2c37bcd6bf576b188de01d8656e970d
mock==2.0.0 \
# --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba
ordereddict==1.1 \
# --hash=sha256:1c35b4ac206cef2d24816c89f89cf289dd3d38cf7c449bb3fab7bf6d43f01b1f
paste==2.0.3 \
# --hash=sha256:2346a347824c32641bf020c17967b49ae74d3310ec1bc9b958d4b84e2d985218
pyflakes==1.3.0 \
# --hash=sha256:a4f93317c97a9d9ed71d6ecfe08b68e3de9fea3f4d94dcd1d9d83ccbf929bc31
pytest==3.0.2 \
# --hash=sha256:64d8937626dd2a4bc15ef0edd307d26636a72a3f3f9664c424d78e40efb1e339
pytest-catchlog==1.2.2 \
# --hash=sha256:4be15dc5ac1750f83960897f591453040dff044b5966fe24a91c2f7d04ecfcf0
pytest-cov==2.3.1 \
# --hash=sha256:fa0a212283cdf52e2eecc24dd6459bb7687cc29adb60cb84258fab73be8dda0f
pytest-xdist==1.15.0 \
# --hash=sha256:6238395f8bd050f9288a3b10f34330edece80f4424cf2b4204d6e7d622f0f00b
View
@@ -0,0 +1,8 @@
pluggy==0.3.1 \
# --hash=sha256:159cc783e056c07da6552aa5aef6b1e6c0064b4f18bd49c531fd2d40aafb0ea3
py==1.4.31 \
# --hash=sha256:a6501963c725fc2554dabfece8ae9a8fb5e149c0ac0a42fd2b02c5c1c57fc114
tox==2.3.1 \
# --hash=sha256:bf7fcc140863820700d3ccd65b33820ba747b61c5fe4e2b91bb8c64cb21a47ee
virtualenv==15.0.1 \
# --hash=sha256:1a74278b8adb383ce4c7619e33c753b1eb7b58dc1e449601c096ca4b76125f84
View
@@ -2,5 +2,7 @@
# TODO: When we can run docker-compose in Taskcluster, we should use
# docker-compose-test.yml instead of running docker directly.
docker build --pull -t balrogtest .
docker run --rm --entrypoint /app/scripts/test-entrypoint.sh balrogtest $@
docker build -t balrogtest -f Dockerfile.dev .
# Using a volume mount here ensures that the tox cache dir ends up on the host,
# which avoids re-installing test dependencies every single time.
docker run --rm balrogtest test $@
View
@@ -1,20 +1,20 @@
set -xe
apt-get -q update
apt-get -q --yes install mysql-client
apt-get clean
if [ ! -e /app/.cache/mysql/db.done ]; then
sleep 30
# We need to sleep awhile for fresh databases because mysql will take longer to initialize.
# Ideally, this would find some better way to probe for mysql-readyness.
sleep 45
echo "Initializing DB..."
python scripts/manage-db.py -d mysql://balrogadmin:balrogadmin@balrogdb/balrog create
bunzip2 -c /app/scripts/sample-data.sql.bz2 | mysql -h balrogdb -u balrogadmin --password=balrogadmin balrog
mysql -h balrogdb -u balrogadmin --password=balrogadmin -e "insert into permissions (username, permission, data_version) values (\"balrogadmin\", \"admin\", 1)" balrog
touch /app/.cache/mysql/db.done
echo "Done"
else
# We also should sleep for existing databases, but we don't need for nearly as long.
sleep 10
python scripts/manage-db.py -d mysql://balrogadmin:balrogadmin@balrogdb/balrog upgrade
fi
# run the command passed from docker
/app/uwsgi/run.sh $@
/app/scripts/run.sh $@
@@ -7,17 +7,22 @@ build_front_end() {
cd -
}
run_back_end_tests() {
cd /app
tox $@
}
run_front_end_tests() {
build_front_end
cd /app/ui/
npm test
}
if [ $1 == "public" ]; then
exec uwsgi --ini /app/uwsgi/public.ini --python-autoreload 1
elif [ $1 == "admin" ]; then
exec uwsgi --ini /app/uwsgi/admin.ini --python-autoreload 1
elif [ $1 == "admin-dev" ]; then
# When running the dev instance, we override /app by the developer's folder on his host
# machine. Building the frontend here makes sure we always serve the most up to date
# revision at each `docker-compose up`. You can still build it manually with, though
build_front_end
exec uwsgi --ini /app/uwsgi/admin.dev.ini --ini /app/uwsgi/admin.ini --python-autoreload 1
elif [ $1 == "upgrade-db" ]; then
if [ -z "${DBURI}" ]; then
@@ -40,6 +45,28 @@ elif [ $1 == "cleanup-db" ]; then
fi
exec scripts/run-batch-deletes.sh $DBURI $MAX_AGE $DELETE_RUN_TIME
elif [ $1 == "test" ]; then
shift
if [[ $1 == "backend" ]]; then
shift
run_back_end_tests $@
elif [[ $1 == "frontend" ]]; then
run_front_end_tests
else
run_back_end_tests $@
backend_rc=$?
run_front_end_tests
frontend_rc=$?
echo
if [[ $backend_rc == 0 && $frontend_rc == 0 ]]; then
echo "All tests pass!!!"
exit 0
else
echo "FAIL FAIL FAIL FAIL FAIL FAIL FAIL FAIL. Some tests failed, see above for details."
exit 1
fi
fi
else
echo "unknown mode: $1"
exit 1
View
@@ -1,23 +0,0 @@
#!/bin/bash
run_back_end_tests() {
pip install -r /app/requirements-test.txt
tox $@
}
run_front_end_tests() {
cd ui/
npm test
}
type_of_tests="$1"
if [[ $type_of_tests == "backend" ]]; then
shift
run_back_end_tests $@
elif [[ $type_of_tests == "frontend" ]]; then
run_front_end_tests
else
run_back_end_tests $@
run_front_end_tests
fi
View
15 tox.ini
@@ -7,20 +7,7 @@ envlist = py27
ignore_errors=True
setenv =
PYTHONDONTWRITEBYTECODE=1
PYTHONPATH={toxinidir}/vendor/lib/python
deps =
# test deps
certifi
coverage
flake8
mock
ordereddict
Paste
pyflakes
pytest
pytest-catchlog
pytest-cov
pytest-xdist
deps = -rrequirements-test.txt
commands=
flake8 auslib scripts uwsgi
View
@@ -35,8 +35,8 @@ module.exports = function(lineman) {
apiProxy: {
prefix: 'api',
enabled: true,
host: 'localhost',
port: 8080
host: 'balrogadmin',
port: 7070
}
},

1 comment on commit d75baaf

Please sign in to comment.