Merge pull request #4601 from willkg/1468815-highlander

fix bug 1468815, 1466974: switch to a single docker image

.circleci/config.yml

@@ -50,7 +50,7 @@ jobs:
       - run:
           name: Lint
           working_directory: /socorro
-          command: docker run local/socorro_webapp ./docker/run_lint.sh
+          command: docker run local/socorro_app shell ./docker/run_lint.sh
 
       - run:
           name: Run tests
@@ -91,10 +91,8 @@ jobs:
             # push on master or git tag
             if [ "${CIRCLE_BRANCH}" == "master" ] || [ -n "${CIRCLE_TAG}" ]; then
               echo "${DOCKER_PASSWORD}" | docker login -u="${DOCKER_USERNAME}" --password-stdin
-              for image in "socorro_processor" "socorro_webapp" "socorro_crontabber"; do
-                retry docker tag "local/${image}:latest" "mozilla/${image}:${DOCKER_TAG}"
-                retry docker push "mozilla/${image}:${DOCKER_TAG}"
-              done
+              retry docker tag "local/socorro_app:latest" "mozilla/socorro_app:${DOCKER_TAG}"
+              retry docker push "mozilla/socorro_app:${DOCKER_TAG}"
             fi
 
 workflows:

.env

@@ -1 +1,8 @@
+# Sets the base name for containers and networks for docker-compose to
+# "socorro_". This is normally set by the name of this directory, but
+# if you clone the repository with a different directory name, then
+# you end up with a different project name and then everything is hosed.
+# Setting it here fixes that.
+COMPOSE_PROJECT_NAME=socorro
+
 PYTHONUNBUFFERED=true

Makefile

@@ -2,8 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-.PHONY: clean default docs help lint
-
 # Include my.env and export it so variables set in there are available
 # in the Makefile.
 include my.env
@@ -15,17 +13,23 @@ export
 SOCORRO_UID ?= 10001
 SOCORRO_GID ?= 10001
 
-default:
-	@echo "You need to specify a subcommand. See \"make help\" for options."
-	@exit 1
+DC := $(shell which docker-compose)
+
+
+.PHONY: help
+help: default
 
-help:
+.PHONY: default
+default:
+	@echo "Usage: make RULE"
+	@echo ""
 	@echo "Socorro make rules:"
 	@echo ""
 	@echo "  build            - build docker containers"
 	@echo "  run              - docker-compose up the entire system for dev"
+	@echo "  stop             - stop all service containers"
 	@echo ""
-	@echo "  shell            - open a shell in the processor container"
+	@echo "  shell            - open a shell in the app container"
 	@echo "  clean            - remove all build, test, coverage and Python artifacts"
 	@echo "  lint             - check style with flake8"
 	@echo "  test             - run unit tests"
@@ -34,24 +38,24 @@ help:
 	@echo ""
 	@echo "  setup            - set up Postgres, Elasticsearch, and local S3"
 	@echo "  updatedata       - add/update necessary database data"
+	@echo "  help             - see this text"
 	@echo ""
 	@echo "See https://socorro.readthedocs.io/ for more documentation."
 
+.PHONY: clean
 clean:
 	-rm .docker-build*
 	-rm -rf build breakpad stackwalk google-breakpad breakpad.tar.gz
 	-rm -rf .cache
 	cd minidump-stackwalk && make clean
 
+.PHONY: docs
 docs: my.env .docker-build-docs
 	${DC} run docs ./docker/run_build_docs.sh
 
+.PHONY: lint
 lint: my.env
-	${DC} run webapp ./docker/run_lint.sh
-
-.PHONY: build setup test testshell run dependencycheck stop
-
-DC := $(shell which docker-compose)
+	${DC} run app shell ./docker/run_lint.sh
 
 my.env:
 	@if [ ! -f my.env ]; \
@@ -63,62 +67,71 @@ my.env:
 .docker-build:
 	make build
 
+.PHONY: build
 build: my.env
-	${DC} build --build-arg userid=${SOCORRO_UID} --build-arg groupid=${SOCORRO_GID} base
-	${DC} build webapp  # crontabber is based off of the webapp image
-	${DC} build processor crontabber oidcprovider
+	${DC} build --build-arg userid=${SOCORRO_UID} --build-arg groupid=${SOCORRO_GID} app
+	${DC} build oidcprovider
 	touch .docker-build
 
 .docker-build-docs:
 	make build-docs
 
+.PHONY: build-docs
 build-docs: my.env
 	${DC} build docs
 	touch .docker-build-docs
 
-# NOTE(willkg): We run setup in the webapp container because the webapp will own
-# postgres going forward and has the needed environment variables.
+.PHONY: setup
 setup: my.env .docker-build
-	${DC} run webapp /app/docker/run_setup.sh
+	${DC} run app shell bash -c /app/docker/run_setup.sh
+
+.PHONY: updatedata
+updatedata: my.env
+	./docker/run_update_data.sh
 
+.PHONY: shell
 shell: my.env .docker-build
-	${DC} run processor bash
+	${DC} run app shell
 
+.PHONY: test
 test: my.env .docker-build
 	./docker/run_tests_in_docker.sh ${ARGS}
 
+.PHONY: testshell
 testshell: my.env .docker-build
 	./docker/run_tests_in_docker.sh --shell
 
-updatedata: my.env
-	./docker/run_update_data.sh
-
+.PHONY: run
 run: my.env
-	${DC} up webapp processor webpack
+	${DC} up processor webapp webpack oidcprovider
 
+.PHONY: stop
 stop: my.env
 	${DC} stop
 
+.PHONY: dependencycheck
 dependencycheck: my.env
-	${DC} run crontabber ./docker/run_dependency_checks.sh
+	${DC} run crontabber shell ./docker/run_dependency_checks.sh
 
 
 # Python 3 migration related things--remove after we've finished migrating
 
-.PHONY: build3 test3 testshell3
-
 .docker-build3:
 	make build3
 
+.PHONY: build3
 build3: my.env
-	${DC} build testpython3
+	${DC} build --build-arg userid=${SOCORRO_UID} --build-arg groupid=${SOCORRO_GID} testpython3
 	touch .docker-build3
 
+.PHONY: lint3
 lint3: my.env .docker-build3
-	${DC} run testpython3 ./docker/run_lint.sh
+	${DC} run --no-deps testpython3 ./docker/run_lint.sh
 
+.PHONY: test3
 test3: my.env .docker-build3
 	USEPYTHON=3 ./docker/run_tests_in_docker.sh ${ARGS}
 
+.PHONY: testshell3
 testshell3: my.env .docker-build3
 	USEPYTHON=3 ./docker/run_tests_in_docker.sh --shell

docker-compose.yml

@@ -3,28 +3,23 @@
 # Note: Requires docker-comopse 1.10+.
 version: "2"
 services:
-  # Socorro base image
-  base:
+  # Socorro app image
+  app:
     build:
       context: .
       dockerfile: ./docker/Dockerfile
-    image: local/socorro_base
-
-  # Docs building container
-  docs:
-    build:
-      context: .
-      dockerfile: ./docker/Dockerfile.docs
-    image: local/socorro_docs
+    image: local/socorro_app
+    env_file:
+      - docker/config/local_dev.env
+      - docker/config/never_on_a_server.env
+      - my.env
     volumes:
       - .:/app
 
-  # Processor app
+  # For development
+
   processor:
-    build:
-      context: .
-      dockerfile: ./docker/Dockerfile.processor
-    image: local/socorro_processor
+    image: local/socorro_app
     env_file:
       - docker/config/local_dev.env
       - docker/config/never_on_a_server.env
@@ -34,15 +29,26 @@ services:
       - localstack-s3
       - elasticsearch
       - rabbitmq
+    command: ["processor"]
+    volumes:
+      - .:/app
+
+  crontabber:
+    image: local/socorro_app
+    env_file:
+      - docker/config/local_dev.env
+      - docker/config/never_on_a_server.env
+      - my.env
+    depends_on:
+      - statsd
+      - postgresql
+      - elasticsearch
+    command: ["crontabber"]
     volumes:
       - .:/app
 
-  # Webapp app
   webapp:
-    build:
-      context: .
-      dockerfile: ./docker/Dockerfile.webapp
-    image: local/socorro_webapp
+    image: local/socorro_app
     env_file:
       - docker/config/local_dev.env
       - docker/config/never_on_a_server.env
@@ -55,12 +61,20 @@ services:
       - rabbitmq
       - memcached
       - oidcprovider
-    command: ["/app/docker/run_webapp.sh", "--dev"]
+    command: ["webapp", "--dev"]
     ports:
       - "8000:8000"
     volumes:
       - .:/app
 
+  # Webpack watcher
+  webpack:
+    image: local/socorro_app
+    working_dir: /app/webapp-django/
+    command: ["webpack"]
+    volumes:
+      - .:/app
+
   # OpenID Connect provider
   oidcprovider:
     build:
@@ -70,33 +84,12 @@ services:
     ports:
       - "8080:8080"
 
-  # Webpack watcher
-  webpack:
-    image: local/socorro_webapp
-    env_file:
-      - docker/config/local_dev.env
-      - docker/config/never_on_a_server.env
-      - my.env
-    working_dir: /app/webapp-django/
-    command: ["/webapp-frontend-deps/node_modules/.bin/webpack", "--watch"]
-    volumes:
-      - .:/app
-
-  # Crontabber app
-  crontabber:
+  # Docs building container
+  docs:
     build:
       context: .
-      dockerfile: ./docker/Dockerfile.crontabber
-    image: local/socorro_crontabber
-    env_file:
-      - docker/config/local_dev.env
-      - docker/config/never_on_a_server.env
-      - my.env
-    depends_on:
-      - localstack-s3
-      - postgresql
-      - elasticsearch
-      - rabbitmq
+      dockerfile: ./docker/Dockerfile.docs
+    image: local/socorro_docs
     volumes:
       - .:/app
 

docker/Dockerfile

@@ -1,5 +1,6 @@
 FROM python:2.7.14-slim
 
+# Set up user and group
 ARG groupid=10001
 ARG userid=10001
 
@@ -11,6 +12,22 @@ RUN groupadd --gid $groupid app && \
 COPY ./docker/set_up_ubuntu.sh /tmp/
 RUN DEBIAN_FRONTEND=noninteractive /tmp/set_up_ubuntu.sh
 
+# Install breakpad and stackwalk bits
+COPY ./scripts/build-breakpad.sh /tmp/scripts/
+COPY ./scripts/build-stackwalker.sh /tmp/scripts/
+COPY ./minidump-stackwalk/ /tmp/minidump-stackwalk/
+RUN STACKWALKDIR=/stackwalk SRCDIR=/tmp /tmp/scripts/build-stackwalker.sh
+
+# Install frontend JS deps
+COPY ./webapp-django/package.json /webapp-frontend-deps/package.json
+COPY ./webapp-django/package-lock.json /webapp-frontend-deps/package-lock.json
+RUN cd /webapp-frontend-deps/ && npm install
+
+# Install crontabber JS deps
+COPY ./crontabber/package.json /crontabber-deps/package.json
+COPY ./crontabber/package-lock.json /crontabber-deps/package-lock.json
+RUN cd /crontabber-deps/ && npm install
+
 # Install Socorro Python requirements
 COPY ./requirements /app/requirements
 RUN pip install -U 'pip>=8' && \
@@ -19,3 +36,30 @@ RUN pip install -U 'pip>=8' && \
 ENV PYTHONUNBUFFERED 1
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONPATH /app
+
+ENV WEBPACK_BINARY /webapp-frontend-deps/node_modules/.bin/webpack
+ENV LESS_BINARY /webapp-frontend-deps/node_modules/.bin/lessc
+ENV UGLIFYJS_BINARY /webapp-frontend-deps/node_modules/.bin/uglifyjs
+ENV CSSMIN_BINARY /webapp-frontend-deps/node_modules/.bin/cssmin
+ENV NPM_ROOT_PATH /webapp-frontend-deps/
+ENV NODE_PATH /webapp-frontend-deps/node_modules/
+
+# Copy everything over
+COPY . /app/
+
+# Build webpack-defined frontend bundles
+RUN cd /app/webapp-django/ && $WEBPACK_BINARY --mode=production --bail
+
+# Run collectstatic in container which puts files in the default place for
+# static files
+RUN cd /app/webapp-django/ && python manage.py collectstatic --noinput
+
+USER app
+
+# Build tmp directories for minidump stackwalker--owned by app
+RUN mkdir -p /tmp/symbols/cache
+RUN mkdir -p /tmp/symbols/tmp
+
+# Set entrypoint for this image. The entrypoint script takes a service
+# to run as the first argument. See the script for available arguments.
+ENTRYPOINT ["/app/docker/socorro_entrypoint.sh"]