Merge pull request #295 from willkg/1531865-redo-tests

fix bug 1531865: redo tests and configuration

.circleci/config.yml

@@ -3,45 +3,65 @@ jobs:
   build_test_push:
     docker:
       - image: docker:stable
+    working_directory: /
     environment:
       APP_NAME: "socorro_collector"
     steps:
       - run:
-          name: Install checkout deps
+          name: Install essential packages
           command: |
-            set -x
             apk update
-            apk add git openssh
+            apk add --update --no-cache \
+                    ca-certificates \
+                    build-base \
+                    bash \
+                    make \
+                    git \
+                    openssh \
+                    docker \
+                    py-pip
+            pip install docker-compose
 
-      - checkout
-      - setup_remote_docker
+      - checkout:
+          path: /antenna
 
       - run:
           name: Create version.json
+          working_directory: /antenna
           command: |
             # create a version.json per https://github.com/mozilla-services/Dockerflow/blob/master/docs/version_object.md
             printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \
             "$CIRCLE_SHA1" \
             "$CIRCLE_TAG" \
             "$CIRCLE_PROJECT_USERNAME" \
             "$CIRCLE_PROJECT_REPONAME" \
-            "$CIRCLE_BUILD_URL" > version.json
+            "$CIRCLE_BUILD_URL" > /antenna/version.json
+
+      - store_artifacts:
+          path: /antenna/version.json
+
+      - setup_remote_docker
 
       - run:
           name: Build Docker image
+          working_directory: /antenna
           command: |
             docker info
-            docker build -t ${APP_NAME}:build .
+            make build
 
       - run:
-          name: Run tests in Docker image
-          command: |
-            docker run ${APP_NAME}:build py.test
-            docker run ${APP_NAME}:build flake8
-            docker run ${APP_NAME}:build bandit -r antenna/
+          name: Lint
+          working_directory: /antenna
+          command: docker run local/antenna_deploy_base /bin/bash ./docker/run_lint.sh
+
+      - run:
+          name: Run tests
+          working_directory: /antenna
+          command: make test
 
       - run:
           name: Push to Dockerhub
+          working_directory: /antenna
           command: |
             function retry {
                 set +e

.gitignore

@@ -52,3 +52,5 @@ target/
 # Docker things
 /.docker-build
 /fakes3_root/
+docker-compose.override.yml
+my.env

Makefile

@@ -1,6 +1,25 @@
-ANTENNA_ENV ?= "dev.env"
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Include my.env and export it so variables set in there are available
+# in Makefile.
+include my.env
+export
+
+# Set these in the environment to override them. This is helpful for
+# development if you have file ownership problems because the user
+# in the container doesn't match the user on your host.
+ANTENNA_UID ?= 10001
+ANTENNA_GID ?= 10001
+
+# Set this in the environment to force --no-cache docker builds.
+DOCKER_BUILD_OPTS :=
+ifeq (1, ${NOCACHE})
+DOCKER_BUILD_OPTS := --no-cache
+endif
+
 DC := $(shell which docker-compose)
-HOSTUSER := $(shell id -u):$(shell id -g)
 
 .PHONY: help
 help: default
@@ -14,31 +33,39 @@ default:
 	@echo "clean            - remove all build, test, coverage and Python artifacts"
 	@echo "lint             - check style with flake8"
 	@echo "test             - run unit tests"
+	@echo "testshell        - open a shell in the test container"
 	@echo "systemtest       - run system tests against a running Antenna instance"
 	@echo "systemtest-shell - open a shell in the systemtest container"
 	@echo "test-coverage    - run tests and generate coverage report in cover/"
 	@echo "docs             - generate Sphinx HTML documentation, including API docs"
 	@echo ""
-	@echo "Set ANTENNA_ENV=/path/to/env/file for configuration."
+	@echo "Adjust your my.env file to set configuration."
 
 # Dev configuration steps
 .docker-build:
 	make build
 
+my.env:
+	@if [ ! -f my.env ]; \
+	then \
+	echo "Copying my.env.dist to my.env..."; \
+	cp docker/config/my.env.dist my.env; \
+	fi
+
 .PHONY: build
-build:
-	ANTENNA_ENV=empty.env ${DC} build deploy-base
+build: my.env
+	${DC} build ${DOCKER_BUILD_OPTS} --build-arg userid=${ANTENNA_UID} --build-arg groupid=${ANTENNA_GID} deploy-base
 	touch .docker-build
 
 .PHONY: run
-run: .docker-build
-	ANTENNA_ENV=${ANTENNA_ENV} ${DC} up web
+run: my.env .docker-build
+	${DC} up web
 
 .PHONY: shell
-shell: .docker-build
-	ANTENNA_ENV=empty.env ${DC} run base bash
+shell: my.env .docker-build
+	${DC} run base bash
 
-.PHONY: clean
+.PHONY: my.env clean
 clean:
 	# python related things
 	-rm -rf build/
@@ -60,26 +87,29 @@ clean:
 	-rm .docker-build
 
 .PHONY: lint
-lint: .docker-build
-	ANTENNA_ENV=empty.env ${DC} run base flake8 --statistics antenna tests/unittest/
-	ANTENNA_ENV=empty.env ${DC} run base bandit -r antenna/
+lint: my.env .docker-build
+	${DC} run --rm --no-deps base  /bin/bash ./docker/run_lint.sh
 
 .PHONY: test
-test: .docker-build
-	ANTENNA_ENV=empty.env ${DC} run base py.test
+test: my.env .docker-build
+	./docker/run_tests_in_docker.sh ${ARGS}
+
+.PHONY: testshell
+testshell: my.env .docker-build
+	./docker/run_tests_in_docker.sh --shell
 
 .PHONY: systemtest
-systemtest: .docker-build
-	ANTENNA_ENV=dev.env ${DC} run systemtest tests/systemtest/run_tests.sh
+systemtest: my.env .docker-build
+	${DC} run systemtest tests/systemtest/run_tests.sh
 
 .PHONY: systemtest-shell
-systemtest-shell: .docker-build
-	ANTENNA_ENV=dev.env ${DC} run systemtest bash
+systemtest-shell: my.env .docker-build
+	${DC} run systemtest bash
 
 .PHONY: test-coverage
-test-coverage: .docker-build
-	ANTENNA_ENV=empty.env ${DC} run base py.test --cov=antenna --cov-report term-missing
+test-coverage: my.env .docker-build
+	${DC} run base py.test --cov=antenna --cov-report term-missing
 
 .PHONY: docs
-docs: .docker-build
-	ANTENNA_ENV=empty.env ${DC} run -u ${HOSTUSER} base ./bin/build_docs.sh
+docs: my.env .docker-build
+	${DC} run -u ${ANTENNA_UID} base ./bin/build_docs.sh

README.rst

@@ -55,7 +55,7 @@ production, see docs_.
 
       You should see a lot of output. It'll start out with something like this::
 
-         ANTENNA_ENV="dev.env" /usr/bin/docker-compose up web
+         /usr/bin/docker-compose up web
          antenna_statsd_1 is up-to-date
          antenna_localstack-s3_1 is up-to-date
          Recreating antenna_web_1
@@ -146,15 +146,10 @@ production, see docs_.
       docker web container.
 
 
-   If you want to run with a different Antenna configuration, put the
-   configuration in an env file and then set ``ANTENNA_ENV``. For example:
+   If you want to run with a different Antenna configuration in the local
+   dev environment, adjust your ``my.env`` file.
 
-   .. code-block:: shell
-
-      $ ANTENNA_ENV=my.env make run
-
-
-   See ``dev.env`` and the docs_ for configuration options.
+   See docs_ for configuration options.
 
 5. Run tests:
 

antenna/app.py

@@ -4,11 +4,10 @@
 
 import logging
 import logging.config
-import os
 from pathlib import Path
 import socket
 
-from everett.manager import ConfigManager, ConfigEnvFileEnv, ConfigOSEnv, ListOf, parse_class
+from everett.manager import ConfigManager, ConfigOSEnv, ListOf, parse_class
 from everett.component import ConfigOptions, RequiredConfigMixin
 import falcon
 import markus
@@ -292,8 +291,6 @@ def get_app(config=None):
     if config is None:
         config = ConfigManager(
             environments=[
-                # Pull configuration from env file specified as ANTENNA_ENV
-                ConfigEnvFileEnv([os.environ.get('ANTENNA_ENV')]),
                 # Pull configuration from environment variables
                 ConfigOSEnv()
             ],

bin/create_s3_bucket.py

@@ -48,9 +48,6 @@ def _log_everything():
 def main(args):
     # Build configuration object just like we do in Antenna.
     config = ConfigManager([
-        # Pull configuration from env file specified as ANTENNA_ENV
-        ConfigEnvFileEnv([os.environ.get('ANTENNA_ENV')]),
-
         # Pull configuration from environment variables
         ConfigOSEnv()
     ])

docker-compose.yml

@@ -8,7 +8,7 @@ services:
   deploy-base:
     build:
       context: .
-      dockerfile: Dockerfile
+      dockerfile: docker/Dockerfile
     image: local/antenna_deploy_base
 
   # ------------------------------------------------------------------
@@ -28,8 +28,9 @@ services:
   web:
     extends:
       service: base
-    environment:
-      - ANTENNA_ENV=${ANTENNA_ENV}
+    env_file:
+      - docker/config/local_dev.env
+      - my.env
     ports:
       - "8000:8000"
     command: ./bin/run_web.sh
@@ -41,8 +42,10 @@ services:
   systemtest:
     extends:
       service: base
+    env_file:
+      - docker/config/local_dev.env
+      - my.env
     environment:
-      - ANTENNA_ENV=${ANTENNA_ENV}
       - POSTURL=http://web:8000/submit
       - NONGINX=1
     links:

Dockerfile → docker/Dockerfile

@@ -1,7 +1,12 @@
-FROM python:3.6.2-slim
+FROM python:3.6.8-slim
+
+# Set up user and group
+ARG groupid=10001
+ARG userid=10001
 
 WORKDIR /app/
-RUN groupadd --gid 10001 app && useradd -g app --uid 10001 --shell /usr/sbin/nologin app
+RUN groupadd --gid $groupid app && \
+    useradd -g app --uid $userid --shell /usr/sbin/nologin app
 
 RUN apt-get update && \
     apt-get install -y gcc apt-transport-https build-essential graphviz

docker/config/my.env.dist

@@ -0,0 +1,12 @@
+# This file is for settings that are specific to your docker local development
+# environment. Since these are specific to your local development environment,
+# don't check them in.
+
+# ---------------------------------------------
+# Docker settings
+# ---------------------------------------------
+
+# If you want to set the uid and gid of the app user that we use in the
+# containers, you can do that with these two variables.
+# ANTENNA_UID=
+# ANTENNA_GID=

docker/run_lint.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Runs linting.
+
+# This should be called from inside a container
+
+set -e
+
+echo ">>> flake8"
+flake8 --statistics antenna tests/unittest/
+
+echo ">>> bandit"
+bandit -r antenna/

docker/run_tests.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Runs tests.
+#
+# This should be called from inside a container and after the dependent
+# services have been launched. It depends on:
+#
+# * elasticsearch
+# * postgresql
+# * rabbitmq
+
+# Failures should cause setup to fail
+set -v -e -x
+
+echo ">>> pytest"
+# Set up environment variables
+
+LOCALSTACK_S3_URL=http://localstack-s3:5000
+
+export PYTHONPATH=/app/:$PYTHONPATH
+PYTEST="$(which pytest)"
+PYTHON="$(which python)"
+
+# Wait for services to be ready
+urlwait "${LOCALSTACK_S3_URL}" 10
+
+# Run tests
+"${PYTEST}"