Revert "Celery 4"

Makefile

@@ -2,6 +2,7 @@
 SHELL := /bin/bash
 DATE = $(shell date +%Y-%m-%d:%H:%M:%S)
 
+PIP_ACCEL_CACHE ?= ${CURDIR}/cache/pip-accel
 APP_VERSION_FILE = app/version.py
 
 GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached")
@@ -33,19 +34,11 @@ help:
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
 .PHONY: venv
-# if there's a virtualenv already active, we don't want to do anything
-ifeq ($(VIRTUAL_ENV),)
 venv: venv/bin/activate ## Create virtualenv if it does not exist
 
 venv/bin/activate:
 	test -d venv || virtualenv venv -p python3
-
-VENV_BIN_DIR = $(shell . venv/bin/activate && echo $$VIRTUAL_ENV/bin)
-else
-venv:
-
-VENV_BIN_DIR = $(VIRTUAL_ENV)/bin
-endif
+	. venv/bin/activate && pip install pip-accel
 
 .PHONY: check-env-vars
 check-env-vars: ## Check mandatory environment variables
@@ -72,17 +65,17 @@ production: ## Set environment to production
 	@true
 
 .PHONY: dependencies
-dependencies: venv
-	$(call install-pycurl, $(VENV_BIN_DIR)/pip)
-	$(VENV_BIN_DIR)/pip install -r requirements_for_test.txt
+dependencies: venv ## Install build dependencies
+	mkdir -p ${PIP_ACCEL_CACHE}
+	. venv/bin/activate && PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} pip-accel install -r requirements_for_test.txt
 
 .PHONY: generate-version-file
 generate-version-file: ## Generates the app version file
 	@echo -e "__travis_commit__ = \"${GIT_COMMIT}\"\n__time__ = \"${DATE}\"\n__travis_job_number__ = \"${BUILD_NUMBER}\"\n__travis_job_url__ = \"${BUILD_URL}\"" > ${APP_VERSION_FILE}
 
 .PHONY: build
 build: dependencies generate-version-file ## Build project
-
+	. venv/bin/activate && PIP_ACCEL_CACHE=${PIP_ACCEL_CACHE} pip-accel install -r requirements.txt
 
 .PHONY: build-paas-artifact
 build-paas-artifact:  ## Build the deploy artifact for PaaS
@@ -104,7 +97,6 @@ test: venv generate-version-file ## Run tests
 freeze-requirements:
 	rm -rf venv-freeze
 	virtualenv -p python3 venv-freeze
-	$(call install-pycurl, $$(pwd)/venv-freeze/bin/pip)
 	$$(pwd)/venv-freeze/bin/pip install -r requirements-app.txt
 	echo '# pyup: ignore file' > requirements.txt
 	echo '# This file is autogenerated. Do not edit it manually.' >> requirements.txt
@@ -113,15 +105,6 @@ freeze-requirements:
 	$$(pwd)/venv-freeze/bin/pip freeze -r <(sed '/^--/d' requirements-app.txt) | sed -n '/The following requirements were added by pip freeze/,$$p' >> requirements.txt
 	rm -rf venv-freeze
 
-define install-pycurl
-	# install pycurl separately to avoid flags disabling wheels for other packages
-	PYCURL_SSL_LIBRARY=openssl ${1} install pycurl==7.43.0.2 --global-option="build_ext" --global-option="-I/usr/local/opt/openssl/include"
-endef
-
-.PHONY: install-pycurl
-install-pycurl:
-	$(call install-pycurl, pip)
-
 .PHONY: test-requirements
 test-requirements:
 	@diff requirements-app.txt requirements.txt | grep '<' \
@@ -131,17 +114,19 @@ test-requirements:
 
 .PHONY: coverage
 coverage: venv ## Create coverage report
-	coveralls
+	. venv/bin/activate && coveralls
 
 .PHONY: prepare-docker-build-image
 prepare-docker-build-image: ## Prepare the Docker builder image
+	mkdir -p ${PIP_ACCEL_CACHE}
 	make -C docker build
 
 .PHONY: build-with-docker
 build-with-docker: prepare-docker-build-image ## Build inside a Docker container
 	@docker run -i${DOCKER_TTY} --rm \
 		--name "${DOCKER_CONTAINER_PREFIX}-build" \
 		-v "`pwd`:/var/project" \
+		-v "${PIP_ACCEL_CACHE}:/var/project/cache/pip-accel" \
 		-e UID=$(shell id -u) \
 		-e GID=$(shell id -g) \
 		-e GIT_COMMIT=${GIT_COMMIT} \

app/config.py

@@ -150,9 +150,9 @@ class Config(object):
     BROKER_URL = 'sqs://'
     BROKER_TRANSPORT_OPTIONS = {
         'region': AWS_REGION,
+        'polling_interval': 1,  # 1 second
         'visibility_timeout': 310,
-        'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX,
-        'wait_time_seconds': 20  # enable long polling, with a wait time of 20 seconds
+        'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX
     }
     CELERY_ENABLE_UTC = True
     CELERY_TIMEZONE = 'Europe/London'
@@ -270,9 +270,7 @@ class Config(object):
             'options': {'queue': QueueNames.PERIODIC}
         }
     }
-
-    # this is overriden by the -Q command, but locally, we should read from all queues
-    CELERY_QUEUES = [Queue(queue, Exchange('default'), routing_key=queue) for queue in QueueNames.all_queues()]
+    CELERY_QUEUES = []
 
     NOTIFICATIONS_ALERT = 5  # five mins
     FROM_NUMBER = 'development'
@@ -360,6 +358,11 @@ class Development(Config):
     STATSD_PORT = 1000
     STATSD_PREFIX = "stats-prefix"
 
+    for queue in QueueNames.all_queues():
+        Config.CELERY_QUEUES.append(
+            Queue(queue, Exchange('default'), routing_key=queue)
+        )
+
     API_HOST_NAME = "http://localhost:6011"
     API_RATE_LIMIT_ENABLED = True
 
@@ -382,6 +385,11 @@ class Test(Development):
 
     BROKER_URL = 'you-forgot-to-mock-celery-in-your-tests://'
 
+    for queue in QueueNames.all_queues():
+        Config.CELERY_QUEUES.append(
+            Queue(queue, Exchange('default'), routing_key=queue)
+        )
+
     API_RATE_LIMIT_ENABLED = True
     API_HOST_NAME = "http://localhost:6011"
 

app/dao/notifications_dao.py

@@ -5,7 +5,7 @@
     timedelta,
 )
 
-from botocore.exceptions import ClientError as BotoClientError
+from boto.exception import BotoClientError
 from flask import current_app
 
 from notifications_utils.recipients import (

docker/Dockerfile

@@ -22,7 +22,6 @@ RUN \
 		libffi-dev \
 		python-dev \
 		jq \
-		libcurl4-openssl-dev \
 	&& echo "Clean up" \
 	&& rm -rf /var/lib/apt/lists/* /tmp/*
 

requirements-app.txt

@@ -2,7 +2,7 @@
 # with package version changes made in requirements-app.txt
 
 cffi==1.11.5
-celery[sqs]==4.2.1
+celery==3.1.26.post2 # pyup: <4
 docopt==0.6.2
 Flask-Bcrypt==0.7.1
 flask-marshmallow==0.9.0
@@ -29,4 +29,4 @@ botocore<1.11.0
 
 git+https://github.com/alphagov/notifications-utils.git@30.5.3#egg=notifications-utils==30.5.3
 
-# if you want to update pycurl please do so in makefile
+git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3

requirements.txt

@@ -4,7 +4,7 @@
 # with package version changes made in requirements-app.txt
 
 cffi==1.11.5
-celery[sqs]==4.2.1
+celery==3.1.26.post2 # pyup: <4
 docopt==0.6.2
 Flask-Bcrypt==0.7.1
 flask-marshmallow==0.9.0
@@ -31,15 +31,16 @@ botocore<1.11.0
 
 git+https://github.com/alphagov/notifications-utils.git@30.5.3#egg=notifications-utils==30.5.3
 
-# if you want to update pycurl please do so in makefile
+git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3
 
 ## The following requirements were added by pip freeze:
 alembic==1.0.0
-amqp==2.3.2
+amqp==1.4.9
+anyjson==0.3.3
 bcrypt==3.1.4
-billiard==3.5.0.4
+billiard==3.3.0.23
 bleach==2.1.3
-boto3==1.9.17
+boto3==1.6.16
 certifi==2018.8.24
 chardet==3.0.4
 Click==7.0
@@ -53,7 +54,7 @@ idna==2.7
 itsdangerous==0.24
 Jinja2==2.10
 jmespath==0.9.3
-kombu==4.2.1
+kombu==3.0.37
 Mako==1.0.7
 MarkupSafe==1.0
 mistune==0.8.3
@@ -62,7 +63,6 @@ orderedset==2.0.1
 phonenumbers==8.9.4
 pyasn1==0.4.4
 pycparser==2.19
-pycurl==7.43.0.2
 PyPDF2==1.26.0
 python-dateutil==2.7.3
 python-editor==1.0.3
@@ -77,6 +77,5 @@ six==1.11.0
 smartypants==2.0.1
 statsd==3.2.2
 urllib3==1.23
-vine==1.1.4
 webencodings==0.5.1
 Werkzeug==0.14.1

scripts/bootstrap.sh

@@ -20,7 +20,16 @@ function display_result {
   fi
 }
 
-make build
+if [ ! $VIRTUAL_ENV ]; then
+  virtualenv -p python3 ./venv
+  . ./venv/bin/activate
+fi
+
+# we need the version file to exist otherwise the app will blow up
+make generate-version-file
+
+# Install Python development dependencies
+pip3 install -r requirements_for_test.txt
 
 # Create Postgres databases
 createdb notification_api