diff --git a/.gitignore b/.gitignore index 6e3c78a1..fea4119d 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ tmp/* # vim swap files .*.swp +venv/ diff --git a/.python-version b/.python-version new file mode 100644 index 00000000..ecc17b8e --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +2.7.13 diff --git a/.travis.yml b/.travis.yml index 663e8205..7985bdeb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,27 +3,61 @@ language: python python: - "2.7" cache: pip -sudo: false +sudo: true +notifications: + email: false + # Need mongodb for testing services: mongodb # command to install dependencies + install: - pip install --upgrade pip - easy_install distribute - pip install --upgrade distribute - pip install --upgrade setuptools - pip install -q -r requirements_for_tests.txt -# command to run tests + env: - - SKIP_VIRUS_SCAN=1 SKIP_SPLINTER_TESTS=1 MONGO_REPLICA_SET='' NO_AUTOPEP8=1 + global: + - PAAS_USER: pp-deploy@digital.cabinet-office.gov.uk + # PAAS_PASSWORD + - secure: HNo6rnNiXmNYAV1TvkcMxeniXpmHoZkNEiT+X9EgyInCOnVm4mz89qi/AElY/ESE5HdYIegJ5l1M5tAWCei4dSDSTSfkbBBiHE6dfpf1OSU857oQ6T6161NRg4V2oxNdWSRgAX+G8rpxBShaXCDU9q9iBxhsqrmTX+UGlzKloDk= + - REDIS_DATABASE_NUMBER_STAGING: 4 + - REDIS_DATABASE_NUMBER_PRODUCTION: 5 + # APP_SIGNON_API_USER_TOKEN_STAGING + - secure: PYLrm6uH6uhNA69xgI5lSXLrX0ouPBdxlMlQXER0K2BkHzxTowNDFyf5LDa2kwIueJ/YXFHAgm7eBN9gr3JIXAN8Mqb15LTHJVg6hc2aLwFZKES4fuPa/da3Pn24xrdTesIedUAv2vSkuacaktZwKxsv6LENw53uA+8kOT8l9SQ= + # APP_SIGNON_API_USER_TOKEN_PRODUCTION + - secure: SBk+7zw2GMkgbl8p7hwXZw5VkDf2FQE4kBklD40c7goORyEF3/FjTVJTVlREN4Rwb3a81N5lkQDl0fLPhGaDv+PyTr+l69IU64w1Whu9e9fL/wMC3zo5Eer3Btptar5jdCsEg+V4woKKXv6j4B4xebHN6sU7C6dX9TGKs+R1aBo= + # APP_SECRET_KEY_STAGING + - secure: Cl7cvdzviF3hCepM7n3XMz9q1D/FCEQkcDZOyRTPkHlm9V2GKi+e9ECuAuWQVRiRDTb7bAZZ5kl+zTj00nMb0BKds3QoisZ5oB8UwYFOQSrYV/NTpge0wsvZZvfqXp5sN3CLVXE2XynLsZZSzOTIte8UKD4giBFsuHo2sBcVbb8= + # APP_SECRET_KEY_PRODUCTION + - secure: ifAsLdVTrzDT8IIEVdW1TJon8VYdmDdzxxE/CtCgc0hcBP9eneKF7qvBOy6TFMnE9KSz4zzyYx+yZI8s5sAsjknw/HEnHYMCYSjRcM3GMWP27/QFM2983CgWkoEn5xGj3WEXra1WJGltzqbeoYfZWmFA9pBGSU6IbR0IzjLC6+g= + # APP_STAGECRAFT_COLLECTION_ENDPOINT_TOKEN + # Value too large to be encrypted. + # APP_STAGECRAFT_OAUTH_TOKEN_STAGING + - secure: JkoKVPblX1sHICKxJMT0kPi5Nax3dn3cG1bzmAmpMDX+rd37I4XtA96G17ibUmUrp2CVrOo4is5WjI9hPDwmC8/o7/6MOHqJo7jeflY+Iq8bWShcja0khlb4aBlpWfgMS+vniLCBPkQtUe2eE87jFLYJXmsuBGfp4r4L88fhgOk= + # APP_STAGECRAFT_OAUTH_TOKEN_PRODUCTION + - secure: HZPkW81a+OsfFrCCpcuYYBKNIEmejsQlIreVo2AJ1NipBVK8DniqIK+bXGUIQraaqeKXApXknfPepVpyqG3uyzHvkboylooWa7fuoJB4bs9Eb91Qo2AeWjP2fts0EmUX8R76K51n7EwUgk9xemM/bE2e+0FmG5e35JpXEwfm/5I= + script: - ./run_tests.sh + after_script: - coveralls -branches: - except: - - release - - /^release_[0-9]+$/ - - /^deployed-to-(preview|staging|production)$/ -notifications: - email: false + +before_deploy: + - chmod +x etc/deploy.sh + +deploy: + - provider: script + script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_STAGING APP_SECRET_KEY=$APP_SECRET_KEY_STAGING REDIS_DATABASE_NUMBER=$REDIS_DATABASE_NUMBER_STAGING APP_STAGECRAFT_OAUTH_TOKEN=$APP_STAGECRAFT_OAUTH_TOKEN_STAGING etc/deploy.sh staging + skip_cleanup: true + on: + branch: staging + + - provider: script + script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_PRODUCTION APP_SECRET_KEY=$APP_SECRET_KEY_PRODUCTION REDIS_DATABASE_NUMBER=$REDIS_DATABASE_NUMBER_PRODUCTION APP_STAGECRAFT_OAUTH_TOKEN=$APP_STAGECRAFT_OAUTH_TOKEN_PRODUCTION etc/deploy.sh production + skip_cleanup: true + on: + branch: production diff --git a/Procfile b/Procfile deleted file mode 100644 index 5e28441f..00000000 --- a/Procfile +++ /dev/null @@ -1,2 +0,0 @@ -web: venv/bin/gunicorn -c /etc/gds/${APP_NAME}/gunicorn ${APP_MODULE} -worker: venv/bin/celery -A backdrop.transformers.worker worker -l debug diff --git a/backdrop/core/config/__init__.py b/backdrop/core/config/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backdrop/core/config/common.py b/backdrop/core/config/common.py new file mode 100644 index 00000000..85076bbf --- /dev/null +++ b/backdrop/core/config/common.py @@ -0,0 +1,24 @@ +import os +import json +from base64 import b64decode + + +def load_paas_settings(): + paas = {} + if 'VCAP_SERVICES' in os.environ: + vcap = json.loads(os.environ['VCAP_SERVICES']) + if 'mongodb' in vcap: + for service in vcap['mongodb']: + if service['name'] == 'gds-performance-platform-mongodb-service': + credentials = service['credentials'] + paas['DATABASE_URL'] = credentials['uri'] + ca_cert = b64decode(credentials['ca_certificate_base64']) + paas['CA_CERTIFICATE'] = ca_cert + if 'REDIS_DATABASE_NUMBER' in os.environ: + for service in vcap['user-provided']: + if service['name'] == 'redis-poc': + database_number = os.environ['REDIS_DATABASE_NUMBER'] + url = service['credentials']['url'] + url += '/' + database_number + paas['REDIS_URL'] = url + return paas diff --git a/backdrop/core/storage/mongo.py b/backdrop/core/storage/mongo.py index 7f1fe536..1c384d88 100644 --- a/backdrop/core/storage/mongo.py +++ b/backdrop/core/storage/mongo.py @@ -39,23 +39,6 @@ def convert_datetimes_to_utc(result): return dict((key, time_as_utc(value)) for key, value in result.items()) -def get_mongo_client(hosts, port): - """Return an appropriate mongo client - """ - client_list = ','.join(':'.join([host, str(port)]) for host in hosts) - - # We can't always guarantee we'll be on 'production' - # so we allow jenkins to add the set as a variable - # Some test environments / other envs have their own sets e.g. 'gds-ci' - replica_set = os.getenv('MONGO_REPLICA_SET', 'production') - - if replica_set == '': - return pymongo.MongoClient(client_list) - else: - return pymongo.MongoReplicaSetClient( - client_list, replicaSet=replica_set) - - def reconnecting_save(collection, record, tries=3): """Save to mongo, retrying if necesarry """ @@ -89,18 +72,32 @@ def reconnecting_save(collection, record, tries=3): class MongoStorageEngine(object): @classmethod - def create(cls, hosts, port, database): - return cls(get_mongo_client(hosts, port), database) + def create(cls, database_url, ca_certificate=None): + if ca_certificate is not None: + dir = os.path.dirname(__file__) + filename = os.path.join(dir, 'mongodb.crt') + f = open(filename, 'w') + f.write(ca_certificate) + f.close() + + mongo_client = pymongo.MongoClient( + database_url, + ssl_ca_certs=filename + ) + else: + mongo_client = pymongo.MongoClient(database_url) + + return cls(mongo_client) - def __init__(self, mongo, database): - self._mongo = mongo - self._db = mongo[database] + def __init__(self, mongo_client): + self._mongo_client = mongo_client + self._db = mongo_client.get_database() def _collection(self, data_set_id): return self._db[data_set_id] def alive(self): - return self._mongo.alive() + return self._mongo_client.alive() def data_set_exists(self, data_set_id): return data_set_id in self._db.collection_names() diff --git a/backdrop/read/api.py b/backdrop/read/api.py index 46741db6..dd7d49a1 100644 --- a/backdrop/read/api.py +++ b/backdrop/read/api.py @@ -19,7 +19,7 @@ from ..core.storage.mongo import MongoStorageEngine from ..core.timeutils import as_utc -GOVUK_ENV = getenv("GOVUK_ENV", "development") +ENVIRONMENT = getenv("ENVIRONMENT", "development") app = Flask("backdrop.read.api") @@ -27,12 +27,12 @@ # Configuration app.config.from_object( - "backdrop.read.config.{}".format(GOVUK_ENV)) + "backdrop.read.config.{}".format(ENVIRONMENT)) storage = MongoStorageEngine.create( - app.config['MONGO_HOSTS'], - app.config['MONGO_PORT'], - app.config['DATABASE_NAME']) + app.config['DATABASE_URL'], + app.config.get('CA_CERTIFICATE') +) admin_api = client.AdminAPI( app.config['STAGECRAFT_URL'], @@ -46,7 +46,7 @@ DEFAULT_DATA_SET_PUBLISHED = True DEFAULT_DATA_SET_REALTIME = False -log_handler.set_up_logging(app, GOVUK_ENV) +log_handler.set_up_logging(app, ENVIRONMENT) class JsonEncoder(json.JSONEncoder): @@ -91,12 +91,7 @@ def http_error_handler(e): @cache_control.nocache @statsd.timer('read.route.heath_check.status') def health_check(): - - if not storage.alive(): - return jsonify(status='error', - message='cannot connect to database'), 500 - - return jsonify(status='ok', message='database is up') + return jsonify(status='ok', message='app is up') @app.route('/_status/data-sets', methods=['GET']) diff --git a/backdrop/read/config/development.py b/backdrop/read/config/development.py index f90ee0b6..613e305e 100644 --- a/backdrop/read/config/development.py +++ b/backdrop/read/config/development.py @@ -1,6 +1,4 @@ -DATABASE_NAME = "backdrop" -MONGO_HOSTS = ['localhost'] -MONGO_PORT = 27017 +DATABASE_URL = 'mongodb://localhost:27017/backdrop_development' LOG_LEVEL = "DEBUG" STAGECRAFT_URL = 'http://localhost:3103' diff --git a/backdrop/read/config/production.py b/backdrop/read/config/production.py new file mode 100644 index 00000000..3ded5859 --- /dev/null +++ b/backdrop/read/config/production.py @@ -0,0 +1,10 @@ +import os +from ...core.config.common import load_paas_settings + +PAAS = load_paas_settings() +DATABASE_URL = PAAS.get('DATABASE_URL') +CA_CERTIFICATE = PAAS.get('CA_CERTIFICATE') +STAGECRAFT_URL = os.getenv('STAGECRAFT_URL') +SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN') +LOG_LEVEL = "INFO" +SESSION_COOKIE_SECURE = True diff --git a/backdrop/read/config/staging.py b/backdrop/read/config/staging.py new file mode 100644 index 00000000..3ded5859 --- /dev/null +++ b/backdrop/read/config/staging.py @@ -0,0 +1,10 @@ +import os +from ...core.config.common import load_paas_settings + +PAAS = load_paas_settings() +DATABASE_URL = PAAS.get('DATABASE_URL') +CA_CERTIFICATE = PAAS.get('CA_CERTIFICATE') +STAGECRAFT_URL = os.getenv('STAGECRAFT_URL') +SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN') +LOG_LEVEL = "INFO" +SESSION_COOKIE_SECURE = True diff --git a/backdrop/read/config/test.py b/backdrop/read/config/test.py index 11afd25d..885720bb 100644 --- a/backdrop/read/config/test.py +++ b/backdrop/read/config/test.py @@ -1,6 +1,4 @@ -DATABASE_NAME = "backdrop_test" -MONGO_HOSTS = ['localhost'] -MONGO_PORT = 27017 +DATABASE_URL = 'mongodb://localhost:27017/backdrop_test' LOG_LEVEL = "ERROR" DATA_SET_RATE_LIMIT = '10000/second' diff --git a/backdrop/transformers/config/development.py b/backdrop/transformers/config/development.py index 788ae55d..41f948d2 100644 --- a/backdrop/transformers/config/development.py +++ b/backdrop/transformers/config/development.py @@ -1,4 +1,4 @@ -TRANSFORMER_AMQP_URL = 'amqp://backdrop_write:backdrop_write@localhost:5672/%2Fbackdrop_write' +BROKER_URL = 'amqp://backdrop_write:backdrop_write@localhost:5672/%2Fbackdrop_write' STAGECRAFT_URL = 'http://localhost:3103' STAGECRAFT_OAUTH_TOKEN = 'development-oauth-access-token' BACKDROP_READ_URL = 'http://backdrop-read.dev.gov.uk/data' diff --git a/backdrop/transformers/config/production.py b/backdrop/transformers/config/production.py new file mode 100644 index 00000000..2249b8b1 --- /dev/null +++ b/backdrop/transformers/config/production.py @@ -0,0 +1,10 @@ +import os +from ...core.config.common import load_paas_settings + +PAAS = load_paas_settings() +BROKER_URL = PAAS.get('REDIS_URL') or os.getenv('REDIS_URL') +BROKER_FAILOVER_STRATEGY = "round-robin" +STAGECRAFT_URL = 'https://performance-platform-stagecraft-production.cloudapps.digital' +STAGECRAFT_OAUTH_TOKEN = os.getenv('STAGECRAFT_OAUTH_TOKEN') +BACKDROP_READ_URL = 'https://performance-platform-backdrop-read-production.cloudapps.digital/data' +BACKDROP_WRITE_URL = 'https://performance-platform-backdrop-write-production.cloudapps.digital/data' diff --git a/backdrop/transformers/config/staging.py b/backdrop/transformers/config/staging.py new file mode 100644 index 00000000..06927d2d --- /dev/null +++ b/backdrop/transformers/config/staging.py @@ -0,0 +1,10 @@ +import os +from ...core.config.common import load_paas_settings + +PAAS = load_paas_settings() +BROKER_URL = PAAS.get('REDIS_URL') or os.getenv('REDIS_URL') +BROKER_FAILOVER_STRATEGY = "round-robin" +STAGECRAFT_URL = 'https://performance-platform-stagecraft-staging.cloudapps.digital' +STAGECRAFT_OAUTH_TOKEN = os.getenv('STAGECRAFT_OAUTH_TOKEN') +BACKDROP_READ_URL = 'https://performance-platform-backdrop-read-staging.cloudapps.digital/data' +BACKDROP_WRITE_URL = 'https://performance-platform-backdrop-write-staging.cloudapps.digital/data' diff --git a/backdrop/transformers/config/test.py b/backdrop/transformers/config/test.py index 843c3366..edb72565 100644 --- a/backdrop/transformers/config/test.py +++ b/backdrop/transformers/config/test.py @@ -1,4 +1,4 @@ -TRANSFORMER_AMQP_URL = 'memory://' +BROKER_URL = 'memory://' STAGECRAFT_URL = 'http://stagecraft' STAGECRAFT_OAUTH_TOKEN = 'development-oauth-access-token' BACKDROP_READ_URL = 'http://backdrop/data' diff --git a/backdrop/transformers/dispatch.py b/backdrop/transformers/dispatch.py index 89f05c20..e2bbb33e 100644 --- a/backdrop/transformers/dispatch.py +++ b/backdrop/transformers/dispatch.py @@ -8,6 +8,7 @@ from statsd import StatsClient from backdrop.core.timeseries import parse_period +from backdrop.core.timeutils import parse_time_as_utc from backdrop.core.log_handler import get_log_file_handler from backdrop.core.errors import incr_on_error from backdrop.transformers.tasks.util import encode_id @@ -16,11 +17,11 @@ from performanceplatform.client import AdminAPI, DataSet -GOVUK_ENV = getenv("GOVUK_ENV", "development") +ENVIRONMENT = getenv("ENVIRONMENT", "development") logger = logging.getLogger() logger.setLevel(logging.DEBUG) logger.addHandler( - get_log_file_handler("log/{}.log".format(GOVUK_ENV), logging.DEBUG)) + get_log_file_handler("log/{}.log".format(ENVIRONMENT), logging.DEBUG)) stats_client = StatsClient(prefix=getenv("GOVUK_STATSD_PREFIX", "pp.apps.backdrop.transformers.worker")) @@ -147,6 +148,9 @@ def run_transform(data_set_config, transform, earliest, latest): data_set_config['data_type'], ) + earliest = parse_time_as_utc(earliest) + latest = parse_time_as_utc(latest) + data = data_set.get( query_parameters=get_query_parameters(transform, earliest, latest) ) diff --git a/backdrop/transformers/worker.py b/backdrop/transformers/worker.py index bb4b37f8..c7c4ed18 100644 --- a/backdrop/transformers/worker.py +++ b/backdrop/transformers/worker.py @@ -4,13 +4,13 @@ import importlib from os import getenv -GOVUK_ENV = getenv("GOVUK_ENV", "development") +ENVIRONMENT = getenv("ENVIRONMENT", "development") config = importlib.import_module( - "backdrop.transformers.config.{}".format(GOVUK_ENV)) + "backdrop.transformers.config.{}".format(ENVIRONMENT)) app = Celery( 'transformations', - broker=config.TRANSFORMER_AMQP_URL, + broker=config.BROKER_URL, include=['backdrop.transformers.dispatch']) diff --git a/backdrop/write/api.py b/backdrop/write/api.py index d9540b9b..592a88e0 100644 --- a/backdrop/write/api.py +++ b/backdrop/write/api.py @@ -18,7 +18,7 @@ from ..core.flaskutils import generate_request_id from ..core.storage.mongo import MongoStorageEngine -GOVUK_ENV = getenv("GOVUK_ENV", "development") +ENVIRONMENT = getenv("ENVIRONMENT", "development") app = Flask("backdrop.write.api") @@ -29,12 +29,12 @@ # Configuration app.config.from_object( - "backdrop.write.config.{}".format(GOVUK_ENV)) + "backdrop.write.config.{}".format(ENVIRONMENT)) storage = MongoStorageEngine.create( - app.config['MONGO_HOSTS'], - app.config['MONGO_PORT'], - app.config['DATABASE_NAME']) + app.config['DATABASE_URL'], + app.config.get('CA_CERTIFICATE') +) admin_api = client.AdminAPI( app.config['STAGECRAFT_URL'], @@ -43,12 +43,12 @@ request_id_fn=generate_request_id, ) -log_handler.set_up_logging(app, GOVUK_ENV) -log_handler.set_up_audit_logging(app, GOVUK_ENV) +log_handler.set_up_logging(app, ENVIRONMENT) +log_handler.set_up_audit_logging(app, ENVIRONMENT) app.url_map.converters["data_set"] = DataSetConverter -celery_app = Celery(broker=app.config['TRANSFORMER_AMQP_URL']) +celery_app = Celery(broker=app.config['BROKER_URL']) app.config['BROKER_FAILOVER_STRATEGY'] = "round-robin" celery_app.conf.update(app.config) @@ -105,10 +105,7 @@ def http_error_handler(e): @cache_control.nocache @statsd.timer('write.route.health_check.status') def health_check(): - if storage.alive(): - return jsonify(status='ok', message='database seems fine') - else: - abort(500, 'cannot connect to database') + return jsonify(status='ok', message='app is up') @app.route('/data//', methods=['POST']) diff --git a/backdrop/write/config/development.py b/backdrop/write/config/development.py index 4f4f868f..775adcb6 100644 --- a/backdrop/write/config/development.py +++ b/backdrop/write/config/development.py @@ -1,6 +1,4 @@ -DATABASE_NAME = "backdrop" -MONGO_HOSTS = ['localhost'] -MONGO_PORT = 27017 +DATABASE_URL = 'mongodb://localhost:27017/backdrop_development' LOG_LEVEL = "DEBUG" DATA_SET_AUTO_ID_KEYS = { "lpa_volumes": ("key", "start_at", "end_at") @@ -17,4 +15,4 @@ SIGNON_API_USER_TOKEN = 'development-oauth-access-token' -TRANSFORMER_AMQP_URL = 'amqp://backdrop_write:backdrop_write@localhost:5672/%2Fbackdrop_write' +BROKER_URL = 'amqp://backdrop_write:backdrop_write@localhost:5672/%2Fbackdrop_write' diff --git a/backdrop/write/config/production.py b/backdrop/write/config/production.py new file mode 100644 index 00000000..74de2b14 --- /dev/null +++ b/backdrop/write/config/production.py @@ -0,0 +1,23 @@ +import os +from ...core.config.common import load_paas_settings + +PAAS = load_paas_settings() +DATABASE_URL = PAAS.get('DATABASE_URL') +CA_CERTIFICATE = PAAS.get('CA_CERTIFICATE') +STAGECRAFT_URL = os.getenv('STAGECRAFT_URL') +BROKER_URL = PAAS.get('REDIS_URL') or os.getenv('REDIS_URL') +BROKER_FAILOVER_STRATEGY = "round-robin" +SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN') +LOG_LEVEL = "INFO" +DATA_SET_UPLOAD_FORMAT = { + "ithc_excel": "excel", +} +DATA_SET_UPLOAD_FILTERS = { + "ithc_excel": [ + "backdrop.core.upload.filters.first_sheet_filter", + ], +} +SESSION_COOKIE_SECURE = True +SECRET_KEY = os.getenv('SECRET_KEY') +STAGECRAFT_COLLECTION_ENDPOINT_TOKEN = os.getenv( + 'STAGECRAFT_COLLECTION_ENDPOINT_TOKEN') diff --git a/backdrop/write/config/staging.py b/backdrop/write/config/staging.py new file mode 100644 index 00000000..cdeb2b26 --- /dev/null +++ b/backdrop/write/config/staging.py @@ -0,0 +1,15 @@ +import os +from ...core.config.common import load_paas_settings + +PAAS = load_paas_settings() +DATABASE_URL = PAAS.get('DATABASE_URL') +CA_CERTIFICATE = PAAS.get('CA_CERTIFICATE') +STAGECRAFT_URL = os.getenv('STAGECRAFT_URL') +BROKER_URL = PAAS.get('REDIS_URL') or os.getenv('REDIS_URL') +BROKER_FAILOVER_STRATEGY = "round-robin" +SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN') +LOG_LEVEL = "INFO" +SESSION_COOKIE_SECURE = True +SECRET_KEY = os.getenv('SECRET_KEY') +STAGECRAFT_COLLECTION_ENDPOINT_TOKEN = os.getenv( + 'STAGECRAFT_COLLECTION_ENDPOINT_TOKEN') diff --git a/backdrop/write/config/test.py b/backdrop/write/config/test.py index 8f156dbe..4bac569a 100644 --- a/backdrop/write/config/test.py +++ b/backdrop/write/config/test.py @@ -1,6 +1,4 @@ -DATABASE_NAME = "backdrop_test" -MONGO_HOSTS = ['localhost'] -MONGO_PORT = 27017 +DATABASE_URL = 'mongodb://localhost:27017/backdrop_test' LOG_LEVEL = "DEBUG" CLIENT_ID = "it's not important here" CLIENT_SECRET = "it's not important here" @@ -9,7 +7,7 @@ "data_set_with_timestamp_auto_id": ["_timestamp", "key"], "evl_volumetrics": ["_timestamp", "service", "transaction"], } -TRANSFORMER_AMQP_URL = 'memory://' +BROKER_URL = 'memory://' from development import (STAGECRAFT_COLLECTION_ENDPOINT_TOKEN, STAGECRAFT_URL, SIGNON_API_USER_TOKEN) diff --git a/etc/deploy.sh b/etc/deploy.sh new file mode 100644 index 00000000..70d3794a --- /dev/null +++ b/etc/deploy.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -e + +if [ -z "$1" ]; then + echo "Missing PAAS space argument" + echo " deploy.sh staging|production" + exit 1 +fi + +PAAS_SPACE=$1 +wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add - +echo "deb http://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list +sudo apt-get update && sudo apt-get install cf-cli + +cf login -u $PAAS_USER -p $PAAS_PASSWORD -a https://api.cloud.service.gov.uk -o gds-performance-platform -s $PAAS_SPACE + +# bind services +cf bind-service performance-platform-backdrop-read gds-performance-platform-mongodb-service +cf bind-service performance-platform-backdrop-write gds-performance-platform-mongodb-service + +# set environmental variables +cf set-env performance-platform-backdrop-read ENVIRONMENT $PAAS_SPACE +cf set-env performance-platform-backdrop-read STAGECRAFT_URL https://performance-platform-stagecraft-$PAAS_SPACE.cloudapps.digital +cf set-env performance-platform-backdrop-read SIGNON_API_USER_TOKEN $APP_SIGNON_API_USER_TOKEN + +cf set-env performance-platform-backdrop-write ENVIRONMENT $PAAS_SPACE +cf set-env performance-platform-backdrop-write STAGECRAFT_URL https://performance-platform-stagecraft-$PAAS_SPACE.cloudapps.digital +cf set-env performance-platform-backdrop-write SIGNON_API_USER_TOKEN $APP_SIGNON_API_USER_TOKEN +cf set-env performance-platform-backdrop-write SECRET_KEY $APP_SECRET_KEY +cf set-env performance-platform-backdrop-write REDIS_DATABASE_NUMBER $REDIS_DATABASE_NUMBER + +cf set-env performance-platform-backdrop-celery-worker ENVIRONMENT $PAAS_SPACE +cf set-env performance-platform-backdrop-celery-worker STAGECRAFT_OAUTH_TOKEN $APP_STAGECRAFT_OAUTH_TOKEN +cf set-env performance-platform-backdrop-celery-worker REDIS_DATABASE_NUMBER $REDIS_DATABASE_NUMBER + +# deploy apps +cf push -f manifest.yml + +# create and map routes +cf map-route performance-platform-backdrop-read cloudapps.digital --hostname performance-platform-backdrop-read-$PAAS_SPACE +cf map-route performance-platform-backdrop-write cloudapps.digital --hostname performance-platform-backdrop-write-$PAAS_SPACE diff --git a/features/environment.py b/features/environment.py index f2b24808..8cf8252a 100644 --- a/features/environment.py +++ b/features/environment.py @@ -10,7 +10,7 @@ os.path.join(os.path.dirname(__file__), '..') ) -os.environ["GOVUK_ENV"] = "test" +os.environ["ENVIRONMENT"] = "test" from support.http_test_client import HTTPTestClient from support.flask_test_client import FlaskTestClient @@ -61,7 +61,7 @@ def create_client(feature): if 'use_write_api_client' in feature.tags: return FlaskTestClient(write_api) if 'use_http_client' in feature.tags: - return HTTPTestClient(config.DATABASE_NAME) + return HTTPTestClient(config.DATABASE_URL) raise AssertionError( "Test client not selected! Please annotate the failing feature with " diff --git a/features/steps/read_api.py b/features/steps/read_api.py index a2e71acb..e72a3bc9 100644 --- a/features/steps/read_api.py +++ b/features/steps/read_api.py @@ -32,7 +32,7 @@ def step(context, fixture_name, data_set_name): '_week_start_at', '_month_start_at']: if key in obj: obj[key] = parser.parse(obj[key]).astimezone(pytz.utc) - context.client.mongo()[data_set_name].save(obj) + context.client.mongo()[data_set_name].insert_one(obj) def get_data_set_settings_from_context_table(table): @@ -56,7 +56,7 @@ def step(context, fixture_name, data_set_name): '_week_start_at', '_month_start_at']: if key in obj: obj[key] = parser.parse(obj[key]).astimezone(pytz.utc) - context.client.mongo()[data_set_name].save(obj) + context.client.mongo()[data_set_name].insert_one(obj) @given('I have a record updated "{timespan}" ago in the "{data_set_name}" data_set') diff --git a/features/support/http_test_client.py b/features/support/http_test_client.py index 581d8ffd..fe9e9809 100644 --- a/features/support/http_test_client.py +++ b/features/support/http_test_client.py @@ -5,11 +5,10 @@ class HTTPTestClient(BaseClient): - def __init__(self, database_name): - self.database_name = database_name + def __init__(self, database_url): self._read_api = FlaskApp("read", "5000") self._write_api = FlaskApp("write", "5001") - self._mongo_db = MongoClient('localhost', 27017)[self.database_name] + self._mongo_db = MongoClient(database_url).get_database() self._start() def get(self, url, headers=None): diff --git a/features/support/support.py b/features/support/support.py index 42e7f81c..bf1bcb4b 100644 --- a/features/support/support.py +++ b/features/support/support.py @@ -18,7 +18,7 @@ def mongo(self): return self._mongo_db def clean_mongo(self): - self._mongo_db.connection.drop_database( + self._mongo_db.client.drop_database( self._mongo_db.name) def before_scenario(self): diff --git a/manifest.yml b/manifest.yml new file mode 100644 index 00000000..52d33cd4 --- /dev/null +++ b/manifest.yml @@ -0,0 +1,30 @@ +--- +buildpack: python_buildpack +services: + - redis-poc + +applications: + - name: performance-platform-backdrop-read + command: python start.py read 8080 + services: + - gds-performance-platform-mongodb-service + instances: 1 + memory: 512M + no-route: true + health-check-type: none + + - name: performance-platform-backdrop-write + command: python start.py write 8080 + services: + - gds-performance-platform-mongodb-service + instances: 1 + memory: 512M + no-route: true + health-check-type: none + + - name: performance-platform-backdrop-celery-worker + command: celery -A backdrop.transformers.worker worker --loglevel=info + instances: 1 + memory: 1G + no-route: true + health-check-type: none diff --git a/requirements.txt b/requirements.txt index 13fd7e01..fb705286 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,13 @@ argh==0.25 -Celery==3.1.17 +celery[redis]==4.1.0 Flask==0.10.1 Flask-FeatureFlags==0.4 -gunicorn==19.1.1 invoke isodate==0.5.0 jsonschema==2.4 performanceplatform-client==0.6.0 #pip==1.5.6 -pymongo==2.8 +pymongo==3.5.1 python-dateutil==2.2 pytz==2013b pyopenssl==17.0.0 diff --git a/tests/__init__.py b/tests/__init__.py index 49c91719..143708ac 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -5,4 +5,4 @@ os.path.join(os.path.dirname(__file__), '..') ) -os.environ["GOVUK_ENV"] = "test" +os.environ["ENVIRONMENT"] = "test" diff --git a/tests/core/integration/test_data_set_integration.py b/tests/core/integration/test_data_set_integration.py index 95585396..66243216 100644 --- a/tests/core/integration/test_data_set_integration.py +++ b/tests/core/integration/test_data_set_integration.py @@ -10,16 +10,14 @@ from backdrop.core.query import Query from tests.support.test_helpers import d_tz -HOSTS = ['localhost'] -PORT = 27017 -DB_NAME = 'performance_platform_test' +DATABASE_URL = 'mongodb://localhost:27017/backdrop_test' DATA_SET = 'data_set_integration_test' class TestDataSetIntegration(unittest.TestCase): def setUp(self): - self.storage = MongoStorageEngine.create(HOSTS, PORT, DB_NAME) + self.storage = MongoStorageEngine.create(DATABASE_URL) self.config = { 'name': DATA_SET, @@ -30,7 +28,8 @@ def setUp(self): self.data_set = DataSet(self.storage, self.config) - self.mongo_collection = MongoClient(HOSTS, PORT)[DB_NAME][DATA_SET] + database = MongoClient(DATABASE_URL).get_database() + self.mongo_collection = database[DATA_SET] def setup__timestamp_data(self): self.mongo_collection.save({ diff --git a/tests/core/storage/test_mongo.py b/tests/core/storage/test_mongo.py index 6ff1f4d1..51db4fd2 100644 --- a/tests/core/storage/test_mongo.py +++ b/tests/core/storage/test_mongo.py @@ -19,11 +19,12 @@ from .test_storage import BaseStorageTest +DATABASE_URL = 'mongodb://localhost:27017/backdrop_test' + class TestMongoStorageEngine(BaseStorageTest): def setup(self): - self.engine = MongoStorageEngine.create( - ['localhost'], 27017, 'backdrop_test') + self.engine = MongoStorageEngine.create(DATABASE_URL) def test_create_data_set(self): self.engine.create_data_set('should_have_index', 0) @@ -53,7 +54,9 @@ def test_batch_last_updated(self): assert_that(last_updated.second, is_(timestamp.second)) def teardown(self): - self.engine._mongo.drop_database('backdrop_test') + mongo_client = self.engine._mongo_client + database_name = mongo_client.get_database().name + mongo_client.drop_database(database_name) class TestReconnectingSave(object): diff --git a/tests/core/storage/test_storage.py b/tests/core/storage/test_storage.py index c99c1948..30b8bcc4 100644 --- a/tests/core/storage/test_storage.py +++ b/tests/core/storage/test_storage.py @@ -37,9 +37,6 @@ def _save_all_with_periods(self, data_set_id, *records): self._save_all(data_set_id, *records) - def test_is_alive(self): - assert_that(self.engine.alive(), is_(True)) - def test_does_not_exist(self): assert_that(self.engine.data_set_exists('foo_bar'), is_(False)) diff --git a/tests/read/test_read_api_service_data_endpoint.py b/tests/read/test_read_api_service_data_endpoint.py index b6b2931f..8eed4066 100644 --- a/tests/read/test_read_api_service_data_endpoint.py +++ b/tests/read/test_read_api_service_data_endpoint.py @@ -115,7 +115,9 @@ class PreflightChecksApiTestCase(unittest.TestCase): def setUp(self): self.app = api.app.test_client() - api.storage._mongo.drop_database(api.app.config['DATABASE_NAME']) + mongo_client = api.storage._mongo_client + database_name = mongo_client.get_database().name + mongo_client.drop_database(database_name) @fake_data_set_exists("data_set", data_group="some-group", data_type="some-type") def test_cors_preflight_requests_have_empty_body(self): diff --git a/tests/write/test_flask_integration.py b/tests/write/test_flask_integration.py index 3d40fa61..330ba095 100644 --- a/tests/write/test_flask_integration.py +++ b/tests/write/test_flask_integration.py @@ -205,18 +205,6 @@ def test_api_exposes_a_healthcheck(self): entity = json.loads(response.data) assert_that(entity["status"], is_("ok")) - @patch("backdrop.write.api.statsd") - @patch("backdrop.write.api.storage") - def test_exception_handling(self, storage, statsd): - storage.alive.side_effect = ValueError("BOOM") - - response = self.app.get("/_status") - - assert_that(response, has_status(500)) - assert_that(response, is_error_response()) - - statsd.incr.assert_called_with("write.error", data_set="/_status") - class UploadPageTestCase(unittest.TestCase): def setUp(self):