diff --git a/.gitignore b/.gitignore index 6e3c78a1..fea4119d 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ tmp/* # vim swap files .*.swp +venv/ diff --git a/.python-version b/.python-version new file mode 100644 index 00000000..ecc17b8e --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +2.7.13 diff --git a/.travis.yml b/.travis.yml index 9e39a1e4..3feedecd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,15 +19,26 @@ install: - pip install -q -r requirements_for_tests.txt env: - global: - - SKIP_VIRUS_SCAN=1 SKIP_SPLINTER_TESTS=1 MONGO_REPLICA_SET='' NO_AUTOPEP8=1 - - PAAS_USER=pp-deploy@digital.cabinet-office.gov.uk + global: + - PAAS_USER: pp-deploy@digital.cabinet-office.gov.uk # PAAS_PASSWORD - secure: HNo6rnNiXmNYAV1TvkcMxeniXpmHoZkNEiT+X9EgyInCOnVm4mz89qi/AElY/ESE5HdYIegJ5l1M5tAWCei4dSDSTSfkbBBiHE6dfpf1OSU857oQ6T6161NRg4V2oxNdWSRgAX+G8rpxBShaXCDU9q9iBxhsqrmTX+UGlzKloDk= + - REDIS_DATABASE_NUMBER_STAGING: 4 + - REDIS_DATABASE_NUMBER_PRODUCTION: 5 # APP_SIGNON_API_USER_TOKEN_STAGING - secure: PYLrm6uH6uhNA69xgI5lSXLrX0ouPBdxlMlQXER0K2BkHzxTowNDFyf5LDa2kwIueJ/YXFHAgm7eBN9gr3JIXAN8Mqb15LTHJVg6hc2aLwFZKES4fuPa/da3Pn24xrdTesIedUAv2vSkuacaktZwKxsv6LENw53uA+8kOT8l9SQ= # APP_SIGNON_API_USER_TOKEN_PRODUCTION - secure: SBk+7zw2GMkgbl8p7hwXZw5VkDf2FQE4kBklD40c7goORyEF3/FjTVJTVlREN4Rwb3a81N5lkQDl0fLPhGaDv+PyTr+l69IU64w1Whu9e9fL/wMC3zo5Eer3Btptar5jdCsEg+V4woKKXv6j4B4xebHN6sU7C6dX9TGKs+R1aBo= + # APP_SECRET_KEY_STAGING + - secure: Cl7cvdzviF3hCepM7n3XMz9q1D/FCEQkcDZOyRTPkHlm9V2GKi+e9ECuAuWQVRiRDTb7bAZZ5kl+zTj00nMb0BKds3QoisZ5oB8UwYFOQSrYV/NTpge0wsvZZvfqXp5sN3CLVXE2XynLsZZSzOTIte8UKD4giBFsuHo2sBcVbb8= + # APP_SECRET_KEY_PRODUCTION + - secure: ifAsLdVTrzDT8IIEVdW1TJon8VYdmDdzxxE/CtCgc0hcBP9eneKF7qvBOy6TFMnE9KSz4zzyYx+yZI8s5sAsjknw/HEnHYMCYSjRcM3GMWP27/QFM2983CgWkoEn5xGj3WEXra1WJGltzqbeoYfZWmFA9pBGSU6IbR0IzjLC6+g= + # APP_STAGECRAFT_COLLECTION_ENDPOINT_TOKEN + # Value too large to be encrypted. + # APP_STAGECRAFT_OAUTH_TOKEN_STAGING + - secure: JkoKVPblX1sHICKxJMT0kPi5Nax3dn3cG1bzmAmpMDX+rd37I4XtA96G17ibUmUrp2CVrOo4is5WjI9hPDwmC8/o7/6MOHqJo7jeflY+Iq8bWShcja0khlb4aBlpWfgMS+vniLCBPkQtUe2eE87jFLYJXmsuBGfp4r4L88fhgOk= + # APP_STAGECRAFT_OAUTH_TOKEN_PRODUCTION + - secure: HZPkW81a+OsfFrCCpcuYYBKNIEmejsQlIreVo2AJ1NipBVK8DniqIK+bXGUIQraaqeKXApXknfPepVpyqG3uyzHvkboylooWa7fuoJB4bs9Eb91Qo2AeWjP2fts0EmUX8R76K51n7EwUgk9xemM/bE2e+0FmG5e35JpXEwfm/5I= script: - ./run_tests.sh @@ -40,13 +51,13 @@ before_deploy: deploy: - provider: script - script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_STAGING etc/deploy.sh staging + script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_STAGING APP_SECRET_KEY=$APP_SECRET_KEY_STAGING REDIS_DATABASE_NUMBER=$REDIS_DATABASE_NUMBER_STAGING APP_STAGECRAFT_OAUTH_TOKEN=$APP_STAGECRAFT_OAUTH_TOKEN_STAGING etc/deploy.sh staging skip_cleanup: true on: branch: staging - provider: script - script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_PRODUCTION etc/deploy.sh production + script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_PRODUCTION APP_SECRET_KEY=$APP_SECRET_KEY_PRODUCTION REDIS_DATABASE_NUMBER=$REDIS_DATABASE_NUMBER_PRODUCTION APP_STAGECRAFT_OAUTH_TOKEN=$APP_STAGECRAFT_OAUTH_TOKEN_PRODUCTION etc/deploy.sh production skip_cleanup: true on: branch: production diff --git a/Procfile b/Procfile deleted file mode 100644 index 5e28441f..00000000 --- a/Procfile +++ /dev/null @@ -1,2 +0,0 @@ -web: venv/bin/gunicorn -c /etc/gds/${APP_NAME}/gunicorn ${APP_MODULE} -worker: venv/bin/celery -A backdrop.transformers.worker worker -l debug diff --git a/backdrop/core/config/common.py b/backdrop/core/config/common.py index c475452e..85076bbf 100644 --- a/backdrop/core/config/common.py +++ b/backdrop/core/config/common.py @@ -2,14 +2,23 @@ import json from base64 import b64decode + def load_paas_settings(): paas = {} if 'VCAP_SERVICES' in os.environ: vcap = json.loads(os.environ['VCAP_SERVICES']) - for service in vcap['mongodb']: - if service['name'] == 'gds-performance-platform-mongodb-service': - credentials = service['credentials'] - paas['DATABASE_URL'] = credentials['uri'] - ca_cert = b64decode(credentials['ca_certificate_base64']) - paas['CA_CERTIFICATE'] = ca_cert + if 'mongodb' in vcap: + for service in vcap['mongodb']: + if service['name'] == 'gds-performance-platform-mongodb-service': + credentials = service['credentials'] + paas['DATABASE_URL'] = credentials['uri'] + ca_cert = b64decode(credentials['ca_certificate_base64']) + paas['CA_CERTIFICATE'] = ca_cert + if 'REDIS_DATABASE_NUMBER' in os.environ: + for service in vcap['user-provided']: + if service['name'] == 'redis-poc': + database_number = os.environ['REDIS_DATABASE_NUMBER'] + url = service['credentials']['url'] + url += '/' + database_number + paas['REDIS_URL'] = url return paas diff --git a/backdrop/core/storage/mongo.py b/backdrop/core/storage/mongo.py index beca31b9..1c384d88 100644 --- a/backdrop/core/storage/mongo.py +++ b/backdrop/core/storage/mongo.py @@ -72,7 +72,7 @@ def reconnecting_save(collection, record, tries=3): class MongoStorageEngine(object): @classmethod - def create(cls, database_url, ca_certificate): + def create(cls, database_url, ca_certificate=None): if ca_certificate is not None: dir = os.path.dirname(__file__) filename = os.path.join(dir, 'mongodb.crt') diff --git a/backdrop/read/api.py b/backdrop/read/api.py index aa3542db..dd7d49a1 100644 --- a/backdrop/read/api.py +++ b/backdrop/read/api.py @@ -91,12 +91,7 @@ def http_error_handler(e): @cache_control.nocache @statsd.timer('read.route.heath_check.status') def health_check(): - - if not storage.alive(): - return jsonify(status='error', - message='cannot connect to database'), 500 - - return jsonify(status='ok', message='database is up') + return jsonify(status='ok', message='app is up') @app.route('/_status/data-sets', methods=['GET']) diff --git a/backdrop/read/config/production.py b/backdrop/read/config/production.py index 652238c0..3ded5859 100644 --- a/backdrop/read/config/production.py +++ b/backdrop/read/config/production.py @@ -7,4 +7,4 @@ STAGECRAFT_URL = os.getenv('STAGECRAFT_URL') SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN') LOG_LEVEL = "INFO" -SESSION_COOKIE_SECURE=True +SESSION_COOKIE_SECURE = True diff --git a/backdrop/read/config/staging.py b/backdrop/read/config/staging.py index 652238c0..3ded5859 100644 --- a/backdrop/read/config/staging.py +++ b/backdrop/read/config/staging.py @@ -7,4 +7,4 @@ STAGECRAFT_URL = os.getenv('STAGECRAFT_URL') SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN') LOG_LEVEL = "INFO" -SESSION_COOKIE_SECURE=True +SESSION_COOKIE_SECURE = True diff --git a/backdrop/transformers/config/development.py b/backdrop/transformers/config/development.py index 788ae55d..41f948d2 100644 --- a/backdrop/transformers/config/development.py +++ b/backdrop/transformers/config/development.py @@ -1,4 +1,4 @@ -TRANSFORMER_AMQP_URL = 'amqp://backdrop_write:backdrop_write@localhost:5672/%2Fbackdrop_write' +BROKER_URL = 'amqp://backdrop_write:backdrop_write@localhost:5672/%2Fbackdrop_write' STAGECRAFT_URL = 'http://localhost:3103' STAGECRAFT_OAUTH_TOKEN = 'development-oauth-access-token' BACKDROP_READ_URL = 'http://backdrop-read.dev.gov.uk/data' diff --git a/backdrop/transformers/config/production.py b/backdrop/transformers/config/production.py new file mode 100644 index 00000000..2249b8b1 --- /dev/null +++ b/backdrop/transformers/config/production.py @@ -0,0 +1,10 @@ +import os +from ...core.config.common import load_paas_settings + +PAAS = load_paas_settings() +BROKER_URL = PAAS.get('REDIS_URL') or os.getenv('REDIS_URL') +BROKER_FAILOVER_STRATEGY = "round-robin" +STAGECRAFT_URL = 'https://performance-platform-stagecraft-production.cloudapps.digital' +STAGECRAFT_OAUTH_TOKEN = os.getenv('STAGECRAFT_OAUTH_TOKEN') +BACKDROP_READ_URL = 'https://performance-platform-backdrop-read-production.cloudapps.digital/data' +BACKDROP_WRITE_URL = 'https://performance-platform-backdrop-write-production.cloudapps.digital/data' diff --git a/backdrop/transformers/config/staging.py b/backdrop/transformers/config/staging.py new file mode 100644 index 00000000..06927d2d --- /dev/null +++ b/backdrop/transformers/config/staging.py @@ -0,0 +1,10 @@ +import os +from ...core.config.common import load_paas_settings + +PAAS = load_paas_settings() +BROKER_URL = PAAS.get('REDIS_URL') or os.getenv('REDIS_URL') +BROKER_FAILOVER_STRATEGY = "round-robin" +STAGECRAFT_URL = 'https://performance-platform-stagecraft-staging.cloudapps.digital' +STAGECRAFT_OAUTH_TOKEN = os.getenv('STAGECRAFT_OAUTH_TOKEN') +BACKDROP_READ_URL = 'https://performance-platform-backdrop-read-staging.cloudapps.digital/data' +BACKDROP_WRITE_URL = 'https://performance-platform-backdrop-write-staging.cloudapps.digital/data' diff --git a/backdrop/transformers/config/test.py b/backdrop/transformers/config/test.py index 843c3366..edb72565 100644 --- a/backdrop/transformers/config/test.py +++ b/backdrop/transformers/config/test.py @@ -1,4 +1,4 @@ -TRANSFORMER_AMQP_URL = 'memory://' +BROKER_URL = 'memory://' STAGECRAFT_URL = 'http://stagecraft' STAGECRAFT_OAUTH_TOKEN = 'development-oauth-access-token' BACKDROP_READ_URL = 'http://backdrop/data' diff --git a/backdrop/transformers/dispatch.py b/backdrop/transformers/dispatch.py index 89f05c20..e2bbb33e 100644 --- a/backdrop/transformers/dispatch.py +++ b/backdrop/transformers/dispatch.py @@ -8,6 +8,7 @@ from statsd import StatsClient from backdrop.core.timeseries import parse_period +from backdrop.core.timeutils import parse_time_as_utc from backdrop.core.log_handler import get_log_file_handler from backdrop.core.errors import incr_on_error from backdrop.transformers.tasks.util import encode_id @@ -16,11 +17,11 @@ from performanceplatform.client import AdminAPI, DataSet -GOVUK_ENV = getenv("GOVUK_ENV", "development") +ENVIRONMENT = getenv("ENVIRONMENT", "development") logger = logging.getLogger() logger.setLevel(logging.DEBUG) logger.addHandler( - get_log_file_handler("log/{}.log".format(GOVUK_ENV), logging.DEBUG)) + get_log_file_handler("log/{}.log".format(ENVIRONMENT), logging.DEBUG)) stats_client = StatsClient(prefix=getenv("GOVUK_STATSD_PREFIX", "pp.apps.backdrop.transformers.worker")) @@ -147,6 +148,9 @@ def run_transform(data_set_config, transform, earliest, latest): data_set_config['data_type'], ) + earliest = parse_time_as_utc(earliest) + latest = parse_time_as_utc(latest) + data = data_set.get( query_parameters=get_query_parameters(transform, earliest, latest) ) diff --git a/backdrop/transformers/worker.py b/backdrop/transformers/worker.py index bb4b37f8..c7c4ed18 100644 --- a/backdrop/transformers/worker.py +++ b/backdrop/transformers/worker.py @@ -4,13 +4,13 @@ import importlib from os import getenv -GOVUK_ENV = getenv("GOVUK_ENV", "development") +ENVIRONMENT = getenv("ENVIRONMENT", "development") config = importlib.import_module( - "backdrop.transformers.config.{}".format(GOVUK_ENV)) + "backdrop.transformers.config.{}".format(ENVIRONMENT)) app = Celery( 'transformations', - broker=config.TRANSFORMER_AMQP_URL, + broker=config.BROKER_URL, include=['backdrop.transformers.dispatch']) diff --git a/backdrop/write/api.py b/backdrop/write/api.py index b50c4679..592a88e0 100644 --- a/backdrop/write/api.py +++ b/backdrop/write/api.py @@ -48,7 +48,7 @@ app.url_map.converters["data_set"] = DataSetConverter -celery_app = Celery(broker=app.config['TRANSFORMER_AMQP_URL']) +celery_app = Celery(broker=app.config['BROKER_URL']) app.config['BROKER_FAILOVER_STRATEGY'] = "round-robin" celery_app.conf.update(app.config) @@ -105,10 +105,7 @@ def http_error_handler(e): @cache_control.nocache @statsd.timer('write.route.health_check.status') def health_check(): - if storage.alive(): - return jsonify(status='ok', message='database seems fine') - else: - abort(500, 'cannot connect to database') + return jsonify(status='ok', message='app is up') @app.route('/data//', methods=['POST']) diff --git a/backdrop/write/config/development.py b/backdrop/write/config/development.py index 2be9d5ea..775adcb6 100644 --- a/backdrop/write/config/development.py +++ b/backdrop/write/config/development.py @@ -15,4 +15,4 @@ SIGNON_API_USER_TOKEN = 'development-oauth-access-token' -TRANSFORMER_AMQP_URL = 'amqp://backdrop_write:backdrop_write@localhost:5672/%2Fbackdrop_write' +BROKER_URL = 'amqp://backdrop_write:backdrop_write@localhost:5672/%2Fbackdrop_write' diff --git a/backdrop/write/config/production.py b/backdrop/write/config/production.py index 652238c0..74de2b14 100644 --- a/backdrop/write/config/production.py +++ b/backdrop/write/config/production.py @@ -5,6 +5,19 @@ DATABASE_URL = PAAS.get('DATABASE_URL') CA_CERTIFICATE = PAAS.get('CA_CERTIFICATE') STAGECRAFT_URL = os.getenv('STAGECRAFT_URL') +BROKER_URL = PAAS.get('REDIS_URL') or os.getenv('REDIS_URL') +BROKER_FAILOVER_STRATEGY = "round-robin" SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN') LOG_LEVEL = "INFO" -SESSION_COOKIE_SECURE=True +DATA_SET_UPLOAD_FORMAT = { + "ithc_excel": "excel", +} +DATA_SET_UPLOAD_FILTERS = { + "ithc_excel": [ + "backdrop.core.upload.filters.first_sheet_filter", + ], +} +SESSION_COOKIE_SECURE = True +SECRET_KEY = os.getenv('SECRET_KEY') +STAGECRAFT_COLLECTION_ENDPOINT_TOKEN = os.getenv( + 'STAGECRAFT_COLLECTION_ENDPOINT_TOKEN') diff --git a/backdrop/write/config/staging.py b/backdrop/write/config/staging.py index 652238c0..cdeb2b26 100644 --- a/backdrop/write/config/staging.py +++ b/backdrop/write/config/staging.py @@ -5,6 +5,11 @@ DATABASE_URL = PAAS.get('DATABASE_URL') CA_CERTIFICATE = PAAS.get('CA_CERTIFICATE') STAGECRAFT_URL = os.getenv('STAGECRAFT_URL') +BROKER_URL = PAAS.get('REDIS_URL') or os.getenv('REDIS_URL') +BROKER_FAILOVER_STRATEGY = "round-robin" SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN') LOG_LEVEL = "INFO" -SESSION_COOKIE_SECURE=True +SESSION_COOKIE_SECURE = True +SECRET_KEY = os.getenv('SECRET_KEY') +STAGECRAFT_COLLECTION_ENDPOINT_TOKEN = os.getenv( + 'STAGECRAFT_COLLECTION_ENDPOINT_TOKEN') diff --git a/backdrop/write/config/test.py b/backdrop/write/config/test.py index 8f156dbe..4bac569a 100644 --- a/backdrop/write/config/test.py +++ b/backdrop/write/config/test.py @@ -1,6 +1,4 @@ -DATABASE_NAME = "backdrop_test" -MONGO_HOSTS = ['localhost'] -MONGO_PORT = 27017 +DATABASE_URL = 'mongodb://localhost:27017/backdrop_test' LOG_LEVEL = "DEBUG" CLIENT_ID = "it's not important here" CLIENT_SECRET = "it's not important here" @@ -9,7 +7,7 @@ "data_set_with_timestamp_auto_id": ["_timestamp", "key"], "evl_volumetrics": ["_timestamp", "service", "transaction"], } -TRANSFORMER_AMQP_URL = 'memory://' +BROKER_URL = 'memory://' from development import (STAGECRAFT_COLLECTION_ENDPOINT_TOKEN, STAGECRAFT_URL, SIGNON_API_USER_TOKEN) diff --git a/etc/deploy.sh b/etc/deploy.sh index a106cd74..3a5931f1 100644 --- a/etc/deploy.sh +++ b/etc/deploy.sh @@ -22,11 +22,17 @@ cf bind-service performance-platform-backdrop-write gds-performance-platform-mon # set environmental variables cf set-env performance-platform-backdrop-read ENVIRONMENT $PAAS_SPACE cf set-env performance-platform-backdrop-read STAGECRAFT_URL https://performance-platform-stagecraft-$PAAS_SPACE.cloudapps.digital -cf set-env performance-platform-backdrop-read SIGNON_API_USER_TOKEN $APP_SIGNON_API_USER_TOKEN +cf set-env performance-platform-backdrop-read SIGNON_API_USER_TOKEN $APP_SIGNON_API_USER_TOKEN cf set-env performance-platform-backdrop-write ENVIRONMENT $PAAS_SPACE cf set-env performance-platform-backdrop-write STAGECRAFT_URL https://performance-platform-stagecraft-$PAAS_SPACE.cloudapps.digital -cf set-env performance-platform-backdrop-write SIGNON_API_USER_TOKEN $APP_SIGNON_API_USER_TOKEN +cf set-env performance-platform-backdrop-write SIGNON_API_USER_TOKEN $APP_SIGNON_API_USER_TOKEN +cf set-env performance-platform-backdrop-write SECRET_KEY $APP_SECRET_KEY +cf set-env performance-platform-backdrop-write REDIS_DATABASE_NUMBER $REDIS_DATABASE_NUMBER + +cf set-env performance-platform-backdrop-celery-worker ENVIRONMENT $PAAS_SPACE +cf set-env performance-platform-backdrop-celery-worker STAGECRAFT_OAUTH_TOKEN $APP_STAGECRAFT_OAUTH_TOKEN +cf set-env performance-platform-backdrop-celery-worker REDIS_DATABASE_NUMBER $REDIS_DATABASE_NUMBER # deploy apps cf push -f manifest.yml diff --git a/features/environment.py b/features/environment.py index 115d0f25..8cf8252a 100644 --- a/features/environment.py +++ b/features/environment.py @@ -61,7 +61,7 @@ def create_client(feature): if 'use_write_api_client' in feature.tags: return FlaskTestClient(write_api) if 'use_http_client' in feature.tags: - return HTTPTestClient(config.DATABASE_NAME) + return HTTPTestClient(config.DATABASE_URL) raise AssertionError( "Test client not selected! Please annotate the failing feature with " diff --git a/features/steps/read_api.py b/features/steps/read_api.py index a2e71acb..e72a3bc9 100644 --- a/features/steps/read_api.py +++ b/features/steps/read_api.py @@ -32,7 +32,7 @@ def step(context, fixture_name, data_set_name): '_week_start_at', '_month_start_at']: if key in obj: obj[key] = parser.parse(obj[key]).astimezone(pytz.utc) - context.client.mongo()[data_set_name].save(obj) + context.client.mongo()[data_set_name].insert_one(obj) def get_data_set_settings_from_context_table(table): @@ -56,7 +56,7 @@ def step(context, fixture_name, data_set_name): '_week_start_at', '_month_start_at']: if key in obj: obj[key] = parser.parse(obj[key]).astimezone(pytz.utc) - context.client.mongo()[data_set_name].save(obj) + context.client.mongo()[data_set_name].insert_one(obj) @given('I have a record updated "{timespan}" ago in the "{data_set_name}" data_set') diff --git a/features/support/http_test_client.py b/features/support/http_test_client.py index 581d8ffd..fe9e9809 100644 --- a/features/support/http_test_client.py +++ b/features/support/http_test_client.py @@ -5,11 +5,10 @@ class HTTPTestClient(BaseClient): - def __init__(self, database_name): - self.database_name = database_name + def __init__(self, database_url): self._read_api = FlaskApp("read", "5000") self._write_api = FlaskApp("write", "5001") - self._mongo_db = MongoClient('localhost', 27017)[self.database_name] + self._mongo_db = MongoClient(database_url).get_database() self._start() def get(self, url, headers=None): diff --git a/features/support/support.py b/features/support/support.py index 42e7f81c..bf1bcb4b 100644 --- a/features/support/support.py +++ b/features/support/support.py @@ -18,7 +18,7 @@ def mongo(self): return self._mongo_db def clean_mongo(self): - self._mongo_db.connection.drop_database( + self._mongo_db.client.drop_database( self._mongo_db.name) def before_scenario(self): diff --git a/manifest.yml b/manifest.yml index a256e77c..52d33cd4 100644 --- a/manifest.yml +++ b/manifest.yml @@ -1,18 +1,30 @@ --- buildpack: python_buildpack -name: performance-platform-backdrop services: - - gds-performance-platform-mongodb-service -memory: 256M + - redis-poc + applications: - name: performance-platform-backdrop-read command: python start.py read 8080 + services: + - gds-performance-platform-mongodb-service instances: 1 memory: 512M + no-route: true health-check-type: none - name: performance-platform-backdrop-write command: python start.py write 8080 + services: + - gds-performance-platform-mongodb-service instances: 1 memory: 512M + no-route: true + health-check-type: none + + - name: performance-platform-backdrop-celery-worker + command: celery -A backdrop.transformers.worker worker --loglevel=info + instances: 1 + memory: 1G + no-route: true health-check-type: none diff --git a/requirements.txt b/requirements.txt index d3c2e2b8..fb705286 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,7 @@ argh==0.25 -Celery==3.1.17 +celery[redis]==4.1.0 Flask==0.10.1 Flask-FeatureFlags==0.4 -gunicorn==19.1.1 invoke isodate==0.5.0 jsonschema==2.4 diff --git a/tests/core/integration/test_data_set_integration.py b/tests/core/integration/test_data_set_integration.py index 95585396..66243216 100644 --- a/tests/core/integration/test_data_set_integration.py +++ b/tests/core/integration/test_data_set_integration.py @@ -10,16 +10,14 @@ from backdrop.core.query import Query from tests.support.test_helpers import d_tz -HOSTS = ['localhost'] -PORT = 27017 -DB_NAME = 'performance_platform_test' +DATABASE_URL = 'mongodb://localhost:27017/backdrop_test' DATA_SET = 'data_set_integration_test' class TestDataSetIntegration(unittest.TestCase): def setUp(self): - self.storage = MongoStorageEngine.create(HOSTS, PORT, DB_NAME) + self.storage = MongoStorageEngine.create(DATABASE_URL) self.config = { 'name': DATA_SET, @@ -30,7 +28,8 @@ def setUp(self): self.data_set = DataSet(self.storage, self.config) - self.mongo_collection = MongoClient(HOSTS, PORT)[DB_NAME][DATA_SET] + database = MongoClient(DATABASE_URL).get_database() + self.mongo_collection = database[DATA_SET] def setup__timestamp_data(self): self.mongo_collection.save({ diff --git a/tests/core/storage/test_mongo.py b/tests/core/storage/test_mongo.py index 6ff1f4d1..51db4fd2 100644 --- a/tests/core/storage/test_mongo.py +++ b/tests/core/storage/test_mongo.py @@ -19,11 +19,12 @@ from .test_storage import BaseStorageTest +DATABASE_URL = 'mongodb://localhost:27017/backdrop_test' + class TestMongoStorageEngine(BaseStorageTest): def setup(self): - self.engine = MongoStorageEngine.create( - ['localhost'], 27017, 'backdrop_test') + self.engine = MongoStorageEngine.create(DATABASE_URL) def test_create_data_set(self): self.engine.create_data_set('should_have_index', 0) @@ -53,7 +54,9 @@ def test_batch_last_updated(self): assert_that(last_updated.second, is_(timestamp.second)) def teardown(self): - self.engine._mongo.drop_database('backdrop_test') + mongo_client = self.engine._mongo_client + database_name = mongo_client.get_database().name + mongo_client.drop_database(database_name) class TestReconnectingSave(object): diff --git a/tests/core/storage/test_storage.py b/tests/core/storage/test_storage.py index c99c1948..30b8bcc4 100644 --- a/tests/core/storage/test_storage.py +++ b/tests/core/storage/test_storage.py @@ -37,9 +37,6 @@ def _save_all_with_periods(self, data_set_id, *records): self._save_all(data_set_id, *records) - def test_is_alive(self): - assert_that(self.engine.alive(), is_(True)) - def test_does_not_exist(self): assert_that(self.engine.data_set_exists('foo_bar'), is_(False)) diff --git a/tests/read/test_read_api_service_data_endpoint.py b/tests/read/test_read_api_service_data_endpoint.py index b6b2931f..8eed4066 100644 --- a/tests/read/test_read_api_service_data_endpoint.py +++ b/tests/read/test_read_api_service_data_endpoint.py @@ -115,7 +115,9 @@ class PreflightChecksApiTestCase(unittest.TestCase): def setUp(self): self.app = api.app.test_client() - api.storage._mongo.drop_database(api.app.config['DATABASE_NAME']) + mongo_client = api.storage._mongo_client + database_name = mongo_client.get_database().name + mongo_client.drop_database(database_name) @fake_data_set_exists("data_set", data_group="some-group", data_type="some-type") def test_cors_preflight_requests_have_empty_body(self): diff --git a/tests/write/test_flask_integration.py b/tests/write/test_flask_integration.py index 3d40fa61..330ba095 100644 --- a/tests/write/test_flask_integration.py +++ b/tests/write/test_flask_integration.py @@ -205,18 +205,6 @@ def test_api_exposes_a_healthcheck(self): entity = json.loads(response.data) assert_that(entity["status"], is_("ok")) - @patch("backdrop.write.api.statsd") - @patch("backdrop.write.api.storage") - def test_exception_handling(self, storage, statsd): - storage.alive.side_effect = ValueError("BOOM") - - response = self.app.get("/_status") - - assert_that(response, has_status(500)) - assert_that(response, is_error_response()) - - statsd.incr.assert_called_with("write.error", data_set="/_status") - class UploadPageTestCase(unittest.TestCase): def setUp(self):