Fix merge conflicts

.gitignore

@@ -18,3 +18,4 @@ tmp/*
 
 # vim swap files
 .*.swp
+venv/

.python-version

@@ -0,0 +1 @@
+2.7.13

.travis.yml

@@ -19,15 +19,26 @@ install:
   - pip install -q -r requirements_for_tests.txt
 
 env:
-  global: 
-    - SKIP_VIRUS_SCAN=1 SKIP_SPLINTER_TESTS=1 MONGO_REPLICA_SET='' NO_AUTOPEP8=1
-    - PAAS_USER=pp-deploy@digital.cabinet-office.gov.uk
+  global:
+    - PAAS_USER: pp-deploy@digital.cabinet-office.gov.uk
     # PAAS_PASSWORD
     - secure: HNo6rnNiXmNYAV1TvkcMxeniXpmHoZkNEiT+X9EgyInCOnVm4mz89qi/AElY/ESE5HdYIegJ5l1M5tAWCei4dSDSTSfkbBBiHE6dfpf1OSU857oQ6T6161NRg4V2oxNdWSRgAX+G8rpxBShaXCDU9q9iBxhsqrmTX+UGlzKloDk=
+    - REDIS_DATABASE_NUMBER_STAGING: 4
+    - REDIS_DATABASE_NUMBER_PRODUCTION: 5
     # APP_SIGNON_API_USER_TOKEN_STAGING
     - secure: PYLrm6uH6uhNA69xgI5lSXLrX0ouPBdxlMlQXER0K2BkHzxTowNDFyf5LDa2kwIueJ/YXFHAgm7eBN9gr3JIXAN8Mqb15LTHJVg6hc2aLwFZKES4fuPa/da3Pn24xrdTesIedUAv2vSkuacaktZwKxsv6LENw53uA+8kOT8l9SQ=
     # APP_SIGNON_API_USER_TOKEN_PRODUCTION
     - secure: SBk+7zw2GMkgbl8p7hwXZw5VkDf2FQE4kBklD40c7goORyEF3/FjTVJTVlREN4Rwb3a81N5lkQDl0fLPhGaDv+PyTr+l69IU64w1Whu9e9fL/wMC3zo5Eer3Btptar5jdCsEg+V4woKKXv6j4B4xebHN6sU7C6dX9TGKs+R1aBo=
+    # APP_SECRET_KEY_STAGING
+    - secure: Cl7cvdzviF3hCepM7n3XMz9q1D/FCEQkcDZOyRTPkHlm9V2GKi+e9ECuAuWQVRiRDTb7bAZZ5kl+zTj00nMb0BKds3QoisZ5oB8UwYFOQSrYV/NTpge0wsvZZvfqXp5sN3CLVXE2XynLsZZSzOTIte8UKD4giBFsuHo2sBcVbb8=
+    # APP_SECRET_KEY_PRODUCTION
+    - secure: ifAsLdVTrzDT8IIEVdW1TJon8VYdmDdzxxE/CtCgc0hcBP9eneKF7qvBOy6TFMnE9KSz4zzyYx+yZI8s5sAsjknw/HEnHYMCYSjRcM3GMWP27/QFM2983CgWkoEn5xGj3WEXra1WJGltzqbeoYfZWmFA9pBGSU6IbR0IzjLC6+g=
+    # APP_STAGECRAFT_COLLECTION_ENDPOINT_TOKEN
+    # Value too large to be encrypted.
+    # APP_STAGECRAFT_OAUTH_TOKEN_STAGING
+    - secure: JkoKVPblX1sHICKxJMT0kPi5Nax3dn3cG1bzmAmpMDX+rd37I4XtA96G17ibUmUrp2CVrOo4is5WjI9hPDwmC8/o7/6MOHqJo7jeflY+Iq8bWShcja0khlb4aBlpWfgMS+vniLCBPkQtUe2eE87jFLYJXmsuBGfp4r4L88fhgOk=
+    # APP_STAGECRAFT_OAUTH_TOKEN_PRODUCTION
+    - secure: HZPkW81a+OsfFrCCpcuYYBKNIEmejsQlIreVo2AJ1NipBVK8DniqIK+bXGUIQraaqeKXApXknfPepVpyqG3uyzHvkboylooWa7fuoJB4bs9Eb91Qo2AeWjP2fts0EmUX8R76K51n7EwUgk9xemM/bE2e+0FmG5e35JpXEwfm/5I=
 
 script:
   - ./run_tests.sh
@@ -40,13 +51,13 @@ before_deploy:
 
 deploy:
   - provider: script
-    script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_STAGING etc/deploy.sh staging
+    script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_STAGING APP_SECRET_KEY=$APP_SECRET_KEY_STAGING REDIS_DATABASE_NUMBER=$REDIS_DATABASE_NUMBER_STAGING APP_STAGECRAFT_OAUTH_TOKEN=$APP_STAGECRAFT_OAUTH_TOKEN_STAGING etc/deploy.sh staging
     skip_cleanup: true
     on:
       branch: staging
 
   - provider: script
-    script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_PRODUCTION etc/deploy.sh production
+    script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_PRODUCTION APP_SECRET_KEY=$APP_SECRET_KEY_PRODUCTION REDIS_DATABASE_NUMBER=$REDIS_DATABASE_NUMBER_PRODUCTION APP_STAGECRAFT_OAUTH_TOKEN=$APP_STAGECRAFT_OAUTH_TOKEN_PRODUCTION etc/deploy.sh production
     skip_cleanup: true
     on:
       branch: production

backdrop/core/config/common.py

@@ -2,14 +2,23 @@
 import json
 from base64 import b64decode
 
+
 def load_paas_settings():
     paas = {}
     if 'VCAP_SERVICES' in os.environ:
         vcap = json.loads(os.environ['VCAP_SERVICES'])
-        for service in vcap['mongodb']:
-            if service['name'] == 'gds-performance-platform-mongodb-service':
-                credentials = service['credentials']
-                paas['DATABASE_URL'] = credentials['uri']
-                ca_cert = b64decode(credentials['ca_certificate_base64'])
-                paas['CA_CERTIFICATE'] = ca_cert
+        if 'mongodb' in vcap:
+            for service in vcap['mongodb']:
+                if service['name'] == 'gds-performance-platform-mongodb-service':
+                    credentials = service['credentials']
+                    paas['DATABASE_URL'] = credentials['uri']
+                    ca_cert = b64decode(credentials['ca_certificate_base64'])
+                    paas['CA_CERTIFICATE'] = ca_cert
+        if 'REDIS_DATABASE_NUMBER' in os.environ:
+            for service in vcap['user-provided']:
+                if service['name'] == 'redis-poc':
+                    database_number = os.environ['REDIS_DATABASE_NUMBER']
+                    url = service['credentials']['url']
+                    url += '/' + database_number
+                    paas['REDIS_URL'] = url
     return paas

backdrop/core/storage/mongo.py

@@ -72,7 +72,7 @@ def reconnecting_save(collection, record, tries=3):
 class MongoStorageEngine(object):
 
     @classmethod
-    def create(cls, database_url, ca_certificate):
+    def create(cls, database_url, ca_certificate=None):
         if ca_certificate is not None:
             dir = os.path.dirname(__file__)
             filename = os.path.join(dir, 'mongodb.crt')

backdrop/read/api.py

@@ -91,12 +91,7 @@ def http_error_handler(e):
 @cache_control.nocache
 @statsd.timer('read.route.heath_check.status')
 def health_check():
-
-    if not storage.alive():
-        return jsonify(status='error',
-                       message='cannot connect to database'), 500
-
-    return jsonify(status='ok', message='database is up')
+    return jsonify(status='ok', message='app is up')
 
 
 @app.route('/_status/data-sets', methods=['GET'])

backdrop/read/config/production.py

@@ -7,4 +7,4 @@
 STAGECRAFT_URL = os.getenv('STAGECRAFT_URL')
 SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN')
 LOG_LEVEL = "INFO"
-SESSION_COOKIE_SECURE=True
+SESSION_COOKIE_SECURE = True

backdrop/read/config/staging.py

@@ -7,4 +7,4 @@
 STAGECRAFT_URL = os.getenv('STAGECRAFT_URL')
 SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN')
 LOG_LEVEL = "INFO"
-SESSION_COOKIE_SECURE=True
+SESSION_COOKIE_SECURE = True

backdrop/transformers/config/development.py

@@ -1,4 +1,4 @@
-TRANSFORMER_AMQP_URL = 'amqp://backdrop_write:backdrop_write@localhost:5672/%2Fbackdrop_write'
+BROKER_URL = 'amqp://backdrop_write:backdrop_write@localhost:5672/%2Fbackdrop_write'
 STAGECRAFT_URL = 'http://localhost:3103'
 STAGECRAFT_OAUTH_TOKEN = 'development-oauth-access-token'
 BACKDROP_READ_URL = 'http://backdrop-read.dev.gov.uk/data'

backdrop/transformers/config/production.py

@@ -0,0 +1,10 @@
+import os
+from ...core.config.common import load_paas_settings
+
+PAAS = load_paas_settings()
+BROKER_URL = PAAS.get('REDIS_URL') or os.getenv('REDIS_URL')
+BROKER_FAILOVER_STRATEGY = "round-robin"
+STAGECRAFT_URL = 'https://performance-platform-stagecraft-production.cloudapps.digital'
+STAGECRAFT_OAUTH_TOKEN = os.getenv('STAGECRAFT_OAUTH_TOKEN')
+BACKDROP_READ_URL = 'https://performance-platform-backdrop-read-production.cloudapps.digital/data'
+BACKDROP_WRITE_URL = 'https://performance-platform-backdrop-write-production.cloudapps.digital/data'

backdrop/transformers/config/staging.py

@@ -0,0 +1,10 @@
+import os
+from ...core.config.common import load_paas_settings
+
+PAAS = load_paas_settings()
+BROKER_URL = PAAS.get('REDIS_URL') or os.getenv('REDIS_URL')
+BROKER_FAILOVER_STRATEGY = "round-robin"
+STAGECRAFT_URL = 'https://performance-platform-stagecraft-staging.cloudapps.digital'
+STAGECRAFT_OAUTH_TOKEN = os.getenv('STAGECRAFT_OAUTH_TOKEN')
+BACKDROP_READ_URL = 'https://performance-platform-backdrop-read-staging.cloudapps.digital/data'
+BACKDROP_WRITE_URL = 'https://performance-platform-backdrop-write-staging.cloudapps.digital/data'

backdrop/transformers/config/test.py

@@ -1,4 +1,4 @@
-TRANSFORMER_AMQP_URL = 'memory://'
+BROKER_URL = 'memory://'
 STAGECRAFT_URL = 'http://stagecraft'
 STAGECRAFT_OAUTH_TOKEN = 'development-oauth-access-token'
 BACKDROP_READ_URL = 'http://backdrop/data'

backdrop/transformers/dispatch.py

@@ -8,6 +8,7 @@
 from statsd import StatsClient
 
 from backdrop.core.timeseries import parse_period
+from backdrop.core.timeutils import parse_time_as_utc
 from backdrop.core.log_handler import get_log_file_handler
 from backdrop.core.errors import incr_on_error
 from backdrop.transformers.tasks.util import encode_id
@@ -16,11 +17,11 @@
 
 from performanceplatform.client import AdminAPI, DataSet
 
-GOVUK_ENV = getenv("GOVUK_ENV", "development")
+ENVIRONMENT = getenv("ENVIRONMENT", "development")
 logger = logging.getLogger()
 logger.setLevel(logging.DEBUG)
 logger.addHandler(
-    get_log_file_handler("log/{}.log".format(GOVUK_ENV), logging.DEBUG))
+    get_log_file_handler("log/{}.log".format(ENVIRONMENT), logging.DEBUG))
 
 stats_client = StatsClient(prefix=getenv("GOVUK_STATSD_PREFIX",
                                          "pp.apps.backdrop.transformers.worker"))
@@ -147,6 +148,9 @@ def run_transform(data_set_config, transform, earliest, latest):
         data_set_config['data_type'],
     )
 
+    earliest = parse_time_as_utc(earliest)
+    latest = parse_time_as_utc(latest)
+
     data = data_set.get(
         query_parameters=get_query_parameters(transform, earliest, latest)
     )

backdrop/transformers/worker.py

@@ -4,13 +4,13 @@
 import importlib
 from os import getenv
 
-GOVUK_ENV = getenv("GOVUK_ENV", "development")
+ENVIRONMENT = getenv("ENVIRONMENT", "development")
 config = importlib.import_module(
-    "backdrop.transformers.config.{}".format(GOVUK_ENV))
+    "backdrop.transformers.config.{}".format(ENVIRONMENT))
 
 app = Celery(
     'transformations',
-    broker=config.TRANSFORMER_AMQP_URL,
+    broker=config.BROKER_URL,
     include=['backdrop.transformers.dispatch'])
 
 

backdrop/write/api.py

@@ -48,7 +48,7 @@
 
 app.url_map.converters["data_set"] = DataSetConverter
 
-celery_app = Celery(broker=app.config['TRANSFORMER_AMQP_URL'])
+celery_app = Celery(broker=app.config['BROKER_URL'])
 app.config['BROKER_FAILOVER_STRATEGY'] = "round-robin"
 celery_app.conf.update(app.config)
 
@@ -105,10 +105,7 @@ def http_error_handler(e):
 @cache_control.nocache
 @statsd.timer('write.route.health_check.status')
 def health_check():
-    if storage.alive():
-        return jsonify(status='ok', message='database seems fine')
-    else:
-        abort(500, 'cannot connect to database')
+    return jsonify(status='ok', message='app is up')
 
 
 @app.route('/data/<data_group>/<data_type>', methods=['POST'])

backdrop/write/config/development.py

@@ -15,4 +15,4 @@
 
 SIGNON_API_USER_TOKEN = 'development-oauth-access-token'
 
-TRANSFORMER_AMQP_URL = 'amqp://backdrop_write:backdrop_write@localhost:5672/%2Fbackdrop_write'
+BROKER_URL = 'amqp://backdrop_write:backdrop_write@localhost:5672/%2Fbackdrop_write'

backdrop/write/config/production.py

@@ -5,6 +5,19 @@
 DATABASE_URL = PAAS.get('DATABASE_URL')
 CA_CERTIFICATE = PAAS.get('CA_CERTIFICATE')
 STAGECRAFT_URL = os.getenv('STAGECRAFT_URL')
+BROKER_URL = PAAS.get('REDIS_URL') or os.getenv('REDIS_URL')
+BROKER_FAILOVER_STRATEGY = "round-robin"
 SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN')
 LOG_LEVEL = "INFO"
-SESSION_COOKIE_SECURE=True
+DATA_SET_UPLOAD_FORMAT = {
+    "ithc_excel": "excel",
+}
+DATA_SET_UPLOAD_FILTERS = {
+    "ithc_excel": [
+        "backdrop.core.upload.filters.first_sheet_filter",
+    ],
+}
+SESSION_COOKIE_SECURE = True
+SECRET_KEY = os.getenv('SECRET_KEY')
+STAGECRAFT_COLLECTION_ENDPOINT_TOKEN = os.getenv(
+    'STAGECRAFT_COLLECTION_ENDPOINT_TOKEN')

backdrop/write/config/staging.py

@@ -5,6 +5,11 @@
 DATABASE_URL = PAAS.get('DATABASE_URL')
 CA_CERTIFICATE = PAAS.get('CA_CERTIFICATE')
 STAGECRAFT_URL = os.getenv('STAGECRAFT_URL')
+BROKER_URL = PAAS.get('REDIS_URL') or os.getenv('REDIS_URL')
+BROKER_FAILOVER_STRATEGY = "round-robin"
 SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN')
 LOG_LEVEL = "INFO"
-SESSION_COOKIE_SECURE=True
+SESSION_COOKIE_SECURE = True
+SECRET_KEY = os.getenv('SECRET_KEY')
+STAGECRAFT_COLLECTION_ENDPOINT_TOKEN = os.getenv(
+    'STAGECRAFT_COLLECTION_ENDPOINT_TOKEN')

backdrop/write/config/test.py

@@ -1,6 +1,4 @@
-DATABASE_NAME = "backdrop_test"
-MONGO_HOSTS = ['localhost']
-MONGO_PORT = 27017
+DATABASE_URL = 'mongodb://localhost:27017/backdrop_test'
 LOG_LEVEL = "DEBUG"
 CLIENT_ID = "it's not important here"
 CLIENT_SECRET = "it's not important here"
@@ -9,7 +7,7 @@
     "data_set_with_timestamp_auto_id": ["_timestamp", "key"],
     "evl_volumetrics": ["_timestamp", "service", "transaction"],
 }
-TRANSFORMER_AMQP_URL = 'memory://'
+BROKER_URL = 'memory://'
 
 from development import (STAGECRAFT_COLLECTION_ENDPOINT_TOKEN, STAGECRAFT_URL,
                          SIGNON_API_USER_TOKEN)

etc/deploy.sh

@@ -22,11 +22,17 @@ cf bind-service performance-platform-backdrop-write gds-performance-platform-mon
 # set environmental variables
 cf set-env performance-platform-backdrop-read ENVIRONMENT $PAAS_SPACE
 cf set-env performance-platform-backdrop-read STAGECRAFT_URL https://performance-platform-stagecraft-$PAAS_SPACE.cloudapps.digital
-cf set-env performance-platform-backdrop-read SIGNON_API_USER_TOKEN $APP_SIGNON_API_USER_TOKEN 
+cf set-env performance-platform-backdrop-read SIGNON_API_USER_TOKEN $APP_SIGNON_API_USER_TOKEN
 
 cf set-env performance-platform-backdrop-write ENVIRONMENT $PAAS_SPACE
 cf set-env performance-platform-backdrop-write STAGECRAFT_URL https://performance-platform-stagecraft-$PAAS_SPACE.cloudapps.digital
-cf set-env performance-platform-backdrop-write SIGNON_API_USER_TOKEN $APP_SIGNON_API_USER_TOKEN 
+cf set-env performance-platform-backdrop-write SIGNON_API_USER_TOKEN $APP_SIGNON_API_USER_TOKEN
+cf set-env performance-platform-backdrop-write SECRET_KEY $APP_SECRET_KEY
+cf set-env performance-platform-backdrop-write REDIS_DATABASE_NUMBER $REDIS_DATABASE_NUMBER
+
+cf set-env performance-platform-backdrop-celery-worker ENVIRONMENT $PAAS_SPACE
+cf set-env performance-platform-backdrop-celery-worker STAGECRAFT_OAUTH_TOKEN $APP_STAGECRAFT_OAUTH_TOKEN
+cf set-env performance-platform-backdrop-celery-worker REDIS_DATABASE_NUMBER $REDIS_DATABASE_NUMBER
 
 # deploy apps
 cf push -f manifest.yml

features/environment.py

@@ -61,7 +61,7 @@ def create_client(feature):
     if 'use_write_api_client' in feature.tags:
         return FlaskTestClient(write_api)
     if 'use_http_client' in feature.tags:
-        return HTTPTestClient(config.DATABASE_NAME)
+        return HTTPTestClient(config.DATABASE_URL)
 
     raise AssertionError(
         "Test client not selected! Please annotate the failing feature with "

features/steps/read_api.py

@@ -32,7 +32,7 @@ def step(context, fixture_name, data_set_name):
                         '_week_start_at', '_month_start_at']:
                 if key in obj:
                     obj[key] = parser.parse(obj[key]).astimezone(pytz.utc)
-            context.client.mongo()[data_set_name].save(obj)
+            context.client.mongo()[data_set_name].insert_one(obj)
 
 
 def get_data_set_settings_from_context_table(table):
@@ -56,7 +56,7 @@ def step(context, fixture_name, data_set_name):
                         '_week_start_at', '_month_start_at']:
                 if key in obj:
                     obj[key] = parser.parse(obj[key]).astimezone(pytz.utc)
-            context.client.mongo()[data_set_name].save(obj)
+            context.client.mongo()[data_set_name].insert_one(obj)
 
 
 @given('I have a record updated "{timespan}" ago in the "{data_set_name}" data_set')

features/support/http_test_client.py

@@ -5,11 +5,10 @@
 
 
 class HTTPTestClient(BaseClient):
-    def __init__(self, database_name):
-        self.database_name = database_name
+    def __init__(self, database_url):
         self._read_api = FlaskApp("read", "5000")
         self._write_api = FlaskApp("write", "5001")
-        self._mongo_db = MongoClient('localhost', 27017)[self.database_name]
+        self._mongo_db = MongoClient(database_url).get_database()
         self._start()
 
     def get(self, url, headers=None):

features/support/support.py

@@ -18,7 +18,7 @@ def mongo(self):
         return self._mongo_db
 
     def clean_mongo(self):
-        self._mongo_db.connection.drop_database(
+        self._mongo_db.client.drop_database(
             self._mongo_db.name)
 
     def before_scenario(self):

manifest.yml

@@ -1,18 +1,30 @@
 ---
 buildpack: python_buildpack
-name: performance-platform-backdrop
 services:
-  - gds-performance-platform-mongodb-service
-memory: 256M
+  - redis-poc
+
 applications:
   - name: performance-platform-backdrop-read
     command: python start.py read 8080
+    services:
+      - gds-performance-platform-mongodb-service
     instances: 1
     memory: 512M
+    no-route: true
     health-check-type: none
 
   - name: performance-platform-backdrop-write
     command: python start.py write 8080
+    services:
+      - gds-performance-platform-mongodb-service
     instances: 1
     memory: 512M
+    no-route: true
+    health-check-type: none
+
+  - name: performance-platform-backdrop-celery-worker
+    command: celery -A backdrop.transformers.worker worker --loglevel=info
+    instances: 1
+    memory: 1G
+    no-route: true
     health-check-type: none

requirements.txt

@@ -1,8 +1,7 @@
 argh==0.25
-Celery==3.1.17
+celery[redis]==4.1.0
 Flask==0.10.1
 Flask-FeatureFlags==0.4
-gunicorn==19.1.1
 invoke
 isodate==0.5.0
 jsonschema==2.4