Merge pull request #475 from alphagov/paas-migration

Paas migration

.gitignore

@@ -18,3 +18,4 @@ tmp/*
 
 # vim swap files
 .*.swp
+venv/

.python-version

@@ -0,0 +1 @@
+2.7.13

.travis.yml

@@ -3,27 +3,61 @@ language: python
 python:
   - "2.7"
 cache: pip
-sudo: false
+sudo: true
+notifications:
+  email: false
+
 # Need mongodb for testing
 services: mongodb
 # command to install dependencies
+
 install:
   - pip install --upgrade pip
   - easy_install distribute
   - pip install --upgrade distribute
   - pip install --upgrade setuptools
   - pip install -q -r requirements_for_tests.txt
-# command to run tests
+
 env:
-  - SKIP_VIRUS_SCAN=1 SKIP_SPLINTER_TESTS=1 MONGO_REPLICA_SET='' NO_AUTOPEP8=1
+  global: 
+    - PAAS_USER: pp-deploy@digital.cabinet-office.gov.uk
+    # PAAS_PASSWORD
+    - secure: HNo6rnNiXmNYAV1TvkcMxeniXpmHoZkNEiT+X9EgyInCOnVm4mz89qi/AElY/ESE5HdYIegJ5l1M5tAWCei4dSDSTSfkbBBiHE6dfpf1OSU857oQ6T6161NRg4V2oxNdWSRgAX+G8rpxBShaXCDU9q9iBxhsqrmTX+UGlzKloDk=
+    - REDIS_DATABASE_NUMBER_STAGING: 4
+    - REDIS_DATABASE_NUMBER_PRODUCTION: 5
+    # APP_SIGNON_API_USER_TOKEN_STAGING
+    - secure: PYLrm6uH6uhNA69xgI5lSXLrX0ouPBdxlMlQXER0K2BkHzxTowNDFyf5LDa2kwIueJ/YXFHAgm7eBN9gr3JIXAN8Mqb15LTHJVg6hc2aLwFZKES4fuPa/da3Pn24xrdTesIedUAv2vSkuacaktZwKxsv6LENw53uA+8kOT8l9SQ=
+    # APP_SIGNON_API_USER_TOKEN_PRODUCTION
+    - secure: SBk+7zw2GMkgbl8p7hwXZw5VkDf2FQE4kBklD40c7goORyEF3/FjTVJTVlREN4Rwb3a81N5lkQDl0fLPhGaDv+PyTr+l69IU64w1Whu9e9fL/wMC3zo5Eer3Btptar5jdCsEg+V4woKKXv6j4B4xebHN6sU7C6dX9TGKs+R1aBo=
+    # APP_SECRET_KEY_STAGING
+    - secure: Cl7cvdzviF3hCepM7n3XMz9q1D/FCEQkcDZOyRTPkHlm9V2GKi+e9ECuAuWQVRiRDTb7bAZZ5kl+zTj00nMb0BKds3QoisZ5oB8UwYFOQSrYV/NTpge0wsvZZvfqXp5sN3CLVXE2XynLsZZSzOTIte8UKD4giBFsuHo2sBcVbb8=
+    # APP_SECRET_KEY_PRODUCTION
+    - secure: ifAsLdVTrzDT8IIEVdW1TJon8VYdmDdzxxE/CtCgc0hcBP9eneKF7qvBOy6TFMnE9KSz4zzyYx+yZI8s5sAsjknw/HEnHYMCYSjRcM3GMWP27/QFM2983CgWkoEn5xGj3WEXra1WJGltzqbeoYfZWmFA9pBGSU6IbR0IzjLC6+g=
+    # APP_STAGECRAFT_COLLECTION_ENDPOINT_TOKEN
+    # Value too large to be encrypted.
+    # APP_STAGECRAFT_OAUTH_TOKEN_STAGING
+    - secure: JkoKVPblX1sHICKxJMT0kPi5Nax3dn3cG1bzmAmpMDX+rd37I4XtA96G17ibUmUrp2CVrOo4is5WjI9hPDwmC8/o7/6MOHqJo7jeflY+Iq8bWShcja0khlb4aBlpWfgMS+vniLCBPkQtUe2eE87jFLYJXmsuBGfp4r4L88fhgOk=
+    # APP_STAGECRAFT_OAUTH_TOKEN_PRODUCTION
+    - secure: HZPkW81a+OsfFrCCpcuYYBKNIEmejsQlIreVo2AJ1NipBVK8DniqIK+bXGUIQraaqeKXApXknfPepVpyqG3uyzHvkboylooWa7fuoJB4bs9Eb91Qo2AeWjP2fts0EmUX8R76K51n7EwUgk9xemM/bE2e+0FmG5e35JpXEwfm/5I=
+
 script:
   - ./run_tests.sh
+
 after_script:
   - coveralls
-branches:
-  except:
-    - release
-    - /^release_[0-9]+$/
-    - /^deployed-to-(preview|staging|production)$/
-notifications:
-  email: false
+
+before_deploy:
+  - chmod +x etc/deploy.sh
+
+deploy:
+  - provider: script
+    script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_STAGING APP_SECRET_KEY=$APP_SECRET_KEY_STAGING REDIS_DATABASE_NUMBER=$REDIS_DATABASE_NUMBER_STAGING APP_STAGECRAFT_OAUTH_TOKEN=$APP_STAGECRAFT_OAUTH_TOKEN_STAGING etc/deploy.sh staging
+    skip_cleanup: true
+    on:
+      branch: staging
+
+  - provider: script
+    script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_PRODUCTION APP_SECRET_KEY=$APP_SECRET_KEY_PRODUCTION REDIS_DATABASE_NUMBER=$REDIS_DATABASE_NUMBER_PRODUCTION APP_STAGECRAFT_OAUTH_TOKEN=$APP_STAGECRAFT_OAUTH_TOKEN_PRODUCTION etc/deploy.sh production
+    skip_cleanup: true
+    on:
+      branch: production

backdrop/core/config/common.py

@@ -0,0 +1,24 @@
+import os
+import json
+from base64 import b64decode
+
+
+def load_paas_settings():
+    paas = {}
+    if 'VCAP_SERVICES' in os.environ:
+        vcap = json.loads(os.environ['VCAP_SERVICES'])
+        if 'mongodb' in vcap:
+            for service in vcap['mongodb']:
+                if service['name'] == 'gds-performance-platform-mongodb-service':
+                    credentials = service['credentials']
+                    paas['DATABASE_URL'] = credentials['uri']
+                    ca_cert = b64decode(credentials['ca_certificate_base64'])
+                    paas['CA_CERTIFICATE'] = ca_cert
+        if 'REDIS_DATABASE_NUMBER' in os.environ:
+            for service in vcap['user-provided']:
+                if service['name'] == 'redis-poc':
+                    database_number = os.environ['REDIS_DATABASE_NUMBER']
+                    url = service['credentials']['url']
+                    url += '/' + database_number
+                    paas['REDIS_URL'] = url
+    return paas

backdrop/core/storage/mongo.py

@@ -39,23 +39,6 @@ def convert_datetimes_to_utc(result):
     return dict((key, time_as_utc(value)) for key, value in result.items())
 
 
-def get_mongo_client(hosts, port):
-    """Return an appropriate mongo client
-    """
-    client_list = ','.join(':'.join([host, str(port)]) for host in hosts)
-
-    # We can't always guarantee we'll be on 'production'
-    # so we allow jenkins to add the set as a variable
-    # Some test environments / other envs have their own sets e.g. 'gds-ci'
-    replica_set = os.getenv('MONGO_REPLICA_SET', 'production')
-
-    if replica_set == '':
-        return pymongo.MongoClient(client_list)
-    else:
-        return pymongo.MongoReplicaSetClient(
-            client_list, replicaSet=replica_set)
-
-
 def reconnecting_save(collection, record, tries=3):
     """Save to mongo, retrying if necesarry
     """
@@ -89,18 +72,32 @@ def reconnecting_save(collection, record, tries=3):
 class MongoStorageEngine(object):
 
     @classmethod
-    def create(cls, hosts, port, database):
-        return cls(get_mongo_client(hosts, port), database)
+    def create(cls, database_url, ca_certificate=None):
+        if ca_certificate is not None:
+            dir = os.path.dirname(__file__)
+            filename = os.path.join(dir, 'mongodb.crt')
+            f = open(filename, 'w')
+            f.write(ca_certificate)
+            f.close()
+
+            mongo_client = pymongo.MongoClient(
+                database_url,
+                ssl_ca_certs=filename
+            )
+        else:
+            mongo_client = pymongo.MongoClient(database_url)
+
+        return cls(mongo_client)
 
-    def __init__(self, mongo, database):
-        self._mongo = mongo
-        self._db = mongo[database]
+    def __init__(self, mongo_client):
+        self._mongo_client = mongo_client
+        self._db = mongo_client.get_database()
 
     def _collection(self, data_set_id):
         return self._db[data_set_id]
 
     def alive(self):
-        return self._mongo.alive()
+        return self._mongo_client.alive()
 
     def data_set_exists(self, data_set_id):
         return data_set_id in self._db.collection_names()

backdrop/read/api.py

@@ -19,20 +19,20 @@
 from ..core.storage.mongo import MongoStorageEngine
 from ..core.timeutils import as_utc
 
-GOVUK_ENV = getenv("GOVUK_ENV", "development")
+ENVIRONMENT = getenv("ENVIRONMENT", "development")
 
 app = Flask("backdrop.read.api")
 
 feature_flags = FeatureFlag(app)
 
 # Configuration
 app.config.from_object(
-    "backdrop.read.config.{}".format(GOVUK_ENV))
+    "backdrop.read.config.{}".format(ENVIRONMENT))
 
 storage = MongoStorageEngine.create(
-    app.config['MONGO_HOSTS'],
-    app.config['MONGO_PORT'],
-    app.config['DATABASE_NAME'])
+    app.config['DATABASE_URL'],
+    app.config.get('CA_CERTIFICATE')
+)
 
 admin_api = client.AdminAPI(
     app.config['STAGECRAFT_URL'],
@@ -46,7 +46,7 @@
 DEFAULT_DATA_SET_PUBLISHED = True
 DEFAULT_DATA_SET_REALTIME = False
 
-log_handler.set_up_logging(app, GOVUK_ENV)
+log_handler.set_up_logging(app, ENVIRONMENT)
 
 
 class JsonEncoder(json.JSONEncoder):
@@ -91,12 +91,7 @@ def http_error_handler(e):
 @cache_control.nocache
 @statsd.timer('read.route.heath_check.status')
 def health_check():
-
-    if not storage.alive():
-        return jsonify(status='error',
-                       message='cannot connect to database'), 500
-
-    return jsonify(status='ok', message='database is up')
+    return jsonify(status='ok', message='app is up')
 
 
 @app.route('/_status/data-sets', methods=['GET'])

backdrop/read/config/development.py

@@ -1,6 +1,4 @@
-DATABASE_NAME = "backdrop"
-MONGO_HOSTS = ['localhost']
-MONGO_PORT = 27017
+DATABASE_URL = 'mongodb://localhost:27017/backdrop_development'
 LOG_LEVEL = "DEBUG"
 
 STAGECRAFT_URL = 'http://localhost:3103'

backdrop/read/config/production.py

@@ -0,0 +1,10 @@
+import os
+from ...core.config.common import load_paas_settings
+
+PAAS = load_paas_settings()
+DATABASE_URL = PAAS.get('DATABASE_URL')
+CA_CERTIFICATE = PAAS.get('CA_CERTIFICATE')
+STAGECRAFT_URL = os.getenv('STAGECRAFT_URL')
+SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN')
+LOG_LEVEL = "INFO"
+SESSION_COOKIE_SECURE = True

backdrop/read/config/staging.py

@@ -0,0 +1,10 @@
+import os
+from ...core.config.common import load_paas_settings
+
+PAAS = load_paas_settings()
+DATABASE_URL = PAAS.get('DATABASE_URL')
+CA_CERTIFICATE = PAAS.get('CA_CERTIFICATE')
+STAGECRAFT_URL = os.getenv('STAGECRAFT_URL')
+SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN')
+LOG_LEVEL = "INFO"
+SESSION_COOKIE_SECURE = True

backdrop/read/config/test.py

@@ -1,6 +1,4 @@
-DATABASE_NAME = "backdrop_test"
-MONGO_HOSTS = ['localhost']
-MONGO_PORT = 27017
+DATABASE_URL = 'mongodb://localhost:27017/backdrop_test'
 LOG_LEVEL = "ERROR"
 
 DATA_SET_RATE_LIMIT = '10000/second'

backdrop/transformers/config/development.py

@@ -1,4 +1,4 @@
-TRANSFORMER_AMQP_URL = 'amqp://backdrop_write:backdrop_write@localhost:5672/%2Fbackdrop_write'
+BROKER_URL = 'amqp://backdrop_write:backdrop_write@localhost:5672/%2Fbackdrop_write'
 STAGECRAFT_URL = 'http://localhost:3103'
 STAGECRAFT_OAUTH_TOKEN = 'development-oauth-access-token'
 BACKDROP_READ_URL = 'http://backdrop-read.dev.gov.uk/data'

backdrop/transformers/config/production.py

@@ -0,0 +1,10 @@
+import os
+from ...core.config.common import load_paas_settings
+
+PAAS = load_paas_settings()
+BROKER_URL = PAAS.get('REDIS_URL') or os.getenv('REDIS_URL')
+BROKER_FAILOVER_STRATEGY = "round-robin"
+STAGECRAFT_URL = 'https://performance-platform-stagecraft-production.cloudapps.digital'
+STAGECRAFT_OAUTH_TOKEN = os.getenv('STAGECRAFT_OAUTH_TOKEN')
+BACKDROP_READ_URL = 'https://performance-platform-backdrop-read-production.cloudapps.digital/data'
+BACKDROP_WRITE_URL = 'https://performance-platform-backdrop-write-production.cloudapps.digital/data'

backdrop/transformers/config/staging.py

@@ -0,0 +1,10 @@
+import os
+from ...core.config.common import load_paas_settings
+
+PAAS = load_paas_settings()
+BROKER_URL = PAAS.get('REDIS_URL') or os.getenv('REDIS_URL')
+BROKER_FAILOVER_STRATEGY = "round-robin"
+STAGECRAFT_URL = 'https://performance-platform-stagecraft-staging.cloudapps.digital'
+STAGECRAFT_OAUTH_TOKEN = os.getenv('STAGECRAFT_OAUTH_TOKEN')
+BACKDROP_READ_URL = 'https://performance-platform-backdrop-read-staging.cloudapps.digital/data'
+BACKDROP_WRITE_URL = 'https://performance-platform-backdrop-write-staging.cloudapps.digital/data'

backdrop/transformers/config/test.py

@@ -1,4 +1,4 @@
-TRANSFORMER_AMQP_URL = 'memory://'
+BROKER_URL = 'memory://'
 STAGECRAFT_URL = 'http://stagecraft'
 STAGECRAFT_OAUTH_TOKEN = 'development-oauth-access-token'
 BACKDROP_READ_URL = 'http://backdrop/data'

backdrop/transformers/dispatch.py

@@ -8,6 +8,7 @@
 from statsd import StatsClient
 
 from backdrop.core.timeseries import parse_period
+from backdrop.core.timeutils import parse_time_as_utc
 from backdrop.core.log_handler import get_log_file_handler
 from backdrop.core.errors import incr_on_error
 from backdrop.transformers.tasks.util import encode_id
@@ -16,11 +17,11 @@
 
 from performanceplatform.client import AdminAPI, DataSet
 
-GOVUK_ENV = getenv("GOVUK_ENV", "development")
+ENVIRONMENT = getenv("ENVIRONMENT", "development")
 logger = logging.getLogger()
 logger.setLevel(logging.DEBUG)
 logger.addHandler(
-    get_log_file_handler("log/{}.log".format(GOVUK_ENV), logging.DEBUG))
+    get_log_file_handler("log/{}.log".format(ENVIRONMENT), logging.DEBUG))
 
 stats_client = StatsClient(prefix=getenv("GOVUK_STATSD_PREFIX",
                                          "pp.apps.backdrop.transformers.worker"))
@@ -147,6 +148,9 @@ def run_transform(data_set_config, transform, earliest, latest):
         data_set_config['data_type'],
     )
 
+    earliest = parse_time_as_utc(earliest)
+    latest = parse_time_as_utc(latest)
+
     data = data_set.get(
         query_parameters=get_query_parameters(transform, earliest, latest)
     )

backdrop/transformers/worker.py

@@ -4,13 +4,13 @@
 import importlib
 from os import getenv
 
-GOVUK_ENV = getenv("GOVUK_ENV", "development")
+ENVIRONMENT = getenv("ENVIRONMENT", "development")
 config = importlib.import_module(
-    "backdrop.transformers.config.{}".format(GOVUK_ENV))
+    "backdrop.transformers.config.{}".format(ENVIRONMENT))
 
 app = Celery(
     'transformations',
-    broker=config.TRANSFORMER_AMQP_URL,
+    broker=config.BROKER_URL,
     include=['backdrop.transformers.dispatch'])
 
 

backdrop/write/api.py

@@ -18,7 +18,7 @@
 from ..core.flaskutils import generate_request_id
 from ..core.storage.mongo import MongoStorageEngine
 
-GOVUK_ENV = getenv("GOVUK_ENV", "development")
+ENVIRONMENT = getenv("ENVIRONMENT", "development")
 
 app = Flask("backdrop.write.api")
 
@@ -29,12 +29,12 @@
 
 # Configuration
 app.config.from_object(
-    "backdrop.write.config.{}".format(GOVUK_ENV))
+    "backdrop.write.config.{}".format(ENVIRONMENT))
 
 storage = MongoStorageEngine.create(
-    app.config['MONGO_HOSTS'],
-    app.config['MONGO_PORT'],
-    app.config['DATABASE_NAME'])
+    app.config['DATABASE_URL'],
+    app.config.get('CA_CERTIFICATE')
+)
 
 admin_api = client.AdminAPI(
     app.config['STAGECRAFT_URL'],
@@ -43,12 +43,12 @@
     request_id_fn=generate_request_id,
 )
 
-log_handler.set_up_logging(app, GOVUK_ENV)
-log_handler.set_up_audit_logging(app, GOVUK_ENV)
+log_handler.set_up_logging(app, ENVIRONMENT)
+log_handler.set_up_audit_logging(app, ENVIRONMENT)
 
 app.url_map.converters["data_set"] = DataSetConverter
 
-celery_app = Celery(broker=app.config['TRANSFORMER_AMQP_URL'])
+celery_app = Celery(broker=app.config['BROKER_URL'])
 app.config['BROKER_FAILOVER_STRATEGY'] = "round-robin"
 celery_app.conf.update(app.config)
 
@@ -105,10 +105,7 @@ def http_error_handler(e):
 @cache_control.nocache
 @statsd.timer('write.route.health_check.status')
 def health_check():
-    if storage.alive():
-        return jsonify(status='ok', message='database seems fine')
-    else:
-        abort(500, 'cannot connect to database')
+    return jsonify(status='ok', message='app is up')
 
 
 @app.route('/data/<data_group>/<data_type>', methods=['POST'])