From 9e4b2ff2efdd9074ae68e477fb9c483f1044fd8c Mon Sep 17 00:00:00 2001 From: Lee Longmore Date: Tue, 7 Nov 2017 13:34:51 +0000 Subject: [PATCH 1/8] Deploy via Travis --- .travis.yml | 43 +++++++++++++++++++++++++++++++++---------- etc/deploy.sh | 36 ++++++++++++++++++++++++++++++++++++ manifest.yml | 18 ++++++++++++++++++ 3 files changed, 87 insertions(+), 10 deletions(-) create mode 100644 etc/deploy.sh create mode 100644 manifest.yml diff --git a/.travis.yml b/.travis.yml index 663e8205..9e39a1e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,27 +3,50 @@ language: python python: - "2.7" cache: pip -sudo: false +sudo: true +notifications: + email: false + # Need mongodb for testing services: mongodb # command to install dependencies + install: - pip install --upgrade pip - easy_install distribute - pip install --upgrade distribute - pip install --upgrade setuptools - pip install -q -r requirements_for_tests.txt -# command to run tests + env: - - SKIP_VIRUS_SCAN=1 SKIP_SPLINTER_TESTS=1 MONGO_REPLICA_SET='' NO_AUTOPEP8=1 + global: + - SKIP_VIRUS_SCAN=1 SKIP_SPLINTER_TESTS=1 MONGO_REPLICA_SET='' NO_AUTOPEP8=1 + - PAAS_USER=pp-deploy@digital.cabinet-office.gov.uk + # PAAS_PASSWORD + - secure: HNo6rnNiXmNYAV1TvkcMxeniXpmHoZkNEiT+X9EgyInCOnVm4mz89qi/AElY/ESE5HdYIegJ5l1M5tAWCei4dSDSTSfkbBBiHE6dfpf1OSU857oQ6T6161NRg4V2oxNdWSRgAX+G8rpxBShaXCDU9q9iBxhsqrmTX+UGlzKloDk= + # APP_SIGNON_API_USER_TOKEN_STAGING + - secure: PYLrm6uH6uhNA69xgI5lSXLrX0ouPBdxlMlQXER0K2BkHzxTowNDFyf5LDa2kwIueJ/YXFHAgm7eBN9gr3JIXAN8Mqb15LTHJVg6hc2aLwFZKES4fuPa/da3Pn24xrdTesIedUAv2vSkuacaktZwKxsv6LENw53uA+8kOT8l9SQ= + # APP_SIGNON_API_USER_TOKEN_PRODUCTION + - secure: SBk+7zw2GMkgbl8p7hwXZw5VkDf2FQE4kBklD40c7goORyEF3/FjTVJTVlREN4Rwb3a81N5lkQDl0fLPhGaDv+PyTr+l69IU64w1Whu9e9fL/wMC3zo5Eer3Btptar5jdCsEg+V4woKKXv6j4B4xebHN6sU7C6dX9TGKs+R1aBo= + script: - ./run_tests.sh + after_script: - coveralls -branches: - except: - - release - - /^release_[0-9]+$/ - - /^deployed-to-(preview|staging|production)$/ -notifications: - email: false + +before_deploy: + - chmod +x etc/deploy.sh + +deploy: + - provider: script + script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_STAGING etc/deploy.sh staging + skip_cleanup: true + on: + branch: staging + + - provider: script + script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_PRODUCTION etc/deploy.sh production + skip_cleanup: true + on: + branch: production diff --git a/etc/deploy.sh b/etc/deploy.sh new file mode 100644 index 00000000..a106cd74 --- /dev/null +++ b/etc/deploy.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -e + +if [ -z "$1" ]; then + echo "Missing PAAS space argument" + echo " deploy.sh staging|production" + exit 1 +fi + +PAAS_SPACE=$1 +wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add - +echo "deb http://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list +sudo apt-get update && sudo apt-get install cf-cli + +cf login -u $PAAS_USER -p $PAAS_PASSWORD -a https://api.cloud.service.gov.uk -o gds-performance-platform -s $PAAS_SPACE + +# bind services +cf bind-service performance-platform-backdrop-read gds-performance-platform-mongodb-service +cf bind-service performance-platform-backdrop-write gds-performance-platform-mongodb-service + +# set environmental variables +cf set-env performance-platform-backdrop-read ENVIRONMENT $PAAS_SPACE +cf set-env performance-platform-backdrop-read STAGECRAFT_URL https://performance-platform-stagecraft-$PAAS_SPACE.cloudapps.digital +cf set-env performance-platform-backdrop-read SIGNON_API_USER_TOKEN $APP_SIGNON_API_USER_TOKEN + +cf set-env performance-platform-backdrop-write ENVIRONMENT $PAAS_SPACE +cf set-env performance-platform-backdrop-write STAGECRAFT_URL https://performance-platform-stagecraft-$PAAS_SPACE.cloudapps.digital +cf set-env performance-platform-backdrop-write SIGNON_API_USER_TOKEN $APP_SIGNON_API_USER_TOKEN + +# deploy apps +cf push -f manifest.yml + +# create and map routes +cf map-route performance-platform-backdrop-read cloudapps.digital --hostname performance-platform-backdrop-read-$PAAS_SPACE +cf map-route performance-platform-backdrop-write cloudapps.digital --hostname performance-platform-backdrop-write-$PAAS_SPACE diff --git a/manifest.yml b/manifest.yml new file mode 100644 index 00000000..a256e77c --- /dev/null +++ b/manifest.yml @@ -0,0 +1,18 @@ +--- +buildpack: python_buildpack +name: performance-platform-backdrop +services: + - gds-performance-platform-mongodb-service +memory: 256M +applications: + - name: performance-platform-backdrop-read + command: python start.py read 8080 + instances: 1 + memory: 512M + health-check-type: none + + - name: performance-platform-backdrop-write + command: python start.py write 8080 + instances: 1 + memory: 512M + health-check-type: none From 6e40e51928cd23e24fc8c3c15c22021f96621fe8 Mon Sep 17 00:00:00 2001 From: Lee Longmore Date: Tue, 7 Nov 2017 16:06:13 +0000 Subject: [PATCH 2/8] Linting --- .gitignore | 1 + backdrop/core/config/common.py | 1 + backdrop/read/config/production.py | 2 +- backdrop/read/config/staging.py | 2 +- backdrop/write/config/production.py | 2 +- backdrop/write/config/staging.py | 2 +- 6 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 6e3c78a1..fea4119d 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ tmp/* # vim swap files .*.swp +venv/ diff --git a/backdrop/core/config/common.py b/backdrop/core/config/common.py index c475452e..1a5e324c 100644 --- a/backdrop/core/config/common.py +++ b/backdrop/core/config/common.py @@ -2,6 +2,7 @@ import json from base64 import b64decode + def load_paas_settings(): paas = {} if 'VCAP_SERVICES' in os.environ: diff --git a/backdrop/read/config/production.py b/backdrop/read/config/production.py index 652238c0..3ded5859 100644 --- a/backdrop/read/config/production.py +++ b/backdrop/read/config/production.py @@ -7,4 +7,4 @@ STAGECRAFT_URL = os.getenv('STAGECRAFT_URL') SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN') LOG_LEVEL = "INFO" -SESSION_COOKIE_SECURE=True +SESSION_COOKIE_SECURE = True diff --git a/backdrop/read/config/staging.py b/backdrop/read/config/staging.py index 652238c0..3ded5859 100644 --- a/backdrop/read/config/staging.py +++ b/backdrop/read/config/staging.py @@ -7,4 +7,4 @@ STAGECRAFT_URL = os.getenv('STAGECRAFT_URL') SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN') LOG_LEVEL = "INFO" -SESSION_COOKIE_SECURE=True +SESSION_COOKIE_SECURE = True diff --git a/backdrop/write/config/production.py b/backdrop/write/config/production.py index 652238c0..3ded5859 100644 --- a/backdrop/write/config/production.py +++ b/backdrop/write/config/production.py @@ -7,4 +7,4 @@ STAGECRAFT_URL = os.getenv('STAGECRAFT_URL') SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN') LOG_LEVEL = "INFO" -SESSION_COOKIE_SECURE=True +SESSION_COOKIE_SECURE = True diff --git a/backdrop/write/config/staging.py b/backdrop/write/config/staging.py index 652238c0..3ded5859 100644 --- a/backdrop/write/config/staging.py +++ b/backdrop/write/config/staging.py @@ -7,4 +7,4 @@ STAGECRAFT_URL = os.getenv('STAGECRAFT_URL') SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN') LOG_LEVEL = "INFO" -SESSION_COOKIE_SECURE=True +SESSION_COOKIE_SECURE = True From edc65c2ae1ba248691642dccda4bf3d6ee6b797f Mon Sep 17 00:00:00 2001 From: Lee Longmore Date: Wed, 8 Nov 2017 16:09:13 +0000 Subject: [PATCH 3/8] Fix tests - includes removal of deprecated alive() function (PyMongo) --- backdrop/core/storage/mongo.py | 2 +- backdrop/read/api.py | 7 +------ backdrop/transformers/dispatch.py | 4 ++-- backdrop/transformers/worker.py | 4 ++-- backdrop/write/api.py | 5 +---- backdrop/write/config/test.py | 4 +--- features/environment.py | 2 +- features/steps/read_api.py | 4 ++-- features/support/http_test_client.py | 5 ++--- features/support/support.py | 2 +- tests/core/integration/test_data_set_integration.py | 9 ++++----- tests/core/storage/test_mongo.py | 9 ++++++--- tests/core/storage/test_storage.py | 3 --- tests/read/test_read_api_service_data_endpoint.py | 4 +++- tests/write/test_flask_integration.py | 12 ------------ 15 files changed, 27 insertions(+), 49 deletions(-) diff --git a/backdrop/core/storage/mongo.py b/backdrop/core/storage/mongo.py index beca31b9..1c384d88 100644 --- a/backdrop/core/storage/mongo.py +++ b/backdrop/core/storage/mongo.py @@ -72,7 +72,7 @@ def reconnecting_save(collection, record, tries=3): class MongoStorageEngine(object): @classmethod - def create(cls, database_url, ca_certificate): + def create(cls, database_url, ca_certificate=None): if ca_certificate is not None: dir = os.path.dirname(__file__) filename = os.path.join(dir, 'mongodb.crt') diff --git a/backdrop/read/api.py b/backdrop/read/api.py index aa3542db..dd7d49a1 100644 --- a/backdrop/read/api.py +++ b/backdrop/read/api.py @@ -91,12 +91,7 @@ def http_error_handler(e): @cache_control.nocache @statsd.timer('read.route.heath_check.status') def health_check(): - - if not storage.alive(): - return jsonify(status='error', - message='cannot connect to database'), 500 - - return jsonify(status='ok', message='database is up') + return jsonify(status='ok', message='app is up') @app.route('/_status/data-sets', methods=['GET']) diff --git a/backdrop/transformers/dispatch.py b/backdrop/transformers/dispatch.py index 89f05c20..9555e4d6 100644 --- a/backdrop/transformers/dispatch.py +++ b/backdrop/transformers/dispatch.py @@ -16,11 +16,11 @@ from performanceplatform.client import AdminAPI, DataSet -GOVUK_ENV = getenv("GOVUK_ENV", "development") +ENVIRONMENT = getenv("ENVIRONMENT", "development") logger = logging.getLogger() logger.setLevel(logging.DEBUG) logger.addHandler( - get_log_file_handler("log/{}.log".format(GOVUK_ENV), logging.DEBUG)) + get_log_file_handler("log/{}.log".format(ENVIRONMENT), logging.DEBUG)) stats_client = StatsClient(prefix=getenv("GOVUK_STATSD_PREFIX", "pp.apps.backdrop.transformers.worker")) diff --git a/backdrop/transformers/worker.py b/backdrop/transformers/worker.py index bb4b37f8..d107582a 100644 --- a/backdrop/transformers/worker.py +++ b/backdrop/transformers/worker.py @@ -4,9 +4,9 @@ import importlib from os import getenv -GOVUK_ENV = getenv("GOVUK_ENV", "development") +ENVIRONMENT = getenv("ENVIRONMENT", "development") config = importlib.import_module( - "backdrop.transformers.config.{}".format(GOVUK_ENV)) + "backdrop.transformers.config.{}".format(ENVIRONMENT)) app = Celery( 'transformations', diff --git a/backdrop/write/api.py b/backdrop/write/api.py index b50c4679..99909385 100644 --- a/backdrop/write/api.py +++ b/backdrop/write/api.py @@ -105,10 +105,7 @@ def http_error_handler(e): @cache_control.nocache @statsd.timer('write.route.health_check.status') def health_check(): - if storage.alive(): - return jsonify(status='ok', message='database seems fine') - else: - abort(500, 'cannot connect to database') + return jsonify(status='ok', message='app is up') @app.route('/data//', methods=['POST']) diff --git a/backdrop/write/config/test.py b/backdrop/write/config/test.py index 8f156dbe..85719089 100644 --- a/backdrop/write/config/test.py +++ b/backdrop/write/config/test.py @@ -1,6 +1,4 @@ -DATABASE_NAME = "backdrop_test" -MONGO_HOSTS = ['localhost'] -MONGO_PORT = 27017 +DATABASE_URL = 'mongodb://localhost:27017/backdrop_test' LOG_LEVEL = "DEBUG" CLIENT_ID = "it's not important here" CLIENT_SECRET = "it's not important here" diff --git a/features/environment.py b/features/environment.py index 115d0f25..8cf8252a 100644 --- a/features/environment.py +++ b/features/environment.py @@ -61,7 +61,7 @@ def create_client(feature): if 'use_write_api_client' in feature.tags: return FlaskTestClient(write_api) if 'use_http_client' in feature.tags: - return HTTPTestClient(config.DATABASE_NAME) + return HTTPTestClient(config.DATABASE_URL) raise AssertionError( "Test client not selected! Please annotate the failing feature with " diff --git a/features/steps/read_api.py b/features/steps/read_api.py index a2e71acb..e72a3bc9 100644 --- a/features/steps/read_api.py +++ b/features/steps/read_api.py @@ -32,7 +32,7 @@ def step(context, fixture_name, data_set_name): '_week_start_at', '_month_start_at']: if key in obj: obj[key] = parser.parse(obj[key]).astimezone(pytz.utc) - context.client.mongo()[data_set_name].save(obj) + context.client.mongo()[data_set_name].insert_one(obj) def get_data_set_settings_from_context_table(table): @@ -56,7 +56,7 @@ def step(context, fixture_name, data_set_name): '_week_start_at', '_month_start_at']: if key in obj: obj[key] = parser.parse(obj[key]).astimezone(pytz.utc) - context.client.mongo()[data_set_name].save(obj) + context.client.mongo()[data_set_name].insert_one(obj) @given('I have a record updated "{timespan}" ago in the "{data_set_name}" data_set') diff --git a/features/support/http_test_client.py b/features/support/http_test_client.py index 581d8ffd..fe9e9809 100644 --- a/features/support/http_test_client.py +++ b/features/support/http_test_client.py @@ -5,11 +5,10 @@ class HTTPTestClient(BaseClient): - def __init__(self, database_name): - self.database_name = database_name + def __init__(self, database_url): self._read_api = FlaskApp("read", "5000") self._write_api = FlaskApp("write", "5001") - self._mongo_db = MongoClient('localhost', 27017)[self.database_name] + self._mongo_db = MongoClient(database_url).get_database() self._start() def get(self, url, headers=None): diff --git a/features/support/support.py b/features/support/support.py index 42e7f81c..bf1bcb4b 100644 --- a/features/support/support.py +++ b/features/support/support.py @@ -18,7 +18,7 @@ def mongo(self): return self._mongo_db def clean_mongo(self): - self._mongo_db.connection.drop_database( + self._mongo_db.client.drop_database( self._mongo_db.name) def before_scenario(self): diff --git a/tests/core/integration/test_data_set_integration.py b/tests/core/integration/test_data_set_integration.py index 95585396..66243216 100644 --- a/tests/core/integration/test_data_set_integration.py +++ b/tests/core/integration/test_data_set_integration.py @@ -10,16 +10,14 @@ from backdrop.core.query import Query from tests.support.test_helpers import d_tz -HOSTS = ['localhost'] -PORT = 27017 -DB_NAME = 'performance_platform_test' +DATABASE_URL = 'mongodb://localhost:27017/backdrop_test' DATA_SET = 'data_set_integration_test' class TestDataSetIntegration(unittest.TestCase): def setUp(self): - self.storage = MongoStorageEngine.create(HOSTS, PORT, DB_NAME) + self.storage = MongoStorageEngine.create(DATABASE_URL) self.config = { 'name': DATA_SET, @@ -30,7 +28,8 @@ def setUp(self): self.data_set = DataSet(self.storage, self.config) - self.mongo_collection = MongoClient(HOSTS, PORT)[DB_NAME][DATA_SET] + database = MongoClient(DATABASE_URL).get_database() + self.mongo_collection = database[DATA_SET] def setup__timestamp_data(self): self.mongo_collection.save({ diff --git a/tests/core/storage/test_mongo.py b/tests/core/storage/test_mongo.py index 6ff1f4d1..51db4fd2 100644 --- a/tests/core/storage/test_mongo.py +++ b/tests/core/storage/test_mongo.py @@ -19,11 +19,12 @@ from .test_storage import BaseStorageTest +DATABASE_URL = 'mongodb://localhost:27017/backdrop_test' + class TestMongoStorageEngine(BaseStorageTest): def setup(self): - self.engine = MongoStorageEngine.create( - ['localhost'], 27017, 'backdrop_test') + self.engine = MongoStorageEngine.create(DATABASE_URL) def test_create_data_set(self): self.engine.create_data_set('should_have_index', 0) @@ -53,7 +54,9 @@ def test_batch_last_updated(self): assert_that(last_updated.second, is_(timestamp.second)) def teardown(self): - self.engine._mongo.drop_database('backdrop_test') + mongo_client = self.engine._mongo_client + database_name = mongo_client.get_database().name + mongo_client.drop_database(database_name) class TestReconnectingSave(object): diff --git a/tests/core/storage/test_storage.py b/tests/core/storage/test_storage.py index c99c1948..30b8bcc4 100644 --- a/tests/core/storage/test_storage.py +++ b/tests/core/storage/test_storage.py @@ -37,9 +37,6 @@ def _save_all_with_periods(self, data_set_id, *records): self._save_all(data_set_id, *records) - def test_is_alive(self): - assert_that(self.engine.alive(), is_(True)) - def test_does_not_exist(self): assert_that(self.engine.data_set_exists('foo_bar'), is_(False)) diff --git a/tests/read/test_read_api_service_data_endpoint.py b/tests/read/test_read_api_service_data_endpoint.py index b6b2931f..8eed4066 100644 --- a/tests/read/test_read_api_service_data_endpoint.py +++ b/tests/read/test_read_api_service_data_endpoint.py @@ -115,7 +115,9 @@ class PreflightChecksApiTestCase(unittest.TestCase): def setUp(self): self.app = api.app.test_client() - api.storage._mongo.drop_database(api.app.config['DATABASE_NAME']) + mongo_client = api.storage._mongo_client + database_name = mongo_client.get_database().name + mongo_client.drop_database(database_name) @fake_data_set_exists("data_set", data_group="some-group", data_type="some-type") def test_cors_preflight_requests_have_empty_body(self): diff --git a/tests/write/test_flask_integration.py b/tests/write/test_flask_integration.py index 3d40fa61..330ba095 100644 --- a/tests/write/test_flask_integration.py +++ b/tests/write/test_flask_integration.py @@ -205,18 +205,6 @@ def test_api_exposes_a_healthcheck(self): entity = json.loads(response.data) assert_that(entity["status"], is_("ok")) - @patch("backdrop.write.api.statsd") - @patch("backdrop.write.api.storage") - def test_exception_handling(self, storage, statsd): - storage.alive.side_effect = ValueError("BOOM") - - response = self.app.get("/_status") - - assert_that(response, has_status(500)) - assert_that(response, is_error_response()) - - statsd.incr.assert_called_with("write.error", data_set="/_status") - class UploadPageTestCase(unittest.TestCase): def setUp(self): From 9f0aab2cb8a3081434a34452252c621acea03f0d Mon Sep 17 00:00:00 2001 From: Lee Longmore Date: Thu, 9 Nov 2017 11:35:23 +0000 Subject: [PATCH 4/8] Set remaining 'write' app settings --- .travis.yml | 10 ++++++++-- backdrop/write/config/production.py | 10 ++++++++++ backdrop/write/config/staging.py | 2 ++ etc/deploy.sh | 1 + 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9e39a1e4..94e03461 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,12 @@ env: - secure: PYLrm6uH6uhNA69xgI5lSXLrX0ouPBdxlMlQXER0K2BkHzxTowNDFyf5LDa2kwIueJ/YXFHAgm7eBN9gr3JIXAN8Mqb15LTHJVg6hc2aLwFZKES4fuPa/da3Pn24xrdTesIedUAv2vSkuacaktZwKxsv6LENw53uA+8kOT8l9SQ= # APP_SIGNON_API_USER_TOKEN_PRODUCTION - secure: SBk+7zw2GMkgbl8p7hwXZw5VkDf2FQE4kBklD40c7goORyEF3/FjTVJTVlREN4Rwb3a81N5lkQDl0fLPhGaDv+PyTr+l69IU64w1Whu9e9fL/wMC3zo5Eer3Btptar5jdCsEg+V4woKKXv6j4B4xebHN6sU7C6dX9TGKs+R1aBo= + # APP_SECRET_KEY_STAGING + - secure: Cl7cvdzviF3hCepM7n3XMz9q1D/FCEQkcDZOyRTPkHlm9V2GKi+e9ECuAuWQVRiRDTb7bAZZ5kl+zTj00nMb0BKds3QoisZ5oB8UwYFOQSrYV/NTpge0wsvZZvfqXp5sN3CLVXE2XynLsZZSzOTIte8UKD4giBFsuHo2sBcVbb8= + # APP_SECRET_KEY_PRODUCTION + - secure: ifAsLdVTrzDT8IIEVdW1TJon8VYdmDdzxxE/CtCgc0hcBP9eneKF7qvBOy6TFMnE9KSz4zzyYx+yZI8s5sAsjknw/HEnHYMCYSjRcM3GMWP27/QFM2983CgWkoEn5xGj3WEXra1WJGltzqbeoYfZWmFA9pBGSU6IbR0IzjLC6+g= + # APP_STAGECRAFT_COLLECTION_ENDPOINT_TOKEN + # Value too large to be encrypted. script: - ./run_tests.sh @@ -40,13 +46,13 @@ before_deploy: deploy: - provider: script - script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_STAGING etc/deploy.sh staging + script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_STAGING APP_SECRET_KEY=$APP_SECRET_KEY_STAGING etc/deploy.sh staging skip_cleanup: true on: branch: staging - provider: script - script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_PRODUCTION etc/deploy.sh production + script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_PRODUCTION APP_SECRET_KEY=$APP_SECRET_KEY_PRODUCTION etc/deploy.sh production skip_cleanup: true on: branch: production diff --git a/backdrop/write/config/production.py b/backdrop/write/config/production.py index 3ded5859..ab253cad 100644 --- a/backdrop/write/config/production.py +++ b/backdrop/write/config/production.py @@ -7,4 +7,14 @@ STAGECRAFT_URL = os.getenv('STAGECRAFT_URL') SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN') LOG_LEVEL = "INFO" +DATA_SET_UPLOAD_FORMAT = { + "ithc_excel": "excel", +} +DATA_SET_UPLOAD_FILTERS = { + "ithc_excel": [ + "backdrop.core.upload.filters.first_sheet_filter", + ], +} SESSION_COOKIE_SECURE = True +SECRET_KEY = os.getenv('SECRET_KEY') +STAGECRAFT_COLLECTION_ENDPOINT_TOKEN = os.getenv('STAGECRAFT_COLLECTION_ENDPOINT_TOKEN') diff --git a/backdrop/write/config/staging.py b/backdrop/write/config/staging.py index 3ded5859..b01b3e16 100644 --- a/backdrop/write/config/staging.py +++ b/backdrop/write/config/staging.py @@ -8,3 +8,5 @@ SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN') LOG_LEVEL = "INFO" SESSION_COOKIE_SECURE = True +SECRET_KEY = os.getenv('SECRET_KEY') +STAGECRAFT_COLLECTION_ENDPOINT_TOKEN = os.getenv('STAGECRAFT_COLLECTION_ENDPOINT_TOKEN') diff --git a/etc/deploy.sh b/etc/deploy.sh index a106cd74..884a8b5c 100644 --- a/etc/deploy.sh +++ b/etc/deploy.sh @@ -27,6 +27,7 @@ cf set-env performance-platform-backdrop-read SIGNON_API_USER_TOKEN $APP_SIGNON_ cf set-env performance-platform-backdrop-write ENVIRONMENT $PAAS_SPACE cf set-env performance-platform-backdrop-write STAGECRAFT_URL https://performance-platform-stagecraft-$PAAS_SPACE.cloudapps.digital cf set-env performance-platform-backdrop-write SIGNON_API_USER_TOKEN $APP_SIGNON_API_USER_TOKEN +cf set-env performance-platform-backdrop-write SECRET_KEY $APP_SECRET_KEY # deploy apps cf push -f manifest.yml From a6a29c236e08593a3cf26ac040da0d41fec58921 Mon Sep 17 00:00:00 2001 From: Lee Longmore Date: Thu, 9 Nov 2017 14:28:54 +0000 Subject: [PATCH 5/8] Configure running of transforms --- .python-version | 1 + .travis.yml | 13 +++++++++---- Procfile | 2 -- backdrop/core/config/common.py | 20 ++++++++++++++------ backdrop/transformers/config/development.py | 2 +- backdrop/transformers/config/production.py | 10 ++++++++++ backdrop/transformers/config/staging.py | 10 ++++++++++ backdrop/transformers/config/test.py | 2 +- backdrop/transformers/worker.py | 2 +- backdrop/write/api.py | 2 +- backdrop/write/config/development.py | 2 +- backdrop/write/config/production.py | 2 ++ backdrop/write/config/staging.py | 2 ++ backdrop/write/config/test.py | 2 +- etc/deploy.sh | 5 +++++ manifest.yml | 18 +++++++++++++++--- requirements.txt | 3 +-- 17 files changed, 75 insertions(+), 23 deletions(-) create mode 100644 .python-version delete mode 100644 Procfile create mode 100644 backdrop/transformers/config/production.py create mode 100644 backdrop/transformers/config/staging.py diff --git a/.python-version b/.python-version new file mode 100644 index 00000000..ecc17b8e --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +2.7.13 diff --git a/.travis.yml b/.travis.yml index 94e03461..7985bdeb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,10 +20,11 @@ install: env: global: - - SKIP_VIRUS_SCAN=1 SKIP_SPLINTER_TESTS=1 MONGO_REPLICA_SET='' NO_AUTOPEP8=1 - - PAAS_USER=pp-deploy@digital.cabinet-office.gov.uk + - PAAS_USER: pp-deploy@digital.cabinet-office.gov.uk # PAAS_PASSWORD - secure: HNo6rnNiXmNYAV1TvkcMxeniXpmHoZkNEiT+X9EgyInCOnVm4mz89qi/AElY/ESE5HdYIegJ5l1M5tAWCei4dSDSTSfkbBBiHE6dfpf1OSU857oQ6T6161NRg4V2oxNdWSRgAX+G8rpxBShaXCDU9q9iBxhsqrmTX+UGlzKloDk= + - REDIS_DATABASE_NUMBER_STAGING: 4 + - REDIS_DATABASE_NUMBER_PRODUCTION: 5 # APP_SIGNON_API_USER_TOKEN_STAGING - secure: PYLrm6uH6uhNA69xgI5lSXLrX0ouPBdxlMlQXER0K2BkHzxTowNDFyf5LDa2kwIueJ/YXFHAgm7eBN9gr3JIXAN8Mqb15LTHJVg6hc2aLwFZKES4fuPa/da3Pn24xrdTesIedUAv2vSkuacaktZwKxsv6LENw53uA+8kOT8l9SQ= # APP_SIGNON_API_USER_TOKEN_PRODUCTION @@ -34,6 +35,10 @@ env: - secure: ifAsLdVTrzDT8IIEVdW1TJon8VYdmDdzxxE/CtCgc0hcBP9eneKF7qvBOy6TFMnE9KSz4zzyYx+yZI8s5sAsjknw/HEnHYMCYSjRcM3GMWP27/QFM2983CgWkoEn5xGj3WEXra1WJGltzqbeoYfZWmFA9pBGSU6IbR0IzjLC6+g= # APP_STAGECRAFT_COLLECTION_ENDPOINT_TOKEN # Value too large to be encrypted. + # APP_STAGECRAFT_OAUTH_TOKEN_STAGING + - secure: JkoKVPblX1sHICKxJMT0kPi5Nax3dn3cG1bzmAmpMDX+rd37I4XtA96G17ibUmUrp2CVrOo4is5WjI9hPDwmC8/o7/6MOHqJo7jeflY+Iq8bWShcja0khlb4aBlpWfgMS+vniLCBPkQtUe2eE87jFLYJXmsuBGfp4r4L88fhgOk= + # APP_STAGECRAFT_OAUTH_TOKEN_PRODUCTION + - secure: HZPkW81a+OsfFrCCpcuYYBKNIEmejsQlIreVo2AJ1NipBVK8DniqIK+bXGUIQraaqeKXApXknfPepVpyqG3uyzHvkboylooWa7fuoJB4bs9Eb91Qo2AeWjP2fts0EmUX8R76K51n7EwUgk9xemM/bE2e+0FmG5e35JpXEwfm/5I= script: - ./run_tests.sh @@ -46,13 +51,13 @@ before_deploy: deploy: - provider: script - script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_STAGING APP_SECRET_KEY=$APP_SECRET_KEY_STAGING etc/deploy.sh staging + script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_STAGING APP_SECRET_KEY=$APP_SECRET_KEY_STAGING REDIS_DATABASE_NUMBER=$REDIS_DATABASE_NUMBER_STAGING APP_STAGECRAFT_OAUTH_TOKEN=$APP_STAGECRAFT_OAUTH_TOKEN_STAGING etc/deploy.sh staging skip_cleanup: true on: branch: staging - provider: script - script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_PRODUCTION APP_SECRET_KEY=$APP_SECRET_KEY_PRODUCTION etc/deploy.sh production + script: APP_SIGNON_API_USER_TOKEN=$APP_SIGNON_API_USER_TOKEN_PRODUCTION APP_SECRET_KEY=$APP_SECRET_KEY_PRODUCTION REDIS_DATABASE_NUMBER=$REDIS_DATABASE_NUMBER_PRODUCTION APP_STAGECRAFT_OAUTH_TOKEN=$APP_STAGECRAFT_OAUTH_TOKEN_PRODUCTION etc/deploy.sh production skip_cleanup: true on: branch: production diff --git a/Procfile b/Procfile deleted file mode 100644 index 5e28441f..00000000 --- a/Procfile +++ /dev/null @@ -1,2 +0,0 @@ -web: venv/bin/gunicorn -c /etc/gds/${APP_NAME}/gunicorn ${APP_MODULE} -worker: venv/bin/celery -A backdrop.transformers.worker worker -l debug diff --git a/backdrop/core/config/common.py b/backdrop/core/config/common.py index 1a5e324c..85076bbf 100644 --- a/backdrop/core/config/common.py +++ b/backdrop/core/config/common.py @@ -7,10 +7,18 @@ def load_paas_settings(): paas = {} if 'VCAP_SERVICES' in os.environ: vcap = json.loads(os.environ['VCAP_SERVICES']) - for service in vcap['mongodb']: - if service['name'] == 'gds-performance-platform-mongodb-service': - credentials = service['credentials'] - paas['DATABASE_URL'] = credentials['uri'] - ca_cert = b64decode(credentials['ca_certificate_base64']) - paas['CA_CERTIFICATE'] = ca_cert + if 'mongodb' in vcap: + for service in vcap['mongodb']: + if service['name'] == 'gds-performance-platform-mongodb-service': + credentials = service['credentials'] + paas['DATABASE_URL'] = credentials['uri'] + ca_cert = b64decode(credentials['ca_certificate_base64']) + paas['CA_CERTIFICATE'] = ca_cert + if 'REDIS_DATABASE_NUMBER' in os.environ: + for service in vcap['user-provided']: + if service['name'] == 'redis-poc': + database_number = os.environ['REDIS_DATABASE_NUMBER'] + url = service['credentials']['url'] + url += '/' + database_number + paas['REDIS_URL'] = url return paas diff --git a/backdrop/transformers/config/development.py b/backdrop/transformers/config/development.py index 788ae55d..41f948d2 100644 --- a/backdrop/transformers/config/development.py +++ b/backdrop/transformers/config/development.py @@ -1,4 +1,4 @@ -TRANSFORMER_AMQP_URL = 'amqp://backdrop_write:backdrop_write@localhost:5672/%2Fbackdrop_write' +BROKER_URL = 'amqp://backdrop_write:backdrop_write@localhost:5672/%2Fbackdrop_write' STAGECRAFT_URL = 'http://localhost:3103' STAGECRAFT_OAUTH_TOKEN = 'development-oauth-access-token' BACKDROP_READ_URL = 'http://backdrop-read.dev.gov.uk/data' diff --git a/backdrop/transformers/config/production.py b/backdrop/transformers/config/production.py new file mode 100644 index 00000000..acc59d3c --- /dev/null +++ b/backdrop/transformers/config/production.py @@ -0,0 +1,10 @@ +import os +from ...core.config.common import load_paas_settings + +PAAS = load_paas_settings() +BROKER_URL = PAAS.get('REDIS_URL') or os.getenv('REDIS_URL') +BROKER_FAILOVER_STRATEGY = "round-robin" +STAGECRAFT_URL = 'https://performance-platform-stagecraft-production.cloudapps.digital' +STAGECRAFT_OAUTH_TOKEN = os.getenv('STAGECRAFT_OAUTH_TOKEN') +BACKDROP_READ_URL = 'https://performance-platform-backdrop-read-production.cloudapps.digital' +BACKDROP_WRITE_URL = 'https://performance-platform-backdrop-write-production.cloudapps.digital' diff --git a/backdrop/transformers/config/staging.py b/backdrop/transformers/config/staging.py new file mode 100644 index 00000000..d93b37b1 --- /dev/null +++ b/backdrop/transformers/config/staging.py @@ -0,0 +1,10 @@ +import os +from ...core.config.common import load_paas_settings + +PAAS = load_paas_settings() +BROKER_URL = PAAS.get('REDIS_URL') or os.getenv('REDIS_URL') +BROKER_FAILOVER_STRATEGY = "round-robin" +STAGECRAFT_URL = 'https://performance-platform-stagecraft-staging.cloudapps.digital' +STAGECRAFT_OAUTH_TOKEN = os.getenv('STAGECRAFT_OAUTH_TOKEN') +BACKDROP_READ_URL = 'https://performance-platform-backdrop-read-staging.cloudapps.digital' +BACKDROP_WRITE_URL = 'https://performance-platform-backdrop-write-staging.cloudapps.digital' diff --git a/backdrop/transformers/config/test.py b/backdrop/transformers/config/test.py index 843c3366..edb72565 100644 --- a/backdrop/transformers/config/test.py +++ b/backdrop/transformers/config/test.py @@ -1,4 +1,4 @@ -TRANSFORMER_AMQP_URL = 'memory://' +BROKER_URL = 'memory://' STAGECRAFT_URL = 'http://stagecraft' STAGECRAFT_OAUTH_TOKEN = 'development-oauth-access-token' BACKDROP_READ_URL = 'http://backdrop/data' diff --git a/backdrop/transformers/worker.py b/backdrop/transformers/worker.py index d107582a..c7c4ed18 100644 --- a/backdrop/transformers/worker.py +++ b/backdrop/transformers/worker.py @@ -10,7 +10,7 @@ app = Celery( 'transformations', - broker=config.TRANSFORMER_AMQP_URL, + broker=config.BROKER_URL, include=['backdrop.transformers.dispatch']) diff --git a/backdrop/write/api.py b/backdrop/write/api.py index 99909385..592a88e0 100644 --- a/backdrop/write/api.py +++ b/backdrop/write/api.py @@ -48,7 +48,7 @@ app.url_map.converters["data_set"] = DataSetConverter -celery_app = Celery(broker=app.config['TRANSFORMER_AMQP_URL']) +celery_app = Celery(broker=app.config['BROKER_URL']) app.config['BROKER_FAILOVER_STRATEGY'] = "round-robin" celery_app.conf.update(app.config) diff --git a/backdrop/write/config/development.py b/backdrop/write/config/development.py index 2be9d5ea..775adcb6 100644 --- a/backdrop/write/config/development.py +++ b/backdrop/write/config/development.py @@ -15,4 +15,4 @@ SIGNON_API_USER_TOKEN = 'development-oauth-access-token' -TRANSFORMER_AMQP_URL = 'amqp://backdrop_write:backdrop_write@localhost:5672/%2Fbackdrop_write' +BROKER_URL = 'amqp://backdrop_write:backdrop_write@localhost:5672/%2Fbackdrop_write' diff --git a/backdrop/write/config/production.py b/backdrop/write/config/production.py index ab253cad..729e0000 100644 --- a/backdrop/write/config/production.py +++ b/backdrop/write/config/production.py @@ -5,6 +5,8 @@ DATABASE_URL = PAAS.get('DATABASE_URL') CA_CERTIFICATE = PAAS.get('CA_CERTIFICATE') STAGECRAFT_URL = os.getenv('STAGECRAFT_URL') +BROKER_URL = PAAS.get('REDIS_URL') or os.getenv('REDIS_URL') +BROKER_FAILOVER_STRATEGY = "round-robin" SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN') LOG_LEVEL = "INFO" DATA_SET_UPLOAD_FORMAT = { diff --git a/backdrop/write/config/staging.py b/backdrop/write/config/staging.py index b01b3e16..7cec13c4 100644 --- a/backdrop/write/config/staging.py +++ b/backdrop/write/config/staging.py @@ -5,6 +5,8 @@ DATABASE_URL = PAAS.get('DATABASE_URL') CA_CERTIFICATE = PAAS.get('CA_CERTIFICATE') STAGECRAFT_URL = os.getenv('STAGECRAFT_URL') +BROKER_URL = PAAS.get('REDIS_URL') or os.getenv('REDIS_URL') +BROKER_FAILOVER_STRATEGY = "round-robin" SIGNON_API_USER_TOKEN = os.getenv('SIGNON_API_USER_TOKEN') LOG_LEVEL = "INFO" SESSION_COOKIE_SECURE = True diff --git a/backdrop/write/config/test.py b/backdrop/write/config/test.py index 85719089..4bac569a 100644 --- a/backdrop/write/config/test.py +++ b/backdrop/write/config/test.py @@ -7,7 +7,7 @@ "data_set_with_timestamp_auto_id": ["_timestamp", "key"], "evl_volumetrics": ["_timestamp", "service", "transaction"], } -TRANSFORMER_AMQP_URL = 'memory://' +BROKER_URL = 'memory://' from development import (STAGECRAFT_COLLECTION_ENDPOINT_TOKEN, STAGECRAFT_URL, SIGNON_API_USER_TOKEN) diff --git a/etc/deploy.sh b/etc/deploy.sh index 884a8b5c..70d3794a 100644 --- a/etc/deploy.sh +++ b/etc/deploy.sh @@ -28,6 +28,11 @@ cf set-env performance-platform-backdrop-write ENVIRONMENT $PAAS_SPACE cf set-env performance-platform-backdrop-write STAGECRAFT_URL https://performance-platform-stagecraft-$PAAS_SPACE.cloudapps.digital cf set-env performance-platform-backdrop-write SIGNON_API_USER_TOKEN $APP_SIGNON_API_USER_TOKEN cf set-env performance-platform-backdrop-write SECRET_KEY $APP_SECRET_KEY +cf set-env performance-platform-backdrop-write REDIS_DATABASE_NUMBER $REDIS_DATABASE_NUMBER + +cf set-env performance-platform-backdrop-celery-worker ENVIRONMENT $PAAS_SPACE +cf set-env performance-platform-backdrop-celery-worker STAGECRAFT_OAUTH_TOKEN $APP_STAGECRAFT_OAUTH_TOKEN +cf set-env performance-platform-backdrop-celery-worker REDIS_DATABASE_NUMBER $REDIS_DATABASE_NUMBER # deploy apps cf push -f manifest.yml diff --git a/manifest.yml b/manifest.yml index a256e77c..52d33cd4 100644 --- a/manifest.yml +++ b/manifest.yml @@ -1,18 +1,30 @@ --- buildpack: python_buildpack -name: performance-platform-backdrop services: - - gds-performance-platform-mongodb-service -memory: 256M + - redis-poc + applications: - name: performance-platform-backdrop-read command: python start.py read 8080 + services: + - gds-performance-platform-mongodb-service instances: 1 memory: 512M + no-route: true health-check-type: none - name: performance-platform-backdrop-write command: python start.py write 8080 + services: + - gds-performance-platform-mongodb-service instances: 1 memory: 512M + no-route: true + health-check-type: none + + - name: performance-platform-backdrop-celery-worker + command: celery -A backdrop.transformers.worker worker --loglevel=info + instances: 1 + memory: 1G + no-route: true health-check-type: none diff --git a/requirements.txt b/requirements.txt index d3c2e2b8..fb705286 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,7 @@ argh==0.25 -Celery==3.1.17 +celery[redis]==4.1.0 Flask==0.10.1 Flask-FeatureFlags==0.4 -gunicorn==19.1.1 invoke isodate==0.5.0 jsonschema==2.4 From 5f415001fe0889d35c6e5629d5d6440487e44f26 Mon Sep 17 00:00:00 2001 From: Lee Longmore Date: Tue, 14 Nov 2017 09:37:16 +0000 Subject: [PATCH 6/8] Set STAGECRAFT_COLLECTION_ENDPOINT_TOKEN --- backdrop/write/config/production.py | 3 ++- backdrop/write/config/staging.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/backdrop/write/config/production.py b/backdrop/write/config/production.py index 729e0000..74de2b14 100644 --- a/backdrop/write/config/production.py +++ b/backdrop/write/config/production.py @@ -19,4 +19,5 @@ } SESSION_COOKIE_SECURE = True SECRET_KEY = os.getenv('SECRET_KEY') -STAGECRAFT_COLLECTION_ENDPOINT_TOKEN = os.getenv('STAGECRAFT_COLLECTION_ENDPOINT_TOKEN') +STAGECRAFT_COLLECTION_ENDPOINT_TOKEN = os.getenv( + 'STAGECRAFT_COLLECTION_ENDPOINT_TOKEN') diff --git a/backdrop/write/config/staging.py b/backdrop/write/config/staging.py index 7cec13c4..cdeb2b26 100644 --- a/backdrop/write/config/staging.py +++ b/backdrop/write/config/staging.py @@ -11,4 +11,5 @@ LOG_LEVEL = "INFO" SESSION_COOKIE_SECURE = True SECRET_KEY = os.getenv('SECRET_KEY') -STAGECRAFT_COLLECTION_ENDPOINT_TOKEN = os.getenv('STAGECRAFT_COLLECTION_ENDPOINT_TOKEN') +STAGECRAFT_COLLECTION_ENDPOINT_TOKEN = os.getenv( + 'STAGECRAFT_COLLECTION_ENDPOINT_TOKEN') From 04b6aa25fb747320cd8946d5d407aa1843d58d09 Mon Sep 17 00:00:00 2001 From: Lee Longmore Date: Tue, 14 Nov 2017 09:41:37 +0000 Subject: [PATCH 7/8] Convert period dates to datetime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ‘entrypoint’ is invoked through REDIS and receives the dates as strings --- backdrop/transformers/dispatch.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backdrop/transformers/dispatch.py b/backdrop/transformers/dispatch.py index 9555e4d6..e2bbb33e 100644 --- a/backdrop/transformers/dispatch.py +++ b/backdrop/transformers/dispatch.py @@ -8,6 +8,7 @@ from statsd import StatsClient from backdrop.core.timeseries import parse_period +from backdrop.core.timeutils import parse_time_as_utc from backdrop.core.log_handler import get_log_file_handler from backdrop.core.errors import incr_on_error from backdrop.transformers.tasks.util import encode_id @@ -147,6 +148,9 @@ def run_transform(data_set_config, transform, earliest, latest): data_set_config['data_type'], ) + earliest = parse_time_as_utc(earliest) + latest = parse_time_as_utc(latest) + data = data_set.get( query_parameters=get_query_parameters(transform, earliest, latest) ) From 35768cda2f9c5a40931dbbae1e6219aae242737f Mon Sep 17 00:00:00 2001 From: Lee Longmore Date: Tue, 14 Nov 2017 11:14:31 +0000 Subject: [PATCH 8/8] Add missing path to urls --- backdrop/transformers/config/production.py | 4 ++-- backdrop/transformers/config/staging.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backdrop/transformers/config/production.py b/backdrop/transformers/config/production.py index acc59d3c..2249b8b1 100644 --- a/backdrop/transformers/config/production.py +++ b/backdrop/transformers/config/production.py @@ -6,5 +6,5 @@ BROKER_FAILOVER_STRATEGY = "round-robin" STAGECRAFT_URL = 'https://performance-platform-stagecraft-production.cloudapps.digital' STAGECRAFT_OAUTH_TOKEN = os.getenv('STAGECRAFT_OAUTH_TOKEN') -BACKDROP_READ_URL = 'https://performance-platform-backdrop-read-production.cloudapps.digital' -BACKDROP_WRITE_URL = 'https://performance-platform-backdrop-write-production.cloudapps.digital' +BACKDROP_READ_URL = 'https://performance-platform-backdrop-read-production.cloudapps.digital/data' +BACKDROP_WRITE_URL = 'https://performance-platform-backdrop-write-production.cloudapps.digital/data' diff --git a/backdrop/transformers/config/staging.py b/backdrop/transformers/config/staging.py index d93b37b1..06927d2d 100644 --- a/backdrop/transformers/config/staging.py +++ b/backdrop/transformers/config/staging.py @@ -6,5 +6,5 @@ BROKER_FAILOVER_STRATEGY = "round-robin" STAGECRAFT_URL = 'https://performance-platform-stagecraft-staging.cloudapps.digital' STAGECRAFT_OAUTH_TOKEN = os.getenv('STAGECRAFT_OAUTH_TOKEN') -BACKDROP_READ_URL = 'https://performance-platform-backdrop-read-staging.cloudapps.digital' -BACKDROP_WRITE_URL = 'https://performance-platform-backdrop-write-staging.cloudapps.digital' +BACKDROP_READ_URL = 'https://performance-platform-backdrop-read-staging.cloudapps.digital/data' +BACKDROP_WRITE_URL = 'https://performance-platform-backdrop-write-staging.cloudapps.digital/data'