Merge pull request #258 from alphagov/master-stagecraft

Master stagecraft

backdrop/admin/app.py

@@ -8,8 +8,6 @@
 from ..core import cache_control, log_handler, database
 from ..core.bucket import Bucket
 from ..core.errors import ParseError, ValidationError
-from ..core.log_handler \
-    import create_request_logger, create_response_logger
 from ..core.repository \
     import BucketConfigRepository, UserConfigRepository
 from ..core.flaskutils import BucketConverter
@@ -35,7 +33,10 @@
     app.config['DATABASE_NAME']
 )
 
-bucket_repository = BucketConfigRepository(db)
+bucket_repository = BucketConfigRepository(
+    app.config['STAGECRAFT_URL'],
+    app.config['STAGECRAFT_DATA_SET_QUERY_TOKEN'])
+
 user_repository = UserConfigRepository(db)
 
 

backdrop/admin/config/development.py

@@ -11,3 +11,6 @@
     from development_environment import *
 except ImportError:
     from development_environment_sample import *
+
+STAGECRAFT_URL = 'http://localhost:8080'
+STAGECRAFT_DATA_SET_QUERY_TOKEN = 'stagecraft-data-set-query-token-fake'

backdrop/admin/config/test.py

@@ -9,3 +9,5 @@
 MONGO_PORT = 27017
 
 from test_environment import *
+
+from development import STAGECRAFT_URL, STAGECRAFT_DATA_SET_QUERY_TOKEN

backdrop/core/database.py

@@ -60,7 +60,7 @@ def collection_exists(self, collection_name):
 
     def create_uncapped_collection(self, collection_name):
         return self.mongo_database.create_collection(
-            name=collection_name, capped=False, size=0)
+            name=collection_name, capped=False)
 
     def create_capped_collection(self, collection_name, capped_size):
         return self.mongo_database.create_collection(name=collection_name,

backdrop/core/repository.py

@@ -1,6 +1,14 @@
+
+import json
+import logging
+
+import requests
+
 from backdrop.core.bucket import BucketConfig
 from backdrop.core.user import UserConfig
 
+logger = logging.getLogger(__name__)
+
 
 class _Repository(object):
 
@@ -42,27 +50,84 @@ def _create_model(self, doc):
 
 class BucketConfigRepository(object):
 
-    def __init__(self, db):
-        self._db = db
-        self._repository = _Repository(db, BucketConfig, "buckets", "name")
+    def __init__(self, stagecraft_url, stagecraft_token):
+        self._stagecraft_url = stagecraft_url
+        self._stagecraft_token = stagecraft_token
 
-    def save(self, bucket_config, create_bucket=True):
-        self._repository.save(bucket_config)
+    def get_all(self):
+        data_set_url = '{url}/data-sets'.format(url=self._stagecraft_url)
 
-        if bucket_config.realtime and create_bucket:
-            self._db.create_capped_collection(bucket_config.name,
-                                              bucket_config.capped_size)
+        # Note: Don't catch HTTP 404 - that should never happen on this URL.
+        json_response = _get_json_url(data_set_url, self._stagecraft_token)
+        data_sets = _decode_json(json_response)
 
-    def get_all(self):
-        return self._repository.get_all()
+        return [_make_bucket_config(data_set) for data_set in data_sets]
 
     def retrieve(self, name):
-        return self._repository.retrieve(name)
+        if len(name) == 0:
+            raise ValueError('Name must not be empty')
+        data_set_url = ('{url}/data-sets/{data_set_name}'.format(
+                        url=self._stagecraft_url,
+                        data_set_name=name))
+
+        try:
+            json_response = _get_json_url(data_set_url, self._stagecraft_token)
+        except requests.HTTPError as e:
+            if e.response.status_code == 404:
+                return None
+            else:
+                raise
+
+        return _make_bucket_config(_decode_json(json_response))
 
     def get_bucket_for_query(self, data_group, data_type):
-        return self._repository.find_first_instance_of(
-            {"data_group": data_group,
-             "data_type": data_type})
+        empty_vars = []
+        if len(data_group) == 0:
+            empty_vars += ['Data Group']
+        if len(data_type) == 0:
+            empty_vars += ['Data Type']
+        if len(empty_vars) > 0:
+            raise ValueError(' and '.join(empty_vars) + 'must not be empty')
+        data_set_url = ('{url}/data-sets?data-group={data_group_name}'
+                        '&data-type={data_type_name}'.format(
+                            url=self._stagecraft_url,
+                            data_group_name=data_group,
+                            data_type_name=data_type))
+
+        json_response = _get_json_url(
+            data_set_url, self._stagecraft_token)
+
+        data_sets = _decode_json(json_response)
+        if len(data_sets) > 0:
+            return _make_bucket_config(data_sets[0])
+
+        return None
+
+
+def _make_bucket_config(stagecraft_dict):
+    if stagecraft_dict is None:
+        return None
+    return BucketConfig(**stagecraft_dict)
+
+
+def _decode_json(string):
+    return json.loads(string) if string is not None else None
+
+
+def _get_json_url(url, token):
+    auth_header = (
+        'Authorization',
+        'Bearer {}'.format(token))
+    response = requests.get(url, headers=dict([
+        ('content-type', 'application/json'),
+        auth_header]))
+    try:
+        response.raise_for_status()
+    except requests.HTTPError as e:
+        logger.exception(e)
+        logger.error('Stagecraft said: {}'.format(response.content))
+        raise
+    return response.content
 
 
 class UserConfigRepository(object):

backdrop/read/api.py

@@ -3,10 +3,8 @@
 from os import getenv
 from bson import ObjectId
 
-from flask import Flask, jsonify, request, redirect
+from flask import Flask, jsonify, request
 from flask_featureflags import FeatureFlag
-from backdrop.core.log_handler \
-    import create_request_logger, create_response_logger
 from backdrop.read.query import Query
 
 from .validation import validate_request_args
@@ -32,7 +30,9 @@
     app.config['DATABASE_NAME']
 )
 
-bucket_repository = BucketConfigRepository(db)
+bucket_repository = BucketConfigRepository(
+    app.config['STAGECRAFT_URL'],
+    app.config['STAGECRAFT_DATA_SET_QUERY_TOKEN'])
 
 log_handler.set_up_logging(app, GOVUK_ENV)
 

backdrop/read/config/development.py

@@ -3,44 +3,47 @@
 MONGO_PORT = 27017
 LOG_LEVEL = "DEBUG"
 RAW_QUERIES_ALLOWED = {
-  "government_annotations": True,
-  "govuk_realtime": True,
-  "licence_finder_monitoring": True,
-  "licensing": False,
-  "licensing_journey": True,
-  "licensing_monitoring": True,
-  "licensing_realtime": True,
-  "lpa_volumes": True,
-  "lpa_monitoring": True,
-  # electronic vehicle licensing
-  "electronic_vehicle_licensing_monitoring": True,
-  "evl_customer_satisfaction": True,
-  "evl_volumetrics": True,
-  "sorn_monitoring": True,
-  "sorn_realtime": True,
-  "tax_disc_monitoring": True,
-  "tax_disc_realtime": True,
-  # fco
-  "deposit_foreign_marriage_journey": True,
-  "deposit_foreign_marriage_monitoring": True,
-  "deposit_foreign_marriage_realtime": True,
-  "pay_foreign_marriage_certificates_journey": True,
-  "pay_foreign_marriage_certificates_monitoring": True,
-  "pay_foreign_marriage_certificates_realtime": True,
-  "pay_legalisation_drop_off_journey": True,
-  "pay_legalisation_drop_off_monitoring": True,
-  "pay_legalisation_drop_off_realtime": True,
-  "pay_legalisation_post_journey": True,
-  "pay_legalisation_post_monitoring": True,
-  "pay_legalisation_post_realtime": True,
-  "pay_register_birth_abroad_journey": True,
-  "pay_register_birth_abroad_monitoring": True,
-  "pay_register_birth_abroad_realtime": True,
-  "pay_register_death_abroad_journey": True,
-  "pay_register_death_abroad_monitoring": True,
-  "pay_register_death_abroad_realtime": True,
-  # HMRC preview
-  "hmrc_preview": True,
-  # LPA / Lasting Power of Attorney
-  "lpa_journey": True,
+    "government_annotations": True,
+    "govuk_realtime": True,
+    "licence_finder_monitoring": True,
+    "licensing": False,
+    "licensing_journey": True,
+    "licensing_monitoring": True,
+    "licensing_realtime": True,
+    "lpa_volumes": True,
+    "lpa_monitoring": True,
+    # electronic vehicle licensing
+    "electronic_vehicle_licensing_monitoring": True,
+    "evl_customer_satisfaction": True,
+    "evl_volumetrics": True,
+    "sorn_monitoring": True,
+    "sorn_realtime": True,
+    "tax_disc_monitoring": True,
+    "tax_disc_realtime": True,
+    # fco
+    "deposit_foreign_marriage_journey": True,
+    "deposit_foreign_marriage_monitoring": True,
+    "deposit_foreign_marriage_realtime": True,
+    "pay_foreign_marriage_certificates_journey": True,
+    "pay_foreign_marriage_certificates_monitoring": True,
+    "pay_foreign_marriage_certificates_realtime": True,
+    "pay_legalisation_drop_off_journey": True,
+    "pay_legalisation_drop_off_monitoring": True,
+    "pay_legalisation_drop_off_realtime": True,
+    "pay_legalisation_post_journey": True,
+    "pay_legalisation_post_monitoring": True,
+    "pay_legalisation_post_realtime": True,
+    "pay_register_birth_abroad_journey": True,
+    "pay_register_birth_abroad_monitoring": True,
+    "pay_register_birth_abroad_realtime": True,
+    "pay_register_death_abroad_journey": True,
+    "pay_register_death_abroad_monitoring": True,
+    "pay_register_death_abroad_realtime": True,
+    # HMRC preview
+    "hmrc_preview": True,
+    # LPA / Lasting Power of Attorney
+    "lpa_journey": True,
 }
+
+STAGECRAFT_URL = 'http://localhost:8080'
+STAGECRAFT_DATA_SET_QUERY_TOKEN = 'stagecraft-data-set-query-token-fake'

backdrop/read/config/test.py

@@ -10,3 +10,5 @@
     "rawr": True,
     "month": True,
 }
+
+from development import STAGECRAFT_URL, STAGECRAFT_DATA_SET_QUERY_TOKEN

backdrop/write/api.py

@@ -31,7 +31,10 @@
     app.config['DATABASE_NAME']
 )
 
-bucket_repository = BucketConfigRepository(db)
+bucket_repository = BucketConfigRepository(
+    app.config['STAGECRAFT_URL'],
+    app.config['STAGECRAFT_DATA_SET_QUERY_TOKEN'])
+
 user_repository = UserConfigRepository(db)
 
 log_handler.set_up_logging(app, GOVUK_ENV)

backdrop/write/config/development.py

@@ -12,3 +12,6 @@
     from development_environment import *
 except ImportError:
     from development_environment_sample import *
+
+STAGECRAFT_URL = 'http://localhost:8080'
+STAGECRAFT_DATA_SET_QUERY_TOKEN = 'stagecraft-data-set-query-token-fake'

backdrop/write/config/test.py

@@ -10,5 +10,6 @@
     "evl_volumetrics": ["_timestamp", "service", "transaction"],
 }
 
-from development import CREATE_COLLECTION_ENDPOINT_TOKEN
+from development import (CREATE_COLLECTION_ENDPOINT_TOKEN, STAGECRAFT_URL,
+                         STAGECRAFT_DATA_SET_QUERY_TOKEN)
 from test_environment import *

features/admin/csv_upload.feature

@@ -2,8 +2,9 @@
 Feature: CSV Upload
 
   Scenario: Upload CSV data
-    Given I have a bucket named "my_bucket"
-    And   bucket setting upload_format is "csv"
+    Given I have a bucket named "my_bucket" with settings
+        | key           | value |
+        | upload_format | "csv" |
     And   I am logged in
     And   I can upload to "my_bucket"
     And   a file named "data.csv"
@@ -28,8 +29,9 @@ Feature: CSV Upload
           city,città
           coffee,caffè
           """
-    And   I have a bucket named "my_bucket"
-    And   bucket setting upload_format is "csv"
+    And   I have a bucket named "my_bucket" with settings
+        | key           | value |
+        | upload_format | "csv" |
     And   I am logged in
     And   I can upload to "my_bucket"
     When  I go to "/my_bucket/upload"
@@ -59,8 +61,9 @@ Feature: CSV Upload
           2013-01-01,2013-01-07,abc,287
           2013-01-01,2013-01-07,def,425
           """
-    And   I have a bucket named "bucket_with_auto_id"
-    And   bucket setting upload_format is "csv"
+    And   I have a bucket named "bucket_with_auto_id" with settings
+        | key           | value |
+        | upload_format | "csv" |
     And   I am logged in
     And   I can upload to "bucket_with_auto_id"
     When  I go to "/bucket_with_auto_id/upload"

features/admin/csv_upload_validation.feature

@@ -8,8 +8,9 @@ Feature: csv upload validation
           Pawel,27,Polish,male
           Max,35,Italian,male
           """
-    And   I have a bucket named "foo"
-    And   bucket setting upload_format is "csv"
+    And   I have a bucket named "foo" with settings
+        | key           | value |
+        | upload_format | "csv" |
     And   I am logged in
     And   I can upload to "foo"
     When  I go to "/foo/upload"
@@ -25,8 +26,9 @@ Feature: csv upload validation
           Pawel,27,Polish,male
           Max,35,Italian
           """
-    And   I have a bucket named "foo"
-    And   bucket setting upload_format is "csv"
+    And   I have a bucket named "foo" with settings
+        | key           | value |
+        | upload_format | "csv" |
     And   I am logged in
     And   I can upload to "foo"
     When  I go to "/foo/upload"
@@ -37,8 +39,9 @@ Feature: csv upload validation
 
   Scenario: file too large
     Given a file named "data.csv" of size "1000000" bytes
-    And   I have a bucket named "foo"
-    And   bucket setting upload_format is "csv"
+    And   I have a bucket named "foo" with settings
+        | key           | value |
+        | upload_format | "csv" |
     And   I am logged in
     And   I can upload to "foo"
     When  I go to "/foo/upload"
@@ -49,8 +52,9 @@ Feature: csv upload validation
 
   Scenario: non UTF8 characters
     Given a file named "data.csv" with fixture "bad-characters.csv"
-    And   I have a bucket named "foo"
-    And   bucket setting upload_format is "csv"
+    And   I have a bucket named "foo" with settings
+        | key           | value |
+        | upload_format | "csv" |
     And   I am logged in
     And   I can upload to "foo"
     When  I go to "/foo/upload"