Backdrop sends Bearer token to Stagecraft

backdrop/core/repository.py

@@ -1,11 +1,14 @@
 
 import json
+import logging
 
 import requests
 
 from backdrop.core.bucket import BucketConfig
 from backdrop.core.user import UserConfig
 
+logger = logging.getLogger(__name__)
+
 
 class _Repository(object):
 
@@ -52,9 +55,12 @@ def __init__(self, stagecraft_url, stagecraft_token):
         self._stagecraft_token = stagecraft_token
 
     def get_all(self):
-        data_set_url = '{url}/data-sets/'.format(url=self._stagecraft_url)
+        data_set_url = '{url}/data-sets'.format(url=self._stagecraft_url)
+
+        # Note: Don't catch HTTP 404 - that should never happen on this URL.
+        json_response = _get_json_url(data_set_url, self._stagecraft_token)
+        data_sets = _decode_json(json_response)
 
-        data_sets = _decode_json(_get_url(data_set_url))
         return [_make_bucket_config(data_set) for data_set in data_sets]
 
     def retrieve(self, name):
@@ -64,8 +70,15 @@ def retrieve(self, name):
                         url=self._stagecraft_url,
                         data_set_name=name))
 
-        data_set = _decode_json(_get_url(data_set_url))
-        return _make_bucket_config(data_set)
+        try:
+            json_response = _get_json_url(data_set_url, self._stagecraft_token)
+        except requests.HTTPError as e:
+            if e.response.status_code == 404:
+                return None
+            else:
+                raise
+
+        return _make_bucket_config(_decode_json(json_response))
 
     def get_bucket_for_query(self, data_group, data_type):
         empty_vars = []
@@ -81,9 +94,13 @@ def get_bucket_for_query(self, data_group, data_type):
                             data_group_name=data_group,
                             data_type_name=data_type))
 
-        data_sets = _decode_json(_get_url(data_set_url))
+        json_response = _get_json_url(
+            data_set_url, self._stagecraft_token)
+
+        data_sets = _decode_json(json_response)
         if len(data_sets) > 0:
             return _make_bucket_config(data_sets[0])
+
         return None
 
 
@@ -97,15 +114,19 @@ def _decode_json(string):
     return json.loads(string) if string is not None else None
 
 
-def _get_url(url):
-    response = requests.get(url)
+def _get_json_url(url, token):
+    auth_header = (
+        'Authorization',
+        'Bearer {}'.format(token))
+    response = requests.get(url, headers=dict([
+        ('content-type', 'application/json'),
+        auth_header]))
     try:
         response.raise_for_status()
     except requests.HTTPError as e:
-        if e.response.status_code == 404:
-            return None
-        raise e
-
+        logger.exception(e)
+        logger.error('Stagecraft said: {}'.format(response.content))
+        raise
     return response.content
 
 

features/steps/read_api.py

@@ -38,7 +38,7 @@ def ensure_bucket_exists(context, bucket_name, settings={}):
 
     url_response_dict = {
         ('GET', u'data-sets/{}'.format(bucket_name)): response,
-        ('GET', u'data-sets/'): [response],
+        ('GET', u'data-sets'): [response],
         ('GET', u'data-sets?data-group={}&data-type={}'.format(
             response['data_group'], response['data_type'])): [response],
     }

tests/core/integration/test_repository.py

@@ -1,58 +1,11 @@
-import unittest
-import mock
-
-from contextlib import contextmanager
-
-from os.path import dirname, join as pjoin
-
 from hamcrest import assert_that, is_, has_entries
-from backdrop.core.bucket import BucketConfig
 from backdrop.core.database import Database
-from backdrop.core.repository import BucketConfigRepository, UserConfigRepository
+from backdrop.core.repository import UserConfigRepository
 from backdrop.core.user import UserConfig
 
 HOST = ['localhost']
 PORT = 27017
 DB_NAME = 'performance_platform_test'
-BUCKET = 'buckets'
-STAGECRAFT_URL = 'fake_url_should_not_be_called'
-STAGECRAFT_DATA_SET_QUERY_TOKEN = 'fake_token_should_not_be_used'
-
-
-@contextmanager
-def fixture(name):
-    filename = pjoin(dirname(__file__), '..', '..', 'fixtures', name)
-    with open(filename, 'r') as f:
-        yield f.read()
-
-
-class TestBucketRepositoryIntegration(unittest.TestCase):
-
-    def setUp(self):
-        self.db = Database(HOST, PORT, DB_NAME)
-        self.db._mongo.drop_database(DB_NAME)
-        self.mongo_collection = self.db.get_collection(BUCKET)
-        self.repository = BucketConfigRepository(
-            STAGECRAFT_URL, STAGECRAFT_DATA_SET_QUERY_TOKEN)
-
-    def test_retrieves_config_by_name(self):
-        with fixture('stagecraft_get_single_data_set.json') as content:
-            with mock.patch('backdrop.core.repository._get_url') as mocked:
-                mocked.return_value = content
-                config = self.repository.retrieve(name="govuk_visitors")
-                mocked.assert_called_once_with(
-                    'fake_url_should_not_be_called/data-sets/govuk_visitors')
-
-        assert_that(config.name, is_('govuk_visitors'))
-
-    def test_retrieves_config_for_service_and_data_type(self):
-        with fixture('stagecraft_query_data_group_type.json') as content:
-            with mock.patch('backdrop.core.repository._get_url') as mocked:
-                mocked.return_value = content
-                config = self.repository.get_bucket_for_query(
-                    data_group="govuk", data_type="realtime")
-
-        assert_that(config.name, is_("govuk_realtime"))
 
 
 class TestUserRepositoryIntegration(object):

tests/core/test_repository.py

@@ -1,58 +1,184 @@
-import json
 import mock
 import unittest
 
 from backdrop.core.bucket import BucketConfig
 from backdrop.core.repository import (BucketConfigRepository,
-                                      UserConfigRepository)
+                                      UserConfigRepository,
+                                      _get_json_url)
 from hamcrest import assert_that, equal_to, is_, has_entries, match_equality
-from mock import Mock
+from mock import Mock, patch
 from nose.tools import assert_raises
 from backdrop.core.user import UserConfig
+from contextlib import contextmanager
+from os.path import dirname, join as pjoin
+import requests
+
+
+@contextmanager
+def fixture(name):
+    filename = pjoin(dirname(__file__), '..', 'fixtures', name)
+    with open(filename, 'r') as f:
+        yield f.read()
+
+_GET_JSON_URL_FUNC = 'backdrop.core.repository._get_json_url'
+
+
+class TestGetJsonUrl(unittest.TestCase):
+
+    @patch('requests.get', spec=True)
+    def test_get_json_url_sends_correct_request(self, mock_get):
+        mock_response = Mock()
+        mock_response.content = '[]'
+        mock_get.return_value = mock_response
+        response_content = _get_json_url("my_url", "some_token")
+        mock_get.assert_called_once_with(
+            'my_url',
+            headers={'content-type': 'application/json',
+                     'Authorization': 'Bearer some_token'})
+        assert_that(response_content, equal_to('[]'))
 
 
 class TestBucketRepository(unittest.TestCase):
     def setUp(self):
-        # This is a bit of a smell. Mongo collection responsibilites should be
-        # split with repo, once we have more than one repository.
-        self.db = Mock()
-        self.mongo_collection = Mock()
-        self.db.get_collection.return_value = self.mongo_collection
         self.bucket_repo = BucketConfigRepository(
             'fake_stagecraft_url', 'fake_stagecraft_token')
 
-    def test_bucket_config_is_created_from_retrieved_data(self):
-        fake_stagecraft_response = json.dumps({
-            "name": "bucket_name",
-            "data_group": "data_group",
-            "data_type": "type",
-            "raw_queries_allowed": False,
-            "bearer_token": "my-bearer-token",
-            "upload_format": "excel"
-        })
-        with mock.patch('backdrop.core.repository._get_url') as mocked:
-            mocked.return_value = fake_stagecraft_response
-
-            bucket = self.bucket_repo.retrieve(name="bucket_name")
-
-        expected_bucket = BucketConfig("bucket_name",
-                                       data_group="data_group",
-                                       data_type="type",
-                                       raw_queries_allowed=False,
-                                       bearer_token="my-bearer-token",
-                                       upload_format="excel")
-
-        assert_that(bucket, equal_to(expected_bucket))
-
-    def test_retrieving_non_existent_bucket_returns_none(self):
-        self.mongo_collection.find_one.return_value = None
-
-        with mock.patch('backdrop.core.repository._get_url') as mocked:
-            mocked.return_value = None
+    def test_retrieve_correctly_decodes_stagecraft_response(self):
+        with fixture('stagecraft_get_single_data_set.json') as content:
+            with mock.patch(_GET_JSON_URL_FUNC) as _get_json_url:
+                _get_json_url.return_value = content
+
+                bucket = self.bucket_repo.retrieve(name="bucket_name")
+
+            expected_bucket = BucketConfig("govuk_visitors",
+                                           data_group="govuk",
+                                           data_type="visitors",
+                                           raw_queries_allowed=True,
+                                           bearer_token="my-bearer-token",
+                                           upload_format="excel",
+                                           upload_filters="",
+                                           auto_ids="",
+                                           queryable=True,
+                                           realtime=False,
+                                           capped_size=None,
+                                           max_age_expected=86400)
+
+            assert_that(bucket, equal_to(expected_bucket))
+
+    def test_get_all_correctly_decodes_stagecraft_response(self):
+        with fixture('stagecraft_list_data_sets.json') as content:
+            with mock.patch(_GET_JSON_URL_FUNC) as _get_json_url:
+                _get_json_url.return_value = content
+
+                buckets = self.bucket_repo.get_all()
+
+            expected_bucket_one = BucketConfig(
+                "govuk_visitors",
+                data_group="govuk",
+                data_type="visitors",
+                raw_queries_allowed=True,
+                bearer_token="my-bearer-token",
+                upload_format="excel",
+                upload_filters="",
+                auto_ids="",
+                queryable=True,
+                realtime=False,
+                capped_size=None,
+                max_age_expected=86400)
+
+            assert_that(len(buckets), equal_to(5))
+            assert_that(buckets[0], equal_to(expected_bucket_one))
+
+    def test_get_bucket_for_query_correctly_decodes_stagecraft_response(self):
+        with fixture('stagecraft_query_data_group_type.json') as content:
+            with mock.patch(_GET_JSON_URL_FUNC) as _get_json_url:
+                _get_json_url.return_value = content
+
+                bucket = self.bucket_repo.get_bucket_for_query(
+                    data_group="govuk", data_type="realtime")
+
+            expected_bucket = BucketConfig("govuk_visitors",
+                                           data_group="govuk",
+                                           data_type="visitors",
+                                           raw_queries_allowed=True,
+                                           bearer_token="my-bearer-token",
+                                           upload_format="excel",
+                                           upload_filters="",
+                                           auto_ids="",
+                                           queryable=True,
+                                           realtime=False,
+                                           capped_size=None,
+                                           max_age_expected=86400)
+
+            assert_that(bucket, equal_to(expected_bucket))
+
+    def test_retrieve_for_non_existent_bucket_returns_none(self):
+        def _mock_raise_http_404(*args, **kwargs):
+            mock_error_response = Mock()
+            mock_error_response.status_code = 404
+            exception = requests.HTTPError()
+            exception.response = mock_error_response
+
+            raise exception
+
+        with mock.patch(_GET_JSON_URL_FUNC) as _get_json_url:
+            _get_json_url.side_effect = _mock_raise_http_404
             bucket = self.bucket_repo.retrieve(name="non_existent")
 
         assert_that(bucket, is_(None))
 
+    def test_retrieve_doesnt_catch_http_errors_other_than_404(self):
+        def _mock_raise_http_503(*args, **kwargs):
+            mock_error_response = Mock()
+            mock_error_response.status_code = 503
+            exception = requests.HTTPError()
+            exception.response = mock_error_response
+
+            raise exception
+
+        with mock.patch(_GET_JSON_URL_FUNC) as _get_json_url:
+            _get_json_url.side_effect = _mock_raise_http_503
+
+            assert_raises(
+                requests.HTTPError,
+                lambda: self.bucket_repo.retrieve(name="non_existent"))
+
+    def test_get_bucket_for_query_for_non_existent_bucket_returns_none(self):
+        with mock.patch(_GET_JSON_URL_FUNC) as _get_json_url:
+            _get_json_url.return_value = '[]'
+            bucket = self.bucket_repo.get_bucket_for_query(
+                data_group="govuk", data_type="realtime")
+
+        assert_that(bucket, is_(None))
+
+    def test_retrieve_calls_correct_url_for_data_set_by_name(self):
+        with fixture('stagecraft_get_single_data_set.json') as content:
+            with mock.patch(_GET_JSON_URL_FUNC) as _get_json_url:
+                _get_json_url.return_value = content
+                self.bucket_repo.retrieve(name="govuk_visitors")
+                _get_json_url.assert_called_once_with(
+                    'fake_stagecraft_url/data-sets/govuk_visitors',
+                    "fake_stagecraft_token")
+
+    def test_get_bucket_for_query_calls_correct_url(self):
+        with fixture('stagecraft_query_data_group_type.json') as content:
+            with mock.patch(_GET_JSON_URL_FUNC) as _get_json_url:
+                _get_json_url.return_value = content
+                self.bucket_repo.get_bucket_for_query(
+                    data_group="govuk", data_type="realtime")
+                _get_json_url.assert_called_once_with(
+                    'fake_stagecraft_url/data-sets?'
+                    'data-group=govuk&data-type=realtime',
+                    "fake_stagecraft_token")
+
+    def test_get_all_calls_correct_url(self):
+        with fixture('stagecraft_query_data_group_type.json') as content:
+            with mock.patch(_GET_JSON_URL_FUNC) as _get_json_url:
+                _get_json_url.return_value = content
+                self.bucket_repo.get_all()
+                _get_json_url.assert_called_once_with(
+                    'fake_stagecraft_url/data-sets', "fake_stagecraft_token")
+
 
 class TestUserConfigRepository(object):
     def setUp(self):

tests/fixtures/stagecraft_get_single_data_set.json

@@ -3,12 +3,12 @@
  "data_group": "govuk", 
  "data_type": "visitors", 
  "raw_queries_allowed": true, 
- "bearer_token": "", 
- "upload_format": "", 
+ "bearer_token": "my-bearer-token", 
+ "upload_format": "excel", 
  "upload_filters": "", 
  "auto_ids": "", 
  "queryable": true, 
  "realtime": false, 
  "capped_size": null, 
  "max_age_expected": 86400
-}
+}