Merge pull request #114 from romanchyla/reuse-cached

Reuse /resources data for remote services

adsws/api/config.py

@@ -36,6 +36,9 @@
     'adsws.solr.app': '/solr',
     'adsws.graphics.app': '/graphics',
 }
+# when defined, the remote resources will be cached (to be reused)
+# in case when the service is temporarily down during a worker startup
+# WEBSERVICES_DISCOVERY_CACHE_DIR='/tmp'
 
 API_PROXYVIEW_HEADERS = {'Cache-Control': 'public, max-age=600'}
 REMOTE_PROXY_ALLOWED_HEADERS = ['Content-Type', 'Content-Disposition']

adsws/api/discoverer/utils.py

@@ -1,5 +1,6 @@
 import os
 import requests
+import json
 from flask.ext.headers import headers
 from flask import request
 from views import ProxyView
@@ -98,20 +99,38 @@ def bootstrap_remote_service(service_uri, deploy_path, app):
             service_uri,
             app.config.get('WEBSERVICES_PUBLISH_ENDPOINT', '/')
         )
-
+
+    cache_key = service_uri.replace('/', '').replace('\\', '').replace('.', '')
+    cache_dir = app.config.get('WEBSERVICES_DISCOVERY_CACHE_DIR', '')
+    cache_path = os.path.join(cache_dir, cache_key)
+    resource_json = {}
+
+    # discover the ratelimits/urls/permissions from the service itself;
+    # if not available, use a cached values (if any)
     try:
         r = requests.get(url, timeout=5)
-    except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
-        app.logger.info('Could not discover {0}'.format(service_uri))
-        return
+        resource_json = r.json()
+        if cache_dir:
+            try:
+                with open(cache_path, 'w') as cf:
+                    cf.write(json.dumps(resource_json))
+            except IOError:
+                app.logger.error('Cant write cached resource {0}'.format(cache_path))
+    except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):        
+        if cache_dir and os.path.exists(cache_path):
+            with open(cache_path, 'r') as cf:
+                resource_json = json.loads(cf.read())
+        else:
+            app.logger.info('Could not discover {0}'.format(service_uri))
+            return
+
 
-    # validate(r.json()) # TODO validate the incoming json
 
     # Start constructing the ProxyViews based on what we got when querying
     # the /resources route.
     # If any part of this procedure fails, log that we couldn't produce this
     # ProxyView, but otherwise continue.
-    for resource, properties in r.json().iteritems():
+    for resource, properties in resource_json.iteritems():
         if resource.startswith('/'):
             resource = resource[1:]
         route = os.path.join(deploy_path, resource)

adsws/tests/test_discoverer.py

@@ -9,6 +9,8 @@
 from sample_microservice import Stubdata
 import mock
 from datetime import datetime
+from unittest import TestCase
+from requests.exceptions import ConnectionError
 
 LIVESERVER_WAIT_SECONDS = 2.5
 
@@ -383,3 +385,62 @@ def create_app(self):
                 time.time()),
         )
         return app
+
+
+import adsws.api.discoverer.utils as discovery_utils
+class TestUnitTests(TestCase):
+
+    @mock.patch.object(discovery_utils, 'requests')
+    def test_bootstrap_remote_caching(self, requests):
+        requests.get.return_value = mock.Mock()
+        requests.get.return_value.json.return_value = {'hey': {'methods': ['IGNORE']}}
+
+        app = mock.Mock()
+        app.app_context = mock.mock_open()
+        app.config = {'WEBSERVICES_DISCOVERY_CACHE_DIR' : '/tmp'}
+
+        with mock.patch("__builtin__.open", mock.mock_open()) as mock_file:
+
+            discovery_utils.bootstrap_remote_service('http://foo.bar-us-east-1.com:8980/tee', '/foo', app)
+            requests.get.assert_called_with('http://foo.bar-us-east-1.com:8980/', timeout=5)
+            mock_file.assert_called_with('/tmp/http:foobar-us-east-1com:8980tee', 'w')
+            file_handle = mock_file.return_value.__enter__.return_value
+            file_handle.write.assert_called_with('{"hey": {"methods": ["IGNORE"]}}')
+
+
+    @mock.patch.object(discovery_utils, 'requests')
+    def test_bootstrap_ignore_remote_caching(self, requests):
+        requests.get.return_value = mock.Mock()
+        requests.get.return_value.json.return_value = {'hey': {'methods': ['IGNORE']}}
+
+        app = mock.Mock()
+        app.app_context = mock.mock_open()
+        app.config = {'WEBSERVICES_DISCOVERY_CACHE_DIR' : ''}
+
+        with mock.patch("__builtin__.open", mock.mock_open()) as mock_file:
+
+            discovery_utils.bootstrap_remote_service('http://foo.bar-us-east-1.com:8980/tee', '/foo', app)
+            requests.get.assert_called_with('http://foo.bar-us-east-1.com:8980/', timeout=5)
+            try:
+                mock_file.assert_called_with('/tmp/http:foobar-us-east-1com:8980tee', 'w')
+                raise Exception('The file was opened!')
+            except AssertionError:
+                pass
+
+    @mock.patch.object(discovery_utils, 'os')
+    @mock.patch.object(discovery_utils, 'requests')
+    @mock.patch.object(discovery_utils, 'json')
+    def test_bootstrap_from_cache(self, m_json, requests, m_os):
+        requests.get.side_effect = ConnectionError()
+        requests.exceptions.ConnectionError = ConnectionError
+        app = mock.Mock()
+        app.app_context = mock.mock_open()
+        app.config = {'WEBSERVICES_DISCOVERY_CACHE_DIR' : '/tmp'}
+        m_os.path.join = os.path.join
+        m_os.path.exists.return_value = True
+        with mock.patch("__builtin__.open", mock.mock_open(read_data='{"hey": {"methods": ["IGNORE"]}}')) as mock_file:
+
+            discovery_utils.bootstrap_remote_service('http://foo.bar-us-east-1.com:8980/tee', '/foo', app)
+            requests.get.assert_called_with('http://foo.bar-us-east-1.com:8980/', timeout=5)
+            m_os.path.exists.called_with('/tmp/http:foobar-us-east-1com:8980tee')
+            m_json.loads.assert_called_with('{"hey": {"methods": ["IGNORE"]}}')