Merge pull request #785 from aaxelb/eng-2740--feed-urls

[ENG-2740] add new feed URLs, make it easy to remove the old

api/urls.py

@@ -24,8 +24,11 @@
     url('^search/', include('api.search.urls'), name='search'),
 
     # TODO refactor non-viewset endpoints to conform to new structure
-    url(r'status/?', views.ServerStatusView.as_view(), name='status'),
-    url(r'rss/?', views.CreativeWorksRSS(), name='rss'),
-    url(r'atom/?', views.CreativeWorksAtom(), name='atom'),
-    url(r'graph/?', GraphQLView.as_view(graphiql=True)),
+    url(r'^status/?', views.ServerStatusView.as_view(), name='status'),
+    url(r'^rss/?', views.LegacyCreativeWorksRSS(), name='rss'),
+    url(r'^atom/?', views.LegacyCreativeWorksAtom(), name='atom'),
+    url(r'^graph/?', GraphQLView.as_view(graphiql=True)),
+
+    url(r'^feeds/rss/?', views.MetadataRecordsRSS(), name='feeds.rss'),
+    url(r'^feeds/atom/?', views.MetadataRecordsAtom(), name='feeds.atom'),
 ]

api/views/feeds.py

@@ -1,12 +1,14 @@
 
 from xml.sax.saxutils import unescape
 import datetime
+from furl import furl
 import json
 import logging
 import re
 import requests
 
 from django.contrib.syndication.views import Feed
+from django.http import HttpResponseGone
 from django.utils.feedgenerator import Atom1Feed
 from django.conf import settings
 
@@ -43,8 +45,9 @@ def parse_date(s):
     return None
 
 
-class CreativeWorksRSS(Feed):
-    link = '{}api/v2/rss/'.format(settings.SHARE_WEB_URL)
+class MetadataRecordsRSS(Feed):
+    _share_index_key = 'BACKCOMPAT_INDEX'  # TODO remove when we drop legacy feeds
+    link = '{}api/v2/feeds/rss/'.format(settings.SHARE_API_URL)
     description = 'Updates to the SHARE open dataset'
     author_name = 'SHARE'
 
@@ -75,7 +78,11 @@ def get_object(self, request):
 
     def items(self, obj):
         headers = {'Content-Type': 'application/json'}
-        search_url = '{}{}/creativeworks/_search'.format(settings.ELASTICSEARCH['URL'], settings.ELASTICSEARCH['PRIMARY_INDEX'])
+        search_url = '{}{}/creativeworks/_search'.format(
+            settings.ELASTICSEARCH['URL'],
+            # TODO as soon as we can drop the legacy feeds, use settings.ELASTICSEARCH['PRIMARY_INDEX']
+            settings.ELASTICSEARCH[self._share_index_key],
+        )
         elastic_response = requests.post(search_url, data=json.dumps(obj), headers=headers)
         json_response = elastic_response.json()
 
@@ -97,7 +104,7 @@ def item_description(self, item):
 
     def item_link(self, item):
         # Link to SHARE curate page
-        return '{}{}/{}'.format(settings.SHARE_WEB_URL, item.get('type').replace(' ', ''), item.get('id'))
+        return '{}{}/{}'.format(settings.SHARE_API_URL, item.get('type').replace(' ', ''), item.get('id'))
 
     def item_author_name(self, item):
         contributor_list = item.get('lists', []).get('contributors', [])
@@ -126,7 +133,36 @@ def item_categories(self, item):
         return [prepare_string(c) for c in categories if c]
 
 
-class CreativeWorksAtom(CreativeWorksRSS):
+class MetadataRecordsAtom(MetadataRecordsRSS):
+    _share_index_key = 'BACKCOMPAT_INDEX'  # TODO remove when we drop legacy feeds
     feed_type = Atom1Feed
-    subtitle = CreativeWorksRSS.description
-    link = '{}api/v2/atom/'.format(settings.SHARE_WEB_URL)
+    subtitle = MetadataRecordsRSS.description
+    link = '{}api/v2/feeds/atom/'.format(settings.SHARE_API_URL)
+
+
+# TODO remove when we drop legacy feeds
+class LegacyCreativeWorksRSS(MetadataRecordsRSS):
+    _share_index_key = 'LEGACY_INDEX'  # TODO remove when we drop legacy feeds
+    link = '{}api/v2/rss/'.format(settings.SHARE_API_URL)
+
+    def __call__(self, request, *args, **kwargs):
+        if not settings.SHARE_LEGACY_PIPELINE:
+            correct_url = furl(MetadataRecordsRSS.link).set(query_params=request.GET)
+            return HttpResponseGone(
+                f'This feed has been removed -- please update to use {correct_url}'
+            )
+        return super().__call__(request, *args, **kwargs)
+
+
+# TODO remove when we drop legacy feeds
+class LegacyCreativeWorksAtom(MetadataRecordsAtom):
+    _share_index_key = 'LEGACY_INDEX'  # TODO remove when we drop legacy feeds
+    link = '{}api/v2/atom/'.format(settings.SHARE_API_URL)
+
+    def __call__(self, request, *args, **kwargs):
+        if not settings.SHARE_LEGACY_PIPELINE:
+            correct_url = furl(MetadataRecordsAtom.link).set(query_params=request.GET)
+            return HttpResponseGone(
+                f'This feed has been removed -- please update to use {correct_url}'
+            )
+        return super().__call__(request, *args, **kwargs)

project/settings.py

@@ -332,6 +332,9 @@ def split(string, delim):
         #     'INDEX_SETUP': 'trove_v0',
         # },
     },
+    # ease the transition
+    'LEGACY_INDEX': 'share_customtax_1',
+    'BACKCOMPAT_INDEX': 'share_postrend_backcompat',
 }
 
 # Seconds, not an actual celery settings

tests/api/test_feeds.py

@@ -44,8 +44,12 @@ def fake_items(self, settings, index_creativeworks):
 
         index_creativeworks(ids)
 
-    def test_get_feed(self, client, fake_items):
-        resp = client.get('/api/v2/atom')
+    @pytest.mark.parametrize('feed_url', [
+        '/api/v2/atom/',
+        '/api/v2/feeds/atom/',
+    ])
+    def test_get_feed(self, client, fake_items, feed_url, settings):
+        resp = client.get(feed_url)
         assert resp.status_code == 200
 
         feed = etree.fromstring(resp.content)
@@ -58,8 +62,12 @@ def test_get_feed(self, client, fake_items):
         'date_updated',
         'date_modified',
     ])
-    def test_order(self, client, order, fake_items):
-        resp = client.get('/api/v2/atom', {'order': order})
+    @pytest.mark.parametrize('feed_url', [
+        '/api/v2/atom/',
+        '/api/v2/feeds/atom/',
+    ])
+    def test_order(self, client, order, fake_items, feed_url):
+        resp = client.get(feed_url, {'order': order})
         assert resp.status_code == 200
 
         feed = etree.fromstring(resp.content)
@@ -94,3 +102,14 @@ def test_order(self, client, order, fake_items):
                 assert entry.find('atom:published', namespaces=NAMESPACES).text == creative_work.date_published.isoformat()
             else:
                 assert entry.find('atom:published', namespaces=NAMESPACES) is None
+
+    @pytest.mark.parametrize('feed_url, expected_status', [
+        ('/api/v2/atom/', 410),
+        ('/api/v2/rss/', 410),
+        ('/api/v2/feeds/atom/', 200),
+        ('/api/v2/feeds/rss/', 200),
+    ])
+    def test_gone(self, client, settings, fake_items, feed_url, expected_status):
+        settings.SHARE_LEGACY_PIPELINE = False
+        resp = client.get(feed_url)
+        assert resp.status_code == expected_status

tests/conftest.py

@@ -239,6 +239,8 @@ def elastic_test_manager(settings, elastic_test_index_name):
         **settings.ELASTICSEARCH,
         'TIMEOUT': 5,
         'PRIMARY_INDEX': elastic_test_index_name,
+        'LEGACY_INDEX': elastic_test_index_name,
+        'BACKCOMPAT_INDEX': elastic_test_index_name,
         'ACTIVE_INDEXES': [elastic_test_index_name],
         'INDEXES': {
             elastic_test_index_name: {