From 0407740e11ec8330361a428d4db2d10da3fd2b2e Mon Sep 17 00:00:00 2001 From: Konstantin Sivakov Date: Thu, 12 Oct 2017 11:25:32 +0200 Subject: [PATCH 01/12] remove pylons routes --- ckan/config/routing.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ckan/config/routing.py b/ckan/config/routing.py index f2a9336e931..08b3a3e3bdc 100644 --- a/ckan/config/routing.py +++ b/ckan/config/routing.py @@ -371,13 +371,6 @@ def make_map(): m.connect('/revision/list', action='list') m.connect('/revision/{id}', action='read') - # feeds - with SubMapper(map, controller='feed') as m: - m.connect('/feeds/group/{id}.atom', action='group') - m.connect('/feeds/organization/{id}.atom', action='organization') - m.connect('/feeds/tag/{id}.atom', action='tag') - m.connect('/feeds/dataset.atom', action='general') - m.connect('/feeds/custom.atom', action='custom') map.connect('ckanadmin_index', '/ckan-admin', controller='admin', action='index', ckan_icon='gavel') From 9768e86dfbf3d26ea59599838cb7358485fc7714 Mon Sep 17 00:00:00 2001 From: Konstantin Sivakov Date: Thu, 12 Oct 2017 11:28:18 +0200 Subject: [PATCH 02/12] fix tests for flask --- ckan/tests/controllers/test_feed.py | 124 ++++++++-------------------- 1 file changed, 36 insertions(+), 88 deletions(-) diff --git a/ckan/tests/controllers/test_feed.py b/ckan/tests/controllers/test_feed.py index ba256d446ba..adaf226ccfc 100644 --- a/ckan/tests/controllers/test_feed.py +++ b/ckan/tests/controllers/test_feed.py @@ -1,147 +1,95 @@ # encoding: utf-8 from ckan.lib.helpers import url_for -from webhelpers.feedgenerator import GeoAtom1Feed - -import ckan.plugins as plugins import ckan.tests.helpers as helpers import ckan.tests.factories as factories class TestFeedNew(helpers.FunctionalTestBase): - @classmethod def teardown_class(cls): helpers.reset_db() def test_atom_feed_page_zero_gives_error(self): group = factories.Group() - offset = url_for(controller='feed', action='group', - id=group['name']) + '?page=0' + offset = url_for(u'feeds.group', id=group['name']) + '?page=0' app = self._get_test_app() + offset = url_for(u'feeds.group', id=group['name']) + u'?page=0' + res = app.get(offset, status=400) assert '"page" parameter must be a positive integer' in res, res def test_atom_feed_page_negative_gives_error(self): group = factories.Group() - offset = url_for(controller='feed', action='group', - id=group['name']) + '?page=-2' + offset = url_for(u'feeds.group', id=group['name']) + '?page=-2' app = self._get_test_app() + offset = url_for(u'feeds.group', id=group['name']) + '?page=-2' res = app.get(offset, status=400) assert '"page" parameter must be a positive integer' in res, res def test_atom_feed_page_not_int_gives_error(self): group = factories.Group() - offset = url_for(controller='feed', action='group', - id=group['name']) + '?page=abc' + offset = url_for(u'feeds.group', id=group['name']) + '?page=abc' app = self._get_test_app() + offset = url_for(u'feeds.group', id=group['name']) + '?page=abc' res = app.get(offset, status=400) assert '"page" parameter must be a positive integer' in res, res def test_general_atom_feed_works(self): dataset = factories.Dataset() - offset = url_for(controller='feed', action='general') + offset = url_for(u'feeds.general') app = self._get_test_app() + offset = url_for(u'feeds.general') res = app.get(offset) - assert '{0}'.format(dataset['title']) in res.body + assert u'{0}'.format( + dataset['title']) in res.body def test_group_atom_feed_works(self): group = factories.Group() dataset = factories.Dataset(groups=[{'id': group['id']}]) - offset = url_for(controller='feed', action='group', - id=group['name']) + offset = url_for(u'feeds.group', id=group['name']) app = self._get_test_app() + offset = url_for(u'feeds.group', id=group['name']) res = app.get(offset) - assert '{0}'.format(dataset['title']) in res.body + assert u'{0}'.format( + dataset['title']) in res.body def test_organization_atom_feed_works(self): group = factories.Organization() dataset = factories.Dataset(owner_org=group['id']) - offset = url_for(controller='feed', action='organization', - id=group['name']) + offset = url_for(u'feeds.organization', id=group['name']) app = self._get_test_app() + offset = url_for(u'feeds.organization', id=group['name']) res = app.get(offset) - assert '{0}'.format(dataset['title']) in res.body + assert u'{0}'.format( + dataset['title']) in res.body def test_custom_atom_feed_works(self): dataset1 = factories.Dataset( - title='Test weekly', - extras=[{'key': 'frequency', 'value': 'weekly'}]) + title=u'Test weekly', + extras=[{ + 'key': 'frequency', + 'value': 'weekly' + }]) dataset2 = factories.Dataset( - title='Test daily', - extras=[{'key': 'frequency', 'value': 'daily'}]) - offset = url_for(controller='feed', action='custom') - params = { - 'q': 'frequency:weekly' - } - app = self._get_test_app() - res = app.get(offset, params=params) - - assert '{0}'.format(dataset1['title']) in res.body - - assert '{0}'.format(dataset2['title']) not in res.body - - -class TestFeedInterface(helpers.FunctionalTestBase): - @classmethod - def setup_class(cls): - super(TestFeedInterface, cls).setup_class() - - if not plugins.plugin_loaded('test_feed_plugin'): - plugins.load('test_feed_plugin') - - @classmethod - def teardown_class(cls): - helpers.reset_db() - plugins.unload('test_feed_plugin') + title=u'Test daily', + extras=[{ + 'key': 'frequency', + 'value': 'daily' + }]) - def test_custom_class_used(self): - - app = self._get_test_app() - with app.flask_app.test_request_context(): - offset = url_for(controller='feed', action='general') app = self._get_test_app() - res = app.get(offset) - - assert 'xmlns:georss="http://www.georss.org/georss"' in res.body, res.body - - def test_additional_fields_added(self): - metadata = { - 'ymin': '-2373790', - 'xmin': '2937940', - 'ymax': '-1681290', - 'xmax': '3567770', - } - - extras = [ - {'key': k, 'value': v} for (k, v) in - metadata.items() - ] - - factories.Dataset(extras=extras) - + offset = url_for('feeds.custom') + params = {'q': 'frequency:weekly'} app = self._get_test_app() - with app.flask_app.test_request_context(): - offset = url_for(controller='feed', action='general') - app = self._get_test_app() - res = app.get(offset) - - assert '-2373790.000000 2937940.000000 -1681290.000000 3567770.000000' in res.body, res.body - - -class MockFeedPlugin(plugins.SingletonPlugin): - plugins.implements(plugins.IFeed) - - def get_feed_class(self): - return GeoAtom1Feed + res = app.get(offset, params=params) - def get_item_additional_fields(self, dataset_dict): - extras = {e['key']: e['value'] for e in dataset_dict['extras']} + assert u'{0}'.format( + dataset1['title']) in res.body - box = tuple(float(extras.get(n)) - for n in ('ymin', 'xmin', 'ymax', 'xmax')) - return {'geometry': box} + assert u'{0}'.format( + dataset2['title']) not in res.body From 2950d87a952b641c1050ee3f1f89c025d94b2c92 Mon Sep 17 00:00:00 2001 From: Konstantin Sivakov Date: Thu, 12 Oct 2017 11:28:43 +0200 Subject: [PATCH 03/12] add feeds blueprint --- ckan/views/feeds.py | 509 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 509 insertions(+) create mode 100644 ckan/views/feeds.py diff --git a/ckan/views/feeds.py b/ckan/views/feeds.py new file mode 100644 index 00000000000..e21b895a2aa --- /dev/null +++ b/ckan/views/feeds.py @@ -0,0 +1,509 @@ +# encoding: utf-8 + +import logging +import urlparse + +from flask import Blueprint +from werkzeug.contrib.atom import AtomFeed + +from ckan.common import _, config, g, request +import ckan.lib.helpers as h +import ckan.lib.base as base +import ckan.model as model +import ckan.logic as logic +import ckan.plugins as plugins + +log = logging.getLogger(__name__) + +feeds = Blueprint(u'feeds', __name__, url_prefix=u'/feeds') + +ITEMS_LIMIT = config.get(u'ckan.feeds.limit', 20) +BASE_URL = config.get(u'ckan.site_url') +SITE_TITLE = config.get(u'ckan.site_title', u'CKAN') + + +def _package_search(data_dict): + """ + Helper method that wraps the package_search action. + + * unless overridden, sorts results by metadata_modified date + * unless overridden, sets a default item limit + """ + context = { + u'model': model, + u'session': model.Session, + u'user': g.user, + u'auth_user_obj': g.userobj + } + if u'sort' not in data_dict or not data_dict['sort']: + data_dict['sort'] = u'metadata_modified desc' + + if u'rows' not in data_dict or not data_dict['rows']: + data_dict['rows'] = ITEMS_LIMIT + + # package_search action modifies the data_dict, so keep our copy intact. + query = logic.get_action(u'package_search')(context, data_dict.copy()) + + return query['count'], query['results'] + + +def output_feed(results, feed_title, feed_description, feed_link, feed_url, + navigation_urls, feed_guid): + author_name = config.get(u'ckan.feeds.author_name', u'').strip() or \ + config.get(u'ckan.site_id', u'').strip() + + # TODO: language + feed_class = None + for plugin in plugins.PluginImplementations(plugins.IFeed): + if hasattr(plugin, u'get_feed_class'): + feed_class = plugin.get_feed_class() + + if not feed_class: + feed_class = _FixedAtomFeed + + feed = feed_class( + title=feed_title, + url=feed_link, + language=u'en', + author={u'name': author_name, + u'uri': BASE_URL}, + id=feed_guid, + feed_url=feed_url, + links=navigation_urls, + generator=(None, None, None)) + + for pkg in results: + additional_fields = {} + + for plugin in plugins.PluginImplementations(plugins.IFeed): + if hasattr(plugin, u'get_item_additional_fields'): + additional_fields = plugin.get_item_additional_fields(pkg) + + feed.add( + title=pkg.get(u'title', u''), + url=h.url_for( + controller=u'package', + action=u'read', + id=pkg['id'], + _external=True), + description=pkg.get(u'notes', u''), + updated=h.date_str_to_datetime(pkg.get(u'metadata_modified')), + published=h.date_str_to_datetime(pkg.get(u'metadata_created')), + unique_id=_create_atom_id(u'/dataset%s' % pkg['id']), + author=pkg.get(u'author', u''), + categories=[{ + 'terms': t['name'] + } for t in pkg.get('tags')], + **additional_fields) + + # response.content_type = feed.get_response() + return feed.get_response() + + +def group(id): + try: + context = { + u'model': model, + u'session': model.Session, + u'user': g.user, + u'auth_user_obj': g.userobj + } + group_dict = logic.get_action(u'group_show')(context, {u'id': id}) + except logic.NotFound: + base.abort(404, _(u'Group not found')) + + return group_or_organization(group_dict, is_org=False) + + +def organization(id): + try: + context = { + u'model': model, + u'session': model.Session, + u'user': g.user, + u'auth_user_obj': g.userobj + } + group_dict = logic.get_action(u'organization_show')(context, { + u'id': id + }) + except logic.NotFound: + base.abort(404, _(u'Organization not found')) + + return group_or_organization(group_dict, is_org=True) + + +def tag(id): + data_dict, params = _parse_url_params() + data_dict['fq'] = u'tags: "%s"' % id + + item_count, results = _package_search(data_dict) + + navigation_urls = _navigation_urls( + params, + item_count=item_count, + limit=data_dict['rows'], + controller=u'feeds', + action=u'tag', + id=id) + + feed_url = _feed_url(params, controller=u'feeds', action=u'tag', id=id) + + alternate_url = _alternate_url(params, tags=id) + + title = u'%s - Tag: "%s"' % (SITE_TITLE, id) + desc = u'Recently created or updated datasets on %s by tag: "%s"' % \ + (SITE_TITLE, id) + guid = _create_atom_id(u'/feeds/tag/%s.atom' % id) + + return output_feed( + results, + feed_title=title, + feed_description=desc, + feed_link=alternate_url, + feed_guid=guid, + feed_url=feed_url, + navigation_urls=navigation_urls) + + +def group_or_organization(obj_dict, is_org): + data_dict, params = _parse_url_params() + if is_org: + key = u'owner_org' + value = obj_dict['id'] + group_type = u'organization' + else: + key = u'groups' + value = obj_dict['name'] + group_type = u'group' + + data_dict['fq'] = u'{0}: "{1}"'.format(key, value) + item_count, results = _package_search(data_dict) + + navigation_urls = _navigation_urls( + params, + item_count=item_count, + limit=data_dict['rows'], + controller=u'feed', + action=group_type, + id=obj_dict['name']) + feed_url = _feed_url( + params, controller=u'feed', action=group_type, id=obj_dict['name']) + # site_title = SITE_TITLE + if is_org: + guid = _create_atom_id( + u'feeds/organization/%s.atom' % obj_dict['name']) + alternate_url = _alternate_url(params, organization=obj_dict['name']) + desc = u'Recently created or updated datasets on %s '\ + 'by organization: "%s"' % (SITE_TITLE, obj_dict['title']) + title = u'%s - Organization: "%s"' % (SITE_TITLE, obj_dict['title']) + + else: + guid = _create_atom_id(u'feeds/group/%s.atom' % obj_dict['name']) + alternate_url = _alternate_url(params, groups=obj_dict['name']) + desc = u'Recently created or updated datasets on %s '\ + 'by group: "%s"' % (SITE_TITLE, obj_dict['title']) + title = u'%s - Group: "%s"' % (SITE_TITLE, obj_dict['title']) + + return output_feed( + results, + feed_title=title, + feed_description=desc, + feed_link=alternate_url, + feed_guid=guid, + feed_url=feed_url, + navigation_urls=navigation_urls) + + +def _parse_url_params(): + """ + Constructs a search-query dict from the URL query parameters. + + Returns the constructed search-query dict, and the valid URL + query parameters. + """ + page = h.get_page_number(request.params) + + limit = ITEMS_LIMIT + data_dict = {u'start': (page - 1) * limit, u'rows': limit} + + # Filter ignored query parameters + valid_params = ['page'] + params = dict((p, request.params.get(p)) for p in valid_params + if p in request.params) + return data_dict, params + + +def general(): + data_dict, params = _parse_url_params() + data_dict['q'] = u'*:*' + + item_count, results = _package_search(data_dict) + + navigation_urls = _navigation_urls( + params, + item_count=item_count, + limit=data_dict['rows'], + controller=u'feeds', + action=u'general') + + feed_url = _feed_url(params, controller=u'feeds', action=u'general') + + alternate_url = _alternate_url(params) + + guid = _create_atom_id(u'/feeds/dataset.atom') + + desc = u'Recently created or updated datasets on %s' % SITE_TITLE + + return output_feed( + results, + feed_title=SITE_TITLE, + feed_description=desc, + feed_link=alternate_url, + feed_guid=guid, + feed_url=feed_url, + navigation_urls=navigation_urls) + + +def custom(): + """ + Custom atom feed + + """ + q = request.params.get(u'q', u'') + fq = u'' + search_params = {} + for (param, value) in request.params.items(): + if param not in [u'q', u'page', u'sort'] \ + and len(value) and not param.startswith(u'_'): + search_params[param] = value + fq += u'%s:"%s"' % (param, value) + + page = h.get_page_number(request.params) + + limit = ITEMS_LIMIT + data_dict = { + u'q': q, + u'fq': fq, + u'start': (page - 1) * limit, + u'rows': limit, + u'sort': request.params.get(u'sort', None) + } + + item_count, results = _package_search(data_dict) + + navigation_urls = _navigation_urls( + request.params, + item_count=item_count, + limit=data_dict['rows'], + controller=u'feeds', + action=u'custom') + + feed_url = _feed_url(request.params, controller=u'feeds', action=u'custom') + + atom_url = h._url_with_params(u'/feeds/custom.atom', search_params.items()) + + alternate_url = _alternate_url(request.params) + + site_title = config.get(u'ckan.site_title', u'CKAN') + + return output_feed( + results, + feed_title=u'%s - Custom query' % site_title, + feed_description=u'Recently created or updated' + ' datasets on %s. Custom query: \'%s\'' % (site_title, q), + feed_link=alternate_url, + feed_guid=_create_atom_id(atom_url), + feed_url=feed_url, + navigation_urls=navigation_urls) + + +def _alternate_url(params, **kwargs): + search_params = params.copy() + search_params.update(kwargs) + + # Can't count on the page sizes being the same on the search results + # view. So provide an alternate link to the first page, regardless + # of the page we're looking at in the feed. + search_params.pop(u'page', None) + return _feed_url(search_params, controller=u'package', action=u'search') + + +def _feed_url(query, controller, action, **kwargs): + """ + Constructs the url for the given action. Encoding the query + parameters. + """ + for item in query.iteritems(): + kwargs['query'] = item + return h.url_for(controller=controller, action=action, **kwargs) + + +def _navigation_urls(query, controller, action, item_count, limit, **kwargs): + """ + Constructs and returns first, last, prev and next links for paging + """ + urls = [] + + page = int(query.get(u'page', 1)) + + # first: remove any page parameter + first_query = query.copy() + first_query.pop(u'page', None) + href = _feed_url(first_query, controller, action, **kwargs) + urls.append({u'rel': u'first', u'href': href}) + + # last: add last page parameter + last_page = (item_count / limit) + min(1, item_count % limit) + last_query = query.copy() + last_query['page'] = last_page + href = _feed_url(last_query, controller, action, **kwargs) + urls.append({u'rel': u'last', u'href': href}) + # previous + if page > 1: + previous_query = query.copy() + previous_query['page'] = page - 1 + href = _feed_url(previous_query, controller, action, **kwargs) + else: + href = None + urls.append({u'rel': u'previous', u'href': href}) + + # next + if page < last_page: + next_query = query.copy() + next_query['page'] = page + 1 + href = _feed_url(next_query, controller, action, **kwargs) + else: + href = None + + urls.append({u'rel': u'next', u'href': href}) + return urls + + +def _create_atom_id(resource_path, authority_name=None, date_string=None): + """ + Helper method that creates an atom id for a feed or entry. + + An id must be unique, and must not change over time. ie - once published, + it represents an atom feed or entry uniquely, and forever. See [4]: + + When an Atom Document is relocated, migrated, syndicated, + republished, exported, or imported, the content of its atom:id + element MUST NOT change. Put another way, an atom:id element + pertains to all instantiations of a particular Atom entry or feed; + revisions retain the same content in their atom:id elements. It is + suggested that the atom:id element be stored along with the + associated resource. + + resource_path + The resource path that uniquely identifies the feed or element. This + mustn't be something that changes over time for a given entry or feed. + And does not necessarily need to be resolvable. + + e.g. ``"/group/933f3857-79fd-4beb-a835-c0349e31ce76"`` could represent + the feed of datasets belonging to the identified group. + + authority_name + The domain name or email address of the publisher of the feed. See [3] + for more details. If ``None`` then the domain name is taken from the + config file. First trying ``ckan.feeds.authority_name``, and failing + that, it uses ``ckan.site_url``. Again, this should not change over + time. + + date_string + A string representing a date on which the authority_name is owned by + the publisher of the feed. + + e.g. ``"2012-03-22"`` + + Again, this should not change over time. + + If date_string is None, then an attempt is made to read the config + option ``ckan.feeds.date``. If that's not available, + then the date_string is not used in the generation of the atom id. + + Following the methods outlined in [1], [2] and [3], this function produces + tagURIs like: + ``"tag:thedatahub.org,2012:/group/933f3857-79fd-4beb-a835-c0349e31ce76"``. + + If not enough information is provide to produce a valid tagURI, then only + the resource_path is used, e.g.: :: + + "http://thedatahub.org/group/933f3857-79fd-4beb-a835-c0349e31ce76" + + or + + "/group/933f3857-79fd-4beb-a835-c0349e31ce76" + + The latter of which is only used if no site_url is available. And it + should be noted will result in an invalid feed. + + [1] http://web.archive.org/web/20110514113830/http://diveintomark.org/\ + archives/2004/05/28/howto-atom-id + [2] http://www.taguri.org/ + [3] http://tools.ietf.org/html/rfc4151#section-2.1 + [4] http://www.ietf.org/rfc/rfc4287 + """ + if authority_name is None: + authority_name = config.get(u'ckan.feeds.authority_name', u'').strip() + if not authority_name: + site_url = config.get(u'ckan.site_url', u'').strip() + authority_name = urlparse.urlparse(site_url).netloc + + if not authority_name: + log.warning(u'No authority_name available for feed generation. ' + 'Generated feed will be invalid.') + + if date_string is None: + date_string = config.get(u'ckan.feeds.date', u'') + + if not date_string: + log.warning(u'No date_string available for feed generation. ' + 'Please set the "ckan.feeds.date" config value.') + + # Don't generate a tagURI without a date as it wouldn't be valid. + # This is best we can do, and if the site_url is not set, then + # this still results in an invalid feed. + site_url = config.get(u'ckan.site_url', u'') + return u''.join([site_url, resource_path]) + + tagging_entity = u','.join([authority_name, date_string]) + return u':'.join(['tag', tagging_entity, resource_path]) + + +class _FixedAtomFeed(AtomFeed): + def add(self, *args, **kwargs): + """ + Drop the pubdate field from the new item. + """ + if u'pubdate' in kwargs: + kwargs.pop(u'pubdate') + if u'generator' in kwargs: + kwargs.pop(u'generator') + defaults = {u'updated': None, u'published': None} + defaults.update(kwargs) + super(_FixedAtomFeed, self).add(*args, **defaults) + + def latest_post_date(self): + """ + Calculates the latest post date from the 'updated' fields, + rather than the 'pubdate' fields. + """ + updates = [ + item['updated'] for item in self.entries + if item['updated'] is not None + ] + if not len(updates): # delegate to parent for default behaviour + return super(_FixedAtomFeed, self).latest_post_date() + return max(updates) + + +# Routing +feeds.add_url_rule(u'/dataset.atom', methods=[u'GET'], view_func=general) +feeds.add_url_rule(u'/custom.atom', methods=[u'GET'], view_func=custom) +feeds.add_url_rule(u'/tag/.atom', methods=[u'GET'], view_func=tag) +feeds.add_url_rule( + u'/group/.atom', methods=[u'GET'], view_func=group) +feeds.add_url_rule( + u'/organization/.atom', + methods=[u'GET'], + view_func=organization) From 9690487330d119bedcafe4f0ac92d6ea24202070 Mon Sep 17 00:00:00 2001 From: Konstantin Sivakov Date: Thu, 12 Oct 2017 11:33:56 +0200 Subject: [PATCH 04/12] rename feed blueprint --- ckan/views/{feeds.py => feed.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ckan/views/{feeds.py => feed.py} (100%) diff --git a/ckan/views/feeds.py b/ckan/views/feed.py similarity index 100% rename from ckan/views/feeds.py rename to ckan/views/feed.py From f088987b8d0917f200269c86ca437ec04053c29b Mon Sep 17 00:00:00 2001 From: Konstantin Sivakov Date: Thu, 12 Oct 2017 14:23:55 +0200 Subject: [PATCH 05/12] update links attribute --- ckan/views/feed.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/ckan/views/feed.py b/ckan/views/feed.py index e21b895a2aa..87424214c9e 100644 --- a/ckan/views/feed.py +++ b/ckan/views/feed.py @@ -12,6 +12,7 @@ import ckan.model as model import ckan.logic as logic import ckan.plugins as plugins +import json log = logging.getLogger(__name__) @@ -47,6 +48,17 @@ def _package_search(data_dict): return query['count'], query['results'] +def _enclosure(pkg): + links = [] + links.append({ + 'href': h.url('api.action', logic_function='package_show', + ver=3, id=pkg['id'], _external=True), + 'rel': '', + 'length': unicode(len(json.dumps(pkg))), + 'type': u'application/json'}) + return links + + def output_feed(results, feed_title, feed_description, feed_link, feed_url, navigation_urls, feed_guid): author_name = config.get(u'ckan.feeds.author_name', u'').strip() or \ @@ -81,19 +93,13 @@ def output_feed(results, feed_title, feed_description, feed_link, feed_url, feed.add( title=pkg.get(u'title', u''), - url=h.url_for( - controller=u'package', - action=u'read', - id=pkg['id'], - _external=True), description=pkg.get(u'notes', u''), updated=h.date_str_to_datetime(pkg.get(u'metadata_modified')), published=h.date_str_to_datetime(pkg.get(u'metadata_created')), - unique_id=_create_atom_id(u'/dataset%s' % pkg['id']), + id=_create_atom_id(u'/dataset%s' % pkg['id']), author=pkg.get(u'author', u''), - categories=[{ - 'terms': t['name'] - } for t in pkg.get('tags')], + categories=[{'terms': t['name']} for t in pkg.get('tags')], + links=_enclosure(pkg), **additional_fields) # response.content_type = feed.get_response() From 7fffe4be33e2c04f9cb37459fbd547346a329d55 Mon Sep 17 00:00:00 2001 From: Konstantin Sivakov Date: Thu, 12 Oct 2017 14:34:01 +0200 Subject: [PATCH 06/12] code cleanup and pep8 --- ckan/views/feed.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ckan/views/feed.py b/ckan/views/feed.py index 87424214c9e..d7cde8f75d2 100644 --- a/ckan/views/feed.py +++ b/ckan/views/feed.py @@ -51,11 +51,11 @@ def _package_search(data_dict): def _enclosure(pkg): links = [] links.append({ - 'href': h.url('api.action', logic_function='package_show', - ver=3, id=pkg['id'], _external=True), - 'rel': '', - 'length': unicode(len(json.dumps(pkg))), - 'type': u'application/json'}) + u'href': h.url(u'api.action', logic_function=u'package_show', + ver=3, id=pkg['id'], _external=True), + u'rel': u'', + u'length': unicode(len(json.dumps(pkg))), + u'type': u'application/json'}) return links @@ -98,7 +98,7 @@ def output_feed(results, feed_title, feed_description, feed_link, feed_url, published=h.date_str_to_datetime(pkg.get(u'metadata_created')), id=_create_atom_id(u'/dataset%s' % pkg['id']), author=pkg.get(u'author', u''), - categories=[{'terms': t['name']} for t in pkg.get('tags')], + categories=[{u'terms': t['name']} for t in pkg.get(u'tags')], links=_enclosure(pkg), **additional_fields) From ee616b400663cb16702f1dd4d3426e3368b1e8e0 Mon Sep 17 00:00:00 2001 From: Konstantin Sivakov Date: Tue, 17 Oct 2017 14:49:06 +0200 Subject: [PATCH 07/12] add custom atom handling --- ckan/tests/controllers/test_feed.py | 80 ++++++++++++++++++++++++++++- ckan/views/feed.py | 15 ++++-- 2 files changed, 90 insertions(+), 5 deletions(-) diff --git a/ckan/tests/controllers/test_feed.py b/ckan/tests/controllers/test_feed.py index adaf226ccfc..a90180f974b 100644 --- a/ckan/tests/controllers/test_feed.py +++ b/ckan/tests/controllers/test_feed.py @@ -4,6 +4,9 @@ import ckan.tests.helpers as helpers import ckan.tests.factories as factories +import ckan.plugins as plugins +from ckan.views.feed import _FixedAtomFeed +from werkzeug.contrib.atom import AtomFeed, FeedEntry class TestFeedNew(helpers.FunctionalTestBase): @@ -82,7 +85,6 @@ def test_custom_atom_feed_works(self): 'value': 'daily' }]) - app = self._get_test_app() offset = url_for('feeds.custom') params = {'q': 'frequency:weekly'} app = self._get_test_app() @@ -93,3 +95,79 @@ def test_custom_atom_feed_works(self): assert u'{0}'.format( dataset2['title']) not in res.body + + +class TestFeedInterface(helpers.FunctionalTestBase): + @classmethod + def setup_class(cls): + super(TestFeedInterface, cls).setup_class() + + if not plugins.plugin_loaded('test_feed_plugin'): + plugins.load('test_feed_plugin') + + @classmethod + def teardown_class(cls): + helpers.reset_db() + plugins.unload('test_feed_plugin') + + def test_custom_class_used(self): + + offset = url_for(u'feeds.custom') + app = self._get_test_app() + res = app.get(offset) + + assert 'xmlns="http://www.w3.org/2005/Atom"' in res.body, res.body + assert 'CKAN - Custom query' in res.body, res.body + + def test_additional_fields_added(self): + metadata = { + 'ymin': '-2373790', + 'xmin': '2937940', + 'ymax': '-1681290', + 'xmax': '3567770', + } + + extras = [{'key': k, 'value': v} for (k, v) in metadata.items()] + + factories.Dataset(extras=extras) + + offset = url_for(u'feeds.custom') + app = self._get_test_app() + res = app.get(offset) + assert '-2373790.000000 2937940.000000 -1681290.000000 3567770.000000' in res.body, res.body + # assert '' in res.body, res.body + + +class MockFeedPlugin(plugins.SingletonPlugin): + plugins.implements(plugins.IFeed) + + def get_feed_class(self): + return _GeoAtomFeed + + def get_item_additional_fields(self, dataset_dict): + extras = {e['key']: e['value'] for e in dataset_dict['extras']} + + box = tuple( + float(extras.get(n)) for n in ('ymin', 'xmin', 'ymax', 'xmax')) + return {'geometry': box} + + +class _GeoAtomFeed(_FixedAtomFeed): + def __init__(self, **kwargs): + _FixedAtomFeed.__init__(self, **kwargs) + + def add(self, *args, **kwargs): + import pdb; pdb.set_trace() + if u'extras' in kwargs: + kwargs.pop(u'extras') + extras = {u'extras': None} + extras.update(kwargs) + entrie = _CustomFeedEntry(**extras) + _FixedAtomFeed.add(self, entrie, id=entrie.id, updated=entrie.updated) + + +class _CustomFeedEntry(FeedEntry): + + def __init__(self, **kwargs): + FeedEntry.__init__(self, **kwargs) + self.extras = kwargs.get('extras') diff --git a/ckan/views/feed.py b/ckan/views/feed.py index d7cde8f75d2..18fd383f9a4 100644 --- a/ckan/views/feed.py +++ b/ckan/views/feed.py @@ -5,7 +5,6 @@ from flask import Blueprint from werkzeug.contrib.atom import AtomFeed - from ckan.common import _, config, g, request import ckan.lib.helpers as h import ckan.lib.base as base @@ -59,6 +58,13 @@ def _enclosure(pkg): return links +def _set_extras(**kw): + extras = [] + for key, value in kw.iteritems(): + extras.append({key: value}) + return extras + + def output_feed(results, feed_title, feed_description, feed_link, feed_url, navigation_urls, feed_guid): author_name = config.get(u'ckan.feeds.author_name', u'').strip() or \ @@ -91,6 +97,7 @@ def output_feed(results, feed_title, feed_description, feed_link, feed_url, if hasattr(plugin, u'get_item_additional_fields'): additional_fields = plugin.get_item_additional_fields(pkg) + import pdb; pdb.set_trace() feed.add( title=pkg.get(u'title', u''), description=pkg.get(u'notes', u''), @@ -100,7 +107,7 @@ def output_feed(results, feed_title, feed_description, feed_link, feed_url, author=pkg.get(u'author', u''), categories=[{u'terms': t['name']} for t in pkg.get(u'tags')], links=_enclosure(pkg), - **additional_fields) + extras=_set_extras(**additional_fields)) # response.content_type = feed.get_response() return feed.get_response() @@ -487,7 +494,7 @@ def add(self, *args, **kwargs): kwargs.pop(u'generator') defaults = {u'updated': None, u'published': None} defaults.update(kwargs) - super(_FixedAtomFeed, self).add(*args, **defaults) + AtomFeed.add(self, *args, **defaults) def latest_post_date(self): """ @@ -501,7 +508,7 @@ def latest_post_date(self): if not len(updates): # delegate to parent for default behaviour return super(_FixedAtomFeed, self).latest_post_date() return max(updates) - + # Routing feeds.add_url_rule(u'/dataset.atom', methods=[u'GET'], view_func=general) From e1df6dc7a546fe80658360437c66a05d51d63795 Mon Sep 17 00:00:00 2001 From: Konstantin Sivakov Date: Wed, 18 Oct 2017 16:43:44 +0200 Subject: [PATCH 08/12] revert back to webhelpers feedgenerator --- ckan/tests/controllers/test_feed.py | 40 ++----- ckan/views/feed.py | 158 +++++++++++++++++++--------- 2 files changed, 120 insertions(+), 78 deletions(-) diff --git a/ckan/tests/controllers/test_feed.py b/ckan/tests/controllers/test_feed.py index a90180f974b..8fc2135b537 100644 --- a/ckan/tests/controllers/test_feed.py +++ b/ckan/tests/controllers/test_feed.py @@ -5,8 +5,7 @@ import ckan.tests.helpers as helpers import ckan.tests.factories as factories import ckan.plugins as plugins -from ckan.views.feed import _FixedAtomFeed -from werkzeug.contrib.atom import AtomFeed, FeedEntry +from webhelpers.feedgenerator import GeoAtom1Feed class TestFeedNew(helpers.FunctionalTestBase): @@ -85,7 +84,7 @@ def test_custom_atom_feed_works(self): 'value': 'daily' }]) - offset = url_for('feeds.custom') + offset = url_for(u'feeds.custom') params = {'q': 'frequency:weekly'} app = self._get_test_app() res = app.get(offset, params=params) @@ -112,12 +111,13 @@ def teardown_class(cls): def test_custom_class_used(self): + import pdb; pdb.set_trace() + app = self._get_test_app() offset = url_for(u'feeds.custom') app = self._get_test_app() res = app.get(offset) - assert 'xmlns="http://www.w3.org/2005/Atom"' in res.body, res.body - assert 'CKAN - Custom query' in res.body, res.body + assert 'xmlns:georss="http://www.georss.org/georss"' in res.body, res.body def test_additional_fields_added(self): metadata = { @@ -131,18 +131,19 @@ def test_additional_fields_added(self): factories.Dataset(extras=extras) - offset = url_for(u'feeds.custom') + app = self._get_test_app() + offset = url_for(controller="feeds", action='general') app = self._get_test_app() res = app.get(offset) - assert '-2373790.000000 2937940.000000 -1681290.000000 3567770.000000' in res.body, res.body - # assert '' in res.body, res.body + + assert '-2373790.000000 2937940.000000 -1681290.000000 3567770.000000' in res.body, res.body class MockFeedPlugin(plugins.SingletonPlugin): plugins.implements(plugins.IFeed) def get_feed_class(self): - return _GeoAtomFeed + return GeoAtom1Feed def get_item_additional_fields(self, dataset_dict): extras = {e['key']: e['value'] for e in dataset_dict['extras']} @@ -150,24 +151,3 @@ def get_item_additional_fields(self, dataset_dict): box = tuple( float(extras.get(n)) for n in ('ymin', 'xmin', 'ymax', 'xmax')) return {'geometry': box} - - -class _GeoAtomFeed(_FixedAtomFeed): - def __init__(self, **kwargs): - _FixedAtomFeed.__init__(self, **kwargs) - - def add(self, *args, **kwargs): - import pdb; pdb.set_trace() - if u'extras' in kwargs: - kwargs.pop(u'extras') - extras = {u'extras': None} - extras.update(kwargs) - entrie = _CustomFeedEntry(**extras) - _FixedAtomFeed.add(self, entrie, id=entrie.id, updated=entrie.updated) - - -class _CustomFeedEntry(FeedEntry): - - def __init__(self, **kwargs): - FeedEntry.__init__(self, **kwargs) - self.extras = kwargs.get('extras') diff --git a/ckan/views/feed.py b/ckan/views/feed.py index 18fd383f9a4..655137c30c3 100644 --- a/ckan/views/feed.py +++ b/ckan/views/feed.py @@ -4,8 +4,8 @@ import urlparse from flask import Blueprint -from werkzeug.contrib.atom import AtomFeed -from ckan.common import _, config, g, request +import webhelpers.feedgenerator +from ckan.common import _, config, g, request, response import ckan.lib.helpers as h import ckan.lib.base as base import ckan.model as model @@ -77,18 +77,20 @@ def output_feed(results, feed_title, feed_description, feed_link, feed_url, feed_class = plugin.get_feed_class() if not feed_class: - feed_class = _FixedAtomFeed + feed_class = _FixedAtom1Feed feed = feed_class( - title=feed_title, - url=feed_link, + feed_title, + feed_link, + feed_description, language=u'en', - author={u'name': author_name, - u'uri': BASE_URL}, - id=feed_guid, + author_name=author_name, + feed_guid=feed_guid, feed_url=feed_url, - links=navigation_urls, - generator=(None, None, None)) + previous_page=navigation_urls['previous'], + next_page=navigation_urls['next'], + first_page=navigation_urls['first'], + last_page=navigation_urls['last'], ) for pkg in results: additional_fields = {} @@ -97,20 +99,33 @@ def output_feed(results, feed_title, feed_description, feed_link, feed_url, if hasattr(plugin, u'get_item_additional_fields'): additional_fields = plugin.get_item_additional_fields(pkg) - import pdb; pdb.set_trace() - feed.add( - title=pkg.get(u'title', u''), - description=pkg.get(u'notes', u''), - updated=h.date_str_to_datetime(pkg.get(u'metadata_modified')), - published=h.date_str_to_datetime(pkg.get(u'metadata_created')), - id=_create_atom_id(u'/dataset%s' % pkg['id']), - author=pkg.get(u'author', u''), - categories=[{u'terms': t['name']} for t in pkg.get(u'tags')], - links=_enclosure(pkg), - extras=_set_extras(**additional_fields)) - - # response.content_type = feed.get_response() - return feed.get_response() + feed.add_item( + title=pkg.get('title', ''), + link=h.url_for( + u'api.action', + logic_function=u'package_read', + id=pkg['id'], + ver=3, + _exteral=True), + description=pkg.get('notes', ''), + updated=h.date_str_to_datetime(pkg.get('metadata_modified')), + published=h.date_str_to_datetime(pkg.get('metadata_created')), + unique_id=_create_atom_id(u'/dataset/%s' % pkg['id']), + author_name=pkg.get('author', ''), + author_email=pkg.get('author_email', ''), + categories=[t['name'] for t in pkg.get('tags', [])], + enclosure=webhelpers.feedgenerator.Enclosure( + h.url_for( + u'api.action', + logic_function=u'package_show', + id=pkg['name'], + ver='3', + _external=True), + unicode(len(json.dumps(pkg))), u'application/json'), + **additional_fields) + + # response.content_type = feed.mime_type + return feed.writeString('utf-8') def group(id): @@ -355,40 +370,42 @@ def _navigation_urls(query, controller, action, item_count, limit, **kwargs): """ Constructs and returns first, last, prev and next links for paging """ - urls = [] - page = int(query.get(u'page', 1)) + urls = dict((rel, None) for rel in 'previous next first last'.split()) + + page = int(query.get('page', 1)) # first: remove any page parameter first_query = query.copy() - first_query.pop(u'page', None) - href = _feed_url(first_query, controller, action, **kwargs) - urls.append({u'rel': u'first', u'href': href}) + first_query.pop('page', None) + urls['first'] = _feed_url(first_query, controller, + action, **kwargs) # last: add last page parameter last_page = (item_count / limit) + min(1, item_count % limit) last_query = query.copy() last_query['page'] = last_page - href = _feed_url(last_query, controller, action, **kwargs) - urls.append({u'rel': u'last', u'href': href}) + urls['last'] = _feed_url(last_query, controller, + action, **kwargs) + # previous if page > 1: previous_query = query.copy() previous_query['page'] = page - 1 - href = _feed_url(previous_query, controller, action, **kwargs) + urls['previous'] = _feed_url(previous_query, controller, + action, **kwargs) else: - href = None - urls.append({u'rel': u'previous', u'href': href}) + urls['previous'] = None # next if page < last_page: next_query = query.copy() next_query['page'] = page + 1 - href = _feed_url(next_query, controller, action, **kwargs) + urls['next'] = _feed_url(next_query, controller, + action, **kwargs) else: - href = None + urls['next'] = None - urls.append({u'rel': u'next', u'href': href}) return urls @@ -483,18 +500,29 @@ def _create_atom_id(resource_path, authority_name=None, date_string=None): return u':'.join(['tag', tagging_entity, resource_path]) -class _FixedAtomFeed(AtomFeed): - def add(self, *args, **kwargs): +class _FixedAtom1Feed(webhelpers.feedgenerator.Atom1Feed): + """ + The Atom1Feed defined in webhelpers doesn't provide all the fields we + might want to publish. + * In Atom1Feed, each is created with identical and + fields. See [1] (webhelpers 1.2) for details. + So, this class fixes that by allow an item to set both an and + field. + * In Atom1Feed, the feed description is not used. So this class uses the + field to publish that. + [1] https://bitbucket.org/bbangert/webhelpers/src/f5867a319abf/\ + webhelpers/feedgenerator.py#cl-373 + """ + + def add_item(self, *args, **kwargs): """ Drop the pubdate field from the new item. """ - if u'pubdate' in kwargs: - kwargs.pop(u'pubdate') - if u'generator' in kwargs: - kwargs.pop(u'generator') - defaults = {u'updated': None, u'published': None} + if 'pubdate' in kwargs: + kwargs.pop('pubdate') + defaults = {'updated': None, 'published': None} defaults.update(kwargs) - AtomFeed.add(self, *args, **defaults) + super(_FixedAtom1Feed, self).add_item(*args, **defaults) def latest_post_date(self): """ @@ -502,13 +530,47 @@ def latest_post_date(self): rather than the 'pubdate' fields. """ updates = [ - item['updated'] for item in self.entries + item['updated'] for item in self.items if item['updated'] is not None ] if not len(updates): # delegate to parent for default behaviour - return super(_FixedAtomFeed, self).latest_post_date() + return super(_FixedAtom1Feed, self).latest_post_date() return max(updates) - + + def add_item_elements(self, handler, item): + """ + Add the and fields to each entry that's written + to the handler. + """ + super(_FixedAtom1Feed, self).add_item_elements(handler, item) + + dfunc = webhelpers.feedgenerator.rfc3339_date + + if (item['updated']): + handler.addQuickElement(u'updated', + dfunc(item['updated']).decode('utf-8')) + + if (item['published']): + handler.addQuickElement(u'published', + dfunc(item['published']).decode('utf-8')) + + def add_root_elements(self, handler): + """ + Add additional feed fields. + * Add the field from the feed description + * Add links other pages of the logical feed. + """ + super(_FixedAtom1Feed, self).add_root_elements(handler) + + handler.addQuickElement(u'subtitle', self.feed['description']) + + for page in ['previous', 'next', 'first', 'last']: + if self.feed.get(page + '_page', None): + handler.addQuickElement(u'link', u'', { + 'rel': page, + 'href': self.feed.get(page + '_page') + }) + # Routing feeds.add_url_rule(u'/dataset.atom', methods=[u'GET'], view_func=general) From 20260cc1eb2ee17be8567bc235789ceafb4450cb Mon Sep 17 00:00:00 2001 From: Konstantin Sivakov Date: Wed, 18 Oct 2017 18:17:26 +0200 Subject: [PATCH 09/12] fix tests --- ckan/tests/controllers/test_feed.py | 15 +++++++-------- ckan/views/feed.py | 12 ++++++------ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/ckan/tests/controllers/test_feed.py b/ckan/tests/controllers/test_feed.py index 8fc2135b537..4d0a3114b50 100644 --- a/ckan/tests/controllers/test_feed.py +++ b/ckan/tests/controllers/test_feed.py @@ -45,7 +45,7 @@ def test_general_atom_feed_works(self): offset = url_for(u'feeds.general') res = app.get(offset) - assert u'{0}'.format( + assert u'{0}'.format( dataset['title']) in res.body def test_group_atom_feed_works(self): @@ -56,7 +56,7 @@ def test_group_atom_feed_works(self): offset = url_for(u'feeds.group', id=group['name']) res = app.get(offset) - assert u'{0}'.format( + assert u'{0}'.format( dataset['title']) in res.body def test_organization_atom_feed_works(self): @@ -67,7 +67,7 @@ def test_organization_atom_feed_works(self): offset = url_for(u'feeds.organization', id=group['name']) res = app.get(offset) - assert u'{0}'.format( + assert u'{0}'.format( dataset['title']) in res.body def test_custom_atom_feed_works(self): @@ -89,10 +89,10 @@ def test_custom_atom_feed_works(self): app = self._get_test_app() res = app.get(offset, params=params) - assert u'{0}'.format( + assert u'{0}'.format( dataset1['title']) in res.body - assert u'{0}'.format( + assert u'{0}'.format( dataset2['title']) not in res.body @@ -111,9 +111,8 @@ def teardown_class(cls): def test_custom_class_used(self): - import pdb; pdb.set_trace() app = self._get_test_app() - offset = url_for(u'feeds.custom') + offset = url_for(u'feeds.general') app = self._get_test_app() res = app.get(offset) @@ -132,7 +131,7 @@ def test_additional_fields_added(self): factories.Dataset(extras=extras) app = self._get_test_app() - offset = url_for(controller="feeds", action='general') + offset = url_for(u'feeds.general') app = self._get_test_app() res = app.get(offset) diff --git a/ckan/views/feed.py b/ckan/views/feed.py index 655137c30c3..204c691dc20 100644 --- a/ckan/views/feed.py +++ b/ckan/views/feed.py @@ -3,7 +3,7 @@ import logging import urlparse -from flask import Blueprint +from flask import Blueprint, make_response import webhelpers.feedgenerator from ckan.common import _, config, g, request, response import ckan.lib.helpers as h @@ -125,7 +125,9 @@ def output_feed(results, feed_title, feed_description, feed_link, feed_url, **additional_fields) # response.content_type = feed.mime_type - return feed.writeString('utf-8') + resp = make_response(feed.writeString('utf-8'), 200) + resp.headers['Content-Type'] = 'text/xml' + return resp def group(id): @@ -332,13 +334,11 @@ def custom(): alternate_url = _alternate_url(request.params) - site_title = config.get(u'ckan.site_title', u'CKAN') - return output_feed( results, - feed_title=u'%s - Custom query' % site_title, + feed_title=u'%s - Custom query' % SITE_TITLE, feed_description=u'Recently created or updated' - ' datasets on %s. Custom query: \'%s\'' % (site_title, q), + ' datasets on %s. Custom query: \'%s\'' % (SITE_TITLE, q), feed_link=alternate_url, feed_guid=_create_atom_id(atom_url), feed_url=feed_url, From f8b83680e1fbb247bdd7416abd07874e20b0c5f7 Mon Sep 17 00:00:00 2001 From: Konstantin Sivakov Date: Wed, 18 Oct 2017 18:38:00 +0200 Subject: [PATCH 10/12] fix typo and pep8 --- ckan/views/feed.py | 54 +++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/ckan/views/feed.py b/ckan/views/feed.py index 204c691dc20..1e6544a0db6 100644 --- a/ckan/views/feed.py +++ b/ckan/views/feed.py @@ -87,10 +87,10 @@ def output_feed(results, feed_title, feed_description, feed_link, feed_url, author_name=author_name, feed_guid=feed_guid, feed_url=feed_url, - previous_page=navigation_urls['previous'], - next_page=navigation_urls['next'], - first_page=navigation_urls['first'], - last_page=navigation_urls['last'], ) + previous_page=navigation_urls[u'previous'], + next_page=navigation_urls[u'next'], + first_page=navigation_urls[u'first'], + last_page=navigation_urls[u'last'], ) for pkg in results: additional_fields = {} @@ -100,33 +100,33 @@ def output_feed(results, feed_title, feed_description, feed_link, feed_url, additional_fields = plugin.get_item_additional_fields(pkg) feed.add_item( - title=pkg.get('title', ''), + title=pkg.get(u'title', u''), link=h.url_for( u'api.action', logic_function=u'package_read', id=pkg['id'], ver=3, - _exteral=True), - description=pkg.get('notes', ''), - updated=h.date_str_to_datetime(pkg.get('metadata_modified')), - published=h.date_str_to_datetime(pkg.get('metadata_created')), + _external=True), + description=pkg.get(u'notes', u''), + updated=h.date_str_to_datetime(pkg.get(u'metadata_modified')), + published=h.date_str_to_datetime(pkg.get(u'metadata_created')), unique_id=_create_atom_id(u'/dataset/%s' % pkg['id']), - author_name=pkg.get('author', ''), - author_email=pkg.get('author_email', ''), - categories=[t['name'] for t in pkg.get('tags', [])], + author_name=pkg.get(u'author', ''), + author_email=pkg.get(u'author_email', ''), + categories=[t['name'] for t in pkg.get(u'tags', [])], enclosure=webhelpers.feedgenerator.Enclosure( h.url_for( u'api.action', logic_function=u'package_show', id=pkg['name'], - ver='3', + ver=3, _external=True), unicode(len(json.dumps(pkg))), u'application/json'), **additional_fields) # response.content_type = feed.mime_type - resp = make_response(feed.writeString('utf-8'), 200) - resp.headers['Content-Type'] = 'text/xml' + resp = make_response(feed.writeString(u'utf-8'), 200) + resp.headers['Content-Type'] = u'text/xml' return resp @@ -371,13 +371,13 @@ def _navigation_urls(query, controller, action, item_count, limit, **kwargs): Constructs and returns first, last, prev and next links for paging """ - urls = dict((rel, None) for rel in 'previous next first last'.split()) + urls = dict((rel, None) for rel in u'previous next first last'.split()) - page = int(query.get('page', 1)) + page = int(query.get(u'page', 1)) # first: remove any page parameter first_query = query.copy() - first_query.pop('page', None) + first_query.pop(u'page', None) urls['first'] = _feed_url(first_query, controller, action, **kwargs) @@ -518,9 +518,9 @@ def add_item(self, *args, **kwargs): """ Drop the pubdate field from the new item. """ - if 'pubdate' in kwargs: - kwargs.pop('pubdate') - defaults = {'updated': None, 'published': None} + if u'pubdate' in kwargs: + kwargs.pop(u'pubdate') + defaults = {u'updated': None, u'published': None} defaults.update(kwargs) super(_FixedAtom1Feed, self).add_item(*args, **defaults) @@ -548,11 +548,11 @@ def add_item_elements(self, handler, item): if (item['updated']): handler.addQuickElement(u'updated', - dfunc(item['updated']).decode('utf-8')) + dfunc(item['updated']).decode(u'utf-8')) if (item['published']): handler.addQuickElement(u'published', - dfunc(item['published']).decode('utf-8')) + dfunc(item['published']).decode(u'utf-8')) def add_root_elements(self, handler): """ @@ -564,11 +564,11 @@ def add_root_elements(self, handler): handler.addQuickElement(u'subtitle', self.feed['description']) - for page in ['previous', 'next', 'first', 'last']: - if self.feed.get(page + '_page', None): + for page in [u'previous', u'next', u'first', u'last']: + if self.feed.get(page + u'_page', None): handler.addQuickElement(u'link', u'', { - 'rel': page, - 'href': self.feed.get(page + '_page') + u'rel': page, + u'href': self.feed.get(page + u'_page') }) From d213032b988600f2b43877d675292ec2e286b245 Mon Sep 17 00:00:00 2001 From: Konstantin Sivakov Date: Wed, 18 Oct 2017 19:12:51 +0200 Subject: [PATCH 11/12] pep8 --- ckan/views/feed.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/views/feed.py b/ckan/views/feed.py index 1e6544a0db6..a09122e2f82 100644 --- a/ckan/views/feed.py +++ b/ckan/views/feed.py @@ -111,8 +111,8 @@ def output_feed(results, feed_title, feed_description, feed_link, feed_url, updated=h.date_str_to_datetime(pkg.get(u'metadata_modified')), published=h.date_str_to_datetime(pkg.get(u'metadata_created')), unique_id=_create_atom_id(u'/dataset/%s' % pkg['id']), - author_name=pkg.get(u'author', ''), - author_email=pkg.get(u'author_email', ''), + author_name=pkg.get(u'author', u''), + author_email=pkg.get(u'author_email', u''), categories=[t['name'] for t in pkg.get(u'tags', [])], enclosure=webhelpers.feedgenerator.Enclosure( h.url_for( From 3c0f38aff9fb6bdef5f5d273f188932db15fa100 Mon Sep 17 00:00:00 2001 From: Konstantin Sivakov Date: Thu, 19 Oct 2017 18:07:48 +0200 Subject: [PATCH 12/12] identation fix and response header --- ckan/views/feed.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ckan/views/feed.py b/ckan/views/feed.py index a09122e2f82..3f483f239ad 100644 --- a/ckan/views/feed.py +++ b/ckan/views/feed.py @@ -124,10 +124,9 @@ def output_feed(results, feed_title, feed_description, feed_link, feed_url, unicode(len(json.dumps(pkg))), u'application/json'), **additional_fields) - # response.content_type = feed.mime_type - resp = make_response(feed.writeString(u'utf-8'), 200) - resp.headers['Content-Type'] = u'text/xml' - return resp + resp = make_response(feed.writeString(u'utf-8'), 200) + resp.headers['Content-Type'] = u'application/atom+xml' + return resp def group(id):