From 9cc1aee00977b75ef8478766740db799031ed0dc Mon Sep 17 00:00:00 2001 From: tobes Date: Mon, 11 Mar 2013 13:51:53 +0000 Subject: [PATCH 1/6] [#267] Fixes issue where incorrectly formated dates broke the indexing --- ckan/lib/search/index.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/ckan/lib/search/index.py b/ckan/lib/search/index.py index fd89ffd4bb6..732d92c1258 100644 --- a/ckan/lib/search/index.py +++ b/ckan/lib/search/index.py @@ -3,6 +3,7 @@ import logging import collections import json +from dateutil.parser import parse import re @@ -156,7 +157,7 @@ def index_package(self, pkg_dict, defer_commit=False): # if there is an owner_org we want to add this to groups for index # purposes - if pkg_dict['owner_org']: + if pkg_dict['organization']: pkg_dict['groups'].append(pkg_dict['organization']['name']) @@ -192,7 +193,21 @@ def index_package(self, pkg_dict, defer_commit=False): # Save dataset type pkg_dict['dataset_type'] = pkg_dict['type'] - pkg_dict = dict([(k.encode('ascii', 'ignore'), v) for (k, v) in pkg_dict.items()]) + # clean the dict fixing keys and dates + # FIXME where are we getting these dirty keys from? can we not just + # fix them in the correct place or is this something that always will + # be needed? For my data not changing the keys seems to not cause a + # problem. + new_dict = {} + for key, value in pkg_dict.items(): + key = key.encode('ascii', 'ignore') + if key.endswith('_date'): + try: + value = parse(value).isoformat() + 'Z' + except ValueError: + continue + new_dict[key] = value + pkg_dict = new_dict for k in ('title', 'notes', 'title_string'): if k in pkg_dict and pkg_dict[k]: From 4a9d9ed8d41fa811d82eeb2269cdf5d5e99d70e6 Mon Sep 17 00:00:00 2001 From: tobes Date: Mon, 11 Mar 2013 14:06:07 +0000 Subject: [PATCH 2/6] [#267] Add dateutils requirement --- pip-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/pip-requirements.txt b/pip-requirements.txt index 9a166fe143c..e80d5aea445 100644 --- a/pip-requirements.txt +++ b/pip-requirements.txt @@ -29,3 +29,4 @@ Jinja2==2.6 fanstatic==0.12 requests==1.1.0 WebTest==1.4.3 +dateutils==0.6.5 From 39b059da512512800aacdfd0d47975ae55aa3792 Mon Sep 17 00:00:00 2001 From: tobes Date: Mon, 11 Mar 2013 14:40:43 +0000 Subject: [PATCH 3/6] [#267] Stop new error related to organisation --- ckan/lib/search/index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/lib/search/index.py b/ckan/lib/search/index.py index 732d92c1258..f608f14e9d3 100644 --- a/ckan/lib/search/index.py +++ b/ckan/lib/search/index.py @@ -157,7 +157,7 @@ def index_package(self, pkg_dict, defer_commit=False): # if there is an owner_org we want to add this to groups for index # purposes - if pkg_dict['organization']: + if pkg_dict.get('organization'): pkg_dict['groups'].append(pkg_dict['organization']['name']) From b928adcb22ff808afff4eb8cf7e340858edb8a7a Mon Sep 17 00:00:00 2001 From: tobes Date: Tue, 12 Mar 2013 17:28:05 +0000 Subject: [PATCH 4/6] [#267] change dependency --- pip-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pip-requirements.txt b/pip-requirements.txt index e80d5aea445..b284cc89dd2 100644 --- a/pip-requirements.txt +++ b/pip-requirements.txt @@ -29,4 +29,4 @@ Jinja2==2.6 fanstatic==0.12 requests==1.1.0 WebTest==1.4.3 -dateutils==0.6.5 +python-dateutil>=1.5.0,<2.0.0 From f962b6f54f0ac034b3814efec3110a8fd9184b00 Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 12 Mar 2013 19:17:27 +0000 Subject: [PATCH 5/6] [#267] Add small test --- ckan/tests/lib/test_solr_search_index.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ckan/tests/lib/test_solr_search_index.py b/ckan/tests/lib/test_solr_search_index.py index 8a14ef2b657..f8c518bc2da 100644 --- a/ckan/tests/lib/test_solr_search_index.py +++ b/ckan/tests/lib/test_solr_search_index.py @@ -65,6 +65,10 @@ def test_index(self): 'owner_org': None, 'metadata_created': datetime_now.isoformat(), 'metadata_modified': datetime_now.isoformat(), + 'extras': [ + {'key': 'test_date', 'value': '2013-03-01'}, + {'key': 'test_wrong_date', 'value': 'Not a date'}, + ] } search.dispatch_by_operation('Package', pkg_dict, 'new') response = self.solr.query('title:penguin', fq=self.fq) From aeece329cd3ddfa3fceae7e1c3391563e4eb1564 Mon Sep 17 00:00:00 2001 From: tobes Date: Wed, 13 Mar 2013 12:01:53 +0000 Subject: [PATCH 6/6] [#604] Add facets to bulk process --- ckan/templates/organization/bulk_process.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ckan/templates/organization/bulk_process.html b/ckan/templates/organization/bulk_process.html index 11c556ff201..1f9acc63b66 100644 --- a/ckan/templates/organization/bulk_process.html +++ b/ckan/templates/organization/bulk_process.html @@ -90,8 +90,9 @@

- {{ h.snippet('snippets/facet_list.html', title='Tags', name='tags', within_tertiary=true, extras={'id':c.group_dict.name}) }} - {{ h.snippet('snippets/facet_list.html', title='Formats', name='res_format', within_tertiary=true, extras={'id':c.group_dict.name}) }} + {% for facet in c.facet_titles %} + {{ h.snippet('snippets/facet_list.html', title=c.facet_titles[facet], name=facet, extras={'id':c.group_dict.id}) }} + {% endfor %} {{ c.page.pager() }}