Merge branch '267-date-indexing-fix'

ckan/lib/search/index.py

@@ -3,6 +3,7 @@
 import logging
 import collections
 import json
+from dateutil.parser import parse
 
 import re
 
@@ -156,7 +157,7 @@ def index_package(self, pkg_dict, defer_commit=False):
 
         # if there is an owner_org we want to add this to groups for index
         # purposes
-        if pkg_dict['owner_org']:
+        if pkg_dict.get('organization'):
             pkg_dict['groups'].append(pkg_dict['organization']['name'])
 
 
@@ -192,7 +193,21 @@ def index_package(self, pkg_dict, defer_commit=False):
         # Save dataset type
         pkg_dict['dataset_type'] = pkg_dict['type']
 
-        pkg_dict = dict([(k.encode('ascii', 'ignore'), v) for (k, v) in pkg_dict.items()])
+        # clean the dict fixing keys and dates
+        # FIXME where are we getting these dirty keys from?  can we not just
+        # fix them in the correct place or is this something that always will
+        # be needed?  For my data not changing the keys seems to not cause a
+        # problem.
+        new_dict = {}
+        for key, value in pkg_dict.items():
+            key = key.encode('ascii', 'ignore')
+            if key.endswith('_date'):
+                try:
+                    value = parse(value).isoformat() + 'Z'
+                except ValueError:
+                    continue
+            new_dict[key] = value
+        pkg_dict = new_dict
 
         for k in ('title', 'notes', 'title_string'):
             if k in pkg_dict and pkg_dict[k]:

ckan/tests/lib/test_solr_search_index.py

@@ -65,6 +65,10 @@ def test_index(self):
             'owner_org': None,
             'metadata_created': datetime_now.isoformat(),
             'metadata_modified': datetime_now.isoformat(),
+            'extras': [
+                {'key': 'test_date', 'value': '2013-03-01'},
+                {'key': 'test_wrong_date', 'value': 'Not a date'},
+            ]
         }
         search.dispatch_by_operation('Package', pkg_dict, 'new')
         response = self.solr.query('title:penguin', fq=self.fq)

pip-requirements.txt

@@ -29,3 +29,4 @@ Jinja2==2.6
 fanstatic==0.12
 requests==1.1.0
 WebTest==1.4.3
+python-dateutil>=1.5.0,<2.0.0