[#846] More fixes for api controller and utf-8

ckan/controllers/api.py

@@ -78,27 +78,27 @@ def _finish(self, status_int, response_data=None,
         '''
         assert(isinstance(status_int, int))
         response.status_int = status_int
-        response_msg = ''
+        out = ''
         if response_data is not None:
-            response.headers['Content-Type'] = CONTENT_TYPES[content_type]
             if content_type == 'json':
-                response_msg = h.json.dumps(response_data)
-                response_msg = response_msg.replace('\"', '\\"')
+                out = h.json.dumps(response_data)
+                # \" gets destroyed here so we need to protect them
+                out = out.replace('\"', '\\"')
                 # we need to sort unicode items like \uxxx
-                response_msg = response_msg.decode('unicode-escape')
-                # \n and \r will have been converted so we undo this
-                response_msg = response_msg.replace('\n', '\\n')
-                response_msg = response_msg.replace('\r', '\\r')
+                out = out.decode('unicode-escape')
+                # fix some chars
+                out = out.replace('\n', '\\n').replace('\r', '\\r')
             else:
-                response_msg = response_data
+                out = response_data
             # Support "JSONP" callback.
             if status_int == 200 and 'callback' in request.params and \
                 (request.method == 'GET' or
                  c.logic_function and request.method == 'POST'):
                 # escape callback to remove '<', '&', '>' chars
                 callback = cgi.escape(request.params['callback'])
-                response_msg = self._wrap_jsonp(callback, response_msg)
-        return response_msg
+                out = self._wrap_jsonp(callback, out)
+            response.headers['Content-Type'] = CONTENT_TYPES[content_type]
+        return out
 
     def _finish_ok(self, response_data=None,
                    content_type='json',
@@ -557,7 +557,12 @@ def search(self, ver=None, register=None):
                     # the search
                     if 'callback' in params:
                         del params['callback']
-                    results = query.run(params)
+                    results = query.run(params, escape=True)
+                    # strip out the data dict if it exists.  This is because it
+                    # breaks unicode output from the api
+                    for x in results['results']:
+                        if 'data_dict' in x:
+                            del x['data_dict']
                 return self._finish_ok(results)
             except search.SearchError, e:
                 log.exception(e)

ckan/lib/search/query.py

@@ -289,7 +289,7 @@ def get_index(self,reference):
             conn.close()
 
 
-    def run(self, query):
+    def run(self, query, escape=False):
         '''
         Performs a dataset search using the given query.
 
@@ -362,6 +362,11 @@ def run(self, query):
             raise SearchError('SOLR returned an error running query: %r Error: %r' %
                               (query, e.reason))
         try:
+            # Extra escaping if requested.  This is mainly to allow us to
+            # output utf-8 encoded data correctly via the api.
+            if escape:
+                solr_response = solr_response.replace('\\n', '\\\\n') \
+                    .replace('\\r', '\\\\r')
             data = json.loads(solr_response)
             response = data['response']
             self.count = response.get('numFound', 0)