[#3467] separate total calculation + include_total param

ckan · Mar 6, 2017 · 609d54b · 609d54b
1 parent efcfe97
commit 609d54b
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 8 deletions.
diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py
@@ -924,6 +924,7 @@ def validate(context, data_dict):
     del data_dict_copy['connection_url']
     del data_dict_copy['resource_id']
     data_dict_copy.pop('id', None)
+    data_dict_copy.pop('include_total', None)
 
     for key, values in data_dict_copy.iteritems():
         if not values:
@@ -993,7 +994,20 @@ def search_data(context, data_dict):
     results = _execute_single_statement(context, sql_string, where_values)
 
     _insert_links(data_dict, limit, offset)
-    return format_results(context, results, data_dict)
+    r = format_results(context, results, data_dict)
+
+    if data_dict.get('include_total', True):
+        count_sql_string = u'''SELECT {distinct} count(*)
+            FROM "{resource}" {ts_query} {where};'''.format(
+            distinct=distinct,
+            resource=resource_id,
+            ts_query=ts_query,
+            where=where_clause)
+        count_result = _execute_single_statement(
+            context, count_sql_string, where_values)
+        data_dict['total'] = count_result.fetchall()[0][0]
+
+    return r
 
 
 def _execute_single_statement(context, sql_string, where_values):
@@ -1014,14 +1028,10 @@ def format_results(context, results, data_dict):
             'id': field[0].decode('utf-8'),
             'type': _get_type(context, field[1])
         })
-    if len(result_fields) and result_fields[-1]['id'] == '_full_count':
-        result_fields.pop()  # remove _full_count
 
     records = []
     for row in results:
         converted_row = {}
-        if '_full_count' in row:
-            data_dict['total'] = row['_full_count']
         for field in result_fields:
             converted_row[field['id']] = convert(row[field['id']],
                                                  field['type'])

diff --git a/ckanext/datastore/logic/schema.py b/ckanext/datastore/logic/schema.py
@@ -16,6 +16,7 @@
 boolean_validator = get_validator('boolean_validator')
 int_validator = get_validator('int_validator')
 OneOf = get_validator('OneOf')
+default = get_validator('default')
 
 
 def rename(old, new):
@@ -148,6 +149,7 @@ def datastore_search_schema():
         'fields': [ignore_missing, list_of_strings_or_string],
         'sort': [ignore_missing, list_of_strings_or_string],
         'distinct': [ignore_missing, boolean_validator],
+        'include_total': [default(True), boolean_validator],
         '__junk': [empty],
         '__before': [rename('id', 'resource_id')]
     }

diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py
@@ -382,8 +382,10 @@ def datastore_search(self, context, data_dict, fields_types, query_dict):
         sort = self._sort(data_dict, fields_types)
         where = self._where(data_dict, fields_types)
 
-        select_cols = [u'"{0}"'.format(field_id) for field_id in field_ids] +\
-                      [u'count(*) over() as "_full_count" %s' % rank_column]
+        select_cols = [
+            datastore_helpers.identifier(field_id) for field_id in field_ids]
+        if rank_column:
+            select_cols.append(rank_column)
 
         query_dict['distinct'] = data_dict.get('distinct', False)
         query_dict['select'] += select_cols
@@ -490,7 +492,7 @@ def _textsearch_query(self, data_dict):
                 rank_columns.append(rank)
 
         statements_str = ', ' + ', '.join(statements)
-        rank_columns_str = ', ' + ', '.join(rank_columns)
+        rank_columns_str = ', '.join(rank_columns)
         return statements_str, rank_columns_str
 
     def _fts_lang(self, lang=None):