From 56fa0970f8d8658a34f9c76f511a4beef38bf559 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Wed, 1 Aug 2012 16:10:59 +0100
Subject: [PATCH 01/53] [#2788] Avoid unnecessary commit request

The commit can be sent on the same request with the dict to index
---
 ckan/lib/search/index.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/ckan/lib/search/index.py b/ckan/lib/search/index.py
index 09ec2eec913..1d912eef1c7 100644
--- a/ckan/lib/search/index.py
+++ b/ckan/lib/search/index.py
@@ -222,8 +222,7 @@ def index_package(self, pkg_dict):
         # send to solr:
         try:
             conn = make_connection()
-            conn.add_many([pkg_dict])
-            conn.commit(wait_flush=False, wait_searcher=False)
+            conn.add_many([pkg_dict], _commit=True)
         except Exception, e:
             log.exception(e)
             raise SearchIndexError(e)

From 4ad7ac00af18f69a4ef27dc7fd6ba18627033ff1 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Thu, 2 Aug 2012 12:06:51 +0100
Subject: [PATCH 02/53] [#2788] Commit only once at the end of a search index
 rebuild

Commiting changes in the Solr search index is a heavy task that takes
significant time. We are currently commiting changes after each update,
which is probably fine for individual updated (ie users editing or
creating a dataset), but when rebuilding the whole index it is
unnecessary. A single commit at the end of the process is needed, and
that speeds the reindexing about a 85%.

A flag has been added (`-e` or `--commit-each`) to allow the old
behaviour (commiting after each edit).
---
 ckan/lib/cli.py             | 21 ++++++++++++++++++---
 ckan/lib/search/__init__.py | 12 +++++++++---
 ckan/lib/search/index.py    | 23 ++++++++++++++++++-----
 ckan/tests/lib/test_cli.py  |  4 ++--
 4 files changed, 47 insertions(+), 13 deletions(-)

diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py
index baaa4674ba9..11272281628 100644
--- a/ckan/lib/cli.py
+++ b/ckan/lib/cli.py
@@ -277,7 +277,7 @@ class SearchIndexCommand(CkanCommand):
     '''Creates a search index for all datasets
 
     Usage:
-      search-index [-i] [-o] [-r] rebuild [dataset-name]     - reindex dataset-name if given, if not then rebuild full search index (all datasets)
+      search-index [-i] [-o] [-r] [-e] rebuild [dataset-name]     - reindex dataset-name if given, if not then rebuild full search index (all datasets)
       search-index check                                     - checks for datasets not indexed
       search-index show {dataset-name}                       - shows index of a dataset
       search-index clear [dataset-name]                      - clears the search index for the provided dataset or for the whole ckan instance
@@ -301,6 +301,13 @@ def __init__(self,name):
         self.parser.add_option('-r', '--refresh', dest='refresh',
             action='store_true', default=False, help='Refresh current index (does not clear the existing one)')
 
+        self.parser.add_option('-e', '--commit-each', dest='commit_each',
+            action='store_true', default=False, help=
+'''Perform a commit after indexing each dataset. This ensures that changes are
+immediately available on the search, but slows significantly the process.
+Default is false.'''
+                    )
+
     def command(self):
         self._load_config()
 
@@ -322,14 +329,22 @@ def command(self):
             print 'Command %s not recognized' % cmd
 
     def rebuild(self):
-        from ckan.lib.search import rebuild
+        from ckan.lib.search import rebuild, commit
+
+        # BY default we don't commit after each request to Solr, as it is
+        # a really heavy operation and slows things a lot
 
         if len(self.args) > 1:
             rebuild(self.args[1])
         else:
             rebuild(only_missing=self.options.only_missing,
                     force=self.options.force,
-                    refresh=self.options.refresh)
+                    refresh=self.options.refresh,
+                    defer_commit=(not self.options.commit_each))
+
+        if not self.options.commit_each:
+            commit()
+
     def check(self):
         from ckan.lib.search import check
 
diff --git a/ckan/lib/search/__init__.py b/ckan/lib/search/__init__.py
index f003efd9350..dfb16e301f8 100644
--- a/ckan/lib/search/__init__.py
+++ b/ckan/lib/search/__init__.py
@@ -131,7 +131,7 @@ def notify(self, entity, operation):
             log.warn("Discarded Sync. indexing for: %s" % entity)
 
 
-def rebuild(package_id=None, only_missing=False, force=False, refresh=False):
+def rebuild(package_id=None, only_missing=False, force=False, refresh=False, defer_commit=False):
     '''
         Rebuilds the search index.
 
@@ -175,12 +175,13 @@ def rebuild(package_id=None, only_missing=False, force=False, refresh=False):
 
         for pkg_id in package_ids:
             try:
-                package_index.insert_dict(
+                package_index.update_dict(
                     get_action('package_show')(
                         {'model': model, 'ignore_auth': True,
                          'validate': False},
                         {'id': pkg_id}
-                    )
+                    ),
+                    defer_commit
                 )
             except Exception, e:
                 log.error('Error while indexing dataset %s: %s' %
@@ -195,6 +196,11 @@ def rebuild(package_id=None, only_missing=False, force=False, refresh=False):
     log.info('Finished rebuilding search index.')
 
 
+def commit():
+    package_index = index_for(model.Package)
+    package_index.commit()
+    log.info('Commited pending changes on the search index')
+
 def check():
     from ckan import model
     package_query = query_for(model.Package)
diff --git a/ckan/lib/search/index.py b/ckan/lib/search/index.py
index 1d912eef1c7..dee7e27c7e8 100644
--- a/ckan/lib/search/index.py
+++ b/ckan/lib/search/index.py
@@ -92,10 +92,10 @@ class PackageSearchIndex(SearchIndex):
     def remove_dict(self, pkg_dict):
         self.delete_package(pkg_dict)
 
-    def update_dict(self, pkg_dict):
-        self.index_package(pkg_dict)
+    def update_dict(self, pkg_dict, defer_commit=False):
+        self.index_package(pkg_dict, defer_commit)
 
-    def index_package(self, pkg_dict):
+    def index_package(self, pkg_dict, defer_commit=False):
         if pkg_dict is None:
             return
         pkg_dict['data_dict'] = json.dumps(pkg_dict)
@@ -222,14 +222,27 @@ def index_package(self, pkg_dict):
         # send to solr:
         try:
             conn = make_connection()
-            conn.add_many([pkg_dict], _commit=True)
+            commit = not defer_commit
+            conn.add_many([pkg_dict], _commit=commit)
+        except Exception, e:
+            log.exception(e)
+            raise SearchIndexError(e)
+        finally:
+            conn.close()
+
+        commit_debug_msg = 'Not commited yet' if defer_commit else 'Commited'
+        log.debug('Updated index for %s [%s]' % (pkg_dict.get('name'), commit_debug_msg))
+
+    def commit(self):
+        try:
+            conn = make_connection()
+            conn.commit(wait_flush=False, wait_searcher=False)
         except Exception, e:
             log.exception(e)
             raise SearchIndexError(e)
         finally:
             conn.close()
 
-        log.debug("Updated index for %s" % pkg_dict.get('name'))
 
     def delete_package(self, pkg_dict):
         conn = make_connection()
diff --git a/ckan/tests/lib/test_cli.py b/ckan/tests/lib/test_cli.py
index 7bea16bb484..5aee50ace1a 100644
--- a/ckan/tests/lib/test_cli.py
+++ b/ckan/tests/lib/test_cli.py
@@ -80,7 +80,7 @@ def test_clear_and_rebuild_index(self):
 
         # Rebuild index
         self.search.args = ()
-        self.search.options = FakeOptions(only_missing=False,force=False,refresh=False)
+        self.search.options = FakeOptions(only_missing=False,force=False,refresh=False,commit_each=False)
         self.search.rebuild()
         pkg_count = model.Session.query(model.Package).filter(model.Package.state==u'active').count()
 
@@ -103,7 +103,7 @@ def test_clear_and_rebuild_only_one(self):
 
         # Rebuild index for annakarenina
         self.search.args = ('rebuild annakarenina').split()
-        self.search.options = FakeOptions(only_missing=False,force=False,refresh=False)
+        self.search.options = FakeOptions(only_missing=False,force=False,refresh=False,commit_each=False)
         self.search.rebuild()
 
         self.query.run({'q':'*:*'})

From dcd013fd6f2e59f1a36574bad6dece1dfe39fc67 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Thu, 2 Aug 2012 16:55:10 +0100
Subject: [PATCH 03/53] [#2788] Allow not returning the package dict when
 creating or updating

This avoids calling package_show
---
 ckan/logic/action/create.py | 8 ++++++--
 ckan/logic/action/update.py | 8 ++++++--
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py
index 94b5f74ce32..d6f98426f9d 100644
--- a/ckan/logic/action/create.py
+++ b/ckan/logic/action/create.py
@@ -98,7 +98,8 @@ def package_create(context, data_dict):
         which groups exist call ``group_list()``
     :type groups: list of dictionaries
 
-    :returns: the newly created dataset
+    :returns: the newly created dataset (if 'return_package_dict' is True in the
+              context, which is the default. Otherwise returns None)
     :rtype: dictionary
 
     '''
@@ -162,7 +163,10 @@ def package_create(context, data_dict):
     ## this is added so that the rest controller can make a new location
     context["id"] = pkg.id
     log.debug('Created object %s' % str(pkg.name))
-    return _get_action('package_show')(context, {'id':context['id']})
+
+    return_package_dict = context.get('return_package_dict',True)
+
+    return _get_action('package_show')(context, {'id':context['id']}) if return_package_dict else None
 
 def package_create_validate(context, data_dict):
     model = context['model']
diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py
index a06274c0714..bf530bd4c07 100644
--- a/ckan/logic/action/update.py
+++ b/ckan/logic/action/update.py
@@ -213,7 +213,8 @@ def package_update(context, data_dict):
     :param id: the name or id of the dataset to update
     :type id: string
 
-    :returns: the updated dataset
+    :returns: the updated dataset (if 'return_package_dict' is True in the
+              context, which is the default. Otherwise returns None)
     :rtype: dictionary
 
     '''
@@ -273,7 +274,10 @@ def package_update(context, data_dict):
         model.repo.commit()
 
     log.debug('Updated object %s' % str(pkg.name))
-    return _get_action('package_show')(context, data_dict)
+
+    return_package_dict = context.get('return_package_dict',True)
+
+    return _get_action('package_show')(context, data_dict) if return_package_dict else None
 
 def package_update_validate(context, data_dict):
     model = context['model']

From c8997b6e20ca040b2146c5eedf88325eb145b7a0 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Tue, 7 Aug 2012 14:54:14 +0100
Subject: [PATCH 04/53] [#2788] Return dataset id if 'return_pacakge_dict' is
 False

---
 ckan/logic/action/create.py | 8 ++++++--
 ckan/logic/action/update.py | 8 ++++++--
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py
index d6f98426f9d..3e0a3ed8d7e 100644
--- a/ckan/logic/action/create.py
+++ b/ckan/logic/action/create.py
@@ -99,7 +99,8 @@ def package_create(context, data_dict):
     :type groups: list of dictionaries
 
     :returns: the newly created dataset (if 'return_package_dict' is True in the
-              context, which is the default. Otherwise returns None)
+              context, which is the default. Otherwise returns just the dataset
+              id)
     :rtype: dictionary
 
     '''
@@ -166,7 +167,10 @@ def package_create(context, data_dict):
 
     return_package_dict = context.get('return_package_dict',True)
 
-    return _get_action('package_show')(context, {'id':context['id']}) if return_package_dict else None
+    output = _get_action('package_show')(context, {'id':context['id']}) \
+            if return_package_dict else context['id']
+
+    return output
 
 def package_create_validate(context, data_dict):
     model = context['model']
diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py
index bf530bd4c07..9a4f9ff0831 100644
--- a/ckan/logic/action/update.py
+++ b/ckan/logic/action/update.py
@@ -214,7 +214,8 @@ def package_update(context, data_dict):
     :type id: string
 
     :returns: the updated dataset (if 'return_package_dict' is True in the
-              context, which is the default. Otherwise returns None)
+              context, which is the default. Otherwise returns just the
+              dataset id)
     :rtype: dictionary
 
     '''
@@ -277,7 +278,10 @@ def package_update(context, data_dict):
 
     return_package_dict = context.get('return_package_dict',True)
 
-    return _get_action('package_show')(context, data_dict) if return_package_dict else None
+    output = _get_action('package_show')(context, {'id': data_dict['id']}) \
+            if return_package_dict else data_dict['id']
+
+    return output
 
 def package_update_validate(context, data_dict):
     model = context['model']

From 583c073aed986a1959a5c5d66ef969923d9677d6 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Tue, 14 Aug 2012 17:33:40 +0100
Subject: [PATCH 05/53] [#2788] Use return_id_only instead of
 return_package_dict

Suggested by @tobes, as it is more backwards compatible
---
 ckan/logic/action/create.py | 11 +++++------
 ckan/logic/action/update.py |  6 +++---
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py
index 3e0a3ed8d7e..8918b69cb2d 100644
--- a/ckan/logic/action/create.py
+++ b/ckan/logic/action/create.py
@@ -98,9 +98,8 @@ def package_create(context, data_dict):
         which groups exist call ``group_list()``
     :type groups: list of dictionaries
 
-    :returns: the newly created dataset (if 'return_package_dict' is True in the
-              context, which is the default. Otherwise returns just the dataset
-              id)
+    :returns: the newly created dataset (unless 'return_id_only' is set to True
+              in the context, in which case just the dataset id will be returned)
     :rtype: dictionary
 
     '''
@@ -165,10 +164,10 @@ def package_create(context, data_dict):
     context["id"] = pkg.id
     log.debug('Created object %s' % str(pkg.name))
 
-    return_package_dict = context.get('return_package_dict',True)
+    return_id_only = context.get('return_id_only', False)
 
-    output = _get_action('package_show')(context, {'id':context['id']}) \
-            if return_package_dict else context['id']
+    output = context['id'] if return_id_only \
+            else _get_action('package_show')(context, {'id':context['id']})
 
     return output
 
diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py
index 9a4f9ff0831..d60c7927728 100644
--- a/ckan/logic/action/update.py
+++ b/ckan/logic/action/update.py
@@ -276,10 +276,10 @@ def package_update(context, data_dict):
 
     log.debug('Updated object %s' % str(pkg.name))
 
-    return_package_dict = context.get('return_package_dict',True)
+    return_id_only = context.get('return_id_only', False)
 
-    output = _get_action('package_show')(context, {'id': data_dict['id']}) \
-            if return_package_dict else data_dict['id']
+    output = data_dict['id'] if return_id_only \
+            else _get_action('package_show')(context, {'id': data_dict['id']})
 
     return output
 

From 80be35cca9995c050e0692b5ba3ffac1c7291461 Mon Sep 17 00:00:00 2001
From: tobes <toby.junk@gmail.com>
Date: Tue, 18 Sep 2012 10:54:42 +0100
Subject: [PATCH 06/53] [#2257] Cleanup deprecated helper functions

---
 ckan/lib/helpers.py                           | 103 ++----------------
 .../functional/api/test_revision_search.py    |   6 +-
 ckan/tests/lib/test_helpers.py                |   4 +-
 test-core.ini                                 |   2 -
 4 files changed, 11 insertions(+), 104 deletions(-)

diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py
index cda89f90fd0..0c1c8440423 100644
--- a/ckan/lib/helpers.py
+++ b/ckan/lib/helpers.py
@@ -37,7 +37,7 @@
 from pylons.i18n import _, ungettext
 
 import ckan.lib.fanstatic_resources as fanstatic_resources
-from lib.maintain import deprecated
+#from lib.maintain import deprecated
 import ckan.model as model
 import ckan.lib.formatters as formatters
 
@@ -295,19 +295,7 @@ def are_there_flash_messages():
     return flash.are_there_messages()
 
 
-def nav_link(*args, **kwargs):
-    # nav_link() used to need c passing as the first arg
-    # this is deprecated as pointless
-    # throws error if ckan.restrict_template_vars is True
-    # When we move to strict helpers then this should be removed as a wrapper
-    if len(args) > 2 or (len(args) > 1 and 'controller' in kwargs):
-        if not asbool(config.get('ckan.restrict_template_vars', 'false')):
-            return _nav_link(*args[1:], **kwargs)
-        raise Exception('nav_link() calling has been changed. remove c in template %s or included one' % _get_template_name())
-    return _nav_link(*args, **kwargs)
-
-
-def _nav_link(text, controller, **kwargs):
+def nav_link(text, controller, **kwargs):
     '''
     params
     class_: pass extra class(s) to add to the <a> tag
@@ -343,20 +331,7 @@ def _link_class(kwargs):
     return kwargs.pop('class_', '') + active
 
 
-def nav_named_link(*args, **kwargs):
-    # subnav_link() used to need c passing as the first arg
-    # this is deprecated as pointless
-    # throws error if ckan.restrict_template_vars is True
-    # When we move to strict helpers then this should be removed as a wrapper
-    if len(args) > 3 or (len(args) > 0 and 'text' in kwargs) or \
-       (len(args) > 1 and 'name' in kwargs):
-        if not asbool(config.get('ckan.restrict_template_vars', 'false')):
-            return _nav_named_link(*args[1:], **kwargs)
-        raise Exception('nav_named_link() calling has been changed. remove c in template %s or included one' % _get_template_name())
-    return _nav_named_link(*args, **kwargs)
-
-
-def _nav_named_link(text, name, **kwargs):
+def nav_named_link(text, name, **kwargs):
     class_ = _link_class(kwargs)
     return link_to(
         text,
@@ -365,19 +340,7 @@ def _nav_named_link(text, name, **kwargs):
     )
 
 
-def subnav_link(*args, **kwargs):
-    # subnav_link() used to need c passing as the first arg
-    # this is deprecated as pointless
-    # throws error if ckan.restrict_template_vars is True
-    # When we move to strict helpers then this should be removed as a wrapper
-    if len(args) > 2 or (len(args) > 1 and 'action' in kwargs):
-        if not asbool(config.get('ckan.restrict_template_vars', 'false')):
-            return _subnav_link(*args[1:], **kwargs)
-        raise Exception('subnav_link() calling has been changed. remove c in template %s or included one' % _get_template_name())
-    return _subnav_link(*args, **kwargs)
-
-
-def _subnav_link(text, action, **kwargs):
+def subnav_link(text, action, **kwargs):
     kwargs['action'] = action
     class_ = _link_class(kwargs)
     return link_to(
@@ -387,20 +350,7 @@ def _subnav_link(text, action, **kwargs):
     )
 
 
-def subnav_named_route(*args, **kwargs):
-    # subnav_link() used to need c passing as the first arg
-    # this is deprecated as pointless
-    # throws error if ckan.restrict_template_vars is True
-    # When we move to strict helpers then this should be removed as a wrapper
-    if len(args) > 2 or (len(args) > 0 and 'text' in kwargs) or \
-       (len(args) > 1 and 'routename' in kwargs):
-        if not asbool(config.get('ckan.restrict_template_vars', 'false')):
-            return _subnav_named_route(*args[1:], **kwargs)
-        raise Exception('subnav_named_route() calling has been changed. remove c in template %s or included one' % _get_template_name())
-    return _subnav_named_route(*args, **kwargs)
-
-
-def _subnav_named_route(text, routename, **kwargs):
+def subnav_named_route(text, routename, **kwargs):
     """ Generate a subnav element based on a named route """
     # FIXME this is the same as _nav_named_link
     # they should be combined
@@ -521,18 +471,6 @@ def unselected_facet_items(facet, limit=10):
     '''
     return get_facet_items_dict(facet, limit=limit, exclude_active=True)
 
-@deprecated('Please use get_facet_title(name) for i18n improvements.')
-def facet_title(name):
-    '''Returns a title for the given facet name.
-    
-    If a mapping is declared in the config, this is used.  Otherwise it falls
-    back to capitalizing the given name.
-
-    This function is deprecated, use `get_facet_title` instead.
-    '''
-    # FIXME this looks like an i18n issue
-    return config.get('search.facets.%s.title' % name, name.capitalize())
-
 def get_facet_title(name):
 
     # if this is set in the config use this
@@ -574,14 +512,6 @@ def sorted_extras(list_):
         output.append((k, v))
     return output
 
-@deprecated('Please use check_access instead.')
-def am_authorized(c, action, domain_object=None):
-    ''' Deprecated. Please use check_access instead'''
-    from ckan.authz import Authorizer
-    if domain_object is None:
-        domain_object = model.System()
-    return Authorizer.am_authorized(c, action, domain_object)
-
 
 def check_access(action, data_dict=None):
     from ckan.logic import check_access as check_access_logic, NotAuthorized
@@ -772,14 +702,6 @@ def render_datetime(datetime_, date_format=None, with_hours=False):
         return ''
 
 
-@deprecated()
-def datetime_to_date_str(datetime_):
-    '''DEPRECATED: Takes a datetime.datetime object and returns a string of it
-    in ISO format.
-    '''
-    return datetime_.isoformat()
-
-
 def date_str_to_datetime(date_str):
     '''Convert ISO-like formatted datestring to datetime object.
 
@@ -965,21 +887,13 @@ def dump_json(obj, **kw):
     return json.dumps(obj, **kw)
 
 
-def auto_log_message(*args):
-    # auto_log_message() used to need c passing as the first arg
-    # this is deprecated as pointless
-    # throws error if ckan.restrict_template_vars is True
-    # When we move to strict helpers then this should be removed as a wrapper
-    if len(args) and asbool(config.get('ckan.restrict_template_vars', 'false')):
-        raise Exception('auto_log_message() calling has been changed. remove c in template %s or included one' % _get_template_name())
-    return _auto_log_message()
-
 def _get_template_name():
     #FIX ME THIS IS BROKEN
     ''' helper function to get the currently/last rendered template name '''
     return c.__debug_info[-1]['template_name']
 
-def _auto_log_message():
+
+def auto_log_message():
     if (c.action == 'new'):
         return _('Created new dataset.')
     elif (c.action == 'editresources'):
@@ -1322,8 +1236,6 @@ def format_resource_items(items):
            'subnav_link',
            'subnav_named_route',
            'default_group_type',
-         # 'facet_title',  # deprecated
-         #  am_authorized, # deprecated
            'check_access',
            'linked_user',
            'group_name_to_title',
@@ -1338,7 +1250,6 @@ def format_resource_items(items):
            'pager_url',
            'render_datetime',
            'date_str_to_datetime',
-           'datetime_to_date_str',
            'parse_rfc_2822_date',
            'time_ago_in_words_from_str',
            'button_attr',
diff --git a/ckan/tests/functional/api/test_revision_search.py b/ckan/tests/functional/api/test_revision_search.py
index f376c36db0c..d223537b1e2 100644
--- a/ckan/tests/functional/api/test_revision_search.py
+++ b/ckan/tests/functional/api/test_revision_search.py
@@ -1,5 +1,3 @@
-from ckan.lib.helpers import datetime_to_date_str
-
 from ckan.tests.functional.api.base import *
 from ckan.tests import TestController as ControllerTestCase
 
@@ -48,7 +46,7 @@ def test_12_search_revision_since_time(self):
         revs = model.Session.query(model.Revision).all()
         # Check since time of first.
         rev_first = revs[-1]
-        params = "?since_time=%s" % datetime_to_date_str(rev_first.timestamp)
+        params = "?since_time=%s" % rev_first.timestamp.isoformat()
         res = self.app.get(offset+params, status=200)
         res_list = self.data_from_res(res)
         assert rev_first.id not in res_list
@@ -56,7 +54,7 @@ def test_12_search_revision_since_time(self):
             assert rev.id in res_list, (rev.id, res_list)
         # Check since time of last.
         rev_last = revs[0]
-        params = "?since_time=%s" % datetime_to_date_str(rev_last.timestamp)
+        params = "?since_time=%s" % rev_last.timestamp.isoformat()
         res = self.app.get(offset+params, status=200)
         res_list = self.data_from_res(res)
         assert res_list == [], res_list
diff --git a/ckan/tests/lib/test_helpers.py b/ckan/tests/lib/test_helpers.py
index d501c8e7837..68859977c62 100644
--- a/ckan/tests/lib/test_helpers.py
+++ b/ckan/tests/lib/test_helpers.py
@@ -35,7 +35,7 @@ def test_render_datetime_blank(self):
         assert_equal(res, '')
 
     def test_datetime_to_date_str(self):
-        res = h.datetime_to_date_str(datetime.datetime(2008, 4, 13, 20, 40, 20, 123456))
+        res = datetime.datetime(2008, 4, 13, 20, 40, 20, 123456).isoformat()
         assert_equal(res, '2008-04-13T20:40:20.123456')
 
     def test_date_str_to_datetime_date_only(self):
@@ -73,7 +73,7 @@ def test_date_str_to_datetime_with_ambiguous_microseconds(self):
 
     def test_time_ago_in_words_from_str(self):
         two_months_ago = datetime.datetime.now() - datetime.timedelta(days=65)
-        two_months_ago_str = h.datetime_to_date_str(two_months_ago)
+        two_months_ago_str = two_months_ago.isoformat()
         res = h.time_ago_in_words_from_str(two_months_ago_str)
         assert_equal(res, '2 months')
 
diff --git a/test-core.ini b/test-core.ini
index c6f8cd7535a..03e997e4568 100644
--- a/test-core.ini
+++ b/test-core.ini
@@ -33,8 +33,6 @@ package_edit_return_url = http://localhost/dataset/<NAME>?test=edit
 rdf_packages = http://test.com/package/
 ckan.extra_resource_fields = alt_url
 
-# disable this so we can test all types of indexing
-ckan.restrict_template_vars = false
 # we need legacy templates for many tests to pass
 ckan.legacy_templates = yes
 # FIXME this is to allow some tests to pass but should be removed when possible

From c3e3a6b981e8665143de535c7f6326ef725c85b3 Mon Sep 17 00:00:00 2001
From: tobes <toby.junk@gmail.com>
Date: Tue, 18 Sep 2012 11:05:22 +0100
Subject: [PATCH 07/53] [#2257] Remove the restrict_template_vars config option

---
 ckan/config/environment.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/ckan/config/environment.py b/ckan/config/environment.py
index 78e647c65bd..0abb2018274 100644
--- a/ckan/config/environment.py
+++ b/ckan/config/environment.py
@@ -32,7 +32,7 @@ class _Helpers(object):
     missing functions from causing template exceptions. Useful if
     templates have helper functions provided by extensions that have
     not been enabled. '''
-    def __init__(self, helpers, restrict=True):
+    def __init__(self, helpers):
         functions = {}
         allowed = helpers.__allowed_functions__[:]
         # list of functions due to be deprecated
@@ -41,8 +41,7 @@ def __init__(self, helpers, restrict=True):
         for helper in dir(helpers):
             if helper not in allowed:
                 self.deprecated.append(helper)
-                if restrict:
-                    continue
+                continue
             functions[helper] = getattr(helpers, helper)
             if helper in allowed:
                 allowed.remove(helper)
@@ -181,9 +180,7 @@ def find_controller(self, controller):
     config['pylons.app_globals']._init()
 
     # add helper functions
-    restrict_helpers = asbool(
-        config.get('ckan.restrict_template_vars', 'true'))
-    helpers = _Helpers(h, restrict_helpers)
+    helpers = _Helpers(h)
     config['pylons.h'] = helpers
 
     ## redo template setup to use genshi.search_path

From ae5d0a7b0d75213f653faeeaf3f6520f6e46e47c Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Tue, 25 Sep 2012 12:16:25 +0100
Subject: [PATCH 08/53] reformat /ckan/ckan/config/middleware.py

---
 ckan/config/middleware.py | 42 +++++++++++++++++++--------------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py
index 9e004a5806a..4702122e3fa 100644
--- a/ckan/config/middleware.py
+++ b/ckan/config/middleware.py
@@ -26,6 +26,7 @@
 from ckan.config.environment import load_environment
 import ckan.lib.app_globals as app_globals
 
+
 def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
     """Create a Pylons WSGI application and return it
 
@@ -64,30 +65,29 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
     app = RoutesMiddleware(app, config['routes.map'])
     app = SessionMiddleware(app, config)
     app = CacheMiddleware(app, config)
-    
+
     # CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares)
     #app = QueueLogMiddleware(app)
-    
+
     # Fanstatic
     if asbool(config.get('debug', False)):
         fanstatic_config = {
-            'versioning' : True,
-            'recompute_hashes' : True,
-            'minified' : False,
-            'bottom' : True,
-            'bundle' : False,
+            'versioning': True,
+            'recompute_hashes': True,
+            'minified': False,
+            'bottom': True,
+            'bundle': False,
         }
     else:
         fanstatic_config = {
-            'versioning' : True,
-            'recompute_hashes' : False,
-            'minified' : True,
-            'bottom' : True,
-            'bundle' : True,
+            'versioning': True,
+            'recompute_hashes': False,
+            'minified': True,
+            'bottom': True,
+            'bundle': True,
         }
     app = Fanstatic(app, **fanstatic_config)
 
-
     if asbool(full_stack):
         # Handle Python exceptions
         app = ErrorHandler(app, global_conf, **config['pylons.errorware'])
@@ -98,7 +98,7 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
             app = StatusCodeRedirect(app, [400, 404])
         else:
             app = StatusCodeRedirect(app, [400, 404, 500])
-    
+
     # Initialize repoze.who
     who_parser = WhoConfig(global_conf['here'])
     who_parser.parse(open(app_conf['who.config_file']))
@@ -106,7 +106,7 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
     if asbool(config.get('openid_enabled', 'true')):
         from repoze.who.plugins.openid.identification import OpenIdIdentificationPlugin
         # Monkey patches for repoze.who.openid
-        # Fixes #1659 - enable log-out when CKAN mounted at non-root URL 
+        # Fixes #1659 - enable log-out when CKAN mounted at non-root URL
         from ckan.lib import repoze_patch
         OpenIdIdentificationPlugin.identify = repoze_patch.identify
         OpenIdIdentificationPlugin.redirect_to_logged_in = repoze_patch.redirect_to_logged_in
@@ -117,7 +117,7 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
                 not isinstance(i, OpenIdIdentificationPlugin)]
         who_parser.challengers = [i for i in who_parser.challengers if \
                 not isinstance(i, OpenIdIdentificationPlugin)]
-    
+
     app = PluggableAuthenticationMiddleware(app,
                 who_parser.identifiers,
                 who_parser.authenticators,
@@ -126,10 +126,10 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
                 who_parser.request_classifier,
                 who_parser.challenge_decider,
                 logging.getLogger('repoze.who'),
-                logging.WARN, # ignored
+                logging.WARN,  # ignored
                 who_parser.remote_user_key,
            )
-    
+
     # Establish the Registry for this application
     app = RegistryManager(app)
 
@@ -152,7 +152,7 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
                     StaticURLParser(public_path.strip(),
                         cache_max_age=static_max_age)
                 )
-        app = Cascade(extra_static_parsers+static_parsers)
+        app = Cascade(extra_static_parsers + static_parsers)
 
     # Page cache
     if asbool(config.get('ckan.page_cache_enabled')):
@@ -164,6 +164,7 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
 
     return app
 
+
 class I18nMiddleware(object):
     """I18n Middleware selects the language based on the url
     eg /fr/home is French"""
@@ -198,7 +199,7 @@ def __call__(self, environ, start_response):
             # Current application url
             path_info = environ['PATH_INFO']
             # sort out weird encodings
-            path_info = '/'.join(urllib.quote(pce,'') for pce in path_info.split('/'))
+            path_info = '/'.join(urllib.quote(pce, '') for pce in path_info.split('/'))
 
             qs = environ.get('QUERY_STRING')
 
@@ -317,7 +318,6 @@ def __init__(self, app, config):
         self.app = app
         self.engine = sa.create_engine(config.get('sqlalchemy.url'))
 
-
     def __call__(self, environ, start_response):
         path = environ['PATH_INFO']
         if path == '/_tracking':

From b838d56089a300653a370e10307faaf79e1efa5b Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Tue, 25 Sep 2012 12:19:18 +0100
Subject: [PATCH 09/53] reformat and removed imports from
 ckan/ckan/lib/plugins.py

---
 ckan/lib/plugins.py | 21 +++++++--------------
 1 file changed, 7 insertions(+), 14 deletions(-)

diff --git a/ckan/lib/plugins.py b/ckan/lib/plugins.py
index 5acc733eeaf..691a4830689 100644
--- a/ckan/lib/plugins.py
+++ b/ckan/lib/plugins.py
@@ -78,7 +78,7 @@ def register_package_plugins(map):
                         controller='package', action='read')
             map.connect('%s_action' % package_type,
                         '/%s/{action}/{id}' % package_type, controller='package',
-                requirements=dict(action='|'.join(['edit', 'authz', 'history' ]))
+                requirements=dict(action='|'.join(['edit', 'authz', 'history']))
             )
 
             if package_type in _package_plugins:
@@ -92,8 +92,6 @@ def register_package_plugins(map):
         _default_package_plugin = DefaultDatasetForm()
 
 
-
-
 def register_group_plugins(map):
     """
     Register the various IGroupForm instances.
@@ -138,7 +136,7 @@ def register_group_plugins(map):
                         controller='group', action='read')
             map.connect('%s_action' % group_type,
                         '/%s/{action}/{id}' % group_type, controller='group',
-                requirements=dict(action='|'.join(['edit', 'authz', 'history' ]))
+                requirements=dict(action='|'.join(['edit', 'authz', 'history']))
             )
 
             if group_type in _group_plugins:
@@ -203,7 +201,6 @@ def history_template(self):
         """
         return 'package/history.html'
 
-
     def package_form(self):
         return 'package/new_package_form.html'
 
@@ -215,7 +212,7 @@ def form_to_db_schema_options(self, options):
         If a context is provided, and it contains a schema, it will be
         returned.
         '''
-        schema = options.get('context',{}).get('schema',None)
+        schema = options.get('context', {}).get('schema', None)
         if schema:
             return schema
 
@@ -248,7 +245,7 @@ def db_to_form_schema_options(self, options):
         If a context is provided, and it contains a schema, it will be
         returned.
         '''
-        schema = options.get('context',{}).get('schema',None)
+        schema = options.get('context', {}).get('schema', None)
         if schema:
             return schema
         return self.db_to_form_schema()
@@ -273,11 +270,9 @@ def check_data_dict(self, data_dict, schema=None):
             raise dictization_functions.DataError(data_dict)
 
     def setup_template_variables(self, context, data_dict):
-        from pylons import config
-
         authz_fn = logic.get_action('group_list_authz')
         c.groups_authz = authz_fn(context, data_dict)
-        data_dict.update({'available_only':True})
+        data_dict.update({'available_only': True})
 
         c.groups_available = authz_fn(context, data_dict)
 
@@ -300,7 +295,6 @@ def setup_template_variables(self, context, data_dict):
                 c.auth_for_change_state = False
 
 
-
 class DefaultGroupForm(object):
     """
     Provides a default implementation of the pluggable Group controller
@@ -352,7 +346,6 @@ def edit_template(self):
         """
         return 'group/edit.html'
 
-
     def group_form(self):
         return 'group/new_group_form.html'
 
@@ -364,7 +357,7 @@ def form_to_db_schema_options(self, options):
         If a context is provided, and it contains a schema, it will be
         returned.
         '''
-        schema = options.get('context',{}).get('schema',None)
+        schema = options.get('context', {}).get('schema', None)
         if schema:
             return schema
 
@@ -396,7 +389,7 @@ def db_to_form_schema_options(self, options):
         If a context is provided, and it contains a schema, it will be
         returned.
         '''
-        schema = options.get('context',{}).get('schema',None)
+        schema = options.get('context', {}).get('schema', None)
         if schema:
             return schema
         return self.db_to_form_schema()

From 00005581cf8d8960fe4d5fbd0958177d45fa9346 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Tue, 25 Sep 2012 12:26:12 +0100
Subject: [PATCH 10/53] reformat and removed double imports from
 ckan/ckan/lib/helpers.py

---
 ckan/lib/helpers.py | 63 ++++++++++++++++++++++++++++++---------------
 1 file changed, 42 insertions(+), 21 deletions(-)

diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py
index cda89f90fd0..8cf03b1f4ea 100644
--- a/ckan/lib/helpers.py
+++ b/ckan/lib/helpers.py
@@ -12,7 +12,6 @@
 import urllib
 import pprint
 import copy
-import logging
 from urllib import urlencode
 
 from paste.deploy.converters import asbool
@@ -110,7 +109,6 @@ def _add_i18n_to_url(url_to_amend, **kw):
     # (as part of the language changing feature).
     # A locale of default will not add locale info to the url.
 
-
     default_locale = False
     locale = kw.pop('locale', None)
     no_root = kw.pop('__ckan_no_root', False)
@@ -177,6 +175,7 @@ def full_current_url():
     for sharing etc '''
     return(url_for(request.environ['CKAN_CURRENT_URL'], qualified=True))
 
+
 def lang():
     ''' Return the language code for the current locale eg `en` '''
     return request.environ.get('CKAN_LANG')
@@ -331,6 +330,7 @@ def _nav_link(text, controller, **kwargs):
         link = ''
     return link
 
+
 def _link_class(kwargs):
     ''' creates classes for the link_to calls '''
     highlight_actions = kwargs.pop('highlight_actions',
@@ -411,6 +411,7 @@ def _subnav_named_route(text, routename, **kwargs):
         class_=class_
     )
 
+
 def build_nav_main(*args):
     ''' build a set of menu items.
 
@@ -446,22 +447,20 @@ def default_group_type():
     return str(config.get('ckan.default.group_type', 'group'))
 
 
-
 _menu_items = {
-    'add dataset' : dict(controller='package', action='new'),
-    'search' : dict(controller='package',
+    'add dataset': dict(controller='package', action='new'),
+    'search': dict(controller='package',
                     action='search',
-                    highlight_actions = 'index search'),
+                    highlight_actions='index search'),
     'default_group': dict(name='%s_index' % default_group_type(),
                           controller='group',
                           highlight_actions='index search'),
-    'about' : dict(controller='home', action='about'),
-    'login' : dict(controller='user', action='login'),
-    'register' : dict(controller='user', action='register'),
+    'about': dict(controller='home', action='about'),
+    'login': dict(controller='user', action='login'),
+    'register': dict(controller='user', action='register'),
 }
 
 
-
 def get_facet_items_dict(facet, limit=10, exclude_active=False):
     '''Return the list of unselected facet items for the given facet, sorted
     by count.
@@ -501,6 +500,7 @@ def get_facet_items_dict(facet, limit=10, exclude_active=False):
     else:
         return facets
 
+
 def unselected_facet_items(facet, limit=10):
     '''Return the list of unselected facet items for the given facet, sorted
     by count.
@@ -521,10 +521,11 @@ def unselected_facet_items(facet, limit=10):
     '''
     return get_facet_items_dict(facet, limit=limit, exclude_active=True)
 
+
 @deprecated('Please use get_facet_title(name) for i18n improvements.')
 def facet_title(name):
     '''Returns a title for the given facet name.
-    
+
     If a mapping is declared in the config, this is used.  Otherwise it falls
     back to capitalizing the given name.
 
@@ -533,6 +534,7 @@ def facet_title(name):
     # FIXME this looks like an i18n issue
     return config.get('search.facets.%s.title' % name, name.capitalize())
 
+
 def get_facet_title(name):
 
     # if this is set in the config use this
@@ -540,15 +542,17 @@ def get_facet_title(name):
     if config_title:
         return config_title
 
-    facet_titles = {'groups' : _('Groups'),
-                  'tags' : _('Tags'),
-                  'res_format' : _('Formats'),
-                  'license' : _('Licence'), }
+    facet_titles = {'groups': _('Groups'),
+                  'tags': _('Tags'),
+                  'res_format': _('Formats'),
+                  'license': _('Licence'), }
     return facet_titles.get(name, name.capitalize())
 
+
 def get_param_int(name, default=10):
     return int(request.params.get(name, default))
 
+
 def _url_with_params(url, params):
     if not params:
         return url
@@ -556,14 +560,16 @@ def _url_with_params(url, params):
                                   for k, v in params]
     return url + u'?' + urlencode(params)
 
+
 def _search_url(params):
     url = url_for(controller='package', action='search')
     return _url_with_params(url, params)
 
+
 def sorted_extras(list_):
     ''' Used for outputting package extras '''
     output = []
-    for extra in sorted(list_, key=lambda x:x['key']):
+    for extra in sorted(list_, key=lambda x: x['key']):
         if extra.get('state') == 'deleted':
             continue
         k, v = extra['key'], extra['value']
@@ -574,6 +580,7 @@ def sorted_extras(list_):
         output.append((k, v))
     return output
 
+
 @deprecated('Please use check_access instead.')
 def am_authorized(c, action, domain_object=None):
     ''' Deprecated. Please use check_access instead'''
@@ -629,6 +636,7 @@ def markdown_extract(text, extract_length=190):
     plain = re.sub(r'<.*?>', '', markdown(text))
     return literal(unicode(truncate(plain, length=extract_length, indicator='...', whole_word=True)))
 
+
 def icon_url(name):
     return url_for_static('/images/icons/%s.png' % name)
 
@@ -667,6 +675,7 @@ def format_icon(_format):
     if ('xml' in _format): return 'page_white_code'
     return 'page_white'
 
+
 def dict_list_reduce(list_, key, unique=True):
     ''' Take a list of dicts and create a new one containing just the
     values for the key with unique values if requested. '''
@@ -703,6 +712,7 @@ def gravatar(email_hash, size=100, default=None):
         % (email_hash, size, default, size, size)
         )
 
+
 def pager_url(page, partial=None, **kwargs):
     routes_dict = _pylons_default_url.environ['pylons.routes_dict']
     kwargs['controller'] = routes_dict['controller']
@@ -721,7 +731,7 @@ def pager(self, *args, **kwargs):
         kwargs.update(
             format=u"<div class='pagination pagination-centered'><ul>$link_previous ~2~ $link_next</ul></div>",
             symbol_previous=u'«', symbol_next=u'»',
-            curpage_attr={'class':'active'}, link_attr={}
+            curpage_attr={'class': 'active'}, link_attr={}
         )
         return super(Page, self).pager(*args, **kwargs)
 
@@ -974,11 +984,13 @@ def auto_log_message(*args):
         raise Exception('auto_log_message() calling has been changed. remove c in template %s or included one' % _get_template_name())
     return _auto_log_message()
 
+
 def _get_template_name():
     #FIX ME THIS IS BROKEN
     ''' helper function to get the currently/last rendered template name '''
     return c.__debug_info[-1]['template_name']
 
+
 def _auto_log_message():
     if (c.action == 'new'):
         return _('Created new dataset.')
@@ -1095,6 +1107,7 @@ def follow_count(obj_type, obj_id):
     context = {'model': model, 'session': model.Session, 'user': c.user}
     return logic.get_action(action)(context, {'id': obj_id})
 
+
 def _create_url_with_params(params=None, controller=None, action=None,
                             extras=None):
     ''' internal function for building urls with parameters. '''
@@ -1123,7 +1136,7 @@ def add_url_param(alternative_url=None, controller=None, action=None,
     via url_for(controller=controller, action=action, **extras)
     controller & action default to the current ones
     '''
-    params_nopage = [(k, v) for k,v in request.params.items() if k != 'page']
+    params_nopage = [(k, v) for k, v in request.params.items() if k != 'page']
     params = set(params_nopage)
     if new_params:
         params |= set(new_params.items())
@@ -1132,6 +1145,7 @@ def add_url_param(alternative_url=None, controller=None, action=None,
     return _create_url_with_params(params=params, controller=controller,
                                    action=action, extras=extras)
 
+
 def remove_url_param(key, value=None, replace=None, controller=None,
                      action=None, extras=None):
     ''' Remove a key from the current parameters. A specific key/value
@@ -1143,25 +1157,28 @@ def remove_url_param(key, value=None, replace=None, controller=None,
     via url_for(controller=controller, action=action, **extras)
     controller & action default to the current ones
     '''
-    params_nopage = [(k, v) for k,v in request.params.items() if k != 'page']
+    params_nopage = [(k, v) for k, v in request.params.items() if k != 'page']
     params = list(params_nopage)
     if value:
         params.remove((key, value))
     else:
-      [params.remove((k, v)) for (k, v) in params[:] if k==key]
+        [params.remove((k, v)) for (k, v) in params[:] if k == key]
     if replace is not None:
         params.append((key, replace))
     return _create_url_with_params(params=params, controller=controller,
                                    action=action, extras=extras)
 
+
 def include_resource(resource):
     r = getattr(fanstatic_resources, resource)
     r.need()
 
+
 def debug_inspect(arg):
     ''' Output pprint.pformat view of supplied arg '''
     return literal('<pre>') + pprint.pformat(arg) + literal('</pre>')
 
+
 def debug_full_info_as_list(debug_info):
     ''' This dumps the template variables for debugging purposes only. '''
     out = []
@@ -1204,6 +1221,7 @@ def popular(type_, number, min=1, title=None):
         raise Exception('popular() did not recieve a valid type_ or title')
     return snippet('snippets/popular.html', title=title, number=number, min=min)
 
+
 def groups_available():
     ''' return a list of available groups '''
     import ckan.logic as logic
@@ -1212,6 +1230,7 @@ def groups_available():
     data_dict = {'available_only': True}
     return logic.get_action('group_list_authz')(context, data_dict)
 
+
 def dashboard_activity_stream(user_id):
     '''Return the dashboard activity stream of the given user.
 
@@ -1240,6 +1259,7 @@ def escape_js(str_to_escape):
                         .replace('\'', '\\\'') \
                         .replace('"', '\\\"')
 
+
 def get_pkg_dict_extra(pkg_dict, key, default=None):
     '''Returns the value for the dataset extra with the provided key.
 
@@ -1259,6 +1279,7 @@ def get_pkg_dict_extra(pkg_dict, key, default=None):
 
     return default
 
+
 def get_request_param(parameter_name, default=None):
     ''' This function allows templates to access query string parameters
     from the request. This is useful for things like sort order in
@@ -1301,7 +1322,7 @@ def format_resource_items(items):
             value = formatters.localised_number(value)
         key = key.replace('_', ' ')
         output.append((key, value))
-    return sorted(output, key=lambda x:x[0])
+    return sorted(output, key=lambda x: x[0])
 
 
 # these are the functions that will end up in `h` template helpers

From c58215e5a2ed475ae5ab82f49358ff274827e08d Mon Sep 17 00:00:00 2001
From: John Martin <me@johnmart.in>
Date: Wed, 3 Oct 2012 14:22:31 +0100
Subject: [PATCH 11/53] Fixes in order to prevent the flickering from happening
 with the data-viewer

---
 .../base/javascript/modules/data-viewer.js    | 34 +++++--------------
 1 file changed, 8 insertions(+), 26 deletions(-)

diff --git a/ckan/public/base/javascript/modules/data-viewer.js b/ckan/public/base/javascript/modules/data-viewer.js
index dedf1e24d39..d9e5235dbc4 100644
--- a/ckan/public/base/javascript/modules/data-viewer.js
+++ b/ckan/public/base/javascript/modules/data-viewer.js
@@ -4,8 +4,7 @@ this.ckan.module('data-viewer', function (jQuery) {
   return {
     options: {
       timeout: 200,
-      minHeight: 400,
-      padding: 42
+      minHeight: 400
     },
 
     initialize: function () {
@@ -19,39 +18,22 @@ this.ckan.module('data-viewer', function (jQuery) {
       var loc = $('body').data('site-root');
       // see if page is in part of the same domain
       if (this.el.attr('src').substring(0, loc.length) === loc) {
+        // hide the scrollbar to prevent jumping
+        $('<style>body{overflow:hidden}</style>').appendTo(this.el.contents().find('body'));
         this._recalibrate();
         setInterval(function() {
-          if (!self.el.is(':animated')) {
-            self._recalibrate();
-          }
+          self._recalibrate();
         }, this.options.timeout);
       } else {
-        this.el.animate({height: 600}, 600);
+        this.el.css('height', 600);
       }
     },
 
-    _showDebugInfo: function() {
-      var iframe = this.el;
-      console.log('=================');
-      console.log($(iframe.get(0), window.top.document).contents().find('body')[0].scrollHeight);
-      console.log($(iframe.get(0).contentWindow.document).height());
-      console.log($(iframe.get(0).contentWindow.document.body).height());
-      console.log($(iframe.contents().height()));
-      console.log($(iframe.contents().innerHeight()));
-      console.log($(iframe.contents().find('html').height()));
-      console.log($(iframe.contents().find('body').height()));
-      console.log($(iframe.contents().find('body').innerHeight()));
-    },
-
     _recalibrate: function() {
       // save reference to this to use in timeout
-      var self = this;
-      var height = self.el.contents().find('body').height();
-      height = Math.max(height, self.options.minHeight);
-      var deltaHeight = height - (self.el.height() - self.options.padding);
-      if (deltaHeight > 1 || deltaHeight < -10) {
-        self.el.animate({height: height+self.options.padding}, Math.min(700, height*2));
-      }
+      var height = this.el.contents().find('body').outerHeight();
+      height = Math.max(height, this.options.minHeight);
+      this.el.css('height', height);
     },
 
     // firefox caches iframes so force it to get fresh content

From dc46a41f163faa9f70f47c0217e4f731c75c097d Mon Sep 17 00:00:00 2001
From: John Martin <me@johnmart.in>
Date: Wed, 3 Oct 2012 14:51:08 +0100
Subject: [PATCH 12/53] Remove fix for `overflow:hidden` on iframe body

It wasn't need as it was a problem that I spotted that only occured when we used `jQuery.animate` and since we moved to `jQuery.css` it doesn't happen any more.
---
 ckan/public/base/javascript/modules/data-viewer.js | 2 --
 1 file changed, 2 deletions(-)

diff --git a/ckan/public/base/javascript/modules/data-viewer.js b/ckan/public/base/javascript/modules/data-viewer.js
index d9e5235dbc4..34d4315da74 100644
--- a/ckan/public/base/javascript/modules/data-viewer.js
+++ b/ckan/public/base/javascript/modules/data-viewer.js
@@ -18,8 +18,6 @@ this.ckan.module('data-viewer', function (jQuery) {
       var loc = $('body').data('site-root');
       // see if page is in part of the same domain
       if (this.el.attr('src').substring(0, loc.length) === loc) {
-        // hide the scrollbar to prevent jumping
-        $('<style>body{overflow:hidden}</style>').appendTo(this.el.contents().find('body'));
         this._recalibrate();
         setInterval(function() {
           self._recalibrate();

From 3a3fc1f1344ef60f68444a84f4cb396c559d0768 Mon Sep 17 00:00:00 2001
From: John Martin <me@johnmart.in>
Date: Wed, 3 Oct 2012 15:45:39 +0100
Subject: [PATCH 13/53] Added deltaHeight back in... as it was the simplest
 solution to the problem

- Tried https://github.com/house9/jquery-iframe-auto-height but it only works onload and there is no api to allow for refiring a resize event
- Tried using `document.scrollHeight` but it was too flaky in returning correct values and could cause the iframe to resize by 1px every timer tick
---
 ckan/public/base/javascript/modules/data-viewer.js | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/ckan/public/base/javascript/modules/data-viewer.js b/ckan/public/base/javascript/modules/data-viewer.js
index 34d4315da74..33809618a69 100644
--- a/ckan/public/base/javascript/modules/data-viewer.js
+++ b/ckan/public/base/javascript/modules/data-viewer.js
@@ -4,7 +4,8 @@ this.ckan.module('data-viewer', function (jQuery) {
   return {
     options: {
       timeout: 200,
-      minHeight: 400
+      minHeight: 400,
+      padding: 30
     },
 
     initialize: function () {
@@ -28,10 +29,12 @@ this.ckan.module('data-viewer', function (jQuery) {
     },
 
     _recalibrate: function() {
-      // save reference to this to use in timeout
-      var height = this.el.contents().find('body').outerHeight();
+      var height = this.el.contents().find('body').outerHeight(true);
       height = Math.max(height, this.options.minHeight);
-      this.el.css('height', height);
+      var deltaHeight = height - (this.el.height() - this.options.padding);
+      if (deltaHeight > 1 || deltaHeight < -10) {
+        this.el.css('height', height + this.options.padding);
+      }
     },
 
     // firefox caches iframes so force it to get fresh content

From 7841c45f8694b32f5f1064ca0de7a9a403e9e0d6 Mon Sep 17 00:00:00 2001
From: John Martin <me@johnmart.in>
Date: Wed, 3 Oct 2012 15:52:33 +0100
Subject: [PATCH 14/53] Removed deltaHeight as it was uneeded, it was only
 really the padding that was really needed

---
 ckan/public/base/javascript/modules/data-viewer.js | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/ckan/public/base/javascript/modules/data-viewer.js b/ckan/public/base/javascript/modules/data-viewer.js
index 33809618a69..f928ed1c0d3 100644
--- a/ckan/public/base/javascript/modules/data-viewer.js
+++ b/ckan/public/base/javascript/modules/data-viewer.js
@@ -31,10 +31,7 @@ this.ckan.module('data-viewer', function (jQuery) {
     _recalibrate: function() {
       var height = this.el.contents().find('body').outerHeight(true);
       height = Math.max(height, this.options.minHeight);
-      var deltaHeight = height - (this.el.height() - this.options.padding);
-      if (deltaHeight > 1 || deltaHeight < -10) {
-        this.el.css('height', height + this.options.padding);
-      }
+      this.el.css('height', height + this.options.padding);
     },
 
     // firefox caches iframes so force it to get fresh content

From ca5882a8f5ecd3b55eb21174223e6c29c26dd19a Mon Sep 17 00:00:00 2001
From: John Martin <me@johnmart.in>
Date: Thu, 4 Oct 2012 11:06:31 +0100
Subject: [PATCH 15/53] Added resources the the side bar of the edit dataset
 page

---
 ckan/templates/package/edit.html               | 4 ++++
 ckan/templates/package/snippets/resources.html | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/ckan/templates/package/edit.html b/ckan/templates/package/edit.html
index c2c6c321221..722df8812a9 100644
--- a/ckan/templates/package/edit.html
+++ b/ckan/templates/package/edit.html
@@ -8,6 +8,10 @@
   <li class="active">{% link_for _('Edit Dataset'), controller='package', action='edit', id=pkg.name %}</li>
 {% endblock %}
 
+{% block resources_module %}
+  {% snippet 'package/snippets/resources.html', pkg=pkg, action='resource_edit' %}
+{% endblock %}
+
 {% block actions_content %}
   {% snippet 'package/snippets/back_to_package_action.html', pkg=pkg %}
 {% endblock %}
diff --git a/ckan/templates/package/snippets/resources.html b/ckan/templates/package/snippets/resources.html
index 9def65b71f0..b6585018dc2 100644
--- a/ckan/templates/package/snippets/resources.html
+++ b/ckan/templates/package/snippets/resources.html
@@ -14,7 +14,7 @@
 {% set resources = pkg.resources or [] %}
 {% if resources %}
   <section class="module module-narrow resources">
-    <h2 class="module-heading">{{ _("Resources") }}</h2>
+    <h2 class="module-heading"><i class="icon-sitemap"></i> {{ _("Resources") }}</h2>
     <ul class="unstyled nav nav-simple">
       {% for resource in resources %}
         <li class="nav-item{{ ' active' if active == resource.id }}">

From ed37c6384a48d9ac411a431449f5a6986b0688c8 Mon Sep 17 00:00:00 2001
From: John Martin <me@johnmart.in>
Date: Thu, 4 Oct 2012 13:49:44 +0100
Subject: [PATCH 16/53] Added left hand nav to the edit/add resource pages to
 allow users to move from editing each different resource easily

---
 ckan/public/base/less/module.less             | 11 +++++++
 ckan/templates/package/edit.html              |  4 +++
 ckan/templates/package/new_resource.html      | 18 ++++++-----
 ckan/templates/package/read.html              |  5 +--
 ckan/templates/package/resource_edit.html     |  6 +++-
 .../snippets/back_to_package_action.html      |  2 +-
 ckan/templates/package/snippets/info.html     | 32 +++++++++++++++++++
 .../templates/package/snippets/resources.html |  2 +-
 8 files changed, 65 insertions(+), 15 deletions(-)
 create mode 100644 ckan/templates/package/snippets/info.html

diff --git a/ckan/public/base/less/module.less b/ckan/public/base/less/module.less
index 63e2cbc8885..471fff25637 100644
--- a/ckan/public/base/less/module.less
+++ b/ckan/public/base/less/module.less
@@ -14,6 +14,11 @@
   #gradient > .vertical(@moduleHeadingBackgroundColorStart, @moduleHeadingBackgroundColorEnd);
 }
 
+.module-heading-secondary {
+  .border-radius(0);
+  border-top: 1px solid @moduleHeadingBorderColor;
+}
+
 .module-heading [class^=icon]:before {
   font-size: 1.2em;
   line-height: 1;
@@ -189,3 +194,9 @@
     overflow: hidden;
   }
 }
+
+.package-info {
+  h4 {
+    margin-bottom: 10px;
+  }
+}
diff --git a/ckan/templates/package/edit.html b/ckan/templates/package/edit.html
index 722df8812a9..e90e7c2a6af 100644
--- a/ckan/templates/package/edit.html
+++ b/ckan/templates/package/edit.html
@@ -15,3 +15,7 @@
 {% block actions_content %}
   {% snippet 'package/snippets/back_to_package_action.html', pkg=pkg %}
 {% endblock %}
+
+{% block secondary_content %}
+  {% snippet 'package/snippets/info.html', pkg=pkg, action='package_edit' %}
+{% endblock %}
diff --git a/ckan/templates/package/new_resource.html b/ckan/templates/package/new_resource.html
index 52a114e39a9..969aa5d086f 100644
--- a/ckan/templates/package/new_resource.html
+++ b/ckan/templates/package/new_resource.html
@@ -11,14 +11,16 @@
 {% block form %}{% snippet 'package/snippets/resource_form.html', data=data, errors=errors, error_summary=error_summary, include_metadata=false, pkg_name=pkg_name, stage=stage, allow_upload=logged_in %}{% endblock %}
 
 {% block secondary_content %}
-  <section class="module module-narrow module-shallow">
-    <h2 class="module-heading"><i class="icon-large icon-info-sign"></i> {{ _('What is data?') }}</h2>
-    <div class="module-content">
-      <p>{{ _('Data can be any file or link to a file containing useful data.') }}</p>
-    </div>
-  </section>
-
-  {% snippet 'package/snippets/resources.html', pkg=pkg_dict, action='resource_edit', active=data.id %}
+  {% if pkg_dict %}
+    {% snippet 'package/snippets/info.html', pkg=pkg_dict, action='resource_new' %}
+  {% else %}  
+    <section class="module module-narrow module-shallow">
+      <h2 class="module-heading"><i class="icon-large icon-info-sign"></i> {{ _('What\'s a resource?') }}</h2>
+      <div class="module-content">
+        <p>{{ _('A resource can be any file or link to a file containing useful data.') }}</p>
+      </div>
+    </section>
+  {% endif %}
 {% endblock %}
 
 {% block scripts %}
diff --git a/ckan/templates/package/read.html b/ckan/templates/package/read.html
index ab2fa6589a8..b59c2604cde 100644
--- a/ckan/templates/package/read.html
+++ b/ckan/templates/package/read.html
@@ -15,10 +15,7 @@
   {# NOTE: Not implemented in stage 1 #}
   {# <li>{% link_for _('History'), controller='package', action='history', id=pkg.name, class_='btn', icon='undo' %}</li> #}
   {% if h.check_access('package_update', {'id':pkg.id }) %}
-    <li>{% link_for _('Edit'), controller='package', action='edit', id=pkg.name, class_='btn', icon='wrench' %}</li>
-  {% endif %}
-  {% if h.check_access('package_update', {'id':pkg.id }) %}
-    <li>{% link_for _('Add resource'), controller='package', action='new_resource', id=pkg.name, class_='btn', icon='plus' %}</li>
+    <li>{% link_for _('Edit'), controller='package', action='edit', id=pkg.name, class_='btn btn-primary', icon='wrench' %}</li>
   {% endif %}
 {% endblock %}
 
diff --git a/ckan/templates/package/resource_edit.html b/ckan/templates/package/resource_edit.html
index b0cde7beb0d..701b751d23d 100644
--- a/ckan/templates/package/resource_edit.html
+++ b/ckan/templates/package/resource_edit.html
@@ -12,8 +12,12 @@
 {% endblock %}
 
 {% block actions_content %}
-  <li>{% link_for _('Back to resource'), controller='package', action='resource_read', id=pkg_dict.name, resource_id=res.id, class_='btn', icon='caret-left' %}</li>
+  <li>{% link_for _('View resource'), controller='package', action='resource_read', id=pkg_dict.name, resource_id=res.id, class_='btn', icon='eye-open' %}</li>
 {% endblock %}
 
 {# logged_in is defined in new_resource.html #}
 {% block form %}{{ h.snippet('package/snippets/resource_edit_form.html', data=data, errors=errors, error_summary=error_summary, pkg_name=pkg_dict.name, form_action=c.form_action, allow_upload=logged_in) }}{% endblock %}
+
+{% block secondary_content %}
+  {% snippet 'package/snippets/info.html', pkg=pkg_dict, active=data.id, action='resource_edit' %}
+{% endblock %}
\ No newline at end of file
diff --git a/ckan/templates/package/snippets/back_to_package_action.html b/ckan/templates/package/snippets/back_to_package_action.html
index 81e0e968a14..c16c20d9ddb 100644
--- a/ckan/templates/package/snippets/back_to_package_action.html
+++ b/ckan/templates/package/snippets/back_to_package_action.html
@@ -1 +1 @@
-<li>{% link_for _('Back to dataset'), controller='package', action='read', id=pkg.name, class_='btn', icon='caret-left' %}</li>
+<li>{% link_for _('View dataset'), controller='package', action='read', id=pkg.name, class_='btn', icon='eye-open' %}</li>
diff --git a/ckan/templates/package/snippets/info.html b/ckan/templates/package/snippets/info.html
new file mode 100644
index 00000000000..52bdc13a462
--- /dev/null
+++ b/ckan/templates/package/snippets/info.html
@@ -0,0 +1,32 @@
+{#
+Displays a sidebard module with information for given package
+
+pkg    - The package dict that owns the resources.
+active - The active resource.
+action - The action that this is coming from.
+
+Example:
+
+  {% snippet "package/snippets/info.html", pkg=pkg %}
+
+#}
+{% if pkg and h.check_access('package_update', {'id':pkg.id }) %}
+  <section class="module module-narrow package-info">
+    <h2 class="module-heading"><i class="icon-sitemap"></i> {{ _("Edit Dataset") }}</h2>
+    <ul class="unstyled nav nav-simple">
+      <li class="nav-item{% if action == 'package_edit' %} active{% endif %}">
+        {% link_for h.dataset_display_name(pkg)|truncate(30), controller='package', action='edit', id=pkg.name %}
+      </li>
+    </ul>
+    {% set resources = pkg.resources or [] %}
+    <h2 class="module-heading module-heading-secondary"><i class="icon-copy"></i> {{ _("Edit Resources") }}</h2>
+    <ul class="unstyled nav nav-simple">
+      {% for resource in resources %}
+        <li class="nav-item{{ ' active' if active == resource.id }}">
+          {% link_for h.resource_display_name(resource)|truncate(30), controller='package', action='resource_edit', id=pkg.name, resource_id=resource.id, inner_span=true %}
+        </li>
+      {% endfor %}
+      <li><a href="{{ h.url_for(controller='package', action='new_resource', id=pkg.name)  }}" class="btn{% if action == 'resource_new' %} disabled{% endif %}"><i class="icon-plus"></i> {{ _('Add new resource') }}</a></li>
+    </ul>
+  </section>
+{% endif %}
\ No newline at end of file
diff --git a/ckan/templates/package/snippets/resources.html b/ckan/templates/package/snippets/resources.html
index b6585018dc2..91939dc4cb3 100644
--- a/ckan/templates/package/snippets/resources.html
+++ b/ckan/templates/package/snippets/resources.html
@@ -14,7 +14,7 @@
 {% set resources = pkg.resources or [] %}
 {% if resources %}
   <section class="module module-narrow resources">
-    <h2 class="module-heading"><i class="icon-sitemap"></i> {{ _("Resources") }}</h2>
+    <h2 class="module-heading"><i class="icon-copy"></i> {{ _("Resources") }}</h2>
     <ul class="unstyled nav nav-simple">
       {% for resource in resources %}
         <li class="nav-item{{ ' active' if active == resource.id }}">

From fa29c2bd9792b8f2dfd2207a1ec6e0e947519197 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Fri, 5 Oct 2012 14:09:20 +0100
Subject: [PATCH 17/53] Fix peer permissions problems with paster command

---
 ckanext/datastore/commands.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py
index 2801ef8f072..d807ddade51 100644
--- a/ckanext/datastore/commands.py
+++ b/ckanext/datastore/commands.py
@@ -114,14 +114,17 @@ def _run_sql(self, sql, as_sql_user, database='postgres'):
         if self.verbose:
             print "Executing: \n#####\n", sql, "\n####\nOn database:", database
         if not self.simulate:
-            self._run_cmd("psql --username='{username}' --dbname='{database}' -W".format(
+            self._run_cmd("sudo -u '{username}' psql --dbname='{database}' -W".format(
                 username=as_sql_user,
                 database=database
             ), inputstring=sql)
 
     def create_db(self):
-        sql = 'create database "{0}"'.format(self.db_write_url_parts['db_name'])
-        self._run_sql(sql, as_sql_user=self.sql_superuser)
+        cmd = "sudo -u {pg_user} createdb -O '{ckan_user}' '{db}'".format(
+        pg_user=self.sql_superuser,
+        db=self.db_write_url_parts['db_name'],
+        ckan_user=self.db_ckan_url_parts['db_user'])
+        self._run_cmd(cmd)
 
     def create_read_only_user(self):
         password = self.db_read_url_parts['db_pass']

From c1a5a640f5a1f73d6468294a4371bf213884c70a Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Fri, 5 Oct 2012 15:10:47 +0100
Subject: [PATCH 18/53] Better error reporting and duplicate alias testing.

---
 ckanext/datastore/db.py                   | 55 +++++++++++----
 ckanext/datastore/tests/test_datastore.py | 83 +++++++++++++++++++----
 2 files changed, 110 insertions(+), 28 deletions(-)

diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py
index fa711d62bb7..24f0d74bb09 100644
--- a/ckanext/datastore/db.py
+++ b/ckanext/datastore/db.py
@@ -244,11 +244,11 @@ def check_fields(context, fields):
     for field in fields:
         if field.get('type') and not _is_valid_pg_type(context, field['type']):
             raise ValidationError({
-                'fields': ['{0} is not a valid field type'.format(field['type'])]
+                'fields': ['"{0}" is not a valid field type'.format(field['type'])]
             })
         elif not _is_valid_field_name(field['id']):
             raise ValidationError({
-                'fields': ['{0} is not a valid field name'.format(field['id'])]
+                'fields': ['"{0}" is not a valid field name'.format(field['id'])]
             })
 
 
@@ -288,7 +288,7 @@ def create_table(context, data_dict):
         if 'type' not in field:
             if not records or field['id'] not in records[0]:
                 raise ValidationError({
-                    'fields': ['{0} type not guessable'.format(field['id'])]
+                    'fields': ['"{0}" type not guessable'.format(field['id'])]
                 })
             field['type'] = _guess_type(records[0][field['id']])
 
@@ -319,12 +319,24 @@ def create_table(context, data_dict):
 
 
 def _get_aliases(context, data_dict):
+    ''' Get a list of aliases for a resource. '''
     res_id = data_dict['resource_id']
-    alias_sql = sqlalchemy.text(u'SELECT name FROM "_table_metadata" WHERE alias_of = :id')
+    alias_sql = sqlalchemy.text(
+        u'SELECT name FROM "_table_metadata" WHERE alias_of = :id')
     results = context['connection'].execute(alias_sql, id=res_id).fetchall()
     return [x[0] for x in results]
 
 
+def _get_resources(context, alias):
+    ''' Get a list of resources for an alias. There could be more than one alias
+    in a resource_dict. '''
+    alias_sql = sqlalchemy.text(
+        u'''SELECT alias_of FROM "_table_metadata"
+        WHERE name = :alias AND alias_of IS NOT NULL''')
+    results = context['connection'].execute(alias_sql, alias=alias).fetchall()
+    return [x[0] for x in results]
+
+
 def create_alias(context, data_dict):
     aliases = _get_list(data_dict.get('aliases'))
     if aliases != None:
@@ -339,6 +351,13 @@ def create_alias(context, data_dict):
                 alias=alias,
                 main=data_dict['resource_id']
             )
+
+            res_ids = _get_resources(context, alias)
+            if res_ids:
+                raise ValidationError({
+                    'alias': [(u'The alias "{0}" already exists.').format(alias)]
+                })
+
             context['connection'].execute(sql_alias_string)
 
 
@@ -384,7 +403,7 @@ def create_indexes(context, data_dict):
         for field in index_fields:
             if field not in field_ids:
                 raise ValidationError({
-                    'index': [('The field {0} is not a valid column name.').format(
+                    'index': [('The field "{0}" is not a valid column name.').format(
                         index)]
                 })
         fields_string = u', '.join([
@@ -450,7 +469,7 @@ def alter_table(context, data_dict):
         if 'type' not in field:
             if not records or field['id'] not in records[0]:
                 raise ValidationError({
-                    'fields': ['{0} type not guessable'.format(field['id'])]
+                    'fields': ['"{0}" type not guessable'.format(field['id'])]
                 })
             field['type'] = _guess_type(records[0][field['id']])
         new_fields.append(field)
@@ -491,7 +510,7 @@ def upsert_data(context, data_dict):
 
     if method not in _methods:
         raise ValidationError({
-            'method': [u'{0} is not defined'.format(method)]
+            'method': [u'"{0}" is not defined'.format(method)]
         })
 
     fields = _get_fields(context, data_dict)
@@ -631,14 +650,14 @@ def _validate_record(record, num, field_names):
     # check record for sanity
     if not isinstance(record, dict):
         raise ValidationError({
-            'records': [u'row {0} is not a json object'.format(num)]
+            'records': [u'row "{0}" is not a json object'.format(num)]
         })
     ## check for extra fields in data
     extra_keys = set(record.keys()) - set(field_names)
 
     if extra_keys:
         raise ValidationError({
-            'records': [u'row {0} has extra keys "{1}"'.format(
+            'records': [u'row "{0}" has extra keys "{1}"'.format(
                 num + 1,
                 ', '.join(list(extra_keys))
             )]
@@ -727,7 +746,7 @@ def _sort(context, data_dict, field_ids):
 
         if field not in field_ids:
             raise ValidationError({
-                'sort': [u'field {0} not it table'.format(
+                'sort': [u'field "{0}" not it table'.format(
                     unicode(field, 'utf-8'))]
             })
         if sort.lower() not in ('asc', 'desc'):
@@ -923,7 +942,7 @@ def create(context, data_dict):
         if ('duplicate key value violates unique constraint' in str(e)
                 or 'could not create unique index' in str(e)):
             raise ValidationError({
-                'constraints': ['Cannot insert records because of uniqueness constraint'],
+                'constraints': ['Cannot insert records or create index because of uniqueness constraint'],
                 'info': {
                     'details': str(e)
                 }
@@ -961,6 +980,16 @@ def upsert(context, data_dict):
         upsert_data(context, data_dict)
         trans.commit()
         return _unrename_json_field(data_dict)
+    except IntegrityError, e:
+        if 'duplicate key value violates unique constraint' in str(e):
+            raise ValidationError({
+                'constraints': ['Cannot insert records because of uniqueness constraint'],
+                'info': {
+                    'details': str(e)
+                }
+            })
+        else:
+            raise
     except Exception, e:
         trans.rollback()
         if 'due to statement timeout' in str(e):
@@ -986,7 +1015,7 @@ def delete(context, data_dict):
         ).fetchone()
         if not result:
             raise ValidationError({
-                'resource_id': [u'table for resource {0} does not exist'.format(
+                'resource_id': [u'table for resource "{0}" does not exist'.format(
                     data_dict['resource_id'])]
             })
         if not 'filters' in data_dict:
@@ -1022,7 +1051,7 @@ def search(context, data_dict):
         ).fetchone()
         if not result:
             raise ValidationError({
-                'resource_id': [u'table for resource {0} does not exist'.format(
+                'resource_id': [u'table for resource "{0}" does not exist'.format(
                     data_dict['resource_id'])]
             })
         return search_data(context, data_dict)
diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py
index 0ba571698aa..a087743b04d 100644
--- a/ckanext/datastore/tests/test_datastore.py
+++ b/ckanext/datastore/tests/test_datastore.py
@@ -258,6 +258,34 @@ def test_create_invalid_unique_index(self):
         res_dict = json.loads(res.body)
         assert res_dict['success'] is False
 
+    def test_create_alias_twice(self):
+        resource = model.Package.get('annakarenina').resources[1]
+        data = {
+            'resource_id': resource.id,
+            'aliases': 'new_alias',
+            'fields': [{'id': 'book', 'type': 'text'},
+                       {'id': 'author', 'type': 'text'}]
+        }
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True, res_dict
+
+        resource = model.Package.get('annakarenina').resources[0]
+        data = {
+            'resource_id': resource.id,
+            'aliases': 'new_alias',
+            'fields': [{'id': 'more books', 'type': 'text'}]
+        }
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is False, res_dict
+
     def test_create_basic(self):
         resource = model.Package.get('annakarenina').resources[0]
         aliases = [u'great_list_of_books', u'another_list_of_b\xfcks']
@@ -487,6 +515,16 @@ def test_create_basic(self):
 
     def test_guess_types(self):
         resource = model.Package.get('annakarenina').resources[1]
+
+        data = {
+            'resource_id': resource.id
+        }
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_delete', params=postparams,
+                            extra_environ=auth, status="*")  # ignore status
+        res_dict = json.loads(res.body)
+
         data = {
             'resource_id': resource.id,
             'fields': [{'id': 'author', 'type': 'json'},
@@ -853,6 +891,36 @@ def setup_class(cls):
     def teardown_class(cls):
         rebuild_all_dbs(cls.Session)
 
+    def test_insert_non_existing_field(self):
+        data = {
+            'resource_id': self.data['resource_id'],
+            'method': 'insert',
+            'records': [{u'b\xfck': 'the hobbit', 'dummy': 'tolkien'}]
+        }
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_upsert', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is False
+
+    def test_insert_with_index_violation(self):
+        data = {
+            'resource_id': self.data['resource_id'],
+            'method': 'insert',
+            'records': [{u'b\xfck': 'annakarenina'}]
+        }
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_upsert', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is False
+
     def test_insert_basic(self):
         hhguide = u"hitchhiker's guide to the galaxy"
         data = {
@@ -879,21 +947,6 @@ def test_insert_basic(self):
 
         assert results.rowcount == 3
 
-    def test_insert_non_existing_field(self):
-        data = {
-            'resource_id': self.data['resource_id'],
-            'method': 'insert',
-            'records': [{u'b\xfck': 'annakarenina', 'dummy': 'tolkien'}]
-        }
-
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_upsert', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is False
-
 
 class TestDatastoreUpdate(tests.WsgiAppCase):
     sysadmin_user = None

From 6fe81ff07ffc2e8a667441d452500f2a66bce62c Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Sat, 6 Oct 2012 18:18:36 +0100
Subject: [PATCH 19/53] Fix setup with separate users for datastore-write and
 ckan. Also make setup easier (one command).

---
 ckanext/datastore/commands.py | 57 ++++++++++++++++++++++++-----------
 1 file changed, 39 insertions(+), 18 deletions(-)

diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py
index d807ddade51..30868a1c34e 100644
--- a/ckanext/datastore/commands.py
+++ b/ckanext/datastore/commands.py
@@ -32,7 +32,7 @@
 GRANT SELECT ON ALL TABLES IN SCHEMA public TO "{readonlyuser}";
 
 -- grant access to new tables and views by default
-ALTER DEFAULT PRIVILEGES FOR USER "{ckanuser}" IN SCHEMA public
+ALTER DEFAULT PRIVILEGES FOR USER "{writeuser}" IN SCHEMA public
    GRANT SELECT ON TABLES TO "{readonlyuser}";
 '''
 
@@ -40,12 +40,16 @@
 class SetupDatastoreCommand(cli.CkanCommand):
     '''Perform commands to set up the datastore.
     Make sure that the datastore urls are set properly before you run these commands.
+    `create-all` will run create-write-user, create-db and create-read-only-user.
 
     Usage::
 
+        paster datastore create-write-user SQL_SUPER_USER
         paster datastore create-db SQL_SUPER_USER
         paster datastore create-read-only-user SQL_SUPER_USER
 
+        paster datastore create-all SQL_SUPER_USER
+
     Where:
         SQL_SUPER_USER is the name of a postgres user with sufficient
                          permissions to create new tables, users, and grant
@@ -77,38 +81,48 @@ def command(self):
 
         assert self.db_write_url_parts['db_name'] == self.db_read_url_parts['db_name'], "write and read db should be the same"
 
+        if len(self.args) != 2:
+            print self.usage
+            return
+        self.sql_superuser = self.args[1]
+
         if cmd == 'create-db':
-            if len(self.args) != 2:
-                print self.usage
-                return
-            self.sql_superuser = self.args[1]
             self.create_db()
             if self.verbose:
                 print 'Creating DB: SUCCESS'
         elif cmd == 'create-read-only-user':
-            if len(self.args) != 2:
-                print self.usage
-                return
-            self.sql_superuser = self.args[1]
             self.create_read_only_user()
             if self.verbose:
                 print 'Creating read-only user: SUCCESS'
+        elif cmd == 'create-write-user':
+            self.create_write_user()
+            if self.verbose:
+                print 'Creating write user: SUCCESS'
+        elif cmd == 'create-all':
+            self.create_db()
+            self.create_write_user()
+            self.create_read_only_user()
+            if self.verbose:
+                print 'Creating db and users for datastore: SUCCESS'
         else:
             print self.usage
             log.error('Command "%s" not recognized' % (cmd,))
             return
 
     def _run_cmd(self, command_line, inputstring=''):
+        if self.verbose:
+            print "Running:", command_line
         import subprocess
-        p = subprocess.Popen(
-            command_line, shell=True,
-            stdin=subprocess.PIPE,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE)
-        stdout_value, stderr_value = p.communicate(input=inputstring)
-        if stderr_value:
-            print '\nAn error occured: {0}'.format(stderr_value)
-            sys.exit(1)
+        if not self.simulate:
+            p = subprocess.Popen(
+                command_line, shell=True,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE)
+            stdout_value, stderr_value = p.communicate(input=inputstring)
+            if stderr_value:
+                print '\nAn error occured: {0}'.format(stderr_value)
+                sys.exit(1)
 
     def _run_sql(self, sql, as_sql_user, database='postgres'):
         if self.verbose:
@@ -126,6 +140,13 @@ def create_db(self):
         ckan_user=self.db_ckan_url_parts['db_user'])
         self._run_cmd(cmd)
 
+    def create_write_user(self):
+        cmd = "sudo -u {pg_user} createuser \
+            --no-createdb --no-createrole --no-superuser '{write_user}'".format(
+        pg_user=self.sql_superuser,
+        write_user=self.db_write_url_parts['db_user'])
+        self._run_cmd(cmd)
+
     def create_read_only_user(self):
         password = self.db_read_url_parts['db_pass']
         self.validate_password(password)

From 7cf6fe87e60d5715df4476512836c9b1e226c666 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Sat, 6 Oct 2012 18:23:42 +0100
Subject: [PATCH 20/53] Consistent naming

---
 ckanext/datastore/commands.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py
index 30868a1c34e..179ec79647c 100644
--- a/ckanext/datastore/commands.py
+++ b/ckanext/datastore/commands.py
@@ -134,17 +134,17 @@ def _run_sql(self, sql, as_sql_user, database='postgres'):
             ), inputstring=sql)
 
     def create_db(self):
-        cmd = "sudo -u {pg_user} createdb -O '{ckan_user}' '{db}'".format(
-        pg_user=self.sql_superuser,
+        cmd = "sudo -u {pguser} createdb -O '{ckanuser}' '{db}'".format(
+        pguser=self.sql_superuser,
         db=self.db_write_url_parts['db_name'],
-        ckan_user=self.db_ckan_url_parts['db_user'])
+        ckanuser=self.db_ckan_url_parts['db_user'])
         self._run_cmd(cmd)
 
     def create_write_user(self):
-        cmd = "sudo -u {pg_user} createuser \
-            --no-createdb --no-createrole --no-superuser '{write_user}'".format(
-        pg_user=self.sql_superuser,
-        write_user=self.db_write_url_parts['db_user'])
+        cmd = "sudo -u {pguser} createuser \
+            --no-createdb --no-createrole --no-superuser '{writeuser}'".format(
+        pguser=self.sql_superuser,
+        writeuser=self.db_write_url_parts['db_user'])
         self._run_cmd(cmd)
 
     def create_read_only_user(self):

From 44fef830e6ce84e5248bd37c8acdf794f5515282 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Sat, 6 Oct 2012 19:36:26 +0100
Subject: [PATCH 21/53] Improved and fixed documentation. Thanks to
 @mwainwright

---
 doc/datastore.rst | 95 +++++++++++++++++++++++------------------------
 1 file changed, 47 insertions(+), 48 deletions(-)

diff --git a/doc/datastore.rst b/doc/datastore.rst
index e41bf1e60ee..0191e860b48 100644
--- a/doc/datastore.rst
+++ b/doc/datastore.rst
@@ -14,9 +14,9 @@ Installation and Configuration
 .. warning:: Make sure that you follow the steps below and make sure that the settings are correct. Wrong settings could lead to serious security issues.
 
 The DataStore in previous lives required a custom setup of ElasticSearch and Nginx,
-but that is no more, as it the relational database management system PostgreSQL.
-However, you should set-up a  separate database for the datastore
-and create a read-only user to make your CKAN installation save.
+but that is no more, as it now uses the relational database management system PostgreSQL.
+However, you should set up a separate database for the DataStore
+and create a read-only user to make your CKAN and DataStore installation safe.
 
 In your ``config`` file ensure that the datastore extension is enabled::
 
@@ -24,26 +24,25 @@ In your ``config`` file ensure that the datastore extension is enabled::
 
 Also ensure that the ``ckan.datastore.write_url`` and ``datastore.read_url`` variables are set::
 
- ckan.datastore.write_url = postgresql://ckanuser:pass@localhost/datastore
+ ckan.datastore.write_url = postgresql://writeuser:pass@localhost/datastore
  ckan.datastore.read_url = postgresql://readonlyuser:pass@localhost/datastore
 
-A few things have to be kept in mind
+A few things have to be kept in mind:
 
-* The datastore cannot be on the CKAN database (except for testing)
-* The write user (i.e. ``ckanuser``) and read-only user (i.e. ``readonlyuser``) cannot be the same
+* The DataStore cannot be on the CKAN database (except for testing)
+* The write user (i.e. ``writeuser``) and read-only user (i.e. ``readonlyuser``) cannot be the same
 
-To create a new database and a read-only user, use the provided paster commands after you have set the right database URLs.::
+To create a new database with a write user and read-only user, use this paster command::
 
- paster datastore create-db SQL_SUPER_USER
- paster datastore create-read-only-user SQL_SUPER_USER
+ paster datastore create-all SQL_SUPER_USER
 
-To test the setup you can create a new datastore, to do so you can run the following command::
+To test the set-up you can create a new DataStore. To do so you can run the following command::
 
  curl -X POST http://127.0.0.1:5000/api/3/action/datastore_create -H "Authorization: {YOUR-API-KEY}" -d '{"resource_id": "{RESOURCE-ID}", "fields": [ {"id": "a"}, {"id": "b"} ], "records": [ { "a": 1, "b": "xyz"}, {"a": 2, "b": "zzz"} ]}'
 
-A table named after the resource id should have been created on your datastore
-database, and visiting this URL should return a response from the datastore with
-the previous records::
+A table named after the resource id should have been created on your DataStore
+database. Visiting the following URL should return a response from the DataStore with
+the records inserted above::
 
  http://127.0.0.1:5000/api/3/action/datastore_search?resource_id={RESOURCE_ID}
 
@@ -55,11 +54,11 @@ The DataStore is distinct but complementary to the FileStore (see
 :doc:`filestore`). In contrast to the the FileStore which provides 'blob'
 storage of whole files with no way to access or query parts of that file, the
 DataStore is like a database in which individual data elements are accessible
-and queryable. To illustrate this distinction consider storing a spreadsheet
-file like a CSV or Excel document. In the FileStore this filed would be stored
+and queryable. To illustrate this distinction, consider storing a spreadsheet
+file like a CSV or Excel document. In the FileStore this file would be stored
 directly. To access it you would download the file as a whole. By contrast, if
-the spreadsheet data is stored in the DataStore one would be able to access
-individual spreadsheet rows via a simple web-api as well as being able to make
+the spreadsheet data is stored in the DataStore, one would be able to access
+individual spreadsheet rows via a simple web API, as well as being able to make
 queries over the spreadsheet contents.
 
 
@@ -71,7 +70,7 @@ uploaded to the :doc:`FileStore <filestore>`) to be automatically added to the
 DataStore. This requires some processing, to extract the data from your files
 and to add it to the DataStore in the format the DataStore can handle.
 
-This task of automatically parsing and then adding data to the datastore is
+This task of automatically parsing and then adding data to the DataStore is
 performed by a DataStorer, a queue process that runs asynchronously and can be
 triggered by uploads or other activities. The DataStorer is an extension and can
 be found, along with installation instructions, at: https://github.com/okfn/ckanext-datastorer
@@ -80,18 +79,18 @@ be found, along with installation instructions, at: https://github.com/okfn/ckan
 The DataStore Data API
 ======================
 
-The DataStore's Data API, which derives from the underlying data-table,
+The DataStore's Data API, which derives from the underlying data table,
 is RESTful and JSON-based with extensive query capabilities.
 
 Each resource in a CKAN instance can have an associated DataStore 'table'. The
-basic API for accessing the DataStore is detailed below. For a detailed
+basic API for accessing the DataStore is outlined below. For a detailed
 tutorial on using this API see :doc:`using-data-api`.
 
 
 API Reference
 -------------
 
-.. note:: Lists can always be expressed in different ways. It is possible to use lists, comma separated strings or single items. These are valid lists: ``['foo', 'bar']``, ``foo, bar``, ``"foo", "bar"`` and ``foo``.
+.. note:: Lists can always be expressed in different ways. It is possible to use lists, comma separated strings or single items. These are valid lists: ``['foo', 'bar']``, ``'foo, bar'``, ``"foo", "bar"`` and ``'foo'``.
 
 
 datastore_create
@@ -103,7 +102,7 @@ The datastore_create API endpoint allows a user to post JSON data to be stored a
     resource_id: resource_id, # the data is going to be stored against.
     aliases: # list of names for read only aliases to the resource
     fields: [] # a list of dictionaries of fields/columns and their extra metadata.
-    records: [] # a list of dictionaries of the data, eg:  [{"dob": "2005", "some_stuff": ['a', b']}, ..]
+    records: [] # a list of dictionaries of the data, eg:  [{"dob": "2005", "some_stuff": ['a', 'b']}, ..]
     primary_key: # list of fields that represent a unique key
     indexes: # indexes on table
  }
@@ -115,12 +114,12 @@ See :ref:`fields` and :ref:`records` for details on how to lay out records.
 datastore_delete
 ~~~~~~~~~~~~~~~~
 
-The datastore_delete API endpoint allows a user to delete from a resource. The JSON for searching must be in the following form::
+The datastore_delete API endpoint allows a user to delete records from a resource. The JSON for searching must be in the following form::
 
  {
     resource_id: resource_id # the data that is going to be deleted.
     filter: # dictionary of matching conditions to delete
-            # e.g  {'key1': 'a. 'key2': 'b'}
+            # e.g  {'key1': 'a', 'key2': 'b'}
             # this will be equivalent to "delete from table where key1 = 'a' and key2 = 'b' "
  }
 
@@ -128,29 +127,29 @@ The datastore_delete API endpoint allows a user to delete from a resource. The J
 datastore_upsert
 ~~~~~~~~~~~~~~~~
 
-The datastore_upsert API endpoint allows a user to add data to an existing datastore resource. In order for the upsert and update to work a unique key has to defined via the datastore_create API endpoint command.
+The datastore_upsert API endpoint allows a user to add or edit records in an existing DataStore resource. In order for the ``upsert`` and ``update`` methods to work, a unique key has to defined via the datastore_create API endpoint command.
 The JSON for searching must be in the following form::
 
  {
     resource_id: resource_id # resource id that the data is going to be stored under.
-    records: [] # a list of dictionaries of the data, eg:  [{"dob": "2005", "some_stuff": ['a', b']}, ..]
+    records: [] # a list of dictionaries of the data, eg:  [{"dob": "2005", "some_stuff": ['a', 'b']}, ..]
     method: # the method to use to put the data into the datastore
             # possible options: upsert (default), insert, update
  }
 
-upsert
+``upsert``
     Update if record with same key already exists, otherwise insert. Requires unique key.
-insert
-    Insert only. This method is faster that upsert because checks are omitted. Does *not* require a unique key.
-update
-    Update only. Exception will occur if the key that should be updated does not exist. Requires unique key.
+``insert``
+    Insert only. This method is faster that upsert, but will fail if any inserted record matches an existing one. Does *not* require a unique key.
+``update``
+    Update only. An exception will occur if the key that should be updated does not exist. Requires unique key.
 
 .. _datastore_search:
 
 datastore_search
 ~~~~~~~~~~~~~~~~
 
-The datastore_search API endpoint allows a user to search data at a resource.
+The datastore_search API endpoint allows a user to search data in a resource.
 The JSON for searching must be in the following form::
 
  {
@@ -171,7 +170,7 @@ The JSON for searching must be in the following form::
 datastore_search_sql
 ~~~~~~~~~~~~~~~~~~~~
 
-The datastore_search_sql API endpoint allows a user to search data at a resource or connect multiple resources with join expressions. The underlying SQL engine is the `PostgreSQL engine <http://www.postgresql.org/docs/9.1/interactive/sql/.html>`_. The JSON for searching must be in the following form::
+The datastore_search_sql API endpoint allows a user to search data in a resource or connect multiple resources with join expressions. The underlying SQL engine is the `PostgreSQL engine <http://www.postgresql.org/docs/9.1/interactive/sql/.html>`_. The JSON for searching must be in the following form::
 
  {
     sql: # a single sql select statement
@@ -183,9 +182,9 @@ The datastore_search_sql API endpoint allows a user to search data at a resource
 datastore_search_htsql
 ~~~~~~~~~~~~~~~~~~~~~~
 
-.. note:: HTSQL is not in the core datastore and has to be installed as an extension. The extension is available on https://github.com/okfn/ckanext-htsql.
+.. note:: HTSQL is not in the core DataStore. To use it, it is necessary to install the ckanext-htsql extension available at https://github.com/okfn/ckanext-htsql.
 
-The datastore_search_htsql API endpoint allows a user to search data at a resource using the `HTSQL <http://htsql.org/doc/>`_ query expression language. The JSON for searching must be in the following form::
+The datastore_search_htsql API endpoint allows a user to search data in a resource using the `HTSQL <http://htsql.org/doc/>`_ query expression language. The JSON for searching must be in the following form::
 
  {
     htsql: # a htsql query statement.
@@ -196,14 +195,14 @@ The datastore_search_htsql API endpoint allows a user to search data at a resour
 Fields
 ~~~~~~
 
-Fields define the column names and the type of the data in a column. They are defined as an array of fields. One field is defined as follows::
+Fields define the column names and the type of the data in a column. A field is defined as follows::
 
     {
         "id": # a string which defines the column name
         "type": # the data type for the column
     }
 
-Field **types are optional** and will be guessed by the provided data. However, setting the types ensures that future inserts to not fail because of wrong types. See :ref:`valid-types` for details on which types are valid.
+Field **types are optional** and will be guessed by the DataStore from the provided data. However, setting the types ensures that future inserts will not fail because of wrong types. See :ref:`valid-types` for details on which types are valid.
 
 Example::
 
@@ -223,7 +222,7 @@ Example::
 Records
 ~~~~~~~
 
-Records are defined as an array of records. One record is the data to be inserted in a table and is defined as follows::
+A record is the data to be inserted in a table and is defined as follows::
 
     {
         "<id>": # data to be set
@@ -235,7 +234,7 @@ Example::
     [
         {
             "foo": 100,
-            "bar": "I'm a text."
+            "bar": "Here's some text"
         },
         {
             "foo": 42
@@ -247,16 +246,16 @@ Example::
 Field types
 -----------
 
-The datastore supports all types supported by PostgreSQL as well as a few additions. A list of the PostgreSQL types can be found in the `type section of the documentation`_. Below you can find a list of the most common data types. The ``json`` type has been added as a storage for nested data.
+The DataStore supports all types supported by PostgreSQL as well as a few additions. A list of the PostgreSQL types can be found in the `type section of the documentation`_. Below you can find a list of the most common data types. The ``json`` type has been added as a storage for nested data.
 
 .. _type section of the documentation: http://www.postgresql.org/docs/9.1/static/datatype.html
 
 
 text
-    Arbitrary text data, e.g. ``I'm a text``.
+    Arbitrary text data, e.g. ``Here's some text``.
 json
     Arbitrary nested json data, e.g ``{"foo": 42, "bar": [1, 2, 3]}``.
-    Please note that this type is a custom type that is wrapped by the datastore.
+    Please note that this type is a custom type that is wrapped by the DataStore.
 date
     Date without time, e.g ``2012-5-25``.
 time
@@ -268,23 +267,23 @@ int
 float
     Floats, e.g. ``1.61803``.
 bool
-    Boolen values, e.g. ``true``, ``0``
+    Boolean values, e.g. ``true``, ``0``
 
 
-You can find more information about the formatting of dates in the `date/time types section of the documentation`_.
+You can find more information about the formatting of dates in the `date/time types section of the PostgreSQL documentation`_.
 
-.. _date/time types section of the documentation: http://www.postgresql.org/docs/9.1/static/datatype-datetime.html
+.. _date/time types section of the PostgreSQL documentation: http://www.postgresql.org/docs/9.1/static/datatype-datetime.html
 
 
 Table aliases
 -------------
 
-Resources in the datastore can have multiple aliases that are easier to remember than the resource id. Aliases can be created and edited with the datastore_create API endpoint. All aliases can be found in a special view called ``_table_metadata``.
+A resource in the DataStore can have multiple aliases that are easier to remember than the resource id. Aliases can be created and edited with the datastore_create API endpoint. All aliases can be found in a special view called ``_table_metadata``.
 
 Comparison of different querying methods
 ----------------------------------------
 
-The datastore supports querying with the datastore_search and datastore_search_sql API endpoint. They are similar but support different features. The following list gives an overview on the different methods.
+The DataStore supports querying with multiple API endpoints. They are similar but support different features. The following list gives an overview of the different methods.
 
 ==============================  =======================  ===========================  =============================
 ..                              :ref:`datastore_search`  :ref:`datastore_search_sql`  :ref:`datastore_search_htsql`

From 6317fc1185eee20275994592cc82861266fdf703 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Sat, 6 Oct 2012 23:11:06 +0100
Subject: [PATCH 22/53] Make docs more readable

---
 doc/datastore.rst | 60 ++++++++++++++++++++++++-----------------------
 1 file changed, 31 insertions(+), 29 deletions(-)

diff --git a/doc/datastore.rst b/doc/datastore.rst
index 0191e860b48..b956210a137 100644
--- a/doc/datastore.rst
+++ b/doc/datastore.rst
@@ -99,12 +99,12 @@ datastore_create
 The datastore_create API endpoint allows a user to post JSON data to be stored against a resource. This endpoint also supports altering tables, aliases and indexes and bulk insertion. The JSON must be in the following form::
 
  {
-    resource_id: resource_id, # the data is going to be stored against.
-    aliases: # list of names for read only aliases to the resource
-    fields: [] # a list of dictionaries of fields/columns and their extra metadata.
-    records: [] # a list of dictionaries of the data, eg:  [{"dob": "2005", "some_stuff": ['a', 'b']}, ..]
-    primary_key: # list of fields that represent a unique key
-    indexes: # indexes on table
+    resource_id:  # the data is going to be stored against.
+    aliases:      # list of names for read only aliases to the resource
+    fields:       # a list of dictionaries of fields/columns and their extra metadata.
+    records:      # a list of dictionaries of the data, eg:  [{"dob": "2005", "some_stuff": ['a', 'b']}, ..]
+    primary_key:  # list of fields that represent a unique key
+    indexes:      # indexes on table
  }
 
 See :ref:`fields` and :ref:`records` for details on how to lay out records.
@@ -117,10 +117,10 @@ datastore_delete
 The datastore_delete API endpoint allows a user to delete records from a resource. The JSON for searching must be in the following form::
 
  {
-    resource_id: resource_id # the data that is going to be deleted.
-    filter: # dictionary of matching conditions to delete
-            # e.g  {'key1': 'a', 'key2': 'b'}
-            # this will be equivalent to "delete from table where key1 = 'a' and key2 = 'b' "
+    resource_id: # the data that is going to be deleted.
+    filter:      # dictionary of matching conditions to delete
+                 # e.g  {'key1': 'a', 'key2': 'b'}
+                 # this will be equivalent to "delete from table where key1 = 'a' and key2 = 'b' "
  }
 
 
@@ -131,10 +131,10 @@ The datastore_upsert API endpoint allows a user to add or edit records in an exi
 The JSON for searching must be in the following form::
 
  {
-    resource_id: resource_id # resource id that the data is going to be stored under.
-    records: [] # a list of dictionaries of the data, eg:  [{"dob": "2005", "some_stuff": ['a', 'b']}, ..]
-    method: # the method to use to put the data into the datastore
-            # possible options: upsert (default), insert, update
+    resource_id: # resource id that the data is going to be stored under.
+    records:     #  a list of dictionaries of the data, eg:  [{"dob": "2005", "some_stuff": ['a', 'b']}, ..]
+    method:      #  the method to use to put the data into the datastore
+                 #  possible options: upsert (default), insert, update
  }
 
 ``upsert``
@@ -153,16 +153,16 @@ The datastore_search API endpoint allows a user to search data in a resource.
 The JSON for searching must be in the following form::
 
  {
-     resource_id: # the resource id to be searched against
-     filters : # dictionary of matching conditions to select e.g  {'key1': 'a. 'key2': 'b'}
-        # this will be equivalent to "select * from table where key1 = 'a' and key2 = 'b' "
-     q: # full text query
-     plain: # treat as plain text query (default: true)
-     language: # language of the full text query (default: english)
-     limit: # limit the amount of rows to size (default: 100)
-     offset: # offset the amount of rows
-     fields:  # list of fields return in that order, defaults (empty or not present) to all fields in fields order.
-     sort: # ordered list of field names as, eg: "fieldname1, fieldname2 desc"
+     resource_id:  # the resource id to be searched against
+     filters :     # dictionary of matching conditions to select e.g  {'key1': 'a. 'key2': 'b'}
+                   # this will be equivalent to "select * from table where key1 = 'a' and key2 = 'b' "
+     q:            # full text query
+     plain:        # treat as plain text query (default: true)
+     language:     # language of the full text query (default: english)
+     limit:        # limit the amount of rows to size (default: 100)
+     offset:       # offset the amount of rows
+     fields:       # list of fields return in that order, defaults (empty or not present) to all fields in fields order.
+     sort:         # ordered list of field names as, eg: "fieldname1, fieldname2 desc"
  }
 
 .. _datastore_search_sql:
@@ -173,7 +173,7 @@ datastore_search_sql
 The datastore_search_sql API endpoint allows a user to search data in a resource or connect multiple resources with join expressions. The underlying SQL engine is the `PostgreSQL engine <http://www.postgresql.org/docs/9.1/interactive/sql/.html>`_. The JSON for searching must be in the following form::
 
  {
-    sql: # a single sql select statement
+    sql:  # a single sql select statement
  }
 
 
@@ -187,7 +187,7 @@ datastore_search_htsql
 The datastore_search_htsql API endpoint allows a user to search data in a resource using the `HTSQL <http://htsql.org/doc/>`_ query expression language. The JSON for searching must be in the following form::
 
  {
-    htsql: # a htsql query statement.
+    htsql:  # a htsql query statement.
  }
 
 .. _fields:
@@ -198,8 +198,8 @@ Fields
 Fields define the column names and the type of the data in a column. A field is defined as follows::
 
     {
-        "id": # a string which defines the column name
-        "type": # the data type for the column
+        "id":    # a string which defines the column name
+        "type":  # the data type for the column
     }
 
 Field **types are optional** and will be guessed by the DataStore from the provided data. However, setting the types ensures that future inserts will not fail because of wrong types. See :ref:`valid-types` for details on which types are valid.
@@ -225,7 +225,7 @@ Records
 A record is the data to be inserted in a table and is defined as follows::
 
     {
-        "<id>": # data to be set
+        "<id>":  # data to be set
         # .. more data
     }
 
@@ -248,6 +248,8 @@ Field types
 
 The DataStore supports all types supported by PostgreSQL as well as a few additions. A list of the PostgreSQL types can be found in the `type section of the documentation`_. Below you can find a list of the most common data types. The ``json`` type has been added as a storage for nested data.
 
+In addition to the listed types below, you can also use array types. They are defines by prepending a ``_`` or appending ``[]`` or ``[n]`` where n denotes the length of the array. An arbitrarily long array of integers would be defined as ``int[]``.
+
 .. _type section of the documentation: http://www.postgresql.org/docs/9.1/static/datatype.html
 
 

From 5614733cb44131b949c35e038f9b9553cc040331 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Sun, 7 Oct 2012 10:43:36 +0100
Subject: [PATCH 23/53] Skip write user creation if ckan user is write user

---
 ckanext/datastore/commands.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py
index 179ec79647c..1a8bcdd06d6 100644
--- a/ckanext/datastore/commands.py
+++ b/ckanext/datastore/commands.py
@@ -100,7 +100,8 @@ def command(self):
                 print 'Creating write user: SUCCESS'
         elif cmd == 'create-all':
             self.create_db()
-            self.create_write_user()
+            if self.db_ckan_url_parts['db_user'] != self.db_write_url_parts['db_user']:
+                self.create_write_user()
             self.create_read_only_user()
             if self.verbose:
                 print 'Creating db and users for datastore: SUCCESS'

From 416def60bbbc324206d7a321f32d0dce351163e4 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Mon, 8 Oct 2012 11:22:39 +0100
Subject: [PATCH 24/53] Add warning to docs that resources in the ds in private
 datasets are not private.

---
 doc/datastore.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/datastore.rst b/doc/datastore.rst
index b956210a137..45192463948 100644
--- a/doc/datastore.rst
+++ b/doc/datastore.rst
@@ -8,6 +8,8 @@ interface and authorization system.
 
 .. note:: The DataStore requires PostgreSQL 9.0 or later.
 
+.. warning:: The DataStore does not support hiding resources in a private dataset.
+
 Installation and Configuration
 ==============================
 

From a18f077887d9fa58c903bfac666ff79e33741180 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Tue, 9 Oct 2012 17:53:58 +0100
Subject: [PATCH 25/53] move sql script

---
 ...te_datastore_db_and_read_only_user.sql => set_permissions.sql} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename ckanext/datastore/bin/{create_datastore_db_and_read_only_user.sql => set_permissions.sql} (100%)

diff --git a/ckanext/datastore/bin/create_datastore_db_and_read_only_user.sql b/ckanext/datastore/bin/set_permissions.sql
similarity index 100%
rename from ckanext/datastore/bin/create_datastore_db_and_read_only_user.sql
rename to ckanext/datastore/bin/set_permissions.sql

From 2a641ac2d470e134c2cc92dc1132333a376f3630 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Tue, 9 Oct 2012 17:55:08 +0100
Subject: [PATCH 26/53] Change the set-up process as recommended by @kindly

---
 ckanext/datastore/bin/__init__.py         |   0
 ckanext/datastore/bin/datastore_setup.py  |  79 ++++++++++++++
 ckanext/datastore/bin/set_permissions.sql |  56 +++++-----
 ckanext/datastore/commands.py             | 127 +++-------------------
 4 files changed, 120 insertions(+), 142 deletions(-)
 create mode 100644 ckanext/datastore/bin/__init__.py
 create mode 100644 ckanext/datastore/bin/datastore_setup.py

diff --git a/ckanext/datastore/bin/__init__.py b/ckanext/datastore/bin/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/ckanext/datastore/bin/datastore_setup.py b/ckanext/datastore/bin/datastore_setup.py
new file mode 100644
index 00000000000..46d015b61aa
--- /dev/null
+++ b/ckanext/datastore/bin/datastore_setup.py
@@ -0,0 +1,79 @@
+'''
+Setup the right permissions on the datastore db
+'''
+
+import sys
+import os
+import logging
+
+
+def _run_cmd(command_line, inputstring=''):
+    logging.info("Running:", command_line)
+    import subprocess
+    p = subprocess.Popen(
+        command_line, shell=True,
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE)
+    stdout_value, stderr_value = p.communicate(input=inputstring)
+    if stderr_value:
+        print '\nAn error occured: {0}'.format(stderr_value)
+        sys.exit(1)
+
+
+def _run_sql(sql, as_sql_user, database='postgres'):
+    logging.debug("Executing: \n#####\n", sql, "\n####\nOn database:", database)
+    _run_cmd("sudo -u '{username}' psql --dbname='{database}' -W".format(
+        username=as_sql_user,
+        database=database
+    ), inputstring=sql)
+
+
+def set_permissions(pguser, ckandb, datastoredb, ckanuser, writeuser, readonlyuser):
+    __dir__ = os.path.dirname(os.path.abspath(__file__))
+    filepath = os.path.join(__dir__, 'set_permissions.sql')
+    with open(filepath) as f:
+        set_permissions_sql = f.read()
+
+        sql = set_permissions_sql.format(
+            ckandb=ckandb,
+            datastoredb=datastoredb,
+            ckanuser=ckanuser,
+            writeuser=writeuser,
+            readonlyuser=readonlyuser)
+
+        _run_sql(sql,
+                  as_sql_user=pguser,
+                  database=datastoredb)
+
+
+if __name__ == '__main__':
+    import argparse
+    argparser = argparse.ArgumentParser(
+        description='Set the permissions on the CKAN datastore. ',
+        epilog='"The ships hung in the sky in much the same way that bricks don\'t."')
+
+    argparser.add_argument('-p', '--pg_super_user', dest='pguser', default='postgres', type=str,
+                       help="the postgres super user")
+
+    argparser.add_argument(dest='ckandb', default='ckan', type=str,
+                       help="the name of the ckan database")
+    argparser.add_argument(dest='datastoredb', default='datastore', type=str,
+                       help="the name of the datastore database")
+    argparser.add_argument(dest='ckanuser', default='ckanuser', type=str,
+                       help="username of the ckan postgres user")
+    argparser.add_argument(dest='writeuser', default='writeuser', type=str,
+                       help="username of the datastore user that can write")
+    argparser.add_argument(dest='readonlyuser', default='readonlyuser',
+                       help="username of the datastore user who has only read permissions")
+
+    args = argparser.parse_args()
+
+    set_permissions(
+        pguser=args.pguser,
+        ckandb=args.ckandb,
+        datastoredb=args.datastoredb,
+        ckanuser=args.ckanuser,
+        writeuser=args.writeuser,
+        readonlyuser=args.readonlyuser
+    )
diff --git a/ckanext/datastore/bin/set_permissions.sql b/ckanext/datastore/bin/set_permissions.sql
index d6db43b5dd8..08206449a3e 100644
--- a/ckanext/datastore/bin/set_permissions.sql
+++ b/ckanext/datastore/bin/set_permissions.sql
@@ -1,56 +1,54 @@
 /*
-This script creates a new datastore database and
+This script sets-up the permissions for the the datastore.
+
+creates a new datastore database and
 a new read-only user for ckan who will only be able
 to select from the datastore database but has no create/write/edit
 permission or any permissions on other databases.
 
-Please set the variables to you current setup. For testing purposes it
+Please set the variables to you current set-up. For testing purposes it
 is possible to set maindb = datastoredb.
 
 To run the script, execute:
-    sudo -u postgres psql postgres -f create_read_only_user.sql
+    sudo -u postgres psql postgres -f set_permissions.sql
 */
 
-\set maindb "ckan"
--- don't quote the datastoredb variable or create the database separately
-\set datastoredb datastore
-\set ckanuser ckanuser
-\set rouser readonlyuser
-\set ropwd 'pass'
-
--- create the datastore database
-create database :datastoredb;
-
--- switch to the new database
-\c :datastoredb;
-
-/*
--- delete the previous users
-REVOKE CONNECT ON DATABASE :datastoredb FROM :rouser;
-DROP OWNED BY :rouser;
-DROP USER :rouser;
---*/
-
--- revoke permissions for the new user
+-- name of the main CKAN database
+\set maindb "{ckandb}"
+-- the name of the datastore database
+\set datastoredb '{datastoredb}'
+-- username of the ckan postgres user
+\set ckanuser '{ckanuser}'
+-- username of the datastore user that can write
+\set wuser '{writeuser}'
+-- username of the datastore user who has only read permissions
+\set rouser '{readonlyuser}'
+
+-- revoke permissions for the read-only user
+---- this step can be ommitted if the datastore not
+---- on the same server as the CKAN database
 REVOKE CREATE ON SCHEMA public FROM PUBLIC;
 REVOKE USAGE ON SCHEMA public FROM PUBLIC;
 
 GRANT CREATE ON SCHEMA public TO :ckanuser;
 GRANT USAGE ON SCHEMA public TO :ckanuser;
 
--- create new read only user
-CREATE USER :rouser WITH PASSWORD :ropwd NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN;
+GRANT CREATE ON SCHEMA public TO :ckanuser;
+GRANT USAGE ON SCHEMA public TO :ckanuser;
 
--- take connect permissions from main db
+-- take connect permissions from main CKAN db
+---- again, this can be ommited if the read-only user can never have
+---- access to the main CKAN database
 REVOKE CONNECT ON DATABASE :maindb FROM :rouser;
 
 -- grant select permissions for read-only user
 GRANT CONNECT ON DATABASE :datastoredb TO :rouser;
 GRANT USAGE ON SCHEMA public TO :rouser;
 
--- grant access to current tables and views
+-- grant access to current tables and views to read-only user
 GRANT SELECT ON ALL TABLES IN SCHEMA public TO :rouser;
 
 -- grant access to new tables and views by default
-ALTER DEFAULT PRIVILEGES FOR USER :ckanuser IN SCHEMA public
+---- the permissions will be set when the write user creates a table
+ALTER DEFAULT PRIVILEGES FOR USER :wuser IN SCHEMA public
    GRANT SELECT ON TABLES TO :rouser;
diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py
index 1a8bcdd06d6..3cc4a402022 100644
--- a/ckanext/datastore/commands.py
+++ b/ckanext/datastore/commands.py
@@ -1,5 +1,4 @@
-import re
-import sys
+import bin.datastore_setup as setup
 import logging
 
 import ckan.lib.cli as cli
@@ -7,48 +6,13 @@
 log = logging.getLogger(__name__)
 
 
-read_only_user_sql = '''
--- revoke permissions for the new user
-REVOKE CREATE ON SCHEMA public FROM PUBLIC;
-REVOKE USAGE ON SCHEMA public FROM PUBLIC;
-
-GRANT CREATE ON SCHEMA public TO "{ckanuser}";
-GRANT USAGE ON SCHEMA public TO "{ckanuser}";
-
-GRANT CREATE ON SCHEMA public TO "{writeuser}";
-GRANT USAGE ON SCHEMA public TO "{writeuser}";
-
--- create new read only user
-CREATE USER "{readonlyuser}" {with_password} NOSUPERUSER NOCREATEDB NOCREATEROLE LOGIN;
-
--- take connect permissions from main db
-REVOKE CONNECT ON DATABASE "{maindb}" FROM "{readonlyuser}";
-
--- grant select permissions for read-only user
-GRANT CONNECT ON DATABASE "{datastore}" TO "{readonlyuser}";
-GRANT USAGE ON SCHEMA public TO "{readonlyuser}";
-
--- grant access to current tables and views
-GRANT SELECT ON ALL TABLES IN SCHEMA public TO "{readonlyuser}";
-
--- grant access to new tables and views by default
-ALTER DEFAULT PRIVILEGES FOR USER "{writeuser}" IN SCHEMA public
-   GRANT SELECT ON TABLES TO "{readonlyuser}";
-'''
-
-
 class SetupDatastoreCommand(cli.CkanCommand):
     '''Perform commands to set up the datastore.
     Make sure that the datastore urls are set properly before you run these commands.
-    `create-all` will run create-write-user, create-db and create-read-only-user.
 
     Usage::
 
-        paster datastore create-write-user SQL_SUPER_USER
-        paster datastore create-db SQL_SUPER_USER
-        paster datastore create-read-only-user SQL_SUPER_USER
-
-        paster datastore create-all SQL_SUPER_USER
+        paster datastore set-permissions SQL_SUPER_USER
 
     Where:
         SQL_SUPER_USER is the name of a postgres user with sufficient
@@ -79,89 +43,26 @@ def command(self):
         self.db_read_url_parts = cli.parse_db_config('ckan.datastore.read_url')
         self.db_ckan_url_parts = cli.parse_db_config('sqlalchemy.url')
 
-        assert self.db_write_url_parts['db_name'] == self.db_read_url_parts['db_name'], "write and read db should be the same"
+        assert self.db_write_url_parts['db_name'] == self.db_read_url_parts['db_name'],\
+            "write and read db have to be the same"
 
         if len(self.args) != 2:
             print self.usage
             return
         self.sql_superuser = self.args[1]
 
-        if cmd == 'create-db':
-            self.create_db()
-            if self.verbose:
-                print 'Creating DB: SUCCESS'
-        elif cmd == 'create-read-only-user':
-            self.create_read_only_user()
+        if cmd == 'set-permissions':
+            setup.set_permissions(
+                pguser=self.sql_superuser,
+                ckandb=self.db_ckan_url_parts['db_name'],
+                datastoredb=self.db_write_url_parts['db_name'],
+                ckanuser=self.db_ckan_url_parts['db_user'],
+                writeuser=self.db_write_url_parts['db_user'],
+                readonlyuser=self.db_read_url_parts['db_user']
+            )
             if self.verbose:
-                print 'Creating read-only user: SUCCESS'
-        elif cmd == 'create-write-user':
-            self.create_write_user()
-            if self.verbose:
-                print 'Creating write user: SUCCESS'
-        elif cmd == 'create-all':
-            self.create_db()
-            if self.db_ckan_url_parts['db_user'] != self.db_write_url_parts['db_user']:
-                self.create_write_user()
-            self.create_read_only_user()
-            if self.verbose:
-                print 'Creating db and users for datastore: SUCCESS'
+                print 'Set permissions for read-only user: SUCCESS'
         else:
             print self.usage
             log.error('Command "%s" not recognized' % (cmd,))
             return
-
-    def _run_cmd(self, command_line, inputstring=''):
-        if self.verbose:
-            print "Running:", command_line
-        import subprocess
-        if not self.simulate:
-            p = subprocess.Popen(
-                command_line, shell=True,
-                stdin=subprocess.PIPE,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE)
-            stdout_value, stderr_value = p.communicate(input=inputstring)
-            if stderr_value:
-                print '\nAn error occured: {0}'.format(stderr_value)
-                sys.exit(1)
-
-    def _run_sql(self, sql, as_sql_user, database='postgres'):
-        if self.verbose:
-            print "Executing: \n#####\n", sql, "\n####\nOn database:", database
-        if not self.simulate:
-            self._run_cmd("sudo -u '{username}' psql --dbname='{database}' -W".format(
-                username=as_sql_user,
-                database=database
-            ), inputstring=sql)
-
-    def create_db(self):
-        cmd = "sudo -u {pguser} createdb -O '{ckanuser}' '{db}'".format(
-        pguser=self.sql_superuser,
-        db=self.db_write_url_parts['db_name'],
-        ckanuser=self.db_ckan_url_parts['db_user'])
-        self._run_cmd(cmd)
-
-    def create_write_user(self):
-        cmd = "sudo -u {pguser} createuser \
-            --no-createdb --no-createrole --no-superuser '{writeuser}'".format(
-        pguser=self.sql_superuser,
-        writeuser=self.db_write_url_parts['db_user'])
-        self._run_cmd(cmd)
-
-    def create_read_only_user(self):
-        password = self.db_read_url_parts['db_pass']
-        self.validate_password(password)
-        sql = read_only_user_sql.format(
-            maindb=self.db_ckan_url_parts['db_name'],
-            datastore=self.db_write_url_parts['db_name'],
-            ckanuser=self.db_ckan_url_parts['db_user'],
-            readonlyuser=self.db_read_url_parts['db_user'],
-            with_password="WITH PASSWORD '{0}'".format(password) if password else "",
-            writeuser=self.db_write_url_parts['db_user'])
-        self._run_sql(sql,
-                      as_sql_user=self.sql_superuser,
-                      database=self.db_write_url_parts['db_name'])
-
-    def validate_password(self, password):
-        if "'" in password:
-            raise ValueError("Passwords cannot contain '")

From 3c874c1b7c11638ad4d49eda207b7af61fa06d9a Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Tue, 9 Oct 2012 17:55:50 +0100
Subject: [PATCH 27/53] Adapt docs to changes from previous commit.

---
 doc/datastore.rst | 88 +++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 81 insertions(+), 7 deletions(-)

diff --git a/doc/datastore.rst b/doc/datastore.rst
index 45192463948..9d61d8fb336 100644
--- a/doc/datastore.rst
+++ b/doc/datastore.rst
@@ -6,7 +6,7 @@ The CKAN DataStore provides a database for structured storage of data together
 with a powerful Web-accessible Data API, all seamlessly integrated into the CKAN
 interface and authorization system.
 
-.. note:: The DataStore requires PostgreSQL 9.0 or later.
+.. note:: The DataStore requires PostgreSQL 9.0 or later. It is possible to use the DataStore on verions prior to 9.0 (for example 8.4). However, the :ref:`datastore_search_sql` will not be available and the set-up is slightly different. See :ref:`old_pg` for mroe details.
 
 .. warning:: The DataStore does not support hiding resources in a private dataset.
 
@@ -20,25 +20,92 @@ but that is no more, as it now uses the relational database management system Po
 However, you should set up a separate database for the DataStore
 and create a read-only user to make your CKAN and DataStore installation safe.
 
+1. Enable the plugin
+--------------------
+
 In your ``config`` file ensure that the datastore extension is enabled::
 
  ckan.plugins = datastore
 
-Also ensure that the ``ckan.datastore.write_url`` and ``datastore.read_url`` variables are set::
+2. Set-up the database
+----------------------
 
- ckan.datastore.write_url = postgresql://writeuser:pass@localhost/datastore
- ckan.datastore.read_url = postgresql://readonlyuser:pass@localhost/datastore
+The datastore requires a separate postgres database to save the resources to.
+
+List existing databases::
+
+ sudo -u postgres psql -l
+
+Check that the encoding of databases is ‘UTF8’, if not internationalisation may be a problem. Since changing the encoding of PostgreSQL may mean deleting existing databases, it is suggested that this is fixed before continuing with the CKAN install.
+
+Next you’ll need to create a two database users for the datastore. One user will be the *write* user that can create, edit and delete resources. The second user will be a *read-only* user who can only read resources.
 
 A few things have to be kept in mind:
 
 * The DataStore cannot be on the CKAN database (except for testing)
 * The write user (i.e. ``writeuser``) and read-only user (i.e. ``readonlyuser``) cannot be the same
 
-To create a new database with a write user and read-only user, use this paster command::
+Create users and databases
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Create a write user called ``writeuser``, and enter pass for the password when prompted::
+
+ sudo -u postgres createuser -S -D -R -P -l writeuser
+
+Create a read-only user called ``readonlyuser``, and enter pass for the password when prompted::
+
+ sudo -u postgres createuser -S -D -R -P -l readonlyuser
+
+Create the database (owned by ``writeuser``), which we’ll call ``datastore``::
+
+ sudo -u postgres createdb -O writeuser datastore
+
+Set URLs
+~~~~~~~~
+
+Now, ensure that the ``ckan.datastore.write_url`` and ``datastore.read_url`` variables are set::
+
+ ckan.datastore.write_url = postgresql://writeuser:pass@localhost/datastore
+ ckan.datastore.read_url = postgresql://readonlyuser:pass@localhost/datastore
+
+Set permissions
+~~~~~~~~~~~~~~~
+
+Once the datastore database and the users are created, the permissions on the datastore and CKAN database have to be set. Since there are different set-ups, there are different ways of setting the permissions. Only one of ther optins should be used.
+
+1. Use the **paster command** if CKAN and Postgres are on the same server
+
+To set the permissions, use this paster command after you've set the database urls::
 
  paster datastore create-all SQL_SUPER_USER
 
-To test the set-up you can create a new DataStore. To do so you can run the following command::
+
+2. Use the **command line tool** in ``datastore/bin/datastore_setup.py``
+
+.. note:: This option should be used if the ckan server is different from the database server.
+
+Copy the content from the ``datastore/bin/`` directory to the database server. Then run the command line tool to set the permissions on the database. To see all available options, run::
+
+ python datastore_setup.py -h
+
+Once you are confident that you know the right names, set the permissions (assuming that the CKAN database is called ``ckan`` and the CKAN PostgreSQL user is called ``ckanuser``)::
+
+ python datastore_setup.py ckan datastore ckanuser writeuser readonlyuser -p postgres
+
+
+3. Run the **SQL commands** manually on the database
+
+.. note:: This option is for more complex set-ups and requires understanding of SQL and PostgreSQL.
+
+Copy the ``set_permissions.sql`` file to the server that the database runs on. Make sure you set all variables in the file correctly and comment out the parts that are not needed for you setup. Then, run the script::
+
+ sudo -u postgres psql postgres -f set_permissions.sql
+
+
+3. Test the set-up
+------------------
+
+The datastore is now set-up. To test the set-up you can create a new DataStore. To do so you can run the following command::
 
  curl -X POST http://127.0.0.1:5000/api/3/action/datastore_create -H "Authorization: {YOUR-API-KEY}" -d '{"resource_id": "{RESOURCE-ID}", "fields": [ {"id": "a"}, {"id": "b"} ], "records": [ { "a": 1, "b": "xyz"}, {"a": 2, "b": "zzz"} ]}'
 
@@ -49,6 +116,13 @@ the records inserted above::
  http://127.0.0.1:5000/api/3/action/datastore_search?resource_id={RESOURCE_ID}
 
 
+.. _old_pg:
+
+Notes on how to install the DataStore on old PostgreSQL versions
+----------------------------------------------------------------
+
+To be continued...
+
 Relationship to FileStore
 =========================
 
@@ -102,7 +176,7 @@ The datastore_create API endpoint allows a user to post JSON data to be stored a
 
  {
     resource_id:  # the data is going to be stored against.
-    aliases:      # list of names for read only aliases to the resource
+    aliases:      # list of names for read-only aliases to the resource
     fields:       # a list of dictionaries of fields/columns and their extra metadata.
     records:      # a list of dictionaries of the data, eg:  [{"dob": "2005", "some_stuff": ['a', 'b']}, ..]
     primary_key:  # list of fields that represent a unique key

From 3a74287bfb17033c3f1fad7251b4ceea1b7f1dbe Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Tue, 9 Oct 2012 20:40:36 +0100
Subject: [PATCH 28/53] Improved the overall script and fixed the grant
 permissions stuff

---
 ckanext/datastore/bin/set_permissions.sql | 12 ++++++------
 ckanext/datastore/commands.py             |  3 +--
 doc/datastore.rst                         |  8 +++++---
 3 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/ckanext/datastore/bin/set_permissions.sql b/ckanext/datastore/bin/set_permissions.sql
index 08206449a3e..1bbc06d203f 100644
--- a/ckanext/datastore/bin/set_permissions.sql
+++ b/ckanext/datastore/bin/set_permissions.sql
@@ -14,15 +14,15 @@ To run the script, execute:
 */
 
 -- name of the main CKAN database
-\set maindb "{ckandb}"
+\set maindb '{ckandb}'
 -- the name of the datastore database
 \set datastoredb '{datastoredb}'
 -- username of the ckan postgres user
-\set ckanuser '{ckanuser}'
+\set ckanuser "{ckanuser}"
 -- username of the datastore user that can write
-\set wuser '{writeuser}'
+\set wuser "{writeuser}"
 -- username of the datastore user who has only read permissions
-\set rouser '{readonlyuser}'
+\set rouser "{readonlyuser}"
 
 -- revoke permissions for the read-only user
 ---- this step can be ommitted if the datastore not
@@ -33,8 +33,8 @@ REVOKE USAGE ON SCHEMA public FROM PUBLIC;
 GRANT CREATE ON SCHEMA public TO :ckanuser;
 GRANT USAGE ON SCHEMA public TO :ckanuser;
 
-GRANT CREATE ON SCHEMA public TO :ckanuser;
-GRANT USAGE ON SCHEMA public TO :ckanuser;
+GRANT CREATE ON SCHEMA public TO :wuser;
+GRANT USAGE ON SCHEMA public TO :wuser;
 
 -- take connect permissions from main CKAN db
 ---- again, this can be ommited if the read-only user can never have
diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py
index 3cc4a402022..364e163eddb 100644
--- a/ckanext/datastore/commands.py
+++ b/ckanext/datastore/commands.py
@@ -49,11 +49,10 @@ def command(self):
         if len(self.args) != 2:
             print self.usage
             return
-        self.sql_superuser = self.args[1]
 
         if cmd == 'set-permissions':
             setup.set_permissions(
-                pguser=self.sql_superuser,
+                pguser=self.args[1],
                 ckandb=self.db_ckan_url_parts['db_name'],
                 datastoredb=self.db_write_url_parts['db_name'],
                 ckanuser=self.db_ckan_url_parts['db_user'],
diff --git a/doc/datastore.rst b/doc/datastore.rst
index 9d61d8fb336..e9bb3bb5faa 100644
--- a/doc/datastore.rst
+++ b/doc/datastore.rst
@@ -6,7 +6,7 @@ The CKAN DataStore provides a database for structured storage of data together
 with a powerful Web-accessible Data API, all seamlessly integrated into the CKAN
 interface and authorization system.
 
-.. note:: The DataStore requires PostgreSQL 9.0 or later. It is possible to use the DataStore on verions prior to 9.0 (for example 8.4). However, the :ref:`datastore_search_sql` will not be available and the set-up is slightly different. See :ref:`old_pg` for mroe details.
+.. note:: The DataStore requires PostgreSQL 9.0 or later. It is possible to use the DataStore on verions prior to 9.0 (for example 8.4). However, the :ref:`datastore_search_sql` will not be available and the set-up is slightly different. Make sure, you read :ref:`old_pg` for more details.
 
 .. warning:: The DataStore does not support hiding resources in a private dataset.
 
@@ -48,6 +48,8 @@ A few things have to be kept in mind:
 Create users and databases
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+.. tip:: The write user does not have to be created since you can also use the CKAN user. However, this might not be possible if the CKAN database and the DataStore database are on different servers. We recommend that you use the same user for CKAN and the write datastore user if possible.
+
 Create a write user called ``writeuser``, and enter pass for the password when prompted::
 
  sudo -u postgres createuser -S -D -R -P -l writeuser
@@ -77,12 +79,12 @@ Once the datastore database and the users are created, the permissions on the da
 
 To set the permissions, use this paster command after you've set the database urls::
 
- paster datastore create-all SQL_SUPER_USER
+ paster datastore set-permissions SQL_SUPER_USER
 
 
 2. Use the **command line tool** in ``datastore/bin/datastore_setup.py``
 
-.. note:: This option should be used if the ckan server is different from the database server.
+.. note:: This option should be used if the CKAN server is different from the database server.
 
 Copy the content from the ``datastore/bin/`` directory to the database server. Then run the command line tool to set the permissions on the database. To see all available options, run::
 

From e80f58e628420ddc5ef777b7e0d5ffbc027a9f88 Mon Sep 17 00:00:00 2001
From: Anton Lundin <anton@dohi.se>
Date: Tue, 9 Oct 2012 14:16:36 +0200
Subject: [PATCH 29/53] Bugfix urls to EU-S3 buckets.

If you try to address EU-buckets with http://s3.amazonaws.com/bucket/
you get:
	The bucket you are attempting to access must be addressed using
	the specified endpoint. Please send all future requests to this
	endpoint.

Thats why we should let underlying infrastructure form the url as it
might know this kind if things.
---
 ckan/controllers/storage.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/ckan/controllers/storage.py b/ckan/controllers/storage.py
index cfee8bd3d78..e52b68a18f9 100644
--- a/ckan/controllers/storage.py
+++ b/ckan/controllers/storage.py
@@ -262,8 +262,9 @@ def get_metadata(self, label):
         if storage_backend in ['google', 's3']:
             if not label.startswith("/"):
                 label = "/" + label
-            url = "https://%s/%s%s" % (self.ofs.conn.server_name(),
-                                       bucket, label)
+            url = "https://%s%s" % (
+                self.ofs.conn.calling_format.build_host(
+                    self.ofs.conn.server_name(), bucket), label)
         else:
             url = h.url_for('storage_file',
                             label=label,

From 097250f35c0a8cd84801eb706b7d770e2495cb19 Mon Sep 17 00:00:00 2001
From: Sean Hammond <seanhammond@lavabit.com>
Date: Wed, 10 Oct 2012 11:00:13 +0200
Subject: [PATCH 30/53] Show dataset author and maintainer names even if they
 have no emails

Dataset author and maintainer names were not being shown on the dataset
read page unless the dataset's author_email and maintainer_email fields
were also filled out (in CKAN 2.0 author and maintainer are simply not
shown, in 1.8 and 1.7 it actually says incorrectly 'No author given' or
'No maintainer given').

Change this behaviour so that:

1. Author/maintainer is not shown if neither author/maintainer nor
author_email/maintainer_email is filled out

2. If only author/maintainer is filled out (no email) just show the name

3. If only email is filled then use the email as both link text and
mailto: href

4. If both name and email are given then use name as link text and email
as mailto: href
---
 ckan/lib/package_saver.py | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/ckan/lib/package_saver.py b/ckan/lib/package_saver.py
index 0ac57df0729..290fb7a6842 100644
--- a/ckan/lib/package_saver.py
+++ b/ckan/lib/package_saver.py
@@ -32,16 +32,33 @@ def render_package(cls, pkg, context):
         url = pkg.get('url', '')
         c.pkg_url_link = h.link_to(url, url, rel='foaf:homepage', target='_blank') \
                 if url else _("No web page given")
-        if pkg.get('author_email', False):
-            c.pkg_author_link = cls._person_email_link(pkg.get('author', ''), pkg.get('author_email', ''), "Author")
+
+        if pkg.get('author') and pkg.get('author_email'):
+            c.pkg_author_link = cls._person_email_link(
+                    name=pkg['author'], email=pkg['author_email'],
+                    reference="Author")
+        elif pkg.get('author'):
+            c.pkg_author_link = pkg['author']
+        elif pkg.get('author_email'):
+            c.pkg_author_link = cls._person_email_link(
+                    name=pkg['author_email'], email=pkg['author_email'],
+                    reference="Author")
         else:
             c.pkg_author_link = _("Author not given")
-        maintainer = pkg.get('maintainer', '')
-        maintainer_email = pkg.get('maintainer_email', '')
-        if maintainer_email:
-            c.pkg_maintainer_link = cls._person_email_link(maintainer, maintainer_email, "Maintainer")
+
+        if pkg.get('maintainer') and pkg.get('maintainer_email'):
+            c.pkg_maintainer_link = cls._person_email_link(
+                    name=pkg['maintainer'], email=pkg['maintainer_email'],
+                    reference="Maintainer")
+        elif pkg.get('maintainer'):
+            c.pkg_maintainer_link = pkg['maintainer']
+        elif pkg.get('maintainer_email'):
+            c.pkg_maintainer_link = cls._person_email_link(
+                    name=pkg['maintainer_email'],
+                    email=pkg['maintainer_email'], reference="Maintainer")
         else:
             c.pkg_maintainer_link = _("Maintainer not given")
+
         c.package_relationships = context['package'].get_relationships_printable()
         c.pkg_extras = []
         for extra in sorted(pkg.get('extras',[]), key=lambda x:x['key']):

From 06285bbc64c3e60142d0aaa5da6bb5aa74fbf5ca Mon Sep 17 00:00:00 2001
From: Sean Hammond <seanhammond@lavabit.com>
Date: Wed, 10 Oct 2012 11:15:29 +0200
Subject: [PATCH 31/53] Factor out some code duplication

---
 ckan/lib/package_saver.py | 40 ++++++++++++---------------------------
 1 file changed, 12 insertions(+), 28 deletions(-)

diff --git a/ckan/lib/package_saver.py b/ckan/lib/package_saver.py
index 290fb7a6842..1a1a9a8e7f6 100644
--- a/ckan/lib/package_saver.py
+++ b/ckan/lib/package_saver.py
@@ -33,31 +33,12 @@ def render_package(cls, pkg, context):
         c.pkg_url_link = h.link_to(url, url, rel='foaf:homepage', target='_blank') \
                 if url else _("No web page given")
 
-        if pkg.get('author') and pkg.get('author_email'):
-            c.pkg_author_link = cls._person_email_link(
-                    name=pkg['author'], email=pkg['author_email'],
-                    reference="Author")
-        elif pkg.get('author'):
-            c.pkg_author_link = pkg['author']
-        elif pkg.get('author_email'):
-            c.pkg_author_link = cls._person_email_link(
-                    name=pkg['author_email'], email=pkg['author_email'],
-                    reference="Author")
-        else:
-            c.pkg_author_link = _("Author not given")
-
-        if pkg.get('maintainer') and pkg.get('maintainer_email'):
-            c.pkg_maintainer_link = cls._person_email_link(
-                    name=pkg['maintainer'], email=pkg['maintainer_email'],
-                    reference="Maintainer")
-        elif pkg.get('maintainer'):
-            c.pkg_maintainer_link = pkg['maintainer']
-        elif pkg.get('maintainer_email'):
-            c.pkg_maintainer_link = cls._person_email_link(
-                    name=pkg['maintainer_email'],
-                    email=pkg['maintainer_email'], reference="Maintainer")
-        else:
-            c.pkg_maintainer_link = _("Maintainer not given")
+        c.pkg_author_link = cls._person_email_link(
+                name=pkg.get('author'), email=pkg.get('author_email'),
+                fallback=_("Author not given"))
+        c.pkg_maintainer_link = cls._person_email_link(
+                name=pkg.get('maintainer'), email=pkg.get('maintainer_email'),
+                fallback=_("Maintainer not given"))
 
         c.package_relationships = context['package'].get_relationships_printable()
         c.pkg_extras = []
@@ -119,9 +100,12 @@ def _revision_validation(cls, log_message):
         return errors
 
     @classmethod
-    def _person_email_link(cls, name, email, reference):
-        assert email
-        return h.mail_to(email_address=email, name=name or email, encode='hex')
+    def _person_email_link(cls, name, email, fallback):
+        if email:
+            return h.mail_to(email_address=email, name=name or email,
+                    encode='hex')
+        else:
+            return name or fallback
 
 class WritePackageFromBoundFieldset(object):
 

From edde347b30bc8bb81795c0c0885b6e3c09931f3e Mon Sep 17 00:00:00 2001
From: Sean Hammond <seanhammond@lavabit.com>
Date: Wed, 10 Oct 2012 14:21:48 +0200
Subject: [PATCH 32/53] Remove duplicate debug setting from deployment.ini_tmpl

deployement.ini_tmpl had debug = true at the top and then set debug =
false further down was overriding it. Set debug only once. Also since
this is deployment.ini_tmpl make the default debug = false and tell
people to set it true for development, not vice-versa.
---
 ckan/config/deployment.ini_tmpl | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl
index b2983190913..709c80fc7d6 100644
--- a/ckan/config/deployment.ini_tmpl
+++ b/ckan/config/deployment.ini_tmpl
@@ -4,7 +4,12 @@
 # The %(here)s variable will be replaced with the parent directory of this file
 #
 [DEFAULT]
-debug = true
+
+# Change debug to true when doing CKAN development, it enables Pylons'
+# interactive debugging tool, makes Fanstatic serve unminified JS and CSS
+# files, and enables CKAN templates' debugging features.
+debug = false
+
 email_to = you@yourdomain.com
 smtp_server = localhost
 error_email_from = paste@localhost
@@ -32,11 +37,6 @@ ckan.plugins = stats
 #beaker.cache.data_dir = %(here)s/data/cache
 #beaker.session.data_dir = %(here)s/data/sessions
 
-# WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*
-# Debug mode will enable the interactive debugging tool, allowing ANYONE to
-# execute malicious code after an exception is raised.
-set debug = false
-
 # Specify the database for SQLAlchemy to use:
 # * Postgres is currently required for a production CKAN deployment
 # * Sqlite (memory or file) can be used as a quick alternative for testing

From e8e8c0144571bce24524b88e214fc03e3a193fd1 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Wed, 10 Oct 2012 17:25:15 +0100
Subject: [PATCH 33/53] added legacy mode

---
 ckanext/datastore/plugin.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py
index 13a62cea896..c0d55869540 100644
--- a/ckanext/datastore/plugin.py
+++ b/ckanext/datastore/plugin.py
@@ -34,6 +34,11 @@ def configure(self, config):
             error_msg = 'ckan.datastore.read_url not found in config'
             raise DatastoreException(error_msg)
 
+        # Legacy mode means that we have no read url. Consequently sql search is not
+        # available and permissions do not have to be changed. In legacy mode, the
+        # datastore runs on PG prior to 9.0 (for example 8.4).
+        self.legacy_mode = 'ckan.datastore.read_url' not in self.config
+
         # Check whether we are running one of the paster commands which means
         # that we should ignore the following tests.
         import sys
@@ -192,11 +197,13 @@ def _create_alias_table(self):
         connection.execute(create_alias_table_sql)
 
     def get_actions(self):
-        return {'datastore_create': action.datastore_create,
+        actions = {'datastore_create': action.datastore_create,
                 'datastore_upsert': action.datastore_upsert,
                 'datastore_delete': action.datastore_delete,
-                'datastore_search': action.datastore_search,
-                'datastore_search_sql': action.datastore_search_sql}
+                'datastore_search': action.datastore_search}
+        if not self.legacy_mode:
+            actions['datastore_search_sql'] = action.datastore_search_sql
+        return actions
 
     def get_auth_functions(self):
         return {'datastore_create': auth.datastore_create,

From b0f38838cd361a685586e96f097b27590eb313d4 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Wed, 10 Oct 2012 20:50:43 +0100
Subject: [PATCH 34/53] Enable legacy mode (no sql search) if read url is
 undefined

---
 ckanext/datastore/logic/action.py         | 11 ++++-------
 ckanext/datastore/plugin.py               | 20 +++++++++++---------
 ckanext/datastore/tests/test_datastore.py |  2 +-
 3 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py
index f15ad891886..5ee3614f90e 100644
--- a/ckanext/datastore/logic/action.py
+++ b/ckanext/datastore/logic/action.py
@@ -76,7 +76,7 @@ def datastore_upsert(context, data_dict):
     '''
     res_id = _get_or_bust(data_dict, 'resource_id')
 
-    data_dict['connection_url'] = pylons.config['ckan.datastore.read_url']
+    data_dict['connection_url'] = pylons.config['ckan.datastore.write_url']
 
     resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata"
                                         WHERE name = :id AND alias_of IS NULL''')
@@ -90,8 +90,6 @@ def datastore_upsert(context, data_dict):
 
     p.toolkit.check_access('datastore_upsert', context, data_dict)
 
-    data_dict['connection_url'] = pylons.config['ckan.datastore.write_url']
-
     result = db.upsert(context, data_dict)
     result.pop('id')
     result.pop('connection_url')
@@ -112,7 +110,7 @@ def datastore_delete(context, data_dict):
     '''
     res_id = _get_or_bust(data_dict, 'resource_id')
 
-    data_dict['connection_url'] = pylons.config['ckan.datastore.read_url']
+    data_dict['connection_url'] = pylons.config['ckan.datastore.write_url']
 
     resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata"
                                         WHERE name = :id AND alias_of IS NULL''')
@@ -126,8 +124,6 @@ def datastore_delete(context, data_dict):
 
     p.toolkit.check_access('datastore_delete', context, data_dict)
 
-    data_dict['connection_url'] = pylons.config['ckan.datastore.write_url']
-
     result = db.delete(context, data_dict)
     result.pop('id')
     result.pop('connection_url')
@@ -172,7 +168,8 @@ def datastore_search(context, data_dict):
     '''
     res_id = _get_or_bust(data_dict, 'resource_id')
 
-    data_dict['connection_url'] = pylons.config['ckan.datastore.read_url']
+    data_dict['connection_url'] = pylons.config.get('ckan.datastore.read_url',
+            pylons.config['ckan.datastore.write_url'])
 
     resources_sql = sqlalchemy.text(u'SELECT 1 FROM "_table_metadata" WHERE name = :id')
     results = db._get_engine(None, data_dict).execute(resources_sql, id=res_id)
diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py
index c0d55869540..696063ca524 100644
--- a/ckanext/datastore/plugin.py
+++ b/ckanext/datastore/plugin.py
@@ -30,10 +30,6 @@ def configure(self, config):
             error_msg = 'ckan.datastore.write_url not found in config'
             raise DatastoreException(error_msg)
 
-        if (not 'ckan.datastore.read_url' in config):
-            error_msg = 'ckan.datastore.read_url not found in config'
-            raise DatastoreException(error_msg)
-
         # Legacy mode means that we have no read url. Consequently sql search is not
         # available and permissions do not have to be changed. In legacy mode, the
         # datastore runs on PG prior to 9.0 (for example 8.4).
@@ -43,20 +39,26 @@ def configure(self, config):
         # that we should ignore the following tests.
         import sys
         if sys.argv[0].split('/')[-1] == 'paster' and "datastore" in [sys.argv[1], sys.argv[2]]:
-            log.warn("Omitting permission checks because you are "
-                        "running paster commands.")
+            log.warn('Omitting permission checks because you are '
+                        'running paster commands.')
             return
 
         self.ckan_url = self.config['sqlalchemy.url']
         self.write_url = self.config['ckan.datastore.write_url']
-        self.read_url = self.config['ckan.datastore.read_url']
+        if self.legacy_mode:
+            self.read_url = self.write_url
+        else:
+            self.read_url = self.config['ckan.datastore.read_url']
 
         if not self._is_read_only_database():
             # Make sure that the right permissions are set
             # so that no harmful queries can be made
             if not ('debug' in config and config['debug']):
                 self._check_separate_db()
-            self._check_read_permissions()
+            if self.legacy_mode:
+                log.warn('Legacy mode active. Thse sql search will not be available.')
+            else:
+                self._check_read_permissions()
 
             self._create_alias_table()
         else:
@@ -74,7 +76,7 @@ def configure(self, config):
         def new_resource_show(context, data_dict):
             engine = db._get_engine(
                 context,
-                {'connection_url': config['ckan.datastore.read_url']}
+                {'connection_url': self.read_url}
             )
             new_data_dict = resource_show(context, data_dict)
             try:
diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py
index a087743b04d..cb37361a2ca 100644
--- a/ckanext/datastore/tests/test_datastore.py
+++ b/ckanext/datastore/tests/test_datastore.py
@@ -33,7 +33,7 @@ def rebuild_all_dbs(Session):
     ''' If the tests are running on the same db, we have to make sure that
     the ckan tables are recrated.
     '''
-    db_read_url_parts = cli.parse_db_config('ckan.datastore.read_url')
+    db_read_url_parts = cli.parse_db_config('ckan.datastore.write_url')
     db_ckan_url_parts = cli.parse_db_config('sqlalchemy.url')
     same_db = db_read_url_parts['db_name'] == db_ckan_url_parts['db_name']
 

From 936605ec54169ebc632d96010907de0701bb6442 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Wed, 10 Oct 2012 21:55:09 +0100
Subject: [PATCH 35/53] Docs on how to install the datastore on pg 8.4

---
 ckanext/datastore/plugin.py |  2 +-
 doc/datastore.rst           | 43 +++++++++++++++++++++++++------------
 2 files changed, 30 insertions(+), 15 deletions(-)

diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py
index 696063ca524..8c4748e3068 100644
--- a/ckanext/datastore/plugin.py
+++ b/ckanext/datastore/plugin.py
@@ -56,7 +56,7 @@ def configure(self, config):
             if not ('debug' in config and config['debug']):
                 self._check_separate_db()
             if self.legacy_mode:
-                log.warn('Legacy mode active. Thse sql search will not be available.')
+                log.warn('Legacy mode active. The sql search will not be available.')
             else:
                 self._check_read_permissions()
 
diff --git a/doc/datastore.rst b/doc/datastore.rst
index e9bb3bb5faa..4c6dee7416f 100644
--- a/doc/datastore.rst
+++ b/doc/datastore.rst
@@ -6,31 +6,36 @@ The CKAN DataStore provides a database for structured storage of data together
 with a powerful Web-accessible Data API, all seamlessly integrated into the CKAN
 interface and authorization system.
 
-.. note:: The DataStore requires PostgreSQL 9.0 or later. It is possible to use the DataStore on verions prior to 9.0 (for example 8.4). However, the :ref:`datastore_search_sql` will not be available and the set-up is slightly different. Make sure, you read :ref:`old_pg` for more details.
+.. note:: The DataStore requires PostgreSQL 9.0 or later. It is possible to use the DataStore on versions prior to 9.0 (for example 8.4). However, the :ref:`datastore_search_sql` will not be available and the set-up is slightly different. Make sure, you read :ref:`old_pg` for more details.
 
 .. warning:: The DataStore does not support hiding resources in a private dataset.
 
+.. _installation:
+
 Installation and Configuration
 ==============================
 
 .. warning:: Make sure that you follow the steps below and make sure that the settings are correct. Wrong settings could lead to serious security issues.
 
-The DataStore in previous lives required a custom setup of ElasticSearch and Nginx,
+.. tip:: If these steps seem to complicated, the legacy mode can be used which does not require you to set the permissions or create a separate user. However, the legacy mode is limited in its capabilities.
+
+
+The DataStore in previous lives required a custom set-up of ElasticSearch and Nginx,
 but that is no more, as it now uses the relational database management system PostgreSQL.
 However, you should set up a separate database for the DataStore
 and create a read-only user to make your CKAN and DataStore installation safe.
 
-1. Enable the plugin
---------------------
+1. Enable the extension
+-----------------------
 
-In your ``config`` file ensure that the datastore extension is enabled::
+In your ``config`` file ensure that the ``datastore`` extension is enabled::
 
  ckan.plugins = datastore
 
 2. Set-up the database
 ----------------------
 
-The datastore requires a separate postgres database to save the resources to.
+The DataStore requires a separate postgres database to save the resources to.
 
 List existing databases::
 
@@ -38,7 +43,7 @@ List existing databases::
 
 Check that the encoding of databases is ‘UTF8’, if not internationalisation may be a problem. Since changing the encoding of PostgreSQL may mean deleting existing databases, it is suggested that this is fixed before continuing with the CKAN install.
 
-Next you’ll need to create a two database users for the datastore. One user will be the *write* user that can create, edit and delete resources. The second user will be a *read-only* user who can only read resources.
+Next you will need to create a two database users for the DataStore. One user will be the *write* user that can create, edit and delete resources. The second user will be a *read-only* user who can only read resources.
 
 A few things have to be kept in mind:
 
@@ -48,7 +53,7 @@ A few things have to be kept in mind:
 Create users and databases
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. tip:: The write user does not have to be created since you can also use the CKAN user. However, this might not be possible if the CKAN database and the DataStore database are on different servers. We recommend that you use the same user for CKAN and the write datastore user if possible.
+.. tip:: The write user does not have to be created since you can also use the CKAN user. However, this might not be possible if the CKAN database and the DataStore database are on different servers. We recommend that you use the same user for CKAN and the write DataStore user if possible.
 
 Create a write user called ``writeuser``, and enter pass for the password when prompted::
 
@@ -73,9 +78,9 @@ Now, ensure that the ``ckan.datastore.write_url`` and ``datastore.read_url`` var
 Set permissions
 ~~~~~~~~~~~~~~~
 
-Once the datastore database and the users are created, the permissions on the datastore and CKAN database have to be set. Since there are different set-ups, there are different ways of setting the permissions. Only one of ther optins should be used.
+Once the DataStore database and the users are created, the permissions on the DataStore and CKAN database have to be set. Since there are different set-ups, there are different ways of setting the permissions. Only one of the options should be used.
 
-1. Use the **paster command** if CKAN and Postgres are on the same server
+1. Use the **paster command** if CKAN and PostgreSQL are on the same server
 
 To set the permissions, use this paster command after you've set the database urls::
 
@@ -99,7 +104,7 @@ Once you are confident that you know the right names, set the permissions (assum
 
 .. note:: This option is for more complex set-ups and requires understanding of SQL and PostgreSQL.
 
-Copy the ``set_permissions.sql`` file to the server that the database runs on. Make sure you set all variables in the file correctly and comment out the parts that are not needed for you setup. Then, run the script::
+Copy the ``set_permissions.sql`` file to the server that the database runs on. Make sure you set all variables in the file correctly and comment out the parts that are not needed for you set-up. Then, run the script::
 
  sudo -u postgres psql postgres -f set_permissions.sql
 
@@ -120,10 +125,20 @@ the records inserted above::
 
 .. _old_pg:
 
-Notes on how to install the DataStore on old PostgreSQL versions
-----------------------------------------------------------------
+Legacy mode: use the DataStore with old PostgreSQL versions
+-----------------------------------------------------------
+
+The DataStore can be used with a PostgreSQL version prior to 9.0 in *legacy mode*. Due to the lack of some functionality, the :ref:`datastore_search_sql` and consequently the :ref:`datastore_search_htsql` cannot be used. The set-up for legacy mode is analogous to the normal set-up as described in :ref:`installation` with a few changes and consists of the following steps:
+
+1. Enable the extension
+#. Set-Up the database
+
+    a) Create a separate database
+    #) Create a write user on the DataStore database (optional since the CKAN user can be used)
+
+#. Test the set-up
 
-To be continued...
+There is no need for a read-only user or special permissions. Therefore the legacy mode can be used for simple set-ups as well.
 
 Relationship to FileStore
 =========================

From aa4bb3f7111e3017a48dc5294532129eda7fc302 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Thu, 11 Oct 2012 00:31:52 +0100
Subject: [PATCH 36/53] Split datastore tests into separate files

---
 ckanext/datastore/plugin.py               |    2 +
 ckanext/datastore/tests/helpers.py        |   31 +
 ckanext/datastore/tests/test_create.py    |  563 +++++++
 ckanext/datastore/tests/test_datastore.py | 1740 ---------------------
 ckanext/datastore/tests/test_delete.py    |  153 ++
 ckanext/datastore/tests/test_search.py    |  479 ++++++
 ckanext/datastore/tests/test_unit.py      |   31 +
 ckanext/datastore/tests/test_upsert.py    |  521 ++++++
 8 files changed, 1780 insertions(+), 1740 deletions(-)
 create mode 100644 ckanext/datastore/tests/helpers.py
 create mode 100644 ckanext/datastore/tests/test_create.py
 delete mode 100644 ckanext/datastore/tests/test_datastore.py
 create mode 100644 ckanext/datastore/tests/test_delete.py
 create mode 100644 ckanext/datastore/tests/test_search.py
 create mode 100644 ckanext/datastore/tests/test_unit.py
 create mode 100644 ckanext/datastore/tests/test_upsert.py

diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py
index 8c4748e3068..ba33e8d7aae 100644
--- a/ckanext/datastore/plugin.py
+++ b/ckanext/datastore/plugin.py
@@ -23,6 +23,8 @@ class DatastorePlugin(p.SingletonPlugin):
     p.implements(p.IActions)
     p.implements(p.IAuthFunctions)
 
+    legacy_mode = False
+
     def configure(self, config):
         self.config = config
         # check for ckan.datastore.write_url and ckan.datastore.read_url
diff --git a/ckanext/datastore/tests/helpers.py b/ckanext/datastore/tests/helpers.py
new file mode 100644
index 00000000000..cf83f57378a
--- /dev/null
+++ b/ckanext/datastore/tests/helpers.py
@@ -0,0 +1,31 @@
+import ckan.model as model
+import ckan.lib.cli as cli
+
+
+def extract(d, keys):
+    return dict((k, d[k]) for k in keys if k in d)
+
+
+def clear_db(Session):
+    drop_tables = u'''select 'drop table "' || tablename || '" cascade;'
+                    from pg_tables where schemaname = 'public' '''
+    c = Session.connection()
+    results = c.execute(drop_tables)
+    for result in results:
+        c.execute(result[0])
+    Session.commit()
+    Session.remove()
+
+
+def rebuild_all_dbs(Session):
+    ''' If the tests are running on the same db, we have to make sure that
+    the ckan tables are recrated.
+    '''
+    db_read_url_parts = cli.parse_db_config('ckan.datastore.write_url')
+    db_ckan_url_parts = cli.parse_db_config('sqlalchemy.url')
+    same_db = db_read_url_parts['db_name'] == db_ckan_url_parts['db_name']
+
+    if same_db:
+        model.repo.tables_created_and_initialised = False
+    clear_db(Session)
+    model.repo.rebuild_db()
diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py
new file mode 100644
index 00000000000..6ee8e21d9f5
--- /dev/null
+++ b/ckanext/datastore/tests/test_create.py
@@ -0,0 +1,563 @@
+import json
+
+import sqlalchemy.orm as orm
+
+import ckan.plugins as p
+import ckan.lib.create_test_data as ctd
+import ckan.model as model
+import ckan.tests as tests
+
+import ckanext.datastore.db as db
+from ckanext.datastore.tests.helpers import rebuild_all_dbs
+
+
+class TestDatastoreCreate(tests.WsgiAppCase):
+    sysadmin_user = None
+    normal_user = None
+    p.load('datastore')
+
+    @classmethod
+    def setup_class(cls):
+        p.load('datastore')
+        ctd.CreateTestData.create()
+        cls.sysadmin_user = model.User.get('testsysadmin')
+        cls.normal_user = model.User.get('annafan')
+        import pylons
+        engine = db._get_engine(
+                None,
+                {'connection_url': pylons.config['ckan.datastore.write_url']}
+            )
+        cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
+
+    @classmethod
+    def teardown_class(cls):
+        rebuild_all_dbs(cls.Session)
+
+    def test_create_requires_auth(self):
+        resource = model.Package.get('annakarenina').resources[0]
+        data = {
+            'resource_id': resource.id
+        }
+        postparams = '%s=1' % json.dumps(data)
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            status=403)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is False
+
+    def test_create_empty_fails(self):
+        postparams = '%s=1' % json.dumps({})
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is False
+
+    def test_create_invalid_alias_name(self):
+        resource = model.Package.get('annakarenina').resources[0]
+        data = {
+            'resource_id': resource.id,
+            'aliases': u'foo"bar',
+            'fields': [{'id': 'book', 'type': 'text'},
+                       {'id': 'author', 'type': 'text'}]
+        }
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is False
+
+        data = {
+            'resource_id': resource.id,
+            'aliases': u'fo%25bar',
+            'fields': [{'id': 'book', 'type': 'text'},
+                       {'id': 'author', 'type': 'text'}]
+        }
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is False
+
+    def test_create_invalid_field_type(self):
+        resource = model.Package.get('annakarenina').resources[0]
+        data = {
+            'resource_id': resource.id,
+            'fields': [{'id': 'book', 'type': 'int['},  # this is invalid
+                       {'id': 'author', 'type': 'INVALID'}]
+        }
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is False
+
+    def test_create_invalid_field_name(self):
+        resource = model.Package.get('annakarenina').resources[0]
+        data = {
+            'resource_id': resource.id,
+            'fields': [{'id': '_book', 'type': 'text'},
+                       {'id': '_author', 'type': 'text'}]
+        }
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is False
+
+        data = {
+            'resource_id': resource.id,
+            'fields': [{'id': '"book"', 'type': 'text'},
+                       {'id': '"author', 'type': 'text'}]
+        }
+        postparams = '%s=1' % json.dumps(data)
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is False
+
+    def test_create_invalid_record_field(self):
+        resource = model.Package.get('annakarenina').resources[0]
+        data = {
+            'resource_id': resource.id,
+            'fields': [{'id': 'book', 'type': 'text'},
+                       {'id': 'author', 'type': 'text'}],
+            'records': [{'book': 'annakarenina', 'author': 'tolstoy'},
+                        {'book': 'warandpeace', 'published': '1869'}]
+        }
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is False
+
+    def test_bad_records(self):
+        resource = model.Package.get('annakarenina').resources[0]
+        data = {
+            'resource_id': resource.id,
+            'fields': [{'id': 'book', 'type': 'text'},
+                       {'id': 'author', 'type': 'text'}],
+            'records': ['bad']  # treat author as null
+        }
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is False
+
+        resource = model.Package.get('annakarenina').resources[0]
+        data = {
+            'resource_id': resource.id,
+            'fields': [{'id': 'book', 'type': 'text'},
+                       {'id': 'author', 'type': 'text'}],
+            'records': [{'book': 'annakarenina', 'author': 'tolstoy'},
+                        [],
+                        {'book': 'warandpeace'}]  # treat author as null
+        }
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is False
+
+    def test_create_invalid_index(self):
+        resource = model.Package.get('annakarenina').resources[0]
+        data = {
+            'resource_id': resource.id,
+            'indexes': 'book, dummy',
+            'fields': [{'id': 'book', 'type': 'text'},
+                       {'id': 'author', 'type': 'text'}]
+        }
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is False
+
+    def test_create_invalid_unique_index(self):
+        resource = model.Package.get('annakarenina').resources[0]
+        data = {
+            'resource_id': resource.id,
+            'primary_key': 'dummy',
+            'fields': [{'id': 'book', 'type': 'text'},
+                       {'id': 'author', 'type': 'text'}]
+        }
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is False
+
+    def test_create_alias_twice(self):
+        resource = model.Package.get('annakarenina').resources[1]
+        data = {
+            'resource_id': resource.id,
+            'aliases': 'new_alias',
+            'fields': [{'id': 'book', 'type': 'text'},
+                       {'id': 'author', 'type': 'text'}]
+        }
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True, res_dict
+
+        resource = model.Package.get('annakarenina').resources[0]
+        data = {
+            'resource_id': resource.id,
+            'aliases': 'new_alias',
+            'fields': [{'id': 'more books', 'type': 'text'}]
+        }
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is False, res_dict
+
+    def test_create_basic(self):
+        resource = model.Package.get('annakarenina').resources[0]
+        aliases = [u'great_list_of_books', u'another_list_of_b\xfcks']
+        data = {
+            'resource_id': resource.id,
+            'aliases': aliases,
+            'fields': [{'id': 'boo%k', 'type': 'text'},
+                       {'id': 'author', 'type': 'json'}],
+            'indexes': [['boo%k', 'author'], 'author'],
+            'records': [
+                        {'boo%k': 'crime', 'author': ['tolstoy', 'dostoevsky']},
+                        {'boo%k': 'annakarenina', 'author': ['tolstoy', 'putin']},
+                        {'boo%k': 'warandpeace'}]  # treat author as null
+        }
+        ### Firstly test to see if resource things it has datastore table
+        postparams = '%s=1' % json.dumps({'id': resource.id})
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/resource_show', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['result']['datastore_active'] == False
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is True
+        res = res_dict['result']
+        assert res['resource_id'] == data['resource_id']
+        assert res['fields'] == data['fields'], res['fields']
+        assert res['records'] == data['records']
+
+        c = self.Session.connection()
+        results = c.execute('select * from "{0}"'.format(resource.id))
+
+        assert results.rowcount == 3
+        for i, row in enumerate(results):
+            assert data['records'][i].get('boo%k') == row['boo%k']
+            assert data['records'][i].get('author') == (
+                json.loads(row['author'][0]) if row['author'] else None)
+
+        results = c.execute('''
+            select * from "{0}" where _full_text @@ to_tsquery('warandpeace')
+            '''.format(resource.id))
+        assert results.rowcount == 1, results.rowcount
+
+        results = c.execute('''
+            select * from "{0}" where _full_text @@ to_tsquery('tolstoy')
+            '''.format(resource.id))
+        assert results.rowcount == 2
+        self.Session.remove()
+
+        # check aliases for resource
+        c = self.Session.connection()
+        for alias in aliases:
+
+            results = [row for row in c.execute(u'select * from "{0}"'.format(resource.id))]
+            results_alias = [row for row in c.execute(u'select * from "{0}"'.format(alias))]
+
+            assert results == results_alias
+
+            sql = (u"select * from _table_metadata "
+                "where alias_of='{0}' and name='{1}'").format(resource.id, alias)
+            results = c.execute(sql)
+            assert results.rowcount == 1
+        self.Session.remove()
+
+        # check to test to see if resource now has a datastore table
+        postparams = '%s=1' % json.dumps({'id': resource.id})
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/resource_show', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['result']['datastore_active'] == True
+
+        #######  insert again simple
+        data2 = {
+            'resource_id': resource.id,
+            'records': [{'boo%k': 'hagji murat', 'author': ['tolstoy']}]
+        }
+
+        postparams = '%s=1' % json.dumps(data2)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is True
+
+        c = self.Session.connection()
+        results = c.execute('select * from "{0}"'.format(resource.id))
+        self.Session.remove()
+
+        assert results.rowcount == 4
+
+        all_data = data['records'] + data2['records']
+        for i, row in enumerate(results):
+            assert all_data[i].get('boo%k') == row['boo%k']
+            assert all_data[i].get('author') == (
+                json.loads(row['author'][0]) if row['author'] else None)
+
+        c = self.Session.connection()
+        results = c.execute('''
+            select * from "{0}" where _full_text @@ 'tolstoy'
+            '''.format(resource.id))
+        self.Session.remove()
+        assert results.rowcount == 3
+
+        #######  insert again extra field
+        data3 = {
+            'resource_id': resource.id,
+            'records': [{'boo%k': 'crime and punsihment',
+                         'author': ['dostoevsky'], 'rating': 2}],
+            'indexes': ['rating']
+        }
+
+        postparams = '%s=1' % json.dumps(data3)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is True
+
+        c = self.Session.connection()
+        results = c.execute('select * from "{0}"'.format(resource.id))
+
+        assert results.rowcount == 5
+
+        all_data = data['records'] + data2['records'] + data3['records']
+        for i, row in enumerate(results):
+            assert all_data[i].get('boo%k') == row['boo%k'], (i, all_data[i].get('boo%k'), row['boo%k'])
+            assert all_data[i].get('author') == (json.loads(row['author'][0]) if row['author'] else None)
+
+        results = c.execute('''select * from "{0}" where _full_text @@ to_tsquery('dostoevsky') '''.format(resource.id))
+        self.Session.remove()
+        assert results.rowcount == 2
+
+        #######  insert again which will fail because of unique book name
+        data4 = {
+            'resource_id': resource.id,
+            'records': [{'boo%k': 'warandpeace'}],
+            'primary_key': 'boo%k'
+        }
+
+        postparams = '%s=1' % json.dumps(data4)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is False
+        assert 'constraints' in res_dict['error']
+
+        #######  insert again which should not fail because constraint is removed
+        data5 = {
+            'resource_id': resource.id,
+            'aliases': 'another_alias',  # replaces aliases
+            'records': [{'boo%k': 'warandpeace'}],
+            'primary_key': ''
+        }
+
+        postparams = '%s=1' % json.dumps(data5)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth, expect_errors=True)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is True, res_dict
+
+        # new aliases should replace old aliases
+        c = self.Session.connection()
+        for alias in aliases:
+            sql = (u"select * from _table_metadata "
+                "where alias_of='{0}' and name='{1}'").format(resource.id, alias)
+            results = c.execute(sql)
+            assert results.rowcount == 0
+
+        sql = (u"select * from _table_metadata "
+            "where alias_of='{0}' and name='{1}'").format(resource.id, 'another_alias')
+        results = c.execute(sql)
+        assert results.rowcount == 1
+        self.Session.remove()
+
+        #######  insert array type
+        data6 = {
+            'resource_id': resource.id,
+            'fields': [{'id': 'boo%k', 'type': 'text'},
+                       {'id': 'author', 'type': 'json'},
+                       {'id': 'rating', 'type': 'int'},
+                       {'id': 'characters', 'type': '_text'}],  # this is an array of strings
+            'records': [{'boo%k': 'the hobbit',
+                         'author': ['tolkien'], 'characters': ['Bilbo', 'Gandalf']}],
+            'indexes': ['characters']
+        }
+
+        postparams = '%s=1' % json.dumps(data6)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth, expect_errors=True)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is True, res_dict
+
+        #######  insert type that requires additional lookup
+        data7 = {
+            'resource_id': resource.id,
+            'fields': [{'id': 'boo%k', 'type': 'text'},
+                       {'id': 'author', 'type': 'json'},
+                       {'id': 'rating', 'type': 'int'},
+                       {'id': 'characters', 'type': '_text'},
+                       {'id': 'location', 'type': 'int[2]'}],
+            'records': [{'boo%k': 'lord of the rings',
+                         'author': ['tolkien'], 'location': [3, -42]}],
+            'indexes': ['characters']
+        }
+
+        postparams = '%s=1' % json.dumps(data7)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth, expect_errors=True)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is True, res_dict
+
+    def test_guess_types(self):
+        resource = model.Package.get('annakarenina').resources[1]
+
+        data = {
+            'resource_id': resource.id
+        }
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_delete', params=postparams,
+                            extra_environ=auth, status="*")  # ignore status
+        res_dict = json.loads(res.body)
+
+        data = {
+            'resource_id': resource.id,
+            'fields': [{'id': 'author', 'type': 'json'},
+                       {'id': 'count'},
+                       {'id': 'book'},
+                       {'id': 'date'}],
+            'records': [{'book': 'annakarenina', 'author': 'tolstoy',
+                         'count': 1, 'date': '2005-12-01', 'count2': 0.5},
+                        {'book': 'crime', 'author': ['tolstoy', 'dostoevsky']},
+                        {'book': 'warandpeace'}]  # treat author as null
+        }
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+
+        c = self.Session.connection()
+        results = c.execute('''select * from "{0}" '''.format(resource.id))
+
+        types = [db._pg_types[field[1]] for field in results.cursor.description]
+
+        assert types == [u'int4', u'tsvector', u'nested', u'int4', u'text', u'timestamp', u'float8'], types
+
+        assert results.rowcount == 3
+        for i, row in enumerate(results):
+            assert data['records'][i].get('book') == row['book']
+            assert data['records'][i].get('author') == (
+                json.loads(row['author'][0]) if row['author'] else None)
+        self.Session.remove()
+
+        ### extend types
+
+        data = {
+            'resource_id': resource.id,
+            'fields': [{'id': 'author', 'type': 'text'},
+                       {'id': 'count'},
+                       {'id': 'book'},
+                       {'id': 'date'},
+                       {'id': 'count2'},
+                       {'id': 'extra', 'type':'text'},
+                       {'id': 'date2'},
+                      ],
+            'records': [{'book': 'annakarenina', 'author': 'tolstoy',
+                         'count': 1, 'date': '2005-12-01', 'count2': 2,
+                         'nested': [1, 2], 'date2': '2005-12-01'}]
+        }
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+
+        c = self.Session.connection()
+        results = c.execute('''select * from "{0}" '''.format(resource.id))
+        self.Session.remove()
+
+        types = [db._pg_types[field[1]] for field in results.cursor.description]
+
+        assert types == [u'int4',  # id
+                         u'tsvector',  # fulltext
+                         u'nested',  # author
+                         u'int4',  # count
+                         u'text',  # book
+                         u'timestamp',  # date
+                         u'float8',  # count2
+                         u'text',  # extra
+                         u'timestamp',  # date2
+                         u'nested',  # count3
+                        ], types
+
+        ### fields resupplied in wrong order
+
+        data = {
+            'resource_id': resource.id,
+            'fields': [{'id': 'author', 'type': 'text'},
+                       {'id': 'count'},
+                       {'id': 'date'},  # date and book in wrong order
+                       {'id': 'book'},
+                       {'id': 'count2'},
+                       {'id': 'extra', 'type':'text'},
+                       {'id': 'date2'},
+                      ],
+            'records': [{'book': 'annakarenina', 'author': 'tolstoy',
+                         'count': 1, 'date': '2005-12-01', 'count2': 2,
+                         'count3': 432, 'date2': '2005-12-01'}]
+        }
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is False
\ No newline at end of file
diff --git a/ckanext/datastore/tests/test_datastore.py b/ckanext/datastore/tests/test_datastore.py
deleted file mode 100644
index cb37361a2ca..00000000000
--- a/ckanext/datastore/tests/test_datastore.py
+++ /dev/null
@@ -1,1740 +0,0 @@
-import unittest
-import json
-import pprint
-import datetime
-
-import sqlalchemy
-import sqlalchemy.orm as orm
-
-import ckan.plugins as p
-import ckan.lib.create_test_data as ctd
-import ckan.model as model
-import ckan.tests as tests
-import ckan.lib.cli as cli
-import ckanext.datastore.db as db
-
-
-def extract(d, keys):
-    return dict((k, d[k]) for k in keys if k in d)
-
-
-def clear_db(Session):
-    drop_tables = u'''select 'drop table "' || tablename || '" cascade;'
-                    from pg_tables where schemaname = 'public' '''
-    c = Session.connection()
-    results = c.execute(drop_tables)
-    for result in results:
-        c.execute(result[0])
-    Session.commit()
-    Session.remove()
-
-
-def rebuild_all_dbs(Session):
-    ''' If the tests are running on the same db, we have to make sure that
-    the ckan tables are recrated.
-    '''
-    db_read_url_parts = cli.parse_db_config('ckan.datastore.write_url')
-    db_ckan_url_parts = cli.parse_db_config('sqlalchemy.url')
-    same_db = db_read_url_parts['db_name'] == db_ckan_url_parts['db_name']
-
-    if same_db:
-        model.repo.tables_created_and_initialised = False
-    clear_db(Session)
-    model.repo.rebuild_db()
-
-
-class TestTypeGetters(unittest.TestCase):
-    def test_list(self):
-        assert db._get_list(None) == None
-        assert db._get_list([]) == []
-        assert db._get_list('') == []
-        assert db._get_list('foo') == ['foo']
-        assert db._get_list('foo, bar') == ['foo', 'bar']
-        assert db._get_list('foo_"bar, baz') == ['foo_"bar', 'baz']
-        assert db._get_list('"foo", "bar"') == ['foo', 'bar']
-        assert db._get_list(u'foo, bar') == ['foo', 'bar']
-        assert db._get_list(['foo', 'bar']) == ['foo', 'bar']
-        assert db._get_list([u'foo', u'bar']) == ['foo', 'bar']
-        assert db._get_list(['foo', ['bar', 'baz']]) == ['foo', ['bar', 'baz']]
-
-    def test_bool(self):
-        assert db._get_bool(None) == False
-        assert db._get_bool(False) == False
-        assert db._get_bool(True) == True
-        assert db._get_bool('', True) == True
-        assert db._get_bool('', False) == False
-        assert db._get_bool('True') == True
-        assert db._get_bool('False') == False
-        assert db._get_bool('1') == True
-        assert db._get_bool('0') == False
-        assert db._get_bool('on') == True
-        assert db._get_bool('off') == False
-
-
-class TestDatastoreCreate(tests.WsgiAppCase):
-    sysadmin_user = None
-    normal_user = None
-    p.load('datastore')
-
-    @classmethod
-    def setup_class(cls):
-        p.load('datastore')
-        ctd.CreateTestData.create()
-        cls.sysadmin_user = model.User.get('testsysadmin')
-        cls.normal_user = model.User.get('annafan')
-        import pylons
-        engine = db._get_engine(
-                None,
-                {'connection_url': pylons.config['ckan.datastore.write_url']}
-            )
-        cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
-
-    @classmethod
-    def teardown_class(cls):
-        rebuild_all_dbs(cls.Session)
-
-    def test_create_requires_auth(self):
-        resource = model.Package.get('annakarenina').resources[0]
-        data = {
-            'resource_id': resource.id
-        }
-        postparams = '%s=1' % json.dumps(data)
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            status=403)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is False
-
-    def test_create_empty_fails(self):
-        postparams = '%s=1' % json.dumps({})
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is False
-
-    def test_create_invalid_alias_name(self):
-        resource = model.Package.get('annakarenina').resources[0]
-        data = {
-            'resource_id': resource.id,
-            'aliases': u'foo"bar',
-            'fields': [{'id': 'book', 'type': 'text'},
-                       {'id': 'author', 'type': 'text'}]
-        }
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is False
-
-        data = {
-            'resource_id': resource.id,
-            'aliases': u'fo%25bar',
-            'fields': [{'id': 'book', 'type': 'text'},
-                       {'id': 'author', 'type': 'text'}]
-        }
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is False
-
-    def test_create_invalid_field_type(self):
-        resource = model.Package.get('annakarenina').resources[0]
-        data = {
-            'resource_id': resource.id,
-            'fields': [{'id': 'book', 'type': 'int['},  # this is invalid
-                       {'id': 'author', 'type': 'INVALID'}]
-        }
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is False
-
-    def test_create_invalid_field_name(self):
-        resource = model.Package.get('annakarenina').resources[0]
-        data = {
-            'resource_id': resource.id,
-            'fields': [{'id': '_book', 'type': 'text'},
-                       {'id': '_author', 'type': 'text'}]
-        }
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is False
-
-        data = {
-            'resource_id': resource.id,
-            'fields': [{'id': '"book"', 'type': 'text'},
-                       {'id': '"author', 'type': 'text'}]
-        }
-        postparams = '%s=1' % json.dumps(data)
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is False
-
-    def test_create_invalid_record_field(self):
-        resource = model.Package.get('annakarenina').resources[0]
-        data = {
-            'resource_id': resource.id,
-            'fields': [{'id': 'book', 'type': 'text'},
-                       {'id': 'author', 'type': 'text'}],
-            'records': [{'book': 'annakarenina', 'author': 'tolstoy'},
-                        {'book': 'warandpeace', 'published': '1869'}]
-        }
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is False
-
-    def test_bad_records(self):
-        resource = model.Package.get('annakarenina').resources[0]
-        data = {
-            'resource_id': resource.id,
-            'fields': [{'id': 'book', 'type': 'text'},
-                       {'id': 'author', 'type': 'text'}],
-            'records': ['bad']  # treat author as null
-        }
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is False
-
-        resource = model.Package.get('annakarenina').resources[0]
-        data = {
-            'resource_id': resource.id,
-            'fields': [{'id': 'book', 'type': 'text'},
-                       {'id': 'author', 'type': 'text'}],
-            'records': [{'book': 'annakarenina', 'author': 'tolstoy'},
-                        [],
-                        {'book': 'warandpeace'}]  # treat author as null
-        }
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is False
-
-    def test_create_invalid_index(self):
-        resource = model.Package.get('annakarenina').resources[0]
-        data = {
-            'resource_id': resource.id,
-            'indexes': 'book, dummy',
-            'fields': [{'id': 'book', 'type': 'text'},
-                       {'id': 'author', 'type': 'text'}]
-        }
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is False
-
-    def test_create_invalid_unique_index(self):
-        resource = model.Package.get('annakarenina').resources[0]
-        data = {
-            'resource_id': resource.id,
-            'primary_key': 'dummy',
-            'fields': [{'id': 'book', 'type': 'text'},
-                       {'id': 'author', 'type': 'text'}]
-        }
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is False
-
-    def test_create_alias_twice(self):
-        resource = model.Package.get('annakarenina').resources[1]
-        data = {
-            'resource_id': resource.id,
-            'aliases': 'new_alias',
-            'fields': [{'id': 'book', 'type': 'text'},
-                       {'id': 'author', 'type': 'text'}]
-        }
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True, res_dict
-
-        resource = model.Package.get('annakarenina').resources[0]
-        data = {
-            'resource_id': resource.id,
-            'aliases': 'new_alias',
-            'fields': [{'id': 'more books', 'type': 'text'}]
-        }
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is False, res_dict
-
-    def test_create_basic(self):
-        resource = model.Package.get('annakarenina').resources[0]
-        aliases = [u'great_list_of_books', u'another_list_of_b\xfcks']
-        data = {
-            'resource_id': resource.id,
-            'aliases': aliases,
-            'fields': [{'id': 'boo%k', 'type': 'text'},
-                       {'id': 'author', 'type': 'json'}],
-            'indexes': [['boo%k', 'author'], 'author'],
-            'records': [
-                        {'boo%k': 'crime', 'author': ['tolstoy', 'dostoevsky']},
-                        {'boo%k': 'annakarenina', 'author': ['tolstoy', 'putin']},
-                        {'boo%k': 'warandpeace'}]  # treat author as null
-        }
-        ### Firstly test to see if resource things it has datastore table
-        postparams = '%s=1' % json.dumps({'id': resource.id})
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/resource_show', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['result']['datastore_active'] == False
-
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is True
-        res = res_dict['result']
-        assert res['resource_id'] == data['resource_id']
-        assert res['fields'] == data['fields'], res['fields']
-        assert res['records'] == data['records']
-
-        c = self.Session.connection()
-        results = c.execute('select * from "{0}"'.format(resource.id))
-
-        assert results.rowcount == 3
-        for i, row in enumerate(results):
-            assert data['records'][i].get('boo%k') == row['boo%k']
-            assert data['records'][i].get('author') == (
-                json.loads(row['author'][0]) if row['author'] else None)
-
-        results = c.execute('''
-            select * from "{0}" where _full_text @@ to_tsquery('warandpeace')
-            '''.format(resource.id))
-        assert results.rowcount == 1, results.rowcount
-
-        results = c.execute('''
-            select * from "{0}" where _full_text @@ to_tsquery('tolstoy')
-            '''.format(resource.id))
-        assert results.rowcount == 2
-        self.Session.remove()
-
-        # check aliases for resource
-        c = self.Session.connection()
-        for alias in aliases:
-
-            results = [row for row in c.execute(u'select * from "{0}"'.format(resource.id))]
-            results_alias = [row for row in c.execute(u'select * from "{0}"'.format(alias))]
-
-            assert results == results_alias
-
-            sql = (u"select * from _table_metadata "
-                "where alias_of='{0}' and name='{1}'").format(resource.id, alias)
-            results = c.execute(sql)
-            assert results.rowcount == 1
-        self.Session.remove()
-
-        # check to test to see if resource now has a datastore table
-        postparams = '%s=1' % json.dumps({'id': resource.id})
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/resource_show', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['result']['datastore_active'] == True
-
-        #######  insert again simple
-        data2 = {
-            'resource_id': resource.id,
-            'records': [{'boo%k': 'hagji murat', 'author': ['tolstoy']}]
-        }
-
-        postparams = '%s=1' % json.dumps(data2)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is True
-
-        c = self.Session.connection()
-        results = c.execute('select * from "{0}"'.format(resource.id))
-        self.Session.remove()
-
-        assert results.rowcount == 4
-
-        all_data = data['records'] + data2['records']
-        for i, row in enumerate(results):
-            assert all_data[i].get('boo%k') == row['boo%k']
-            assert all_data[i].get('author') == (
-                json.loads(row['author'][0]) if row['author'] else None)
-
-        c = self.Session.connection()
-        results = c.execute('''
-            select * from "{0}" where _full_text @@ 'tolstoy'
-            '''.format(resource.id))
-        self.Session.remove()
-        assert results.rowcount == 3
-
-        #######  insert again extra field
-        data3 = {
-            'resource_id': resource.id,
-            'records': [{'boo%k': 'crime and punsihment',
-                         'author': ['dostoevsky'], 'rating': 2}],
-            'indexes': ['rating']
-        }
-
-        postparams = '%s=1' % json.dumps(data3)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is True
-
-        c = self.Session.connection()
-        results = c.execute('select * from "{0}"'.format(resource.id))
-
-        assert results.rowcount == 5
-
-        all_data = data['records'] + data2['records'] + data3['records']
-        for i, row in enumerate(results):
-            assert all_data[i].get('boo%k') == row['boo%k'], (i, all_data[i].get('boo%k'), row['boo%k'])
-            assert all_data[i].get('author') == (json.loads(row['author'][0]) if row['author'] else None)
-
-        results = c.execute('''select * from "{0}" where _full_text @@ to_tsquery('dostoevsky') '''.format(resource.id))
-        self.Session.remove()
-        assert results.rowcount == 2
-
-        #######  insert again which will fail because of unique book name
-        data4 = {
-            'resource_id': resource.id,
-            'records': [{'boo%k': 'warandpeace'}],
-            'primary_key': 'boo%k'
-        }
-
-        postparams = '%s=1' % json.dumps(data4)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is False
-        assert 'constraints' in res_dict['error']
-
-        #######  insert again which should not fail because constraint is removed
-        data5 = {
-            'resource_id': resource.id,
-            'aliases': 'another_alias',  # replaces aliases
-            'records': [{'boo%k': 'warandpeace'}],
-            'primary_key': ''
-        }
-
-        postparams = '%s=1' % json.dumps(data5)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth, expect_errors=True)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is True, res_dict
-
-        # new aliases should replace old aliases
-        c = self.Session.connection()
-        for alias in aliases:
-            sql = (u"select * from _table_metadata "
-                "where alias_of='{0}' and name='{1}'").format(resource.id, alias)
-            results = c.execute(sql)
-            assert results.rowcount == 0
-
-        sql = (u"select * from _table_metadata "
-            "where alias_of='{0}' and name='{1}'").format(resource.id, 'another_alias')
-        results = c.execute(sql)
-        assert results.rowcount == 1
-        self.Session.remove()
-
-        #######  insert array type
-        data6 = {
-            'resource_id': resource.id,
-            'fields': [{'id': 'boo%k', 'type': 'text'},
-                       {'id': 'author', 'type': 'json'},
-                       {'id': 'rating', 'type': 'int'},
-                       {'id': 'characters', 'type': '_text'}],  # this is an array of strings
-            'records': [{'boo%k': 'the hobbit',
-                         'author': ['tolkien'], 'characters': ['Bilbo', 'Gandalf']}],
-            'indexes': ['characters']
-        }
-
-        postparams = '%s=1' % json.dumps(data6)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth, expect_errors=True)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is True, res_dict
-
-        #######  insert type that requires additional lookup
-        data7 = {
-            'resource_id': resource.id,
-            'fields': [{'id': 'boo%k', 'type': 'text'},
-                       {'id': 'author', 'type': 'json'},
-                       {'id': 'rating', 'type': 'int'},
-                       {'id': 'characters', 'type': '_text'},
-                       {'id': 'location', 'type': 'int[2]'}],
-            'records': [{'boo%k': 'lord of the rings',
-                         'author': ['tolkien'], 'location': [3, -42]}],
-            'indexes': ['characters']
-        }
-
-        postparams = '%s=1' % json.dumps(data7)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth, expect_errors=True)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is True, res_dict
-
-    def test_guess_types(self):
-        resource = model.Package.get('annakarenina').resources[1]
-
-        data = {
-            'resource_id': resource.id
-        }
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_delete', params=postparams,
-                            extra_environ=auth, status="*")  # ignore status
-        res_dict = json.loads(res.body)
-
-        data = {
-            'resource_id': resource.id,
-            'fields': [{'id': 'author', 'type': 'json'},
-                       {'id': 'count'},
-                       {'id': 'book'},
-                       {'id': 'date'}],
-            'records': [{'book': 'annakarenina', 'author': 'tolstoy',
-                         'count': 1, 'date': '2005-12-01', 'count2': 0.5},
-                        {'book': 'crime', 'author': ['tolstoy', 'dostoevsky']},
-                        {'book': 'warandpeace'}]  # treat author as null
-        }
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-
-        c = self.Session.connection()
-        results = c.execute('''select * from "{0}" '''.format(resource.id))
-
-        types = [db._pg_types[field[1]] for field in results.cursor.description]
-
-        assert types == [u'int4', u'tsvector', u'nested', u'int4', u'text', u'timestamp', u'float8'], types
-
-        assert results.rowcount == 3
-        for i, row in enumerate(results):
-            assert data['records'][i].get('book') == row['book']
-            assert data['records'][i].get('author') == (
-                json.loads(row['author'][0]) if row['author'] else None)
-        self.Session.remove()
-
-        ### extend types
-
-        data = {
-            'resource_id': resource.id,
-            'fields': [{'id': 'author', 'type': 'text'},
-                       {'id': 'count'},
-                       {'id': 'book'},
-                       {'id': 'date'},
-                       {'id': 'count2'},
-                       {'id': 'extra', 'type':'text'},
-                       {'id': 'date2'},
-                      ],
-            'records': [{'book': 'annakarenina', 'author': 'tolstoy',
-                         'count': 1, 'date': '2005-12-01', 'count2': 2,
-                         'nested': [1, 2], 'date2': '2005-12-01'}]
-        }
-
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-
-        c = self.Session.connection()
-        results = c.execute('''select * from "{0}" '''.format(resource.id))
-        self.Session.remove()
-
-        types = [db._pg_types[field[1]] for field in results.cursor.description]
-
-        assert types == [u'int4',  # id
-                         u'tsvector',  # fulltext
-                         u'nested',  # author
-                         u'int4',  # count
-                         u'text',  # book
-                         u'timestamp',  # date
-                         u'float8',  # count2
-                         u'text',  # extra
-                         u'timestamp',  # date2
-                         u'nested',  # count3
-                        ], types
-
-        ### fields resupplied in wrong order
-
-        data = {
-            'resource_id': resource.id,
-            'fields': [{'id': 'author', 'type': 'text'},
-                       {'id': 'count'},
-                       {'id': 'date'},  # date and book in wrong order
-                       {'id': 'book'},
-                       {'id': 'count2'},
-                       {'id': 'extra', 'type':'text'},
-                       {'id': 'date2'},
-                      ],
-            'records': [{'book': 'annakarenina', 'author': 'tolstoy',
-                         'count': 1, 'date': '2005-12-01', 'count2': 2,
-                         'count3': 432, 'date2': '2005-12-01'}]
-        }
-
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is False
-
-
-class TestDatastoreUpsert(tests.WsgiAppCase):
-    sysadmin_user = None
-    normal_user = None
-    p.load('datastore')
-
-    @classmethod
-    def setup_class(cls):
-        p.load('datastore')
-        ctd.CreateTestData.create()
-        cls.sysadmin_user = model.User.get('testsysadmin')
-        cls.normal_user = model.User.get('annafan')
-        resource = model.Package.get('annakarenina').resources[0]
-        cls.data = {
-            'resource_id': resource.id,
-            'fields': [{'id': u'b\xfck', 'type': 'text'},
-                       {'id': 'author', 'type': 'text'},
-                       {'id': 'nested', 'type': 'json'},
-                       {'id': 'characters', 'type': 'text[]'},
-                       {'id': 'published'}],
-            'primary_key': u'b\xfck',
-            'records': [{u'b\xfck': 'annakarenina', 'author': 'tolstoy',
-                        'published': '2005-03-01', 'nested': ['b', {'moo': 'moo'}]},
-                        {u'b\xfck': 'warandpeace', 'author': 'tolstoy',
-                        'nested': {'a':'b'}}
-                       ]
-            }
-        postparams = '%s=1' % json.dumps(cls.data)
-        auth = {'Authorization': str(cls.sysadmin_user.apikey)}
-        res = cls.app.post('/api/action/datastore_create', params=postparams,
-                           extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-
-        import pylons
-        engine = db._get_engine(
-                None,
-                {'connection_url': pylons.config['ckan.datastore.write_url']}
-            )
-        cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
-
-    @classmethod
-    def teardown_class(cls):
-        rebuild_all_dbs(cls.Session)
-
-    def test_upsert_requires_auth(self):
-        data = {
-            'resource_id': self.data['resource_id']
-        }
-        postparams = '%s=1' % json.dumps(data)
-        res = self.app.post('/api/action/datastore_upsert', params=postparams,
-                            status=403)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is False
-
-    def test_upsert_empty_fails(self):
-        postparams = '%s=1' % json.dumps({})
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_upsert', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is False
-
-    def test_upsert_basic(self):
-        c = self.Session.connection()
-        results = c.execute('select 1 from "{0}"'.format(self.data['resource_id']))
-        assert results.rowcount == 2
-        self.Session.remove()
-
-        hhguide = u"hitchhiker's guide to the galaxy"
-
-        data = {
-            'resource_id': self.data['resource_id'],
-            'method': 'upsert',
-            'records': [{
-                'author': 'adams',
-                'nested': {'a': 2, 'b': {'c': 'd'}},
-                'characters': ['Arthur Dent', 'Marvin'],
-                'nested': {'foo': 'bar'},
-                u'b\xfck': hhguide}]
-        }
-
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_upsert', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is True
-
-        c = self.Session.connection()
-        results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
-        assert results.rowcount == 3
-
-        records = results.fetchall()
-        assert records[2][u'b\xfck'] == hhguide
-        assert records[2].author == 'adams'
-        assert records[2].characters == ['Arthur Dent', 'Marvin']
-        assert json.loads(records[2].nested.json) == {'foo': 'bar'}
-        self.Session.remove()
-
-        c = self.Session.connection()
-        results = c.execute("select * from \"{0}\" where author='{1}'".format(self.data['resource_id'], 'adams'))
-        assert results.rowcount == 1
-        self.Session.remove()
-
-        # upsert only the publish date
-        data = {
-            'resource_id': self.data['resource_id'],
-            'method': 'upsert',
-            'records': [{'published': '1979-1-1', u'b\xfck': hhguide}]
-        }
-
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_upsert', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is True
-
-        c = self.Session.connection()
-        results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
-        assert results.rowcount == 3
-
-        records = results.fetchall()
-        assert records[2][u'b\xfck'] == hhguide
-        assert records[2].author == 'adams'
-        assert records[2].published == datetime.datetime(1979, 1, 1)
-        self.Session.remove()
-
-        # delete publish date
-        data = {
-            'resource_id': self.data['resource_id'],
-            'method': 'upsert',
-            'records': [{u'b\xfck': hhguide, 'published': None}]
-        }
-
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_upsert', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is True
-
-        c = self.Session.connection()
-        results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
-        assert results.rowcount == 3
-
-        records = results.fetchall()
-        assert records[2][u'b\xfck'] == hhguide
-        assert records[2].author == 'adams'
-        assert records[2].published == None
-        self.Session.remove()
-
-        data = {
-            'resource_id': self.data['resource_id'],
-            'method': 'upsert',
-            'records': [{'author': 'tolkien', u'b\xfck': 'the hobbit'}]
-        }
-
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_upsert', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is True
-
-        c = self.Session.connection()
-        results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
-        assert results.rowcount == 4
-
-        records = results.fetchall()
-        assert records[3][u'b\xfck'] == 'the hobbit'
-        assert records[3].author == 'tolkien'
-        self.Session.remove()
-
-        # test % in records
-        data = {
-            'resource_id': self.data['resource_id'],
-            'method': 'upsert',
-            'records': [{'author': 'tol % kien', u'b\xfck': 'the % hobbit'}]
-        }
-
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_upsert', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is True
-
-    def test_upsert_missing_key(self):
-        data = {
-            'resource_id': self.data['resource_id'],
-            'method': 'upsert',
-            'records': [{'author': 'tolkien'}]
-        }
-
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_upsert', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is False
-
-    def test_upsert_non_existing_field(self):
-        data = {
-            'resource_id': self.data['resource_id'],
-            'method': 'upsert',
-            'records': [{u'b\xfck': 'annakarenina', 'dummy': 'tolkien'}]
-        }
-
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_upsert', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is False
-
-
-class TestDatastoreInsert(tests.WsgiAppCase):
-    sysadmin_user = None
-    normal_user = None
-    p.load('datastore')
-
-    @classmethod
-    def setup_class(cls):
-        p.load('datastore')
-        ctd.CreateTestData.create()
-        cls.sysadmin_user = model.User.get('testsysadmin')
-        cls.normal_user = model.User.get('annafan')
-        resource = model.Package.get('annakarenina').resources[0]
-        cls.data = {
-            'resource_id': resource.id,
-            'fields': [{'id': u'b\xfck', 'type': 'text'},
-                       {'id': 'author', 'type': 'text'},
-                       {'id': 'nested', 'type': 'json'},
-                       {'id': 'characters', 'type': 'text[]'},
-                       {'id': 'published'}],
-            'primary_key': u'b\xfck',
-            'records': [{u'b\xfck': 'annakarenina', 'author': 'tolstoy',
-                        'published': '2005-03-01', 'nested': ['b', {'moo': 'moo'}]},
-                        {u'b\xfck': 'warandpeace', 'author': 'tolstoy',
-                        'nested': {'a':'b'}}
-                       ]
-            }
-        postparams = '%s=1' % json.dumps(cls.data)
-        auth = {'Authorization': str(cls.sysadmin_user.apikey)}
-        res = cls.app.post('/api/action/datastore_create', params=postparams,
-                           extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-
-        import pylons
-        engine = db._get_engine(
-                None,
-                {'connection_url': pylons.config['ckan.datastore.write_url']}
-            )
-        cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
-
-    @classmethod
-    def teardown_class(cls):
-        rebuild_all_dbs(cls.Session)
-
-    def test_insert_non_existing_field(self):
-        data = {
-            'resource_id': self.data['resource_id'],
-            'method': 'insert',
-            'records': [{u'b\xfck': 'the hobbit', 'dummy': 'tolkien'}]
-        }
-
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_upsert', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is False
-
-    def test_insert_with_index_violation(self):
-        data = {
-            'resource_id': self.data['resource_id'],
-            'method': 'insert',
-            'records': [{u'b\xfck': 'annakarenina'}]
-        }
-
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_upsert', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is False
-
-    def test_insert_basic(self):
-        hhguide = u"hitchhiker's guide to the galaxy"
-        data = {
-            'resource_id': self.data['resource_id'],
-            'method': 'insert',
-            'records': [{
-                'author': 'adams',
-                'characters': ['Arthur Dent', 'Marvin'],
-                'nested': {'foo': 'bar', 'baz': 3},
-                u'b\xfck': hhguide}]
-        }
-
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_upsert', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is True
-
-        c = self.Session.connection()
-        results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
-        self.Session.remove()
-
-        assert results.rowcount == 3
-
-
-class TestDatastoreUpdate(tests.WsgiAppCase):
-    sysadmin_user = None
-    normal_user = None
-    p.load('datastore')
-
-    @classmethod
-    def setup_class(cls):
-        p.load('datastore')
-        ctd.CreateTestData.create()
-        cls.sysadmin_user = model.User.get('testsysadmin')
-        cls.normal_user = model.User.get('annafan')
-        resource = model.Package.get('annakarenina').resources[0]
-        hhguide = u"hitchhiker's guide to the galaxy"
-        cls.data = {
-            'resource_id': resource.id,
-            'fields': [{'id': u'b\xfck', 'type': 'text'},
-                       {'id': 'author', 'type': 'text'},
-                       {'id': 'nested', 'type': 'json'},
-                       {'id': 'characters', 'type': 'text[]'},
-                       {'id': 'published'}],
-            'primary_key': u'b\xfck',
-            'records': [{u'b\xfck': 'annakarenina', 'author': 'tolstoy',
-                        'published': '2005-03-01', 'nested': ['b', {'moo': 'moo'}]},
-                        {u'b\xfck': 'warandpeace', 'author': 'tolstoy',
-                        'nested': {'a':'b'}},
-                        {'author': 'adams',
-                        'characters': ['Arthur Dent', 'Marvin'],
-                        'nested': {'foo': 'bar'},
-                        u'b\xfck': hhguide}
-                       ]
-            }
-        postparams = '%s=1' % json.dumps(cls.data)
-        auth = {'Authorization': str(cls.sysadmin_user.apikey)}
-        res = cls.app.post('/api/action/datastore_create', params=postparams,
-                           extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-
-        import pylons
-        engine = db._get_engine(
-                None,
-                {'connection_url': pylons.config['ckan.datastore.write_url']}
-            )
-        cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
-
-    @classmethod
-    def teardown_class(cls):
-        rebuild_all_dbs(cls.Session)
-
-    def test_update_basic(self):
-        c = self.Session.connection()
-        results = c.execute('select 1 from "{0}"'.format(self.data['resource_id']))
-        assert results.rowcount == 3, results.rowcount
-        self.Session.remove()
-
-        hhguide = u"hitchhiker's guide to the galaxy"
-        data = {
-            'resource_id': self.data['resource_id'],
-            'method': 'update',
-            'records': [{
-                'author': 'adams',
-                'characters': ['Arthur Dent', 'Marvin'],
-                'nested': {'baz': 3},
-                u'b\xfck': hhguide}]
-        }
-
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_upsert', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is True
-
-        c = self.Session.connection()
-        results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
-        assert results.rowcount == 3
-
-        records = results.fetchall()
-        assert records[2][u'b\xfck'] == hhguide
-        assert records[2].author == 'adams'
-        self.Session.remove()
-
-        c = self.Session.connection()
-        results = c.execute("select * from \"{0}\" where author='{1}'".format(self.data['resource_id'], 'adams'))
-        assert results.rowcount == 1
-        self.Session.remove()
-
-        # update only the publish date
-        data = {
-            'resource_id': self.data['resource_id'],
-            'method': 'update',
-            'records': [{'published': '1979-1-1', u'b\xfck': hhguide}]
-        }
-
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_upsert', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is True
-
-        c = self.Session.connection()
-        results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
-        self.Session.remove()
-        assert results.rowcount == 3
-
-        records = results.fetchall()
-        assert records[2][u'b\xfck'] == hhguide
-        assert records[2].author == 'adams'
-        assert records[2].published == datetime.datetime(1979, 1, 1)
-
-        # delete publish date
-        data = {
-            'resource_id': self.data['resource_id'],
-            'method': 'update',
-            'records': [{u'b\xfck': hhguide, 'published': None}]
-        }
-
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_upsert', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is True
-
-        c = self.Session.connection()
-        results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
-        self.Session.remove()
-        assert results.rowcount == 3
-
-        records = results.fetchall()
-        assert records[2][u'b\xfck'] == hhguide
-        assert records[2].author == 'adams'
-        assert records[2].published == None
-
-    def test_update_missing_key(self):
-        data = {
-            'resource_id': self.data['resource_id'],
-            'method': 'update',
-            'records': [{'author': 'tolkien'}]
-        }
-
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_upsert', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is False
-
-    def test_update_non_existing_key(self):
-        data = {
-            'resource_id': self.data['resource_id'],
-            'method': 'update',
-            'records': [{u'b\xfck': '', 'author': 'tolkien'}]
-        }
-
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_upsert', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is False
-
-    def test_update_non_existing_field(self):
-        data = {
-            'resource_id': self.data['resource_id'],
-            'method': 'update',
-            'records': [{u'b\xfck': 'annakarenina', 'dummy': 'tolkien'}]
-        }
-
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_upsert', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-
-        assert res_dict['success'] is False
-
-
-class TestDatastoreDelete(tests.WsgiAppCase):
-    sysadmin_user = None
-    normal_user = None
-    Session = None
-
-    @classmethod
-    def setup_class(cls):
-        p.load('datastore')
-        ctd.CreateTestData.create()
-        cls.sysadmin_user = model.User.get('testsysadmin')
-        cls.normal_user = model.User.get('annafan')
-        resource = model.Package.get('annakarenina').resources[0]
-        cls.data = {
-            'resource_id': resource.id,
-            'aliases': 'books2',
-            'fields': [{'id': 'book', 'type': 'text'},
-                       {'id': 'author', 'type': 'text'}],
-            'records': [{'book': 'annakarenina', 'author': 'tolstoy'},
-                        {'book': 'warandpeace', 'author': 'tolstoy'}]
-        }
-
-        #model.repo.rebuild_db()
-        #model.repo.clean_db()
-
-        import pylons
-        engine = db._get_engine(
-                None,
-                {'connection_url': pylons.config['ckan.datastore.write_url']}
-            )
-        cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
-
-    @classmethod
-    def teardown_class(cls):
-        rebuild_all_dbs(cls.Session)
-
-    def _create(self):
-        postparams = '%s=1' % json.dumps(self.data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_create', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-        return res_dict
-
-    def _delete(self):
-        data = {'resource_id': self.data['resource_id']}
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_delete', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-        assert res_dict['result'] == data
-        return res_dict
-
-    def test_delete_basic(self):
-        self._create()
-        self._delete()
-        resource_id = self.data['resource_id']
-        c = self.Session.connection()
-
-        # alias should be deleted
-        results = c.execute("select 1 from pg_views where viewname = '{0}'".format(self.data['aliases']))
-        assert results.rowcount == 0
-
-        try:
-            # check that data was actually deleted: this should raise a
-            # ProgrammingError as the table should not exist any more
-            c.execute('select * from "{0}";'.format(resource_id))
-            raise Exception("Data not deleted")
-        except sqlalchemy.exc.ProgrammingError as e:
-            expected_msg = 'relation "{0}" does not exist'.format(resource_id)
-            assert expected_msg in str(e)
-
-        self.Session.remove()
-
-    def test_delete_invalid_resource_id(self):
-        postparams = '%s=1' % json.dumps({'resource_id': 'bad'})
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_delete', params=postparams,
-                            extra_environ=auth, status=404)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is False
-
-    def test_delete_filters(self):
-        self._create()
-        resource_id = self.data['resource_id']
-
-        # try and delete just the 'warandpeace' row
-        data = {'resource_id': resource_id,
-                'filters': {'book': 'warandpeace'}}
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_delete', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-
-        c = self.Session.connection()
-        result = c.execute('select * from "{0}";'.format(resource_id))
-        results = [r for r in result]
-        assert len(results) == 1
-        assert results[0].book == 'annakarenina'
-        self.Session.remove()
-
-        # shouldn't delete anything
-        data = {'resource_id': resource_id,
-                'filters': {'book': 'annakarenina', 'author': 'bad'}}
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_delete', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-
-        c = self.Session.connection()
-        result = c.execute('select * from "{0}";'.format(resource_id))
-        results = [r for r in result]
-        assert len(results) == 1
-        assert results[0].book == 'annakarenina'
-        self.Session.remove()
-
-        # delete the 'annakarenina' row
-        data = {'resource_id': resource_id,
-                'filters': {'book': 'annakarenina', 'author': 'tolstoy'}}
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_delete', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-
-        c = self.Session.connection()
-        result = c.execute('select * from "{0}";'.format(resource_id))
-        results = [r for r in result]
-        assert len(results) == 0
-        self.Session.remove()
-
-        self._delete()
-
-
-class TestDatastoreSearch(tests.WsgiAppCase):
-    sysadmin_user = None
-    normal_user = None
-
-    @classmethod
-    def setup_class(cls):
-        p.load('datastore')
-        ctd.CreateTestData.create()
-        cls.sysadmin_user = model.User.get('testsysadmin')
-        cls.normal_user = model.User.get('annafan')
-        resource = model.Package.get('annakarenina').resources[0]
-        cls.data = {
-            'resource_id': resource.id,
-            'aliases': 'books3',
-            'fields': [{'id': u'b\xfck', 'type': 'text'},
-                       {'id': 'author', 'type': 'text'},
-                       {'id': 'published'},
-                       {'id': u'characters', u'type': u'_text'}],
-            'records': [{u'b\xfck': 'annakarenina', 'author': 'tolstoy',
-                        'published': '2005-03-01', 'nested': ['b', {'moo': 'moo'}],
-                        u'characters': [u'Princess Anna', u'Sergius']},
-                        {u'b\xfck': 'warandpeace', 'author': 'tolstoy',
-                        'nested': {'a':'b'}}
-                       ]
-        }
-        postparams = '%s=1' % json.dumps(cls.data)
-        auth = {'Authorization': str(cls.sysadmin_user.apikey)}
-        res = cls.app.post('/api/action/datastore_create', params=postparams,
-                           extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-
-        cls.expected_records = [{u'published': u'2005-03-01T00:00:00',
-                                 u'_id': 1,
-                                 u'nested': [u'b', {u'moo': u'moo'}],
-                                 u'b\xfck': u'annakarenina',
-                                 u'author': u'tolstoy',
-                                 u'characters': [u'Princess Anna', u'Sergius']},
-                                {u'published': None,
-                                 u'_id': 2,
-                                 u'nested': {u'a': u'b'},
-                                 u'b\xfck': u'warandpeace',
-                                 u'author': u'tolstoy',
-                                 u'characters': None}]
-
-        import pylons
-        engine = db._get_engine(
-                None,
-                {'connection_url': pylons.config['ckan.datastore.write_url']}
-            )
-        cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
-
-    @classmethod
-    def teardown_class(cls):
-        rebuild_all_dbs(cls.Session)
-
-    def test_search_basic(self):
-        data = {'resource_id': self.data['resource_id']}
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_search', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-        result = res_dict['result']
-        assert result['total'] == len(self.data['records'])
-        assert result['records'] == self.expected_records, result['records']
-
-    def test_search_alias(self):
-        data = {'resource_id': self.data['aliases']}
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_search', params=postparams,
-                            extra_environ=auth)
-        res_dict_alias = json.loads(res.body)
-        result = res_dict_alias['result']
-        assert result['total'] == len(self.data['records'])
-        assert result['records'] == self.expected_records, result['records']
-
-    def test_search_invalid_field(self):
-        data = {'resource_id': self.data['resource_id'],
-                'fields': [{'id': 'bad'}]}
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_search', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is False
-
-    def test_search_fields(self):
-        data = {'resource_id': self.data['resource_id'],
-                'fields': [u'b\xfck']}
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_search', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-        result = res_dict['result']
-        assert result['total'] == len(self.data['records'])
-        assert result['records'] == [{u'b\xfck': 'annakarenina'},
-                                     {u'b\xfck': 'warandpeace'}], result['records']
-
-        data = {'resource_id': self.data['resource_id'],
-                'fields': u'b\xfck, author'}
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_search', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-        result = res_dict['result']
-        assert result['total'] == len(self.data['records'])
-        assert result['records'] == [{u'b\xfck': 'annakarenina', 'author': 'tolstoy'},
-                    {u'b\xfck': 'warandpeace', 'author': 'tolstoy'}], result['records']
-
-    def test_search_filters(self):
-        data = {'resource_id': self.data['resource_id'],
-                'filters': {u'b\xfck': 'annakarenina'}}
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_search', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-        result = res_dict['result']
-        assert result['total'] == 1
-        assert result['records'] == [self.expected_records[0]]
-
-    def test_search_array_filters(self):
-        data = {'resource_id': self.data['resource_id'],
-                'filters': {u'characters': [u'Princess Anna', u'Sergius']}}
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_search', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-        result = res_dict['result']
-        assert result['total'] == 1
-        assert result['records'] == [self.expected_records[0]]
-
-    def test_search_sort(self):
-        data = {'resource_id': self.data['resource_id'],
-                'sort': u'b\xfck asc, author desc'}
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_search', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-        result = res_dict['result']
-        assert result['total'] == 2
-
-        assert result['records'] == self.expected_records, result['records']
-
-        data = {'resource_id': self.data['resource_id'],
-                'sort': [u'b\xfck desc', '"author" asc']}
-        postparams = '%s=1' % json.dumps(data)
-        res = self.app.post('/api/action/datastore_search', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-        result = res_dict['result']
-        assert result['total'] == 2
-
-        assert result['records'] == self.expected_records[::-1]
-
-    def test_search_limit(self):
-        data = {'resource_id': self.data['resource_id'],
-                'limit': 1}
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_search', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-        result = res_dict['result']
-        assert result['total'] == 2
-        assert result['records'] == [self.expected_records[0]]
-
-    def test_search_invalid_limit(self):
-        data = {'resource_id': self.data['resource_id'],
-                'limit': 'bad'}
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_search', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is False
-
-    def test_search_offset(self):
-        data = {'resource_id': self.data['resource_id'],
-                'limit': 1,
-                'offset': 1}
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_search', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-        result = res_dict['result']
-        assert result['total'] == 2
-        assert result['records'] == [self.expected_records[1]]
-
-    def test_search_invalid_offset(self):
-        data = {'resource_id': self.data['resource_id'],
-                'offset': 'bad'}
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_search', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is False
-
-    def test_search_full_text(self):
-        data = {'resource_id': self.data['resource_id'],
-                'q': 'annakarenina'}
-
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_search', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-        result = res_dict['result']
-        assert result['total'] == 1
-
-        results = [extract(result['records'][0],
-            [u'_id', u'author', u'b\xfck', u'nested', u'published', u'characters'])]
-        assert results == [self.expected_records[0]], result['records']
-
-        data = {'resource_id': self.data['resource_id'],
-                'q': 'tolstoy'}
-        postparams = '%s=1' % json.dumps(data)
-        res = self.app.post('/api/action/datastore_search', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-        result = res_dict['result']
-        assert result['total'] == 2
-        results = [extract(
-                record,
-                [u'_id', u'author', u'b\xfck', u'nested', u'published', u'characters']
-            ) for record in result['records']]
-        assert results == self.expected_records, result['records']
-
-        expected_fields = [{u'type': u'int4', u'id': u'_id'},
-                        {u'type': u'text', u'id': u'b\xfck'},
-                        {u'type': u'text', u'id': u'author'},
-                        {u'type': u'timestamp', u'id': u'published'},
-                        {u'type': u'json', u'id': u'nested'},
-                        {u'type': u'float4', u'id': u'rank'}]
-        for field in expected_fields:
-            assert field in result['fields'], field
-
-        # test multiple word queries (connected with and)
-        data = {'resource_id': self.data['resource_id'],
-                'plain': True,
-                'q': 'tolstoy annakarenina'}
-        postparams = '%s=1' % json.dumps(data)
-        res = self.app.post('/api/action/datastore_search', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-        result = res_dict['result']
-        assert result['total'] == 1
-        results = [extract(
-                result['records'][0],
-                [u'_id', u'author', u'b\xfck', u'nested', u'published', u'characters']
-            )]
-        assert results == [self.expected_records[0]], result['records']
-
-        for field in expected_fields:
-            assert field in result['fields'], field
-
-    def test_search_table_metadata(self):
-        data = {'resource_id': "_table_metadata"}
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_search', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-
-
-class TestDatastoreFullTextSearch(tests.WsgiAppCase):
-    @classmethod
-    def setup_class(cls):
-        p.load('datastore')
-        ctd.CreateTestData.create()
-        cls.sysadmin_user = model.User.get('testsysadmin')
-        cls.normal_user = model.User.get('annafan')
-        resource = model.Package.get('annakarenina').resources[0]
-        cls.data = dict(
-            resource_id=resource.id,
-            fields=[
-              {'id': 'id'},
-              {'id': 'date', 'type':'date'},
-              {'id': 'x'},
-              {'id': 'y'},
-              {'id': 'z'},
-              {'id': 'country'},
-              {'id': 'title'},
-              {'id': 'lat'},
-              {'id': 'lon'}
-            ],
-            records=[
-              {'id': 0, 'date': '2011-01-01', 'x': 1, 'y': 2, 'z': 3, 'country': 'DE', 'title': 'first', 'lat':52.56, 'lon':13.40},
-              {'id': 1, 'date': '2011-02-02', 'x': 2, 'y': 4, 'z': 24, 'country': 'UK', 'title': 'second', 'lat':54.97, 'lon':-1.60},
-              {'id': 2, 'date': '2011-03-03', 'x': 3, 'y': 6, 'z': 9, 'country': 'US', 'title': 'third', 'lat':40.00, 'lon':-75.5},
-              {'id': 3, 'date': '2011-04-04', 'x': 4, 'y': 8, 'z': 6, 'country': 'UK', 'title': 'fourth', 'lat':57.27, 'lon':-6.20},
-              {'id': 4, 'date': '2011-05-04', 'x': 5, 'y': 10, 'z': 15, 'country': 'UK', 'title': 'fifth', 'lat':51.58, 'lon':0},
-              {'id': 5, 'date': '2011-06-02', 'x': 6, 'y': 12, 'z': 18, 'country': 'DE', 'title': 'sixth', 'lat':51.04, 'lon':7.9}
-            ]
-        )
-        postparams = '%s=1' % json.dumps(cls.data)
-        auth = {'Authorization': str(cls.sysadmin_user.apikey)}
-        res = cls.app.post('/api/action/datastore_create', params=postparams,
-                           extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-
-    @classmethod
-    def teardown_class(cls):
-        model.repo.rebuild_db()
-
-    def test_search_full_text(self):
-        data = {'resource_id': self.data['resource_id'],
-                'q': 'DE'}
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_search', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['result']['total'] == 2, pprint.pformat(res_dict)
-
-    def test_advanced_search_full_text(self):
-        data = {'resource_id': self.data['resource_id'],
-                'plain': 'False',
-                'q': 'DE | UK'}
-        postparams = '%s=1' % json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_search', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['result']['total'] == 5, pprint.pformat(res_dict)
-
-
-class TestDatastoreSQL(tests.WsgiAppCase):
-    sysadmin_user = None
-    normal_user = None
-
-    @classmethod
-    def setup_class(cls):
-        p.load('datastore')
-        ctd.CreateTestData.create()
-        cls.sysadmin_user = model.User.get('testsysadmin')
-        cls.normal_user = model.User.get('annafan')
-        resource = model.Package.get('annakarenina').resources[0]
-        cls.data = {
-            'resource_id': resource.id,
-            'aliases': 'books4',
-            'fields': [{'id': u'b\xfck', 'type': 'text'},
-                       {'id': 'author', 'type': 'text'},
-                       {'id': 'published'}],
-            'records': [{u'b\xfck': 'annakarenina',
-                        'author': 'tolstoy',
-                        'published': '2005-03-01',
-                        'nested': ['b', {'moo': 'moo'}]},
-                        {u'b\xfck': 'warandpeace',
-                        'author': 'tolstoy',
-                        'nested': {'a':'b'}}
-                       ]
-        }
-        postparams = '%s=1' % json.dumps(cls.data)
-        auth = {'Authorization': str(cls.sysadmin_user.apikey)}
-        res = cls.app.post('/api/action/datastore_create', params=postparams,
-                           extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-
-        cls.expected_records = [{u'_full_text': u"'annakarenina':1 'b':3 'moo':4 'tolstoy':2",
-                                  u'_id': 1,
-                                  u'author': u'tolstoy',
-                                  u'b\xfck': u'annakarenina',
-                                  u'nested': [u'b', {u'moo': u'moo'}],
-                                  u'published': u'2005-03-01T00:00:00'},
-                                 {u'_full_text': u"'b':3 'tolstoy':2 'warandpeac':1",
-                                  u'_id': 2,
-                                  u'author': u'tolstoy',
-                                  u'b\xfck': u'warandpeace',
-                                  u'nested': {u'a': u'b'},
-                                  u'published': None}]
-        cls.expected_join_results = [{u'first': 1, u'second': 1}, {u'first': 1, u'second': 2}]
-
-    @classmethod
-    def teardown_class(cls):
-        model.repo.rebuild_db()
-
-    def test_is_single_statement(self):
-        singles = ['SELECT * FROM footable',
-            'SELECT * FROM "bartable"',
-            'SELECT * FROM "bartable";',
-            "select 'foo'||chr(59)||'bar'"]
-
-        for single in singles:
-            assert db._is_single_statement(single) is True
-
-        multiples = ['SELECT * FROM abc; SET LOCAL statement_timeout to'
-            'SET LOCAL statement_timeout to; SELECT * FROM abc',
-            'SELECT * FROM "foo"; SELECT * FROM "abc"']
-
-        for multiple in multiples:
-            assert db._is_single_statement(multiple) is False
-
-    def test_invalid_statement(self):
-        query = 'SELECT ** FROM public.foobar'
-        data = {'sql': query}
-        postparams = json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_search_sql', params=postparams,
-                            extra_environ=auth, status=409)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is False
-
-    def test_select_basic(self):
-        query = 'SELECT * FROM public."{0}"'.format(self.data['resource_id'])
-        data = {'sql': query}
-        postparams = json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_search_sql', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-        result = res_dict['result']
-        assert result['records'] == self.expected_records
-
-        # test alias search
-        query = 'SELECT * FROM public."{0}"'.format(self.data['aliases'])
-        data = {'sql': query}
-        postparams = json.dumps(data)
-        res = self.app.post('/api/action/datastore_search_sql', params=postparams,
-                            extra_environ=auth)
-        res_dict_alias = json.loads(res.body)
-
-        assert result['records'] == res_dict_alias['result']['records']
-
-    def test_self_join(self):
-        query = '''
-            select a._id as first, b._id as second
-            from "{0}" AS a,
-                 "{0}" AS b
-            where a.author = b.author
-            limit 2
-            '''.format(self.data['resource_id'])
-        data = {'sql': query}
-        postparams = json.dumps(data)
-        auth = {'Authorization': str(self.sysadmin_user.apikey)}
-        res = self.app.post('/api/action/datastore_search_sql', params=postparams,
-                            extra_environ=auth)
-        res_dict = json.loads(res.body)
-        assert res_dict['success'] is True
-        result = res_dict['result']
-        assert result['records'] == self.expected_join_results
diff --git a/ckanext/datastore/tests/test_delete.py b/ckanext/datastore/tests/test_delete.py
new file mode 100644
index 00000000000..0649510687a
--- /dev/null
+++ b/ckanext/datastore/tests/test_delete.py
@@ -0,0 +1,153 @@
+import json
+
+import sqlalchemy
+import sqlalchemy.orm as orm
+
+import ckan.plugins as p
+import ckan.lib.create_test_data as ctd
+import ckan.model as model
+import ckan.tests as tests
+
+import ckanext.datastore.db as db
+from ckanext.datastore.tests.helpers import rebuild_all_dbs
+
+
+class TestDatastoreDelete(tests.WsgiAppCase):
+    sysadmin_user = None
+    normal_user = None
+    Session = None
+
+    @classmethod
+    def setup_class(cls):
+        p.load('datastore')
+        ctd.CreateTestData.create()
+        cls.sysadmin_user = model.User.get('testsysadmin')
+        cls.normal_user = model.User.get('annafan')
+        resource = model.Package.get('annakarenina').resources[0]
+        cls.data = {
+            'resource_id': resource.id,
+            'aliases': 'books2',
+            'fields': [{'id': 'book', 'type': 'text'},
+                       {'id': 'author', 'type': 'text'}],
+            'records': [{'book': 'annakarenina', 'author': 'tolstoy'},
+                        {'book': 'warandpeace', 'author': 'tolstoy'}]
+        }
+
+        #model.repo.rebuild_db()
+        #model.repo.clean_db()
+
+        import pylons
+        engine = db._get_engine(
+                None,
+                {'connection_url': pylons.config['ckan.datastore.write_url']}
+            )
+        cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
+
+    @classmethod
+    def teardown_class(cls):
+        rebuild_all_dbs(cls.Session)
+
+    def _create(self):
+        postparams = '%s=1' % json.dumps(self.data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_create', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+        return res_dict
+
+    def _delete(self):
+        data = {'resource_id': self.data['resource_id']}
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_delete', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+        assert res_dict['result'] == data
+        return res_dict
+
+    def test_delete_basic(self):
+        self._create()
+        self._delete()
+        resource_id = self.data['resource_id']
+        c = self.Session.connection()
+
+        # alias should be deleted
+        results = c.execute("select 1 from pg_views where viewname = '{0}'".format(self.data['aliases']))
+        assert results.rowcount == 0
+
+        try:
+            # check that data was actually deleted: this should raise a
+            # ProgrammingError as the table should not exist any more
+            c.execute('select * from "{0}";'.format(resource_id))
+            raise Exception("Data not deleted")
+        except sqlalchemy.exc.ProgrammingError as e:
+            expected_msg = 'relation "{0}" does not exist'.format(resource_id)
+            assert expected_msg in str(e)
+
+        self.Session.remove()
+
+    def test_delete_invalid_resource_id(self):
+        postparams = '%s=1' % json.dumps({'resource_id': 'bad'})
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_delete', params=postparams,
+                            extra_environ=auth, status=404)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is False
+
+    def test_delete_filters(self):
+        self._create()
+        resource_id = self.data['resource_id']
+
+        # try and delete just the 'warandpeace' row
+        data = {'resource_id': resource_id,
+                'filters': {'book': 'warandpeace'}}
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_delete', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+
+        c = self.Session.connection()
+        result = c.execute('select * from "{0}";'.format(resource_id))
+        results = [r for r in result]
+        assert len(results) == 1
+        assert results[0].book == 'annakarenina'
+        self.Session.remove()
+
+        # shouldn't delete anything
+        data = {'resource_id': resource_id,
+                'filters': {'book': 'annakarenina', 'author': 'bad'}}
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_delete', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+
+        c = self.Session.connection()
+        result = c.execute('select * from "{0}";'.format(resource_id))
+        results = [r for r in result]
+        assert len(results) == 1
+        assert results[0].book == 'annakarenina'
+        self.Session.remove()
+
+        # delete the 'annakarenina' row
+        data = {'resource_id': resource_id,
+                'filters': {'book': 'annakarenina', 'author': 'tolstoy'}}
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_delete', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+
+        c = self.Session.connection()
+        result = c.execute('select * from "{0}";'.format(resource_id))
+        results = [r for r in result]
+        assert len(results) == 0
+        self.Session.remove()
+
+        self._delete()
\ No newline at end of file
diff --git a/ckanext/datastore/tests/test_search.py b/ckanext/datastore/tests/test_search.py
new file mode 100644
index 00000000000..1168159032b
--- /dev/null
+++ b/ckanext/datastore/tests/test_search.py
@@ -0,0 +1,479 @@
+import json
+import pprint
+
+import sqlalchemy.orm as orm
+
+import ckan.plugins as p
+import ckan.lib.create_test_data as ctd
+import ckan.model as model
+import ckan.tests as tests
+
+import ckanext.datastore.db as db
+from ckanext.datastore.tests.helpers import extract, rebuild_all_dbs
+
+
+class TestDatastoreSearch(tests.WsgiAppCase):
+    sysadmin_user = None
+    normal_user = None
+
+    @classmethod
+    def setup_class(cls):
+        p.load('datastore')
+        ctd.CreateTestData.create()
+        cls.sysadmin_user = model.User.get('testsysadmin')
+        cls.normal_user = model.User.get('annafan')
+        resource = model.Package.get('annakarenina').resources[0]
+        cls.data = {
+            'resource_id': resource.id,
+            'aliases': 'books3',
+            'fields': [{'id': u'b\xfck', 'type': 'text'},
+                       {'id': 'author', 'type': 'text'},
+                       {'id': 'published'},
+                       {'id': u'characters', u'type': u'_text'}],
+            'records': [{u'b\xfck': 'annakarenina', 'author': 'tolstoy',
+                        'published': '2005-03-01', 'nested': ['b', {'moo': 'moo'}],
+                        u'characters': [u'Princess Anna', u'Sergius']},
+                        {u'b\xfck': 'warandpeace', 'author': 'tolstoy',
+                        'nested': {'a':'b'}}
+                       ]
+        }
+        postparams = '%s=1' % json.dumps(cls.data)
+        auth = {'Authorization': str(cls.sysadmin_user.apikey)}
+        res = cls.app.post('/api/action/datastore_create', params=postparams,
+                           extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+
+        cls.expected_records = [{u'published': u'2005-03-01T00:00:00',
+                                 u'_id': 1,
+                                 u'nested': [u'b', {u'moo': u'moo'}],
+                                 u'b\xfck': u'annakarenina',
+                                 u'author': u'tolstoy',
+                                 u'characters': [u'Princess Anna', u'Sergius']},
+                                {u'published': None,
+                                 u'_id': 2,
+                                 u'nested': {u'a': u'b'},
+                                 u'b\xfck': u'warandpeace',
+                                 u'author': u'tolstoy',
+                                 u'characters': None}]
+
+        import pylons
+        engine = db._get_engine(
+                None,
+                {'connection_url': pylons.config['ckan.datastore.write_url']}
+            )
+        cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
+
+    @classmethod
+    def teardown_class(cls):
+        rebuild_all_dbs(cls.Session)
+
+    def test_search_basic(self):
+        data = {'resource_id': self.data['resource_id']}
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_search', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+        result = res_dict['result']
+        assert result['total'] == len(self.data['records'])
+        assert result['records'] == self.expected_records, result['records']
+
+    def test_search_alias(self):
+        data = {'resource_id': self.data['aliases']}
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_search', params=postparams,
+                            extra_environ=auth)
+        res_dict_alias = json.loads(res.body)
+        result = res_dict_alias['result']
+        assert result['total'] == len(self.data['records'])
+        assert result['records'] == self.expected_records, result['records']
+
+    def test_search_invalid_field(self):
+        data = {'resource_id': self.data['resource_id'],
+                'fields': [{'id': 'bad'}]}
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_search', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is False
+
+    def test_search_fields(self):
+        data = {'resource_id': self.data['resource_id'],
+                'fields': [u'b\xfck']}
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_search', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+        result = res_dict['result']
+        assert result['total'] == len(self.data['records'])
+        assert result['records'] == [{u'b\xfck': 'annakarenina'},
+                                     {u'b\xfck': 'warandpeace'}], result['records']
+
+        data = {'resource_id': self.data['resource_id'],
+                'fields': u'b\xfck, author'}
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_search', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+        result = res_dict['result']
+        assert result['total'] == len(self.data['records'])
+        assert result['records'] == [{u'b\xfck': 'annakarenina', 'author': 'tolstoy'},
+                    {u'b\xfck': 'warandpeace', 'author': 'tolstoy'}], result['records']
+
+    def test_search_filters(self):
+        data = {'resource_id': self.data['resource_id'],
+                'filters': {u'b\xfck': 'annakarenina'}}
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_search', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+        result = res_dict['result']
+        assert result['total'] == 1
+        assert result['records'] == [self.expected_records[0]]
+
+    def test_search_array_filters(self):
+        data = {'resource_id': self.data['resource_id'],
+                'filters': {u'characters': [u'Princess Anna', u'Sergius']}}
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_search', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+        result = res_dict['result']
+        assert result['total'] == 1
+        assert result['records'] == [self.expected_records[0]]
+
+    def test_search_sort(self):
+        data = {'resource_id': self.data['resource_id'],
+                'sort': u'b\xfck asc, author desc'}
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_search', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+        result = res_dict['result']
+        assert result['total'] == 2
+
+        assert result['records'] == self.expected_records, result['records']
+
+        data = {'resource_id': self.data['resource_id'],
+                'sort': [u'b\xfck desc', '"author" asc']}
+        postparams = '%s=1' % json.dumps(data)
+        res = self.app.post('/api/action/datastore_search', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+        result = res_dict['result']
+        assert result['total'] == 2
+
+        assert result['records'] == self.expected_records[::-1]
+
+    def test_search_limit(self):
+        data = {'resource_id': self.data['resource_id'],
+                'limit': 1}
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_search', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+        result = res_dict['result']
+        assert result['total'] == 2
+        assert result['records'] == [self.expected_records[0]]
+
+    def test_search_invalid_limit(self):
+        data = {'resource_id': self.data['resource_id'],
+                'limit': 'bad'}
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_search', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is False
+
+    def test_search_offset(self):
+        data = {'resource_id': self.data['resource_id'],
+                'limit': 1,
+                'offset': 1}
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_search', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+        result = res_dict['result']
+        assert result['total'] == 2
+        assert result['records'] == [self.expected_records[1]]
+
+    def test_search_invalid_offset(self):
+        data = {'resource_id': self.data['resource_id'],
+                'offset': 'bad'}
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_search', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is False
+
+    def test_search_full_text(self):
+        data = {'resource_id': self.data['resource_id'],
+                'q': 'annakarenina'}
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_search', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+        result = res_dict['result']
+        assert result['total'] == 1
+
+        results = [extract(result['records'][0],
+            [u'_id', u'author', u'b\xfck', u'nested', u'published', u'characters'])]
+        assert results == [self.expected_records[0]], result['records']
+
+        data = {'resource_id': self.data['resource_id'],
+                'q': 'tolstoy'}
+        postparams = '%s=1' % json.dumps(data)
+        res = self.app.post('/api/action/datastore_search', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+        result = res_dict['result']
+        assert result['total'] == 2
+        results = [extract(
+                record,
+                [u'_id', u'author', u'b\xfck', u'nested', u'published', u'characters']
+            ) for record in result['records']]
+        assert results == self.expected_records, result['records']
+
+        expected_fields = [{u'type': u'int4', u'id': u'_id'},
+                        {u'type': u'text', u'id': u'b\xfck'},
+                        {u'type': u'text', u'id': u'author'},
+                        {u'type': u'timestamp', u'id': u'published'},
+                        {u'type': u'json', u'id': u'nested'},
+                        {u'type': u'float4', u'id': u'rank'}]
+        for field in expected_fields:
+            assert field in result['fields'], field
+
+        # test multiple word queries (connected with and)
+        data = {'resource_id': self.data['resource_id'],
+                'plain': True,
+                'q': 'tolstoy annakarenina'}
+        postparams = '%s=1' % json.dumps(data)
+        res = self.app.post('/api/action/datastore_search', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+        result = res_dict['result']
+        assert result['total'] == 1
+        results = [extract(
+                result['records'][0],
+                [u'_id', u'author', u'b\xfck', u'nested', u'published', u'characters']
+            )]
+        assert results == [self.expected_records[0]], result['records']
+
+        for field in expected_fields:
+            assert field in result['fields'], field
+
+    def test_search_table_metadata(self):
+        data = {'resource_id': "_table_metadata"}
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_search', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+
+
+class TestDatastoreFullTextSearch(tests.WsgiAppCase):
+    @classmethod
+    def setup_class(cls):
+        p.load('datastore')
+        ctd.CreateTestData.create()
+        cls.sysadmin_user = model.User.get('testsysadmin')
+        cls.normal_user = model.User.get('annafan')
+        resource = model.Package.get('annakarenina').resources[0]
+        cls.data = dict(
+            resource_id=resource.id,
+            fields=[
+              {'id': 'id'},
+              {'id': 'date', 'type':'date'},
+              {'id': 'x'},
+              {'id': 'y'},
+              {'id': 'z'},
+              {'id': 'country'},
+              {'id': 'title'},
+              {'id': 'lat'},
+              {'id': 'lon'}
+            ],
+            records=[
+              {'id': 0, 'date': '2011-01-01', 'x': 1, 'y': 2, 'z': 3, 'country': 'DE', 'title': 'first', 'lat':52.56, 'lon':13.40},
+              {'id': 1, 'date': '2011-02-02', 'x': 2, 'y': 4, 'z': 24, 'country': 'UK', 'title': 'second', 'lat':54.97, 'lon':-1.60},
+              {'id': 2, 'date': '2011-03-03', 'x': 3, 'y': 6, 'z': 9, 'country': 'US', 'title': 'third', 'lat':40.00, 'lon':-75.5},
+              {'id': 3, 'date': '2011-04-04', 'x': 4, 'y': 8, 'z': 6, 'country': 'UK', 'title': 'fourth', 'lat':57.27, 'lon':-6.20},
+              {'id': 4, 'date': '2011-05-04', 'x': 5, 'y': 10, 'z': 15, 'country': 'UK', 'title': 'fifth', 'lat':51.58, 'lon':0},
+              {'id': 5, 'date': '2011-06-02', 'x': 6, 'y': 12, 'z': 18, 'country': 'DE', 'title': 'sixth', 'lat':51.04, 'lon':7.9}
+            ]
+        )
+        postparams = '%s=1' % json.dumps(cls.data)
+        auth = {'Authorization': str(cls.sysadmin_user.apikey)}
+        res = cls.app.post('/api/action/datastore_create', params=postparams,
+                           extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+
+    @classmethod
+    def teardown_class(cls):
+        model.repo.rebuild_db()
+
+    def test_search_full_text(self):
+        data = {'resource_id': self.data['resource_id'],
+                'q': 'DE'}
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_search', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['result']['total'] == 2, pprint.pformat(res_dict)
+
+    def test_advanced_search_full_text(self):
+        data = {'resource_id': self.data['resource_id'],
+                'plain': 'False',
+                'q': 'DE | UK'}
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_search', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['result']['total'] == 5, pprint.pformat(res_dict)
+
+
+class TestDatastoreSQL(tests.WsgiAppCase):
+    sysadmin_user = None
+    normal_user = None
+
+    @classmethod
+    def setup_class(cls):
+        p.load('datastore')
+        ctd.CreateTestData.create()
+        cls.sysadmin_user = model.User.get('testsysadmin')
+        cls.normal_user = model.User.get('annafan')
+        resource = model.Package.get('annakarenina').resources[0]
+        cls.data = {
+            'resource_id': resource.id,
+            'aliases': 'books4',
+            'fields': [{'id': u'b\xfck', 'type': 'text'},
+                       {'id': 'author', 'type': 'text'},
+                       {'id': 'published'}],
+            'records': [{u'b\xfck': 'annakarenina',
+                        'author': 'tolstoy',
+                        'published': '2005-03-01',
+                        'nested': ['b', {'moo': 'moo'}]},
+                        {u'b\xfck': 'warandpeace',
+                        'author': 'tolstoy',
+                        'nested': {'a':'b'}}
+                       ]
+        }
+        postparams = '%s=1' % json.dumps(cls.data)
+        auth = {'Authorization': str(cls.sysadmin_user.apikey)}
+        res = cls.app.post('/api/action/datastore_create', params=postparams,
+                           extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+
+        cls.expected_records = [{u'_full_text': u"'annakarenina':1 'b':3 'moo':4 'tolstoy':2",
+                                  u'_id': 1,
+                                  u'author': u'tolstoy',
+                                  u'b\xfck': u'annakarenina',
+                                  u'nested': [u'b', {u'moo': u'moo'}],
+                                  u'published': u'2005-03-01T00:00:00'},
+                                 {u'_full_text': u"'b':3 'tolstoy':2 'warandpeac':1",
+                                  u'_id': 2,
+                                  u'author': u'tolstoy',
+                                  u'b\xfck': u'warandpeace',
+                                  u'nested': {u'a': u'b'},
+                                  u'published': None}]
+        cls.expected_join_results = [{u'first': 1, u'second': 1}, {u'first': 1, u'second': 2}]
+
+    @classmethod
+    def teardown_class(cls):
+        model.repo.rebuild_db()
+
+    def test_is_single_statement(self):
+        singles = ['SELECT * FROM footable',
+            'SELECT * FROM "bartable"',
+            'SELECT * FROM "bartable";',
+            "select 'foo'||chr(59)||'bar'"]
+
+        for single in singles:
+            assert db._is_single_statement(single) is True
+
+        multiples = ['SELECT * FROM abc; SET LOCAL statement_timeout to'
+            'SET LOCAL statement_timeout to; SELECT * FROM abc',
+            'SELECT * FROM "foo"; SELECT * FROM "abc"']
+
+        for multiple in multiples:
+            assert db._is_single_statement(multiple) is False
+
+    def test_invalid_statement(self):
+        query = 'SELECT ** FROM public.foobar'
+        data = {'sql': query}
+        postparams = json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_search_sql', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is False
+
+    def test_select_basic(self):
+        query = 'SELECT * FROM public."{0}"'.format(self.data['resource_id'])
+        data = {'sql': query}
+        postparams = json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_search_sql', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+        result = res_dict['result']
+        assert result['records'] == self.expected_records
+
+        # test alias search
+        query = 'SELECT * FROM public."{0}"'.format(self.data['aliases'])
+        data = {'sql': query}
+        postparams = json.dumps(data)
+        res = self.app.post('/api/action/datastore_search_sql', params=postparams,
+                            extra_environ=auth)
+        res_dict_alias = json.loads(res.body)
+
+        assert result['records'] == res_dict_alias['result']['records']
+
+    def test_self_join(self):
+        query = '''
+            select a._id as first, b._id as second
+            from "{0}" AS a,
+                 "{0}" AS b
+            where a.author = b.author
+            limit 2
+            '''.format(self.data['resource_id'])
+        data = {'sql': query}
+        postparams = json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_search_sql', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+        result = res_dict['result']
+        assert result['records'] == self.expected_join_results
diff --git a/ckanext/datastore/tests/test_unit.py b/ckanext/datastore/tests/test_unit.py
new file mode 100644
index 00000000000..c01231fb04d
--- /dev/null
+++ b/ckanext/datastore/tests/test_unit.py
@@ -0,0 +1,31 @@
+import unittest
+
+import ckanext.datastore.db as db
+
+
+class TestTypeGetters(unittest.TestCase):
+    def test_list(self):
+        assert db._get_list(None) == None
+        assert db._get_list([]) == []
+        assert db._get_list('') == []
+        assert db._get_list('foo') == ['foo']
+        assert db._get_list('foo, bar') == ['foo', 'bar']
+        assert db._get_list('foo_"bar, baz') == ['foo_"bar', 'baz']
+        assert db._get_list('"foo", "bar"') == ['foo', 'bar']
+        assert db._get_list(u'foo, bar') == ['foo', 'bar']
+        assert db._get_list(['foo', 'bar']) == ['foo', 'bar']
+        assert db._get_list([u'foo', u'bar']) == ['foo', 'bar']
+        assert db._get_list(['foo', ['bar', 'baz']]) == ['foo', ['bar', 'baz']]
+
+    def test_bool(self):
+        assert db._get_bool(None) == False
+        assert db._get_bool(False) == False
+        assert db._get_bool(True) == True
+        assert db._get_bool('', True) == True
+        assert db._get_bool('', False) == False
+        assert db._get_bool('True') == True
+        assert db._get_bool('False') == False
+        assert db._get_bool('1') == True
+        assert db._get_bool('0') == False
+        assert db._get_bool('on') == True
+        assert db._get_bool('off') == False
diff --git a/ckanext/datastore/tests/test_upsert.py b/ckanext/datastore/tests/test_upsert.py
new file mode 100644
index 00000000000..fa8016a6d3c
--- /dev/null
+++ b/ckanext/datastore/tests/test_upsert.py
@@ -0,0 +1,521 @@
+import json
+import datetime
+
+import sqlalchemy.orm as orm
+
+import ckan.plugins as p
+import ckan.lib.create_test_data as ctd
+import ckan.model as model
+import ckan.tests as tests
+
+import ckanext.datastore.db as db
+from ckanext.datastore.tests.helpers import rebuild_all_dbs
+
+
+class TestDatastoreUpsert(tests.WsgiAppCase):
+    sysadmin_user = None
+    normal_user = None
+    p.load('datastore')
+
+    @classmethod
+    def setup_class(cls):
+        p.load('datastore')
+        ctd.CreateTestData.create()
+        cls.sysadmin_user = model.User.get('testsysadmin')
+        cls.normal_user = model.User.get('annafan')
+        resource = model.Package.get('annakarenina').resources[0]
+        cls.data = {
+            'resource_id': resource.id,
+            'fields': [{'id': u'b\xfck', 'type': 'text'},
+                       {'id': 'author', 'type': 'text'},
+                       {'id': 'nested', 'type': 'json'},
+                       {'id': 'characters', 'type': 'text[]'},
+                       {'id': 'published'}],
+            'primary_key': u'b\xfck',
+            'records': [{u'b\xfck': 'annakarenina', 'author': 'tolstoy',
+                        'published': '2005-03-01', 'nested': ['b', {'moo': 'moo'}]},
+                        {u'b\xfck': 'warandpeace', 'author': 'tolstoy',
+                        'nested': {'a':'b'}}
+                       ]
+            }
+        postparams = '%s=1' % json.dumps(cls.data)
+        auth = {'Authorization': str(cls.sysadmin_user.apikey)}
+        res = cls.app.post('/api/action/datastore_create', params=postparams,
+                           extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+
+        import pylons
+        engine = db._get_engine(
+                None,
+                {'connection_url': pylons.config['ckan.datastore.write_url']}
+            )
+        cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
+
+    @classmethod
+    def teardown_class(cls):
+        rebuild_all_dbs(cls.Session)
+
+    def test_upsert_requires_auth(self):
+        data = {
+            'resource_id': self.data['resource_id']
+        }
+        postparams = '%s=1' % json.dumps(data)
+        res = self.app.post('/api/action/datastore_upsert', params=postparams,
+                            status=403)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is False
+
+    def test_upsert_empty_fails(self):
+        postparams = '%s=1' % json.dumps({})
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_upsert', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is False
+
+    def test_upsert_basic(self):
+        c = self.Session.connection()
+        results = c.execute('select 1 from "{0}"'.format(self.data['resource_id']))
+        assert results.rowcount == 2
+        self.Session.remove()
+
+        hhguide = u"hitchhiker's guide to the galaxy"
+
+        data = {
+            'resource_id': self.data['resource_id'],
+            'method': 'upsert',
+            'records': [{
+                'author': 'adams',
+                'nested': {'a': 2, 'b': {'c': 'd'}},
+                'characters': ['Arthur Dent', 'Marvin'],
+                'nested': {'foo': 'bar'},
+                u'b\xfck': hhguide}]
+        }
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_upsert', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is True
+
+        c = self.Session.connection()
+        results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
+        assert results.rowcount == 3
+
+        records = results.fetchall()
+        assert records[2][u'b\xfck'] == hhguide
+        assert records[2].author == 'adams'
+        assert records[2].characters == ['Arthur Dent', 'Marvin']
+        assert json.loads(records[2].nested.json) == {'foo': 'bar'}
+        self.Session.remove()
+
+        c = self.Session.connection()
+        results = c.execute("select * from \"{0}\" where author='{1}'".format(self.data['resource_id'], 'adams'))
+        assert results.rowcount == 1
+        self.Session.remove()
+
+        # upsert only the publish date
+        data = {
+            'resource_id': self.data['resource_id'],
+            'method': 'upsert',
+            'records': [{'published': '1979-1-1', u'b\xfck': hhguide}]
+        }
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_upsert', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is True
+
+        c = self.Session.connection()
+        results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
+        assert results.rowcount == 3
+
+        records = results.fetchall()
+        assert records[2][u'b\xfck'] == hhguide
+        assert records[2].author == 'adams'
+        assert records[2].published == datetime.datetime(1979, 1, 1)
+        self.Session.remove()
+
+        # delete publish date
+        data = {
+            'resource_id': self.data['resource_id'],
+            'method': 'upsert',
+            'records': [{u'b\xfck': hhguide, 'published': None}]
+        }
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_upsert', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is True
+
+        c = self.Session.connection()
+        results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
+        assert results.rowcount == 3
+
+        records = results.fetchall()
+        assert records[2][u'b\xfck'] == hhguide
+        assert records[2].author == 'adams'
+        assert records[2].published == None
+        self.Session.remove()
+
+        data = {
+            'resource_id': self.data['resource_id'],
+            'method': 'upsert',
+            'records': [{'author': 'tolkien', u'b\xfck': 'the hobbit'}]
+        }
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_upsert', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is True
+
+        c = self.Session.connection()
+        results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
+        assert results.rowcount == 4
+
+        records = results.fetchall()
+        assert records[3][u'b\xfck'] == 'the hobbit'
+        assert records[3].author == 'tolkien'
+        self.Session.remove()
+
+        # test % in records
+        data = {
+            'resource_id': self.data['resource_id'],
+            'method': 'upsert',
+            'records': [{'author': 'tol % kien', u'b\xfck': 'the % hobbit'}]
+        }
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_upsert', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is True
+
+    def test_upsert_missing_key(self):
+        data = {
+            'resource_id': self.data['resource_id'],
+            'method': 'upsert',
+            'records': [{'author': 'tolkien'}]
+        }
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_upsert', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is False
+
+    def test_upsert_non_existing_field(self):
+        data = {
+            'resource_id': self.data['resource_id'],
+            'method': 'upsert',
+            'records': [{u'b\xfck': 'annakarenina', 'dummy': 'tolkien'}]
+        }
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_upsert', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is False
+
+
+class TestDatastoreInsert(tests.WsgiAppCase):
+    sysadmin_user = None
+    normal_user = None
+    p.load('datastore')
+
+    @classmethod
+    def setup_class(cls):
+        p.load('datastore')
+        ctd.CreateTestData.create()
+        cls.sysadmin_user = model.User.get('testsysadmin')
+        cls.normal_user = model.User.get('annafan')
+        resource = model.Package.get('annakarenina').resources[0]
+        cls.data = {
+            'resource_id': resource.id,
+            'fields': [{'id': u'b\xfck', 'type': 'text'},
+                       {'id': 'author', 'type': 'text'},
+                       {'id': 'nested', 'type': 'json'},
+                       {'id': 'characters', 'type': 'text[]'},
+                       {'id': 'published'}],
+            'primary_key': u'b\xfck',
+            'records': [{u'b\xfck': 'annakarenina', 'author': 'tolstoy',
+                        'published': '2005-03-01', 'nested': ['b', {'moo': 'moo'}]},
+                        {u'b\xfck': 'warandpeace', 'author': 'tolstoy',
+                        'nested': {'a':'b'}}
+                       ]
+            }
+        postparams = '%s=1' % json.dumps(cls.data)
+        auth = {'Authorization': str(cls.sysadmin_user.apikey)}
+        res = cls.app.post('/api/action/datastore_create', params=postparams,
+                           extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+
+        import pylons
+        engine = db._get_engine(
+                None,
+                {'connection_url': pylons.config['ckan.datastore.write_url']}
+            )
+        cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
+
+    @classmethod
+    def teardown_class(cls):
+        rebuild_all_dbs(cls.Session)
+
+    def test_insert_non_existing_field(self):
+        data = {
+            'resource_id': self.data['resource_id'],
+            'method': 'insert',
+            'records': [{u'b\xfck': 'the hobbit', 'dummy': 'tolkien'}]
+        }
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_upsert', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is False
+
+    def test_insert_with_index_violation(self):
+        data = {
+            'resource_id': self.data['resource_id'],
+            'method': 'insert',
+            'records': [{u'b\xfck': 'annakarenina'}]
+        }
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_upsert', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is False
+
+    def test_insert_basic(self):
+        hhguide = u"hitchhiker's guide to the galaxy"
+        data = {
+            'resource_id': self.data['resource_id'],
+            'method': 'insert',
+            'records': [{
+                'author': 'adams',
+                'characters': ['Arthur Dent', 'Marvin'],
+                'nested': {'foo': 'bar', 'baz': 3},
+                u'b\xfck': hhguide}]
+        }
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_upsert', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is True
+
+        c = self.Session.connection()
+        results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
+        self.Session.remove()
+
+        assert results.rowcount == 3
+
+
+class TestDatastoreUpdate(tests.WsgiAppCase):
+    sysadmin_user = None
+    normal_user = None
+    p.load('datastore')
+
+    @classmethod
+    def setup_class(cls):
+        p.load('datastore')
+        ctd.CreateTestData.create()
+        cls.sysadmin_user = model.User.get('testsysadmin')
+        cls.normal_user = model.User.get('annafan')
+        resource = model.Package.get('annakarenina').resources[0]
+        hhguide = u"hitchhiker's guide to the galaxy"
+        cls.data = {
+            'resource_id': resource.id,
+            'fields': [{'id': u'b\xfck', 'type': 'text'},
+                       {'id': 'author', 'type': 'text'},
+                       {'id': 'nested', 'type': 'json'},
+                       {'id': 'characters', 'type': 'text[]'},
+                       {'id': 'published'}],
+            'primary_key': u'b\xfck',
+            'records': [{u'b\xfck': 'annakarenina', 'author': 'tolstoy',
+                        'published': '2005-03-01', 'nested': ['b', {'moo': 'moo'}]},
+                        {u'b\xfck': 'warandpeace', 'author': 'tolstoy',
+                        'nested': {'a':'b'}},
+                        {'author': 'adams',
+                        'characters': ['Arthur Dent', 'Marvin'],
+                        'nested': {'foo': 'bar'},
+                        u'b\xfck': hhguide}
+                       ]
+            }
+        postparams = '%s=1' % json.dumps(cls.data)
+        auth = {'Authorization': str(cls.sysadmin_user.apikey)}
+        res = cls.app.post('/api/action/datastore_create', params=postparams,
+                           extra_environ=auth)
+        res_dict = json.loads(res.body)
+        assert res_dict['success'] is True
+
+        import pylons
+        engine = db._get_engine(
+                None,
+                {'connection_url': pylons.config['ckan.datastore.write_url']}
+            )
+        cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine))
+
+    @classmethod
+    def teardown_class(cls):
+        rebuild_all_dbs(cls.Session)
+
+    def test_update_basic(self):
+        c = self.Session.connection()
+        results = c.execute('select 1 from "{0}"'.format(self.data['resource_id']))
+        assert results.rowcount == 3, results.rowcount
+        self.Session.remove()
+
+        hhguide = u"hitchhiker's guide to the galaxy"
+        data = {
+            'resource_id': self.data['resource_id'],
+            'method': 'update',
+            'records': [{
+                'author': 'adams',
+                'characters': ['Arthur Dent', 'Marvin'],
+                'nested': {'baz': 3},
+                u'b\xfck': hhguide}]
+        }
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_upsert', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is True
+
+        c = self.Session.connection()
+        results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
+        assert results.rowcount == 3
+
+        records = results.fetchall()
+        assert records[2][u'b\xfck'] == hhguide
+        assert records[2].author == 'adams'
+        self.Session.remove()
+
+        c = self.Session.connection()
+        results = c.execute("select * from \"{0}\" where author='{1}'".format(self.data['resource_id'], 'adams'))
+        assert results.rowcount == 1
+        self.Session.remove()
+
+        # update only the publish date
+        data = {
+            'resource_id': self.data['resource_id'],
+            'method': 'update',
+            'records': [{'published': '1979-1-1', u'b\xfck': hhguide}]
+        }
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_upsert', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is True
+
+        c = self.Session.connection()
+        results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
+        self.Session.remove()
+        assert results.rowcount == 3
+
+        records = results.fetchall()
+        assert records[2][u'b\xfck'] == hhguide
+        assert records[2].author == 'adams'
+        assert records[2].published == datetime.datetime(1979, 1, 1)
+
+        # delete publish date
+        data = {
+            'resource_id': self.data['resource_id'],
+            'method': 'update',
+            'records': [{u'b\xfck': hhguide, 'published': None}]
+        }
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_upsert', params=postparams,
+                            extra_environ=auth)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is True
+
+        c = self.Session.connection()
+        results = c.execute('select * from "{0}"'.format(self.data['resource_id']))
+        self.Session.remove()
+        assert results.rowcount == 3
+
+        records = results.fetchall()
+        assert records[2][u'b\xfck'] == hhguide
+        assert records[2].author == 'adams'
+        assert records[2].published == None
+
+    def test_update_missing_key(self):
+        data = {
+            'resource_id': self.data['resource_id'],
+            'method': 'update',
+            'records': [{'author': 'tolkien'}]
+        }
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_upsert', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is False
+
+    def test_update_non_existing_key(self):
+        data = {
+            'resource_id': self.data['resource_id'],
+            'method': 'update',
+            'records': [{u'b\xfck': '', 'author': 'tolkien'}]
+        }
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_upsert', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is False
+
+    def test_update_non_existing_field(self):
+        data = {
+            'resource_id': self.data['resource_id'],
+            'method': 'update',
+            'records': [{u'b\xfck': 'annakarenina', 'dummy': 'tolkien'}]
+        }
+
+        postparams = '%s=1' % json.dumps(data)
+        auth = {'Authorization': str(self.sysadmin_user.apikey)}
+        res = self.app.post('/api/action/datastore_upsert', params=postparams,
+                            extra_environ=auth, status=409)
+        res_dict = json.loads(res.body)
+
+        assert res_dict['success'] is False

From 7ec092197fdbce3950b6c0ebf5b13406fa362f2c Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Fri, 12 Oct 2012 11:51:22 +0100
Subject: [PATCH 37/53] Better docs for datastore

Link to docstrings via autodoc to avoid duplication. Check api reference
for consistency. Other minor changes.
---
 ckanext/datastore/logic/action.py | 105 +++++++++++++++-------
 doc/apiv3.rst                     |   2 +
 doc/datastore.rst                 | 144 +++++++-----------------------
 3 files changed, 108 insertions(+), 143 deletions(-)

diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py
index 5ee3614f90e..3e004a67c6e 100644
--- a/ckanext/datastore/logic/action.py
+++ b/ckanext/datastore/logic/action.py
@@ -12,13 +12,19 @@
 def datastore_create(context, data_dict):
     '''Adds a new table to the datastore.
 
-    :param resource_id: resource id that the data is going to be stored under.
+    The datastore_create action allows a user to post JSON data to be
+    stored against a resource. This endpoint also supports altering tables,
+    aliases and indexes and bulk insertion.
+
+    See :ref:`fields` and :ref:`records` for details on how to lay out records.
+
+    :param resource_id: resource id that the data is going to be stored against.
     :type resource_id: string
-    :param aliases: names for read only aliases to the resource.
+    :param aliases: names for read only aliases of the resource.
     :type aliases: list or comma separated string
     :param fields: fields/columns and their extra metadata.
     :type fields: list of dictionaries
-    :param records: the data, eg: [{"dob": "2005", "some_stuff": ['a', b']}]
+    :param records: the data, eg: [{"dob": "2005", "some_stuff": ["a", "b"]}]
     :type records: list of dictionaries
     :param primary_key: fields that represent a unique key
     :type primary_key: list or comma separated string
@@ -58,16 +64,29 @@ def datastore_create(context, data_dict):
 def datastore_upsert(context, data_dict):
     '''Updates or inserts into a table in the datastore
 
+    The datastore_upsert API action allows a user to add or edit records to
+    an existing dataStore resource. In order for the *upsert* and *update*
+    methods to work, a unique key has to be defined via the datastore_create
+    action. The available methods are:
+
+    *upsert*
+        Update if record with same key already exists, otherwise insert.
+        Requires unique key.
+    *insert*
+        Insert only. This method is faster that upsert, but will fail if any
+        inserted record matches an existing one. Does *not* require a unique
+        key.
+    *update*
+        Update only. An exception will occur if the key that should be updated
+        does not exist. Requires unique key.
+
+
     :param resource_id: resource id that the data is going to be stored under.
     :type resource_id: string
-    :param records: the data, eg: [{"dob": "2005", "some_stuff": ['a', b']}]
+    :param records: the data, eg: [{"dob": "2005", "some_stuff": ["a","b"]}]
     :type records: list of dictionaries
-    :param method: the method to use to put the data into the datastore
-                    possible options: upsert (default), insert, update
-        :param upsert: update if record with same key already exists,
-                        otherwise insert
-        :param insert: insert only, faster because checks are omitted
-        :param update: update only, exception if key does not exist
+    :param method: the method to use to put the data into the datastore.
+                   Possible options are: upsert (default), insert, update
     :type method: string
 
     :returns: the newly created data object.
@@ -97,12 +116,13 @@ def datastore_upsert(context, data_dict):
 
 
 def datastore_delete(context, data_dict):
-    '''Deletes a table from the datastore.
+    '''Deletes a table or a set of records from the datastore.
 
     :param resource_id: resource id that the data will be deleted from.
     :type resource_id: string
-    :param filter: filter to do deleting on over (eg {'name': 'fred'}).
+    :param filters: filters to apply before deleting (eg {"name": "fred"}).
                    If missing delete whole table and all dependent views.
+    :type filters: dictionary
 
     :returns: original filters sent.
     :rtype: dictionary
@@ -134,9 +154,11 @@ def datastore_delete(context, data_dict):
 def datastore_search(context, data_dict):
     '''Search a datastore table.
 
-    :param resource_id: id or alias of the data that is going to be selected.
+    The datastore_search action allows a user to search data in a resource.
+
+    :param resource_id: id or alias of the resource to be searched against.
     :type resource_id: string
-    :param filters: matching conditions to select.
+    :param filters: matching conditions to select, e.g {"key1": "a", "key2": "b"}
     :type filters: dictionary
     :param q: full text query
     :type q: string
@@ -146,24 +168,31 @@ def datastore_search(context, data_dict):
     :type language: string
     :param limit: maximum number of rows to return (default: 100)
     :type limit: int
-    :param offset: offset the number of rows
+    :param offset: offset this number of rows
     :type offset: int
-    :param fields: fields to return
-                   (default: all fields in original order)
+    :param fields: fields to return (default: all fields in original order)
     :type fields: list or comma separated string
     :param sort: comma separated field names with ordering
-                 eg: "fieldname1, fieldname2 desc"
+                 e.g.: "fieldname1, fieldname2 desc"
     :type sort: string
 
-    :returns: a dictionary containing the search parameters and the
-              search results.
-              keys: fields: same as datastore_create accepts
-                    offset: query offset value
-                    limit: query limit value
-                    filters: query filters
-                    total: number of total matching records
-                    records: list of matching results
-    :rtype: dictionary
+    **Results:**
+
+    The result of this action is a dict with the following keys:
+
+    :rtype: A dictionary with the following keys
+    :param fields: fields/columns and their extra metadata
+    :type fields: list of dictionaries
+    :param offset: query offset value
+    :type offset: int
+    :param limit: query limit value
+    :type limit: int
+    :param filters: query filters
+    :type filters: list of dictionaries
+    :param total: number of total matching records
+    :type total: int
+    :param records: list of matching results
+    :type records: list of dictionaries
 
     '''
     res_id = _get_or_bust(data_dict, 'resource_id')
@@ -190,15 +219,27 @@ def datastore_search(context, data_dict):
 
 @logic.side_effect_free
 def datastore_search_sql(context, data_dict):
-    '''Execute SQL-Queries on the datastore.
+    '''Execute SQL queries on the datastore.
+
+    The datastore_search_sql action allows a user to search data in a resource
+    or connect multiple resources with join expressions. The underlying SQL
+    engine is the
+    `PostgreSQL engine <http://www.postgresql.org/docs/9.1/interactive/sql/.html>`_
+
+    .. note:: This action is only available when using PostgreSQL 9.X and using a read-only user on the database.
 
     :param sql: a single sql select statement
     :type sql: string
 
-    :returns: a dictionary containing the search results.
-              keys: fields: columns for results
-                    records: results from the query
-    :rtype: dictionary
+    **Results:**
+
+    The result of this action is a dict with the following keys:
+
+    :rtype: A dictionary with the following keys
+    :param fields: fields/columns and their extra metadata
+    :type fields: list of dictionaries
+    :param records: list of matching results
+    :type records: list of dictionaries
 
     '''
     sql = _get_or_bust(data_dict, 'sql')
diff --git a/doc/apiv3.rst b/doc/apiv3.rst
index f9b5a580952..59c140fc1f3 100644
--- a/doc/apiv3.rst
+++ b/doc/apiv3.rst
@@ -18,6 +18,8 @@ If you don't specify the version number then you will default to version 1 of th
  * ``http://ckan.net/api/util`` (version 1)
  * ``http://ckan.net/api/action`` (version 3)
 
+.. _action-api:
+
 Action API
 ~~~~~~~~~~
 
diff --git a/doc/datastore.rst b/doc/datastore.rst
index 4c6dee7416f..bee115b0dd1 100644
--- a/doc/datastore.rst
+++ b/doc/datastore.rst
@@ -6,7 +6,8 @@ The CKAN DataStore provides a database for structured storage of data together
 with a powerful Web-accessible Data API, all seamlessly integrated into the CKAN
 interface and authorization system.
 
-.. note:: The DataStore requires PostgreSQL 9.0 or later. It is possible to use the DataStore on versions prior to 9.0 (for example 8.4). However, the :ref:`datastore_search_sql` will not be available and the set-up is slightly different. Make sure, you read :ref:`old_pg` for more details.
+.. note:: The DataStore requires PostgreSQL 9.0 or later. It is possible to use the DataStore on versions prior to 9.0 (for example 8.4). However, the :meth:`~ckanext.datastore.logic.action.datastore_search_sql` will not be available and the set-up is slightly different. Make sure, you read :ref:`old_pg` for more details.
+
 
 .. warning:: The DataStore does not support hiding resources in a private dataset.
 
@@ -82,7 +83,7 @@ Once the DataStore database and the users are created, the permissions on the Da
 
 1. Use the **paster command** if CKAN and PostgreSQL are on the same server
 
-To set the permissions, use this paster command after you've set the database urls::
+To set the permissions, use this paster command after you've set the database urls (make sure to have your virtualenv activated)::
 
  paster datastore set-permissions SQL_SUPER_USER
 
@@ -128,7 +129,7 @@ the records inserted above::
 Legacy mode: use the DataStore with old PostgreSQL versions
 -----------------------------------------------------------
 
-The DataStore can be used with a PostgreSQL version prior to 9.0 in *legacy mode*. Due to the lack of some functionality, the :ref:`datastore_search_sql` and consequently the :ref:`datastore_search_htsql` cannot be used. The set-up for legacy mode is analogous to the normal set-up as described in :ref:`installation` with a few changes and consists of the following steps:
+The DataStore can be used with a PostgreSQL version prior to 9.0 in *legacy mode*. Due to the lack of some functionality, the :meth:`~ckanext.datastore.logic.action.datastore_search_sql` and consequently the :ref:`datastore_search_htsql` cannot be used. The set-up for legacy mode is analogous to the normal set-up as described in :ref:`installation` with a few changes and consists of the following steps:
 
 1. Enable the extension
 #. Set-Up the database
@@ -173,7 +174,7 @@ The DataStore Data API
 ======================
 
 The DataStore's Data API, which derives from the underlying data table,
-is RESTful and JSON-based with extensive query capabilities.
+is JSON-based with extensive query capabilities.
 
 Each resource in a CKAN instance can have an associated DataStore 'table'. The
 basic API for accessing the DataStore is outlined below. For a detailed
@@ -183,105 +184,15 @@ tutorial on using this API see :doc:`using-data-api`.
 API Reference
 -------------
 
-.. note:: Lists can always be expressed in different ways. It is possible to use lists, comma separated strings or single items. These are valid lists: ``['foo', 'bar']``, ``'foo, bar'``, ``"foo", "bar"`` and ``'foo'``.
-
-
-datastore_create
-~~~~~~~~~~~~~~~~
-
-The datastore_create API endpoint allows a user to post JSON data to be stored against a resource. This endpoint also supports altering tables, aliases and indexes and bulk insertion. The JSON must be in the following form::
-
- {
-    resource_id:  # the data is going to be stored against.
-    aliases:      # list of names for read-only aliases to the resource
-    fields:       # a list of dictionaries of fields/columns and their extra metadata.
-    records:      # a list of dictionaries of the data, eg:  [{"dob": "2005", "some_stuff": ['a', 'b']}, ..]
-    primary_key:  # list of fields that represent a unique key
-    indexes:      # indexes on table
- }
-
-See :ref:`fields` and :ref:`records` for details on how to lay out records.
-
-
-
-datastore_delete
-~~~~~~~~~~~~~~~~
-
-The datastore_delete API endpoint allows a user to delete records from a resource. The JSON for searching must be in the following form::
-
- {
-    resource_id: # the data that is going to be deleted.
-    filter:      # dictionary of matching conditions to delete
-                 # e.g  {'key1': 'a', 'key2': 'b'}
-                 # this will be equivalent to "delete from table where key1 = 'a' and key2 = 'b' "
- }
-
-
-datastore_upsert
-~~~~~~~~~~~~~~~~
-
-The datastore_upsert API endpoint allows a user to add or edit records in an existing DataStore resource. In order for the ``upsert`` and ``update`` methods to work, a unique key has to defined via the datastore_create API endpoint command.
-The JSON for searching must be in the following form::
-
- {
-    resource_id: # resource id that the data is going to be stored under.
-    records:     #  a list of dictionaries of the data, eg:  [{"dob": "2005", "some_stuff": ['a', 'b']}, ..]
-    method:      #  the method to use to put the data into the datastore
-                 #  possible options: upsert (default), insert, update
- }
-
-``upsert``
-    Update if record with same key already exists, otherwise insert. Requires unique key.
-``insert``
-    Insert only. This method is faster that upsert, but will fail if any inserted record matches an existing one. Does *not* require a unique key.
-``update``
-    Update only. An exception will occur if the key that should be updated does not exist. Requires unique key.
-
-.. _datastore_search:
-
-datastore_search
-~~~~~~~~~~~~~~~~
+The datastore related API actions are accessed via CKAN's :ref:`action-api`. When POSTing
+requests, parameters should be provided as JSON objects.
 
-The datastore_search API endpoint allows a user to search data in a resource.
-The JSON for searching must be in the following form::
-
- {
-     resource_id:  # the resource id to be searched against
-     filters :     # dictionary of matching conditions to select e.g  {'key1': 'a. 'key2': 'b'}
-                   # this will be equivalent to "select * from table where key1 = 'a' and key2 = 'b' "
-     q:            # full text query
-     plain:        # treat as plain text query (default: true)
-     language:     # language of the full text query (default: english)
-     limit:        # limit the amount of rows to size (default: 100)
-     offset:       # offset the amount of rows
-     fields:       # list of fields return in that order, defaults (empty or not present) to all fields in fields order.
-     sort:         # ordered list of field names as, eg: "fieldname1, fieldname2 desc"
- }
-
-.. _datastore_search_sql:
-
-datastore_search_sql
-~~~~~~~~~~~~~~~~~~~~
-
-The datastore_search_sql API endpoint allows a user to search data in a resource or connect multiple resources with join expressions. The underlying SQL engine is the `PostgreSQL engine <http://www.postgresql.org/docs/9.1/interactive/sql/.html>`_. The JSON for searching must be in the following form::
-
- {
-    sql:  # a single sql select statement
- }
-
-
-.. _datastore_search_htsql:
-
-datastore_search_htsql
-~~~~~~~~~~~~~~~~~~~~~~
+.. note:: Lists can always be expressed in different ways. It is possible to use lists, comma separated strings or single items. These are valid lists: ``['foo', 'bar']``, ``'foo, bar'``, ``"foo", "bar"`` and ``'foo'``.
 
-.. note:: HTSQL is not in the core DataStore. To use it, it is necessary to install the ckanext-htsql extension available at https://github.com/okfn/ckanext-htsql.
+.. automodule:: ckanext.datastore.logic.action
+   :members:
 
-The datastore_search_htsql API endpoint allows a user to search data in a resource using the `HTSQL <http://htsql.org/doc/>`_ query expression language. The JSON for searching must be in the following form::
 
- {
-    htsql:  # a htsql query statement.
- }
 
 .. _fields:
 
@@ -375,19 +286,30 @@ Table aliases
 
 A resource in the DataStore can have multiple aliases that are easier to remember than the resource id. Aliases can be created and edited with the datastore_create API endpoint. All aliases can be found in a special view called ``_table_metadata``.
 
+
+.. _datastore_search_htsql:
+
+HTSQL Support
+=============
+
+
+The `ckanext-htsql <https://github.com/okfn/ckanext-htsql>`_ extension adds an API action that allows a user to search data in a resource using the `HTSQL <http://htsql.org/doc/>`_ query expression language. Please refer to the extension documentation to know more.
+
+
+
 Comparison of different querying methods
-----------------------------------------
+========================================
 
 The DataStore supports querying with multiple API endpoints. They are similar but support different features. The following list gives an overview of the different methods.
 
-==============================  =======================  ===========================  =============================
-..                              :ref:`datastore_search`  :ref:`datastore_search_sql`  :ref:`datastore_search_htsql`
-..                                                       SQL                          HTSQL
-==============================  =======================  ===========================  =============================
-**Status**                      Stable                   Stable                       Available as extension
-**Ease of use**                 Easy                     Complex                      Medium
-**Flexibility**                 Low                      High                         Medium
-**Query language**              Custom (JSON)            SQL                          HTSQL
-**Connect multiple resources**  No                       Yes                          Not yet
-**Use aliases**                 Yes                      Yes                          Yes
-==============================  =======================  ===========================  =============================
+==============================  ========================================================  ============================================================  =============================
+..                              :meth:`~ckanext.datastore.logic.action.datastore_search`  :meth:`~ckanext.datastore.logic.action.datastore_search_sql`  :ref:`datastore_search_htsql`
+..                                                                                        SQL                                                           HTSQL
+==============================  ========================================================  ============================================================  =============================
+**Status**                      Stable                                                    Stable                                                        Available as extension
+**Ease of use**                 Easy                                                      Complex                                                       Medium
+**Flexibility**                 Low                                                       High                                                          Medium
+**Query language**              Custom (JSON)                                             SQL                                                           HTSQL
+**Connect multiple resources**  No                                                        Yes                                                           Not yet
+**Use aliases**                 Yes                                                       Yes                                                           Yes
+==============================  ========================================================  ============================================================  =============================

From caa9c398d7145febba90b0e7c8328f420fef78c1 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Fri, 12 Oct 2012 12:10:41 +0100
Subject: [PATCH 38/53] Add missing reference

---
 doc/datastore.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/datastore.rst b/doc/datastore.rst
index bee115b0dd1..4f1998cdbbe 100644
--- a/doc/datastore.rst
+++ b/doc/datastore.rst
@@ -156,6 +156,8 @@ individual spreadsheet rows via a simple web API, as well as being able to make
 queries over the spreadsheet contents.
 
 
+.. _datastorer:
+
 DataStorer: Automatically Add Data to the DataStore
 ===================================================
 

From 8e87f15294cf3ac26cbf2f6beb70877d50814082 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Fri, 12 Oct 2012 13:47:18 +0200
Subject: [PATCH 39/53] Link how to use api docs to datastore api docs

---
 doc/using-data-api.rst | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/doc/using-data-api.rst b/doc/using-data-api.rst
index 3ab9e0c4748..47e393a6deb 100644
--- a/doc/using-data-api.rst
+++ b/doc/using-data-api.rst
@@ -16,12 +16,18 @@ Quickstart
 
 There are several endpoints into the DataStore API, they are:
 
-* datastore_create: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_create``
-* datastore_delete: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_delete``
-* datastore_delete: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_upsert``
-* datastore_search: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search``
-* datastore_search_sql: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search_sql``
-* datastore_search_htsql: ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search_htsql``
+:meth:`~ckanext.datastore.logic.action.datastore_create`
+  at ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_create``
+:meth:`~ckanext.datastore.logic.action.datastore_delete`
+  at ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_delete``
+:meth:`~ckanext.datastore.logic.action.datastore_upsert`
+  at ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_upsert``
+:meth:`~ckanext.datastore.logic.action.datastore_search`
+  at ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search``
+:meth:`~ckanext.datastore.logic.action.datastore_search_sql`
+  at ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search_sql``
+``datastore_search_htsql()``, see :ref:`datastore_search_htsql`
+  at ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search_htsql``
 
 Before going into detail about the API and the examples, it is useful to create a resource first so that you can store data against it in the Datastore. This can be done either through the CKAN Graphical User Interface or the API.
 

From 779337f1e364382b9902ae316f55f449b971c210 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Fri, 12 Oct 2012 14:59:48 +0200
Subject: [PATCH 40/53] moved file

---
 doc/{using-data-api.rst => datastore-api.rst} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename doc/{using-data-api.rst => datastore-api.rst} (100%)

diff --git a/doc/using-data-api.rst b/doc/datastore-api.rst
similarity index 100%
rename from doc/using-data-api.rst
rename to doc/datastore-api.rst

From ceffd4dfe1a659e77595eaa50b03fcaaec8603e6 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Fri, 12 Oct 2012 15:00:20 +0200
Subject: [PATCH 41/53] Reordered datastore docs to make more sense

---
 ckanext/datastore/logic/action.py |   1 +
 doc/datastore-api.rst             | 153 +++++++++++++++-
 doc/datastore.rst                 | 280 +-----------------------------
 doc/index.rst                     |   3 +-
 4 files changed, 152 insertions(+), 285 deletions(-)

diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py
index 3e004a67c6e..76809f762ed 100644
--- a/ckanext/datastore/logic/action.py
+++ b/ckanext/datastore/logic/action.py
@@ -227,6 +227,7 @@ def datastore_search_sql(context, data_dict):
     `PostgreSQL engine <http://www.postgresql.org/docs/9.1/interactive/sql/.html>`_
 
     .. note:: This action is only available when using PostgreSQL 9.X and using a read-only user on the database.
+        It is not available in :ref:`legacy mode<legacy_mode>`.
 
     :param sql: a single sql select statement
     :type sql: string
diff --git a/doc/datastore-api.rst b/doc/datastore-api.rst
index 47e393a6deb..a30c602975a 100644
--- a/doc/datastore-api.rst
+++ b/doc/datastore-api.rst
@@ -1,9 +1,12 @@
-==================
-Using the Data API
-==================
+======================
+The DataStore Data API
+======================
+
+The following provides an introduction to using the CKAN :doc:`DataStore<datastore>` Data API.
 
-The following provides an introduction to using the CKAN :doc:`DataStore
-<datastore>` Data API.
+The DataStore's Data API, which derives from the underlying data table, is JSON-based with extensive query capabilities.
+
+Each resource in a CKAN instance can have an associated DataStore 'table'. The basic API for accessing the DataStore is outlined below.
 
 Introduction
 ============
@@ -24,11 +27,149 @@ There are several endpoints into the DataStore API, they are:
   at ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_upsert``
 :meth:`~ckanext.datastore.logic.action.datastore_search`
   at ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search``
-:meth:`~ckanext.datastore.logic.action.datastore_search_sql`
+:meth:`~ckanext.datastore.logic.action.datastore_search_sql`, not available in :ref:`legacy_mode`
   at ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search_sql``
 ``datastore_search_htsql()``, see :ref:`datastore_search_htsql`
   at ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search_htsql``
 
+
+API Reference
+=============
+
+The datastore related API actions are accessed via CKAN's :ref:`action-api`. When POSTing
+requests, parameters should be provided as JSON objects.
+
+.. note:: Lists can always be expressed in different ways. It is possible to use lists, comma separated strings or single items. These are valid lists: ``['foo', 'bar']``, ``'foo, bar'``, ``"foo", "bar"`` and ``'foo'``.
+
+.. automodule:: ckanext.datastore.logic.action
+   :members:
+
+
+
+.. _fields:
+
+Fields
+------
+
+Fields define the column names and the type of the data in a column. A field is defined as follows::
+
+    {
+        "id":    # a string which defines the column name
+        "type":  # the data type for the column
+    }
+
+Field **types are optional** and will be guessed by the DataStore from the provided data. However, setting the types ensures that future inserts will not fail because of wrong types. See :ref:`valid-types` for details on which types are valid.
+
+Example::
+
+    [
+        {
+            "id": "foo",
+            "type": "int4"
+        },
+        {
+            "id": "bar"
+            # type is optional
+        }
+    ]
+
+.. _records:
+
+Records
+-------
+
+A record is the data to be inserted in a table and is defined as follows::
+
+    {
+        "<id>":  # data to be set
+        # .. more data
+    }
+
+Example::
+
+    [
+        {
+            "foo": 100,
+            "bar": "Here's some text"
+        },
+        {
+            "foo": 42
+        }
+    ]
+
+.. _valid-types:
+
+Field types
+-----------
+
+The DataStore supports all types supported by PostgreSQL as well as a few additions. A list of the PostgreSQL types can be found in the `type section of the documentation`_. Below you can find a list of the most common data types. The ``json`` type has been added as a storage for nested data.
+
+In addition to the listed types below, you can also use array types. They are defines by prepending a ``_`` or appending ``[]`` or ``[n]`` where n denotes the length of the array. An arbitrarily long array of integers would be defined as ``int[]``.
+
+.. _type section of the documentation: http://www.postgresql.org/docs/9.1/static/datatype.html
+
+
+text
+    Arbitrary text data, e.g. ``Here's some text``.
+json
+    Arbitrary nested json data, e.g ``{"foo": 42, "bar": [1, 2, 3]}``.
+    Please note that this type is a custom type that is wrapped by the DataStore.
+date
+    Date without time, e.g ``2012-5-25``.
+time
+    Time without date, e.g ``12:42``.
+timestamp
+    Date and time, e.g ``2012-10-01T02:43Z``.
+int
+    Integer numbers, e.g ``42``, ``7``.
+float
+    Floats, e.g. ``1.61803``.
+bool
+    Boolean values, e.g. ``true``, ``0``
+
+
+You can find more information about the formatting of dates in the `date/time types section of the PostgreSQL documentation`_.
+
+.. _date/time types section of the PostgreSQL documentation: http://www.postgresql.org/docs/9.1/static/datatype-datetime.html
+
+
+Table aliases
+-------------
+
+A resource in the DataStore can have multiple aliases that are easier to remember than the resource id. Aliases can be created and edited with the datastore_create API endpoint. All aliases can be found in a special view called ``_table_metadata``.
+
+
+.. _datastore_search_htsql:
+
+HTSQL Support
+-------------
+
+
+The `ckanext-htsql <https://github.com/okfn/ckanext-htsql>`_ extension adds an API action that allows a user to search data in a resource using the `HTSQL <http://htsql.org/doc/>`_ query expression language. Please refer to the extension documentation to know more.
+
+
+
+Comparison of different querying methods
+----------------------------------------
+
+The DataStore supports querying with multiple API endpoints. They are similar but support different features. The following list gives an overview of the different methods.
+
+==============================  ========================================================  ============================================================  =============================
+..                              :meth:`~ckanext.datastore.logic.action.datastore_search`  :meth:`~ckanext.datastore.logic.action.datastore_search_sql`  :ref:`datastore_search_htsql`
+..                                                                                        SQL                                                           HTSQL
+==============================  ========================================================  ============================================================  =============================
+**Status**                      Stable                                                    Stable                                                        Available as extension
+**Ease of use**                 Easy                                                      Complex                                                       Medium
+**Flexibility**                 Low                                                       High                                                          Medium
+**Query language**              Custom (JSON)                                             SQL                                                           HTSQL
+**Connect multiple resources**  No                                                        Yes                                                           Not yet
+**Use aliases**                 Yes                                                       Yes                                                           Yes
+==============================  ========================================================  ============================================================  =============================
+
+
+Using the Data API
+==================
+
 Before going into detail about the API and the examples, it is useful to create a resource first so that you can store data against it in the Datastore. This can be done either through the CKAN Graphical User Interface or the API.
 
 Using the data API, we need to send JSON with this structure::
diff --git a/doc/datastore.rst b/doc/datastore.rst
index 4f1998cdbbe..1a4484bcf31 100644
--- a/doc/datastore.rst
+++ b/doc/datastore.rst
@@ -6,140 +6,9 @@ The CKAN DataStore provides a database for structured storage of data together
 with a powerful Web-accessible Data API, all seamlessly integrated into the CKAN
 interface and authorization system.
 
-.. note:: The DataStore requires PostgreSQL 9.0 or later. It is possible to use the DataStore on versions prior to 9.0 (for example 8.4). However, the :meth:`~ckanext.datastore.logic.action.datastore_search_sql` will not be available and the set-up is slightly different. Make sure, you read :ref:`old_pg` for more details.
+The installation and set-up of the DataStore in outlined in :doc:`datastore-setup`.
 
-
-.. warning:: The DataStore does not support hiding resources in a private dataset.
-
-.. _installation:
-
-Installation and Configuration
-==============================
-
-.. warning:: Make sure that you follow the steps below and make sure that the settings are correct. Wrong settings could lead to serious security issues.
-
-.. tip:: If these steps seem to complicated, the legacy mode can be used which does not require you to set the permissions or create a separate user. However, the legacy mode is limited in its capabilities.
-
-
-The DataStore in previous lives required a custom set-up of ElasticSearch and Nginx,
-but that is no more, as it now uses the relational database management system PostgreSQL.
-However, you should set up a separate database for the DataStore
-and create a read-only user to make your CKAN and DataStore installation safe.
-
-1. Enable the extension
------------------------
-
-In your ``config`` file ensure that the ``datastore`` extension is enabled::
-
- ckan.plugins = datastore
-
-2. Set-up the database
-----------------------
-
-The DataStore requires a separate postgres database to save the resources to.
-
-List existing databases::
-
- sudo -u postgres psql -l
-
-Check that the encoding of databases is ‘UTF8’, if not internationalisation may be a problem. Since changing the encoding of PostgreSQL may mean deleting existing databases, it is suggested that this is fixed before continuing with the CKAN install.
-
-Next you will need to create a two database users for the DataStore. One user will be the *write* user that can create, edit and delete resources. The second user will be a *read-only* user who can only read resources.
-
-A few things have to be kept in mind:
-
-* The DataStore cannot be on the CKAN database (except for testing)
-* The write user (i.e. ``writeuser``) and read-only user (i.e. ``readonlyuser``) cannot be the same
-
-Create users and databases
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. tip:: The write user does not have to be created since you can also use the CKAN user. However, this might not be possible if the CKAN database and the DataStore database are on different servers. We recommend that you use the same user for CKAN and the write DataStore user if possible.
-
-Create a write user called ``writeuser``, and enter pass for the password when prompted::
-
- sudo -u postgres createuser -S -D -R -P -l writeuser
-
-Create a read-only user called ``readonlyuser``, and enter pass for the password when prompted::
-
- sudo -u postgres createuser -S -D -R -P -l readonlyuser
-
-Create the database (owned by ``writeuser``), which we’ll call ``datastore``::
-
- sudo -u postgres createdb -O writeuser datastore
-
-Set URLs
-~~~~~~~~
-
-Now, ensure that the ``ckan.datastore.write_url`` and ``datastore.read_url`` variables are set::
-
- ckan.datastore.write_url = postgresql://writeuser:pass@localhost/datastore
- ckan.datastore.read_url = postgresql://readonlyuser:pass@localhost/datastore
-
-Set permissions
-~~~~~~~~~~~~~~~
-
-Once the DataStore database and the users are created, the permissions on the DataStore and CKAN database have to be set. Since there are different set-ups, there are different ways of setting the permissions. Only one of the options should be used.
-
-1. Use the **paster command** if CKAN and PostgreSQL are on the same server
-
-To set the permissions, use this paster command after you've set the database urls (make sure to have your virtualenv activated)::
-
- paster datastore set-permissions SQL_SUPER_USER
-
-
-2. Use the **command line tool** in ``datastore/bin/datastore_setup.py``
-
-.. note:: This option should be used if the CKAN server is different from the database server.
-
-Copy the content from the ``datastore/bin/`` directory to the database server. Then run the command line tool to set the permissions on the database. To see all available options, run::
-
- python datastore_setup.py -h
-
-Once you are confident that you know the right names, set the permissions (assuming that the CKAN database is called ``ckan`` and the CKAN PostgreSQL user is called ``ckanuser``)::
-
- python datastore_setup.py ckan datastore ckanuser writeuser readonlyuser -p postgres
-
-
-3. Run the **SQL commands** manually on the database
-
-.. note:: This option is for more complex set-ups and requires understanding of SQL and PostgreSQL.
-
-Copy the ``set_permissions.sql`` file to the server that the database runs on. Make sure you set all variables in the file correctly and comment out the parts that are not needed for you set-up. Then, run the script::
-
- sudo -u postgres psql postgres -f set_permissions.sql
-
-
-3. Test the set-up
-------------------
-
-The datastore is now set-up. To test the set-up you can create a new DataStore. To do so you can run the following command::
-
- curl -X POST http://127.0.0.1:5000/api/3/action/datastore_create -H "Authorization: {YOUR-API-KEY}" -d '{"resource_id": "{RESOURCE-ID}", "fields": [ {"id": "a"}, {"id": "b"} ], "records": [ { "a": 1, "b": "xyz"}, {"a": 2, "b": "zzz"} ]}'
-
-A table named after the resource id should have been created on your DataStore
-database. Visiting the following URL should return a response from the DataStore with
-the records inserted above::
-
- http://127.0.0.1:5000/api/3/action/datastore_search?resource_id={RESOURCE_ID}
-
-
-.. _old_pg:
-
-Legacy mode: use the DataStore with old PostgreSQL versions
------------------------------------------------------------
-
-The DataStore can be used with a PostgreSQL version prior to 9.0 in *legacy mode*. Due to the lack of some functionality, the :meth:`~ckanext.datastore.logic.action.datastore_search_sql` and consequently the :ref:`datastore_search_htsql` cannot be used. The set-up for legacy mode is analogous to the normal set-up as described in :ref:`installation` with a few changes and consists of the following steps:
-
-1. Enable the extension
-#. Set-Up the database
-
-    a) Create a separate database
-    #) Create a write user on the DataStore database (optional since the CKAN user can be used)
-
-#. Test the set-up
-
-There is no need for a read-only user or special permissions. Therefore the legacy mode can be used for simple set-ups as well.
+If you want to use the API and you are looking for the API documentation, go to :doc:`datastore-api`.
 
 Relationship to FileStore
 =========================
@@ -170,148 +39,3 @@ This task of automatically parsing and then adding data to the DataStore is
 performed by a DataStorer, a queue process that runs asynchronously and can be
 triggered by uploads or other activities. The DataStorer is an extension and can
 be found, along with installation instructions, at: https://github.com/okfn/ckanext-datastorer
-
-
-The DataStore Data API
-======================
-
-The DataStore's Data API, which derives from the underlying data table,
-is JSON-based with extensive query capabilities.
-
-Each resource in a CKAN instance can have an associated DataStore 'table'. The
-basic API for accessing the DataStore is outlined below. For a detailed
-tutorial on using this API see :doc:`using-data-api`.
-
-
-API Reference
--------------
-
-The datastore related API actions are accessed via CKAN's :ref:`action-api`. When POSTing
-requests, parameters should be provided as JSON objects.
-
-.. note:: Lists can always be expressed in different ways. It is possible to use lists, comma separated strings or single items. These are valid lists: ``['foo', 'bar']``, ``'foo, bar'``, ``"foo", "bar"`` and ``'foo'``.
-
-.. automodule:: ckanext.datastore.logic.action
-   :members:
-
-
-
-.. _fields:
-
-Fields
-~~~~~~
-
-Fields define the column names and the type of the data in a column. A field is defined as follows::
-
-    {
-        "id":    # a string which defines the column name
-        "type":  # the data type for the column
-    }
-
-Field **types are optional** and will be guessed by the DataStore from the provided data. However, setting the types ensures that future inserts will not fail because of wrong types. See :ref:`valid-types` for details on which types are valid.
-
-Example::
-
-    [
-        {
-            "id": "foo",
-            "type": "int4"
-        },
-        {
-            "id": "bar"
-            # type is optional
-        }
-    ]
-
-.. _records:
-
-Records
-~~~~~~~
-
-A record is the data to be inserted in a table and is defined as follows::
-
-    {
-        "<id>":  # data to be set
-        # .. more data
-    }
-
-Example::
-
-    [
-        {
-            "foo": 100,
-            "bar": "Here's some text"
-        },
-        {
-            "foo": 42
-        }
-    ]
-
-.. _valid-types:
-
-Field types
------------
-
-The DataStore supports all types supported by PostgreSQL as well as a few additions. A list of the PostgreSQL types can be found in the `type section of the documentation`_. Below you can find a list of the most common data types. The ``json`` type has been added as a storage for nested data.
-
-In addition to the listed types below, you can also use array types. They are defines by prepending a ``_`` or appending ``[]`` or ``[n]`` where n denotes the length of the array. An arbitrarily long array of integers would be defined as ``int[]``.
-
-.. _type section of the documentation: http://www.postgresql.org/docs/9.1/static/datatype.html
-
-
-text
-    Arbitrary text data, e.g. ``Here's some text``.
-json
-    Arbitrary nested json data, e.g ``{"foo": 42, "bar": [1, 2, 3]}``.
-    Please note that this type is a custom type that is wrapped by the DataStore.
-date
-    Date without time, e.g ``2012-5-25``.
-time
-    Time without date, e.g ``12:42``.
-timestamp
-    Date and time, e.g ``2012-10-01T02:43Z``.
-int
-    Integer numbers, e.g ``42``, ``7``.
-float
-    Floats, e.g. ``1.61803``.
-bool
-    Boolean values, e.g. ``true``, ``0``
-
-
-You can find more information about the formatting of dates in the `date/time types section of the PostgreSQL documentation`_.
-
-.. _date/time types section of the PostgreSQL documentation: http://www.postgresql.org/docs/9.1/static/datatype-datetime.html
-
-
-Table aliases
--------------
-
-A resource in the DataStore can have multiple aliases that are easier to remember than the resource id. Aliases can be created and edited with the datastore_create API endpoint. All aliases can be found in a special view called ``_table_metadata``.
-
-
-.. _datastore_search_htsql:
-
-HTSQL Support
-=============
-
-
-The `ckanext-htsql <https://github.com/okfn/ckanext-htsql>`_ extension adds an API action that allows a user to search data in a resource using the `HTSQL <http://htsql.org/doc/>`_ query expression language. Please refer to the extension documentation to know more.
-
-
-
-Comparison of different querying methods
-========================================
-
-The DataStore supports querying with multiple API endpoints. They are similar but support different features. The following list gives an overview of the different methods.
-
-==============================  ========================================================  ============================================================  =============================
-..                              :meth:`~ckanext.datastore.logic.action.datastore_search`  :meth:`~ckanext.datastore.logic.action.datastore_search_sql`  :ref:`datastore_search_htsql`
-..                                                                                        SQL                                                           HTSQL
-==============================  ========================================================  ============================================================  =============================
-**Status**                      Stable                                                    Stable                                                        Available as extension
-**Ease of use**                 Easy                                                      Complex                                                       Medium
-**Flexibility**                 Low                                                       High                                                          Medium
-**Query language**              Custom (JSON)                                             SQL                                                           HTSQL
-**Connect multiple resources**  No                                                        Yes                                                           Not yet
-**Use aliases**                 Yes                                                       Yes                                                           Yes
-==============================  ========================================================  ============================================================  =============================
diff --git a/doc/index.rst b/doc/index.rst
index 3b7dce35d5f..4acc89dc4e7 100644
--- a/doc/index.rst
+++ b/doc/index.rst
@@ -25,6 +25,7 @@ Installation
    post-installation
    deployment
    solr-setup
+   datastore-setup
 
 Customizing and Extending
 =========================
@@ -69,7 +70,7 @@ The CKAN API
 
    api
    api-tutorial
-   using-data-api
+   datastore-api
 
 General Administration
 ======================

From dbebfd25d40601a6923a3d08e877b6161a3ca91a Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Fri, 12 Oct 2012 15:07:13 +0200
Subject: [PATCH 42/53] Improve references

---
 doc/datastore-api.rst | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/doc/datastore-api.rst b/doc/datastore-api.rst
index a30c602975a..9b723c5170d 100644
--- a/doc/datastore-api.rst
+++ b/doc/datastore-api.rst
@@ -27,7 +27,7 @@ There are several endpoints into the DataStore API, they are:
   at ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_upsert``
 :meth:`~ckanext.datastore.logic.action.datastore_search`
   at ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search``
-:meth:`~ckanext.datastore.logic.action.datastore_search_sql`, not available in :ref:`legacy_mode`
+:meth:`~ckanext.datastore.logic.action.datastore_search_sql`, not available in :ref:`legacy mode<legacy_mode>`
   at ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search_sql``
 ``datastore_search_htsql()``, see :ref:`datastore_search_htsql`
   at ``http://{YOUR-CKAN-INSTALLATION}/api/3/action/datastore_search_htsql``
@@ -155,8 +155,7 @@ Comparison of different querying methods
 The DataStore supports querying with multiple API endpoints. They are similar but support different features. The following list gives an overview of the different methods.
 
 ==============================  ========================================================  ============================================================  =============================
-..                              :meth:`~ckanext.datastore.logic.action.datastore_search`  :meth:`~ckanext.datastore.logic.action.datastore_search_sql`  :ref:`datastore_search_htsql`
-..                                                                                        SQL                                                           HTSQL
+..                              :meth:`~ckanext.datastore.logic.action.datastore_search`  :meth:`~ckanext.datastore.logic.action.datastore_search_sql`  :ref:`HTSQL<datastore_search_htsql>`
 ==============================  ========================================================  ============================================================  =============================
 **Status**                      Stable                                                    Stable                                                        Available as extension
 **Ease of use**                 Easy                                                      Complex                                                       Medium

From 119236a5a8a927743880ea8a89c0363e3bdcf14f Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Fri, 12 Oct 2012 15:13:59 +0200
Subject: [PATCH 43/53] Added missing setup docs

---
 doc/datastore-setup.rst | 146 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 146 insertions(+)
 create mode 100644 doc/datastore-setup.rst

diff --git a/doc/datastore-setup.rst b/doc/datastore-setup.rst
new file mode 100644
index 00000000000..a4c2ee09d2a
--- /dev/null
+++ b/doc/datastore-setup.rst
@@ -0,0 +1,146 @@
+========================
+Setting up the DataStore
+========================
+
+.. warning:: Make sure that you follow the steps below and make sure that the settings are correct. Wrong settings could lead to serious security issues.
+
+.. note:: The DataStore requires PostgreSQL 9.0 or later. It is possible to use the DataStore on versions prior to 9.0 (for example 8.4). However, the :meth:`~ckanext.datastore.logic.action.datastore_search_sql` will not be available and the set-up is slightly different. Make sure, you read :ref:`legacy_mode` for more details.
+
+.. warning:: The DataStore does not support hiding resources in a private dataset.
+
+
+The :doc:`DataStore<datastore>` in previous lives required a custom set-up of ElasticSearch and Nginx,
+but that is no more, as it now uses the relational database management system PostgreSQL.
+However, you should set up a separate database for the DataStore
+and create a read-only user to make your CKAN and DataStore installation safe.
+
+1. Enable the extension
+=======================
+
+In your ``config`` file ensure that the ``datastore`` extension is enabled::
+
+ ckan.plugins = datastore
+
+2. Set-up the database
+======================
+
+The DataStore requires a separate postgres database to save the resources to.
+
+List existing databases::
+
+ sudo -u postgres psql -l
+
+Check that the encoding of databases is ‘UTF8’, if not internationalisation may be a problem. Since changing the encoding of PostgreSQL may mean deleting existing databases, it is suggested that this is fixed before continuing with the CKAN install.
+
+Next you will need to create a two database users for the DataStore. One user will be the *write* user that can create, edit and delete resources. The second user will be a *read-only* user who can only read resources.
+
+A few things have to be kept in mind:
+
+* The DataStore cannot be on the CKAN database (except for testing)
+* The write user (i.e. ``writeuser``) and read-only user (i.e. ``readonlyuser``) cannot be the same
+
+Create users and databases
+--------------------------
+
+.. tip:: The write user does not have to be created since you can also use the CKAN user. However, this might not be possible if the CKAN database and the DataStore database are on different servers. We recommend that you use the same user for CKAN and the write DataStore user if possible.
+
+Create a write user called ``writeuser``, and enter pass for the password when prompted::
+
+ sudo -u postgres createuser -S -D -R -P -l writeuser
+
+Create a read-only user called ``readonlyuser``, and enter pass for the password when prompted::
+
+ sudo -u postgres createuser -S -D -R -P -l readonlyuser
+
+Create the database (owned by ``writeuser``), which we’ll call ``datastore``::
+
+ sudo -u postgres createdb -O writeuser datastore
+
+Set URLs
+--------
+
+Now, ensure that the ``ckan.datastore.write_url`` and ``ckan.datastore.read_url`` variables are set::
+
+ ckan.datastore.write_url = postgresql://writeuser:pass@localhost/datastore
+ ckan.datastore.read_url = postgresql://readonlyuser:pass@localhost/datastore
+
+These URLs define how the DataStore connects to the PostgreSQL database. Having a read-only connections makes it possible to use the powerful PostgreSQL database directly.
+
+Set permissions
+---------------
+
+.. tip:: See :ref:`legacy_mode` if these steps continue to fail or seem to complicated for your set-up. However, keep in mind that the legacy mode is limited in its capabilities.
+
+Once the DataStore database and the users are created, the permissions on the DataStore and CKAN database have to be set. Since there are different set-ups, there are different ways of setting the permissions. Only **one** of the options should be used.
+
+Option 1: Paster command
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This option is preferred if CKAN and PostgreSQL are on the same server.
+
+To set the permissions, use this paster command after you've set the database urls (make sure to have your virtualenv activated)::
+
+ paster datastore set-permissions SQL_SUPER_USER
+
+
+Option 2: Command line tool
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This option should be used if the CKAN server is different from the database server.
+
+Copy the content from the ``datastore/bin/`` directory to the database server. Then run the command line tool ``datastore_setup.py`` to set the permissions on the database. To see all available options, run::
+
+ python datastore_setup.py -h
+
+Once you are confident that you know the right names, set the permissions (assuming that the CKAN database is called ``ckan`` and the CKAN PostgreSQL user is called ``ckanuser``)::
+
+ python datastore_setup.py ckan datastore ckanuser writeuser readonlyuser -p postgres
+
+
+Option 3: SQL script
+~~~~~~~~~~~~~~~~~~~~
+
+This option is for more complex set-ups and requires understanding of SQL and PostgreSQL.
+
+Copy the ``set_permissions.sql`` file to the server that the database runs on. Make sure you set all variables in the file correctly and comment out the parts that are not needed for you set-up. Then, run the script::
+
+ sudo -u postgres psql postgres -f set_permissions.sql
+
+
+3. Test the set-up
+==================
+
+The datastore is now set-up. To test the set-up you can create a new DataStore. To do so you can run the following command::
+
+ curl -X POST http://127.0.0.1:5000/api/3/action/datastore_create -H "Authorization: {YOUR-API-KEY}" -d '{"resource_id": "{RESOURCE-ID}", "fields": [ {"id": "a"}, {"id": "b"} ], "records": [ { "a": 1, "b": "xyz"}, {"a": 2, "b": "zzz"} ]}'
+
+A table named after the resource id should have been created on your DataStore
+database. Visiting the following URL should return a response from the DataStore with
+the records inserted above::
+
+ http://127.0.0.1:5000/api/3/action/datastore_search?resource_id={RESOURCE_ID}
+
+To find out more about the DataStore API, go to :doc:`datastore-api`.
+
+
+.. _legacy_mode:
+
+Legacy mode: use the DataStore with old PostgreSQL versions
+===========================================================
+
+.. tip:: The legacy mode can also be used to simplify the set-up since it does not require you to set the permissions or create a separate user.
+
+The DataStore can be used with a PostgreSQL version prior to 9.0 in *legacy mode*. Due to the lack of some functionality, the :meth:`~ckanext.datastore.logic.action.datastore_search_sql` and consequently the :ref:`datastore_search_htsql` cannot be used. To enable the legacy mode, remove the declaration of the ``ckan.datastore.read_url``.
+
+The set-up for legacy mode is analogous to the normal set-up as described in :ref:`installation` with a few changes and consists of the following steps:
+
+1. Enable the extension
+2. Do **not** set the ``ckan.datastore.read_url`` which enables legacy mode
+#. Set-Up the database
+
+    a) Create a separate database
+    #) Create a write user on the DataStore database (optional since the CKAN user can be used)
+
+#. Test the set-up
+
+There is no need for a read-only user or special permissions. Therefore the legacy mode can be used for simple set-ups as well.

From 3cafece798584e302b80fa808c1f24c90ae42fbc Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Fri, 12 Oct 2012 15:20:22 +0200
Subject: [PATCH 44/53] Minor doc and reference fixes and improvements

---
 doc/datastore-setup.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/doc/datastore-setup.rst b/doc/datastore-setup.rst
index a4c2ee09d2a..bd95a952b21 100644
--- a/doc/datastore-setup.rst
+++ b/doc/datastore-setup.rst
@@ -17,14 +17,14 @@ and create a read-only user to make your CKAN and DataStore installation safe.
 1. Enable the extension
 =======================
 
-In your ``config`` file ensure that the ``datastore`` extension is enabled::
+Since the DataStore is an optional extension, it has to be enabled separately. To do so, ensure that the ``datastore`` extension is enabled in your ``config`` file::
 
  ckan.plugins = datastore
 
 2. Set-up the database
 ======================
 
-The DataStore requires a separate postgres database to save the resources to.
+The DataStore requires a separate PostgreSQL database to save the resources to.
 
 List existing databases::
 
@@ -78,7 +78,7 @@ Option 1: Paster command
 
 This option is preferred if CKAN and PostgreSQL are on the same server.
 
-To set the permissions, use this paster command after you've set the database urls (make sure to have your virtualenv activated)::
+To set the permissions, use this paster command after you've set the database URLs (make sure to have your virtualenv activated)::
 
  paster datastore set-permissions SQL_SUPER_USER
 
@@ -132,10 +132,10 @@ Legacy mode: use the DataStore with old PostgreSQL versions
 
 The DataStore can be used with a PostgreSQL version prior to 9.0 in *legacy mode*. Due to the lack of some functionality, the :meth:`~ckanext.datastore.logic.action.datastore_search_sql` and consequently the :ref:`datastore_search_htsql` cannot be used. To enable the legacy mode, remove the declaration of the ``ckan.datastore.read_url``.
 
-The set-up for legacy mode is analogous to the normal set-up as described in :ref:`installation` with a few changes and consists of the following steps:
+The set-up for legacy mode is analogous to the normal set-up as described above with a few changes and consists of the following steps:
 
 1. Enable the extension
-2. Do **not** set the ``ckan.datastore.read_url`` which enables legacy mode
+2. The legacy mode is enabled by **not** setting the ``ckan.datastore.read_url``
 #. Set-Up the database
 
     a) Create a separate database

From dbfae05bec95e364d14d2d8ddae7b6ef15eb8385 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Fri, 12 Oct 2012 15:28:21 +0200
Subject: [PATCH 45/53] Coding style in sql

---
 ckanext/datastore/db.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py
index 24f0d74bb09..84ad63a8ec0 100644
--- a/ckanext/datastore/db.py
+++ b/ckanext/datastore/db.py
@@ -145,7 +145,7 @@ def _is_valid_pg_type(context, type_name):
     else:
         connection = context['connection']
         try:
-            connection.execute('select %s::regtype', type_name)
+            connection.execute('SELECT %s::regtype', type_name)
         except ProgrammingError, e:
             if 'invalid type name' in str(e) or 'does not exist' in str(e):
                 return False

From defd153ef8f43cee2180916e21db698e2e12cdb0 Mon Sep 17 00:00:00 2001
From: Dominik Moritz <domoritz@gmail.com>
Date: Fri, 12 Oct 2012 15:37:41 +0200
Subject: [PATCH 46/53] Changed heading because the datastore doc page does not
 contain the api docs any more

---
 doc/datastore.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/doc/datastore.rst b/doc/datastore.rst
index 1a4484bcf31..c8388edea3b 100644
--- a/doc/datastore.rst
+++ b/doc/datastore.rst
@@ -1,6 +1,6 @@
-==========================
-DataStore and the Data API
-==========================
+=========
+DataStore
+=========
 
 The CKAN DataStore provides a database for structured storage of data together
 with a powerful Web-accessible Data API, all seamlessly integrated into the CKAN

From eeb716cdff6322f0a0c02439b4756ffbd43a7ee2 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Fri, 12 Oct 2012 16:36:16 +0100
Subject: [PATCH 47/53] Add back warning about dangers of enabling debug on
 production

---
 ckan/config/deployment.ini_tmpl | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl
index 709c80fc7d6..bcb468f47c8 100644
--- a/ckan/config/deployment.ini_tmpl
+++ b/ckan/config/deployment.ini_tmpl
@@ -8,6 +8,10 @@
 # Change debug to true when doing CKAN development, it enables Pylons'
 # interactive debugging tool, makes Fanstatic serve unminified JS and CSS
 # files, and enables CKAN templates' debugging features.
+#   
+# WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PRODUCTION ENVIRONMENT*
+# Debug mode will enable the interactive debugging tool, allowing ANYONE to
+# execute malicious code after an exception is raised.
 debug = false
 
 email_to = you@yourdomain.com

From 84382d84350cc4e6b4aacbca702e5be317eb5d42 Mon Sep 17 00:00:00 2001
From: Sean Hammond <seanhammond@lavabit.com>
Date: Fri, 12 Oct 2012 18:36:56 +0200
Subject: [PATCH 48/53] [#2257] Remove leftover references to
 restrict_template_vars

These got in after merging master I think
---
 ckan/config/environment.py | 6 ++----
 ckan/lib/helpers.py        | 1 -
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/ckan/config/environment.py b/ckan/config/environment.py
index c8d6f3bb7b0..c6e13b62af7 100644
--- a/ckan/config/environment.py
+++ b/ckan/config/environment.py
@@ -79,10 +79,8 @@ def __getattr__(self, name):
             return self.functions[name]
         else:
             if name in self.deprecated:
-                msg = 'Template helper function `%s` is not available ' \
-                      'as it has been deprecated.\nYou can enable it ' \
-                      'by setting ckan.restrict_template_vars = true ' \
-                      'in your .ini file.' % name
+                msg = ('Template helper function `{0}` is not available '
+                       'because it has been deprecated.'.format(name))
                 self.log.critical(msg)
             else:
                 msg = 'Helper function `%s` could not be found\n ' \
diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py
index f8fc3cebfc4..a159840a3b4 100644
--- a/ckan/lib/helpers.py
+++ b/ckan/lib/helpers.py
@@ -1311,7 +1311,6 @@ def resource_preview(resource, pkg_id):
 
 
 # these are the functions that will end up in `h` template helpers
-# if config option restrict_template_vars is true
 __allowed_functions__ = [
     # functions defined in ckan.lib.helpers
            'redirect_to',

From 7c9dec2386971ebfb3f4ad34f7109c77d5792ba9 Mon Sep 17 00:00:00 2001
From: Sean Hammond <seanhammond@lavabit.com>
Date: Fri, 12 Oct 2012 18:43:24 +0200
Subject: [PATCH 49/53] [#2257] Remove restrict_template_vars from docs

---
 doc/coding-standards.rst | 12 +++++-------
 doc/configuration.rst    | 19 -------------------
 2 files changed, 5 insertions(+), 26 deletions(-)

diff --git a/doc/coding-standards.rst b/doc/coding-standards.rst
index db2e019e506..731c12cb760 100644
--- a/doc/coding-standards.rst
+++ b/doc/coding-standards.rst
@@ -569,12 +569,11 @@ Guidelines when writing controller actions:
 
     ckan.logic.action.get.group_activity_list_html(...)
 
-- Controllers have access to helper functions in ``ckan.lib.helpers``.  When
-  developing for ckan core, only use the helper functions found in
-  ``ckan.lib.helpers.__allowed_functions__`` because any instance may set the
-  ``ckan.restrict_template_vars`` configuration value to ``True``.
+- Controllers have access to helper functions in ``ckan.lib.helpers``.
+  When developing for ckan core, only use the helper functions found in
+  ``ckan.lib.helpers.__allowed_functions__``.
 
-.. todo:: Anything else for contrllers?
+.. todo:: Anything else for controllers?
 
 Templating
 ----------
@@ -584,8 +583,7 @@ Helper Functions
 
 Templates have access to a set of helper functions in ``ckan.lib.helpers``.
 When developing for ckan core, only use the helper functions found in
-``ckan.lib.helpers.__allowed_functions__`` because any instance may set the
-``ckan.restrict_template_vars`` configuration value to ``True``.
+``ckan.lib.helpers.__allowed_functions__``.
 
 .. todo:: Jinja2 templates
 
diff --git a/doc/configuration.rst b/doc/configuration.rst
index 1e855ab32ad..1cab244f685 100644
--- a/doc/configuration.rst
+++ b/doc/configuration.rst
@@ -681,22 +681,3 @@ Example::
   ckan.backup_dir = /var/backups/ckan/
 
 This is a directory where SQL database backups are to be written, assuming a script has been installed to do this.
-
-
-
-Compatibility
--------------
-
-.. index::
-   single: restrict_template_vars
-
-restrict_template_vars
-^^^^^^^^^^^^^^^^^^^^^^
-
-Example::
-
-  ckan.restrict_template_vars = true
-
-Default value:  ``false``
-
-This is used to limit the functions available via h in templates.  It also forces correct usage of functions as some function signatures have changed.  It's main purpose is to allow transition to a cleaner world.

From 1501aef89d0ee8ee5e702d40563aa794264373bf Mon Sep 17 00:00:00 2001
From: Sean Hammond <seanhammond@lavabit.com>
Date: Fri, 12 Oct 2012 18:48:20 +0200
Subject: [PATCH 50/53] [#2257] Update CHANGELOG

---
 CHANGELOG.txt | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/CHANGELOG.txt b/CHANGELOG.txt
index b8065484ebf..4983e011218 100644
--- a/CHANGELOG.txt
+++ b/CHANGELOG.txt
@@ -1,6 +1,16 @@
 CKAN CHANGELOG
 ++++++++++++++
 
+v2.0
+====
+
+* [#2257] Removed restrict_template_vars config setting.
+* [#2257] Removed deprecated facet_title() template helper function, use
+  get_facet_title() instead.
+* [#2257] Removed deprecated am_authorized() template helper function, use
+  check_access() instead.
+* [#2257] Removed deprecated datetime_to_datestr() template helper function.
+
 v1.8
 ====
 

From 9ae6d0f0a4f1805b527ae9523fead80fca45e63c Mon Sep 17 00:00:00 2001
From: Sean Hammond <seanhammond@lavabit.com>
Date: Mon, 15 Oct 2012 12:57:10 +0200
Subject: [PATCH 51/53] Small docs fix

---
 doc/extension-templating.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/doc/extension-templating.rst b/doc/extension-templating.rst
index 401d28d69fe..21265b95ed3 100644
--- a/doc/extension-templating.rst
+++ b/doc/extension-templating.rst
@@ -26,7 +26,9 @@ templating language:
     <p>This is my map snippet</p>
 
     {# my-snippet-scripts.html #}
-    <script src="{% url_for_static='/javascript/map.js' %}"></script>
+    <script type="text/javascript"
+            src="{{ h.url_for_static('/javascript/map.js') }}">
+    </script>
 
 Theme Extensions
 ----------------

From b625eb5dc0ac659eeea2e06657be6972269fc3de Mon Sep 17 00:00:00 2001
From: Sean Hammond <seanhammond@lavabit.com>
Date: Mon, 15 Oct 2012 19:19:17 +0200
Subject: [PATCH 52/53] Documented some undocumented related_list params

---
 ckan/logic/action/get.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py
index a7d6dd59e7d..637c76204ed 100644
--- a/ckan/logic/action/get.py
+++ b/ckan/logic/action/get.py
@@ -188,6 +188,16 @@ def related_list(context, data_dict=None):
     :type id: string
     :param dataset: dataset dictionary of the dataset (optional)
     :type dataset: dictionary
+    :param type_filter: the type of related item to show (optional,
+      default: None, show all items)
+    :type type_filter: string
+    :param sort: the order to sort the related items in, possible values are
+      'view_count_asc', 'view_count_desc', 'created_asc' or 'created_desc'
+      (optional)
+    :type sort: string
+    :param featured: whether or not to restrict the results to only featured
+      related items (optional, default: False)
+    :type featured: bool
 
     :rtype: list of dictionaries
 

From bab3906edd686fcad9e683347b82a83c1c74797b Mon Sep 17 00:00:00 2001
From: John Martin <me@johnmart.in>
Date: Tue, 16 Oct 2012 17:47:05 +0100
Subject: [PATCH 53/53] Fix for new dataset step 2 showing edit navigation menu
 on left

---
 ckan/templates/package/new_resource.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ckan/templates/package/new_resource.html b/ckan/templates/package/new_resource.html
index 969aa5d086f..bc833bba096 100644
--- a/ckan/templates/package/new_resource.html
+++ b/ckan/templates/package/new_resource.html
@@ -11,7 +11,7 @@
 {% block form %}{% snippet 'package/snippets/resource_form.html', data=data, errors=errors, error_summary=error_summary, include_metadata=false, pkg_name=pkg_name, stage=stage, allow_upload=logged_in %}{% endblock %}
 
 {% block secondary_content %}
-  {% if pkg_dict %}
+  {% if pkg_dict and pkg_dict.state != 'draft' %}
     {% snippet 'package/snippets/info.html', pkg=pkg_dict, action='resource_new' %}
   {% else %}  
     <section class="module module-narrow module-shallow">